1 /* 2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver. 3 * 4 * Copyright (c) 2010-2015 Chelsio Communications, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Written by: Karen Xie (kxie@chelsio.com) 11 * Written by: Rakesh Ranjan (rranjan@chelsio.com) 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 15 16 #include <linux/skbuff.h> 17 #include <linux/crypto.h> 18 #include <linux/scatterlist.h> 19 #include <linux/pci.h> 20 #include <scsi/scsi.h> 21 #include <scsi/scsi_cmnd.h> 22 #include <scsi/scsi_host.h> 23 #include <linux/if_vlan.h> 24 #include <linux/inet.h> 25 #include <net/dst.h> 26 #include <net/route.h> 27 #include <net/ipv6.h> 28 #include <net/ip6_route.h> 29 #include <net/addrconf.h> 30 31 #include <linux/inetdevice.h> /* ip_dev_find */ 32 #include <linux/module.h> 33 #include <net/tcp.h> 34 35 static unsigned int dbg_level; 36 37 #include "libcxgbi.h" 38 39 #define DRV_MODULE_NAME "libcxgbi" 40 #define DRV_MODULE_DESC "Chelsio iSCSI driver library" 41 #define DRV_MODULE_VERSION "0.9.1-ko" 42 #define DRV_MODULE_RELDATE "Apr. 2015" 43 44 static char version[] = 45 DRV_MODULE_DESC " " DRV_MODULE_NAME 46 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 47 48 MODULE_AUTHOR("Chelsio Communications, Inc."); 49 MODULE_DESCRIPTION(DRV_MODULE_DESC); 50 MODULE_VERSION(DRV_MODULE_VERSION); 51 MODULE_LICENSE("GPL"); 52 53 module_param(dbg_level, uint, 0644); 54 MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)"); 55 56 57 /* 58 * cxgbi device management 59 * maintains a list of the cxgbi devices 60 */ 61 static LIST_HEAD(cdev_list); 62 static DEFINE_MUTEX(cdev_mutex); 63 64 static LIST_HEAD(cdev_rcu_list); 65 static DEFINE_SPINLOCK(cdev_rcu_lock); 66 67 static inline void cxgbi_decode_sw_tag(u32 sw_tag, int *idx, int *age) 68 { 69 if (age) 70 *age = sw_tag & 0x7FFF; 71 if (idx) 72 *idx = (sw_tag >> 16) & 0x7FFF; 73 } 74 75 int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base, 76 unsigned int max_conn) 77 { 78 struct cxgbi_ports_map *pmap = &cdev->pmap; 79 80 pmap->port_csk = cxgbi_alloc_big_mem(max_conn * 81 sizeof(struct cxgbi_sock *), 82 GFP_KERNEL); 83 if (!pmap->port_csk) { 84 pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn); 85 return -ENOMEM; 86 } 87 88 pmap->max_connect = max_conn; 89 pmap->sport_base = base; 90 spin_lock_init(&pmap->lock); 91 return 0; 92 } 93 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create); 94 95 void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev) 96 { 97 struct cxgbi_ports_map *pmap = &cdev->pmap; 98 struct cxgbi_sock *csk; 99 int i; 100 101 for (i = 0; i < pmap->max_connect; i++) { 102 if (pmap->port_csk[i]) { 103 csk = pmap->port_csk[i]; 104 pmap->port_csk[i] = NULL; 105 log_debug(1 << CXGBI_DBG_SOCK, 106 "csk 0x%p, cdev 0x%p, offload down.\n", 107 csk, cdev); 108 spin_lock_bh(&csk->lock); 109 cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN); 110 cxgbi_sock_closed(csk); 111 spin_unlock_bh(&csk->lock); 112 cxgbi_sock_put(csk); 113 } 114 } 115 } 116 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup); 117 118 static inline void cxgbi_device_destroy(struct cxgbi_device *cdev) 119 { 120 log_debug(1 << CXGBI_DBG_DEV, 121 "cdev 0x%p, p# %u.\n", cdev, cdev->nports); 122 cxgbi_hbas_remove(cdev); 123 cxgbi_device_portmap_cleanup(cdev); 124 cxgbi_ppm_release(cdev->cdev2ppm(cdev)); 125 if (cdev->pmap.max_connect) 126 cxgbi_free_big_mem(cdev->pmap.port_csk); 127 kfree(cdev); 128 } 129 130 struct cxgbi_device *cxgbi_device_register(unsigned int extra, 131 unsigned int nports) 132 { 133 struct cxgbi_device *cdev; 134 135 cdev = kzalloc(sizeof(*cdev) + extra + nports * 136 (sizeof(struct cxgbi_hba *) + 137 sizeof(struct net_device *)), 138 GFP_KERNEL); 139 if (!cdev) { 140 pr_warn("nport %d, OOM.\n", nports); 141 return NULL; 142 } 143 cdev->ports = (struct net_device **)(cdev + 1); 144 cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports * 145 sizeof(struct net_device *)); 146 if (extra) 147 cdev->dd_data = ((char *)cdev->hbas) + 148 nports * sizeof(struct cxgbi_hba *); 149 spin_lock_init(&cdev->pmap.lock); 150 151 mutex_lock(&cdev_mutex); 152 list_add_tail(&cdev->list_head, &cdev_list); 153 mutex_unlock(&cdev_mutex); 154 155 spin_lock(&cdev_rcu_lock); 156 list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list); 157 spin_unlock(&cdev_rcu_lock); 158 159 log_debug(1 << CXGBI_DBG_DEV, 160 "cdev 0x%p, p# %u.\n", cdev, nports); 161 return cdev; 162 } 163 EXPORT_SYMBOL_GPL(cxgbi_device_register); 164 165 void cxgbi_device_unregister(struct cxgbi_device *cdev) 166 { 167 log_debug(1 << CXGBI_DBG_DEV, 168 "cdev 0x%p, p# %u,%s.\n", 169 cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : ""); 170 171 mutex_lock(&cdev_mutex); 172 list_del(&cdev->list_head); 173 mutex_unlock(&cdev_mutex); 174 175 spin_lock(&cdev_rcu_lock); 176 list_del_rcu(&cdev->rcu_node); 177 spin_unlock(&cdev_rcu_lock); 178 synchronize_rcu(); 179 180 cxgbi_device_destroy(cdev); 181 } 182 EXPORT_SYMBOL_GPL(cxgbi_device_unregister); 183 184 void cxgbi_device_unregister_all(unsigned int flag) 185 { 186 struct cxgbi_device *cdev, *tmp; 187 188 mutex_lock(&cdev_mutex); 189 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 190 if ((cdev->flags & flag) == flag) { 191 mutex_unlock(&cdev_mutex); 192 cxgbi_device_unregister(cdev); 193 mutex_lock(&cdev_mutex); 194 } 195 } 196 mutex_unlock(&cdev_mutex); 197 } 198 EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all); 199 200 struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev) 201 { 202 struct cxgbi_device *cdev, *tmp; 203 204 mutex_lock(&cdev_mutex); 205 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 206 if (cdev->lldev == lldev) { 207 mutex_unlock(&cdev_mutex); 208 return cdev; 209 } 210 } 211 mutex_unlock(&cdev_mutex); 212 213 log_debug(1 << CXGBI_DBG_DEV, 214 "lldev 0x%p, NO match found.\n", lldev); 215 return NULL; 216 } 217 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev); 218 219 struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev, 220 int *port) 221 { 222 struct net_device *vdev = NULL; 223 struct cxgbi_device *cdev, *tmp; 224 int i; 225 226 if (ndev->priv_flags & IFF_802_1Q_VLAN) { 227 vdev = ndev; 228 ndev = vlan_dev_real_dev(ndev); 229 log_debug(1 << CXGBI_DBG_DEV, 230 "vlan dev %s -> %s.\n", vdev->name, ndev->name); 231 } 232 233 mutex_lock(&cdev_mutex); 234 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 235 for (i = 0; i < cdev->nports; i++) { 236 if (ndev == cdev->ports[i]) { 237 cdev->hbas[i]->vdev = vdev; 238 mutex_unlock(&cdev_mutex); 239 if (port) 240 *port = i; 241 return cdev; 242 } 243 } 244 } 245 mutex_unlock(&cdev_mutex); 246 log_debug(1 << CXGBI_DBG_DEV, 247 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name); 248 return NULL; 249 } 250 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev); 251 252 struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev, 253 int *port) 254 { 255 struct net_device *vdev = NULL; 256 struct cxgbi_device *cdev; 257 int i; 258 259 if (ndev->priv_flags & IFF_802_1Q_VLAN) { 260 vdev = ndev; 261 ndev = vlan_dev_real_dev(ndev); 262 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name); 263 } 264 265 rcu_read_lock(); 266 list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) { 267 for (i = 0; i < cdev->nports; i++) { 268 if (ndev == cdev->ports[i]) { 269 cdev->hbas[i]->vdev = vdev; 270 rcu_read_unlock(); 271 if (port) 272 *port = i; 273 return cdev; 274 } 275 } 276 } 277 rcu_read_unlock(); 278 279 log_debug(1 << CXGBI_DBG_DEV, 280 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name); 281 return NULL; 282 } 283 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu); 284 285 #if IS_ENABLED(CONFIG_IPV6) 286 static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev, 287 int *port) 288 { 289 struct net_device *vdev = NULL; 290 struct cxgbi_device *cdev, *tmp; 291 int i; 292 293 if (ndev->priv_flags & IFF_802_1Q_VLAN) { 294 vdev = ndev; 295 ndev = vlan_dev_real_dev(ndev); 296 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name); 297 } 298 299 mutex_lock(&cdev_mutex); 300 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 301 for (i = 0; i < cdev->nports; i++) { 302 if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr, 303 MAX_ADDR_LEN)) { 304 cdev->hbas[i]->vdev = vdev; 305 mutex_unlock(&cdev_mutex); 306 if (port) 307 *port = i; 308 return cdev; 309 } 310 } 311 } 312 mutex_unlock(&cdev_mutex); 313 log_debug(1 << CXGBI_DBG_DEV, 314 "ndev 0x%p, %s, NO match mac found.\n", 315 ndev, ndev->name); 316 return NULL; 317 } 318 #endif 319 320 void cxgbi_hbas_remove(struct cxgbi_device *cdev) 321 { 322 int i; 323 struct cxgbi_hba *chba; 324 325 log_debug(1 << CXGBI_DBG_DEV, 326 "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 327 328 for (i = 0; i < cdev->nports; i++) { 329 chba = cdev->hbas[i]; 330 if (chba) { 331 cdev->hbas[i] = NULL; 332 iscsi_host_remove(chba->shost); 333 pci_dev_put(cdev->pdev); 334 iscsi_host_free(chba->shost); 335 } 336 } 337 } 338 EXPORT_SYMBOL_GPL(cxgbi_hbas_remove); 339 340 int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun, 341 unsigned int max_id, struct scsi_host_template *sht, 342 struct scsi_transport_template *stt) 343 { 344 struct cxgbi_hba *chba; 345 struct Scsi_Host *shost; 346 int i, err; 347 348 log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 349 350 for (i = 0; i < cdev->nports; i++) { 351 shost = iscsi_host_alloc(sht, sizeof(*chba), 1); 352 if (!shost) { 353 pr_info("0x%p, p%d, %s, host alloc failed.\n", 354 cdev, i, cdev->ports[i]->name); 355 err = -ENOMEM; 356 goto err_out; 357 } 358 359 shost->transportt = stt; 360 shost->max_lun = max_lun; 361 shost->max_id = max_id; 362 shost->max_channel = 0; 363 shost->max_cmd_len = 16; 364 365 chba = iscsi_host_priv(shost); 366 chba->cdev = cdev; 367 chba->ndev = cdev->ports[i]; 368 chba->shost = shost; 369 370 log_debug(1 << CXGBI_DBG_DEV, 371 "cdev 0x%p, p#%d %s: chba 0x%p.\n", 372 cdev, i, cdev->ports[i]->name, chba); 373 374 pci_dev_get(cdev->pdev); 375 err = iscsi_host_add(shost, &cdev->pdev->dev); 376 if (err) { 377 pr_info("cdev 0x%p, p#%d %s, host add failed.\n", 378 cdev, i, cdev->ports[i]->name); 379 pci_dev_put(cdev->pdev); 380 scsi_host_put(shost); 381 goto err_out; 382 } 383 384 cdev->hbas[i] = chba; 385 } 386 387 return 0; 388 389 err_out: 390 cxgbi_hbas_remove(cdev); 391 return err; 392 } 393 EXPORT_SYMBOL_GPL(cxgbi_hbas_add); 394 395 /* 396 * iSCSI offload 397 * 398 * - source port management 399 * To find a free source port in the port allocation map we use a very simple 400 * rotor scheme to look for the next free port. 401 * 402 * If a source port has been specified make sure that it doesn't collide with 403 * our normal source port allocation map. If it's outside the range of our 404 * allocation/deallocation scheme just let them use it. 405 * 406 * If the source port is outside our allocation range, the caller is 407 * responsible for keeping track of their port usage. 408 */ 409 410 static struct cxgbi_sock *find_sock_on_port(struct cxgbi_device *cdev, 411 unsigned char port_id) 412 { 413 struct cxgbi_ports_map *pmap = &cdev->pmap; 414 unsigned int i; 415 unsigned int used; 416 417 if (!pmap->max_connect || !pmap->used) 418 return NULL; 419 420 spin_lock_bh(&pmap->lock); 421 used = pmap->used; 422 for (i = 0; used && i < pmap->max_connect; i++) { 423 struct cxgbi_sock *csk = pmap->port_csk[i]; 424 425 if (csk) { 426 if (csk->port_id == port_id) { 427 spin_unlock_bh(&pmap->lock); 428 return csk; 429 } 430 used--; 431 } 432 } 433 spin_unlock_bh(&pmap->lock); 434 435 return NULL; 436 } 437 438 static int sock_get_port(struct cxgbi_sock *csk) 439 { 440 struct cxgbi_device *cdev = csk->cdev; 441 struct cxgbi_ports_map *pmap = &cdev->pmap; 442 unsigned int start; 443 int idx; 444 __be16 *port; 445 446 if (!pmap->max_connect) { 447 pr_err("cdev 0x%p, p#%u %s, NO port map.\n", 448 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 449 return -EADDRNOTAVAIL; 450 } 451 452 if (csk->csk_family == AF_INET) 453 port = &csk->saddr.sin_port; 454 else /* ipv6 */ 455 port = &csk->saddr6.sin6_port; 456 457 if (*port) { 458 pr_err("source port NON-ZERO %u.\n", 459 ntohs(*port)); 460 return -EADDRINUSE; 461 } 462 463 spin_lock_bh(&pmap->lock); 464 if (pmap->used >= pmap->max_connect) { 465 spin_unlock_bh(&pmap->lock); 466 pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n", 467 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 468 return -EADDRNOTAVAIL; 469 } 470 471 start = idx = pmap->next; 472 do { 473 if (++idx >= pmap->max_connect) 474 idx = 0; 475 if (!pmap->port_csk[idx]) { 476 pmap->used++; 477 *port = htons(pmap->sport_base + idx); 478 pmap->next = idx; 479 pmap->port_csk[idx] = csk; 480 spin_unlock_bh(&pmap->lock); 481 cxgbi_sock_get(csk); 482 log_debug(1 << CXGBI_DBG_SOCK, 483 "cdev 0x%p, p#%u %s, p %u, %u.\n", 484 cdev, csk->port_id, 485 cdev->ports[csk->port_id]->name, 486 pmap->sport_base + idx, pmap->next); 487 return 0; 488 } 489 } while (idx != start); 490 spin_unlock_bh(&pmap->lock); 491 492 /* should not happen */ 493 pr_warn("cdev 0x%p, p#%u %s, next %u?\n", 494 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 495 pmap->next); 496 return -EADDRNOTAVAIL; 497 } 498 499 static void sock_put_port(struct cxgbi_sock *csk) 500 { 501 struct cxgbi_device *cdev = csk->cdev; 502 struct cxgbi_ports_map *pmap = &cdev->pmap; 503 __be16 *port; 504 505 if (csk->csk_family == AF_INET) 506 port = &csk->saddr.sin_port; 507 else /* ipv6 */ 508 port = &csk->saddr6.sin6_port; 509 510 if (*port) { 511 int idx = ntohs(*port) - pmap->sport_base; 512 513 *port = 0; 514 if (idx < 0 || idx >= pmap->max_connect) { 515 pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n", 516 cdev, csk->port_id, 517 cdev->ports[csk->port_id]->name, 518 ntohs(*port)); 519 return; 520 } 521 522 spin_lock_bh(&pmap->lock); 523 pmap->port_csk[idx] = NULL; 524 pmap->used--; 525 spin_unlock_bh(&pmap->lock); 526 527 log_debug(1 << CXGBI_DBG_SOCK, 528 "cdev 0x%p, p#%u %s, release %u.\n", 529 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 530 pmap->sport_base + idx); 531 532 cxgbi_sock_put(csk); 533 } 534 } 535 536 /* 537 * iscsi tcp connection 538 */ 539 void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk) 540 { 541 if (csk->cpl_close) { 542 kfree_skb(csk->cpl_close); 543 csk->cpl_close = NULL; 544 } 545 if (csk->cpl_abort_req) { 546 kfree_skb(csk->cpl_abort_req); 547 csk->cpl_abort_req = NULL; 548 } 549 if (csk->cpl_abort_rpl) { 550 kfree_skb(csk->cpl_abort_rpl); 551 csk->cpl_abort_rpl = NULL; 552 } 553 } 554 EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs); 555 556 static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev) 557 { 558 struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO); 559 560 if (!csk) { 561 pr_info("alloc csk %zu failed.\n", sizeof(*csk)); 562 return NULL; 563 } 564 565 if (cdev->csk_alloc_cpls(csk) < 0) { 566 pr_info("csk 0x%p, alloc cpls failed.\n", csk); 567 kfree(csk); 568 return NULL; 569 } 570 571 spin_lock_init(&csk->lock); 572 kref_init(&csk->refcnt); 573 skb_queue_head_init(&csk->receive_queue); 574 skb_queue_head_init(&csk->write_queue); 575 setup_timer(&csk->retry_timer, NULL, (unsigned long)csk); 576 rwlock_init(&csk->callback_lock); 577 csk->cdev = cdev; 578 csk->flags = 0; 579 cxgbi_sock_set_state(csk, CTP_CLOSED); 580 581 log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk); 582 583 return csk; 584 } 585 586 static struct rtable *find_route_ipv4(struct flowi4 *fl4, 587 __be32 saddr, __be32 daddr, 588 __be16 sport, __be16 dport, u8 tos) 589 { 590 struct rtable *rt; 591 592 rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr, 593 dport, sport, IPPROTO_TCP, tos, 0); 594 if (IS_ERR(rt)) 595 return NULL; 596 597 return rt; 598 } 599 600 static struct cxgbi_sock *cxgbi_check_route(struct sockaddr *dst_addr) 601 { 602 struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr; 603 struct dst_entry *dst; 604 struct net_device *ndev; 605 struct cxgbi_device *cdev; 606 struct rtable *rt = NULL; 607 struct neighbour *n; 608 struct flowi4 fl4; 609 struct cxgbi_sock *csk = NULL; 610 unsigned int mtu = 0; 611 int port = 0xFFFF; 612 int err = 0; 613 614 rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0, daddr->sin_port, 0); 615 if (!rt) { 616 pr_info("no route to ipv4 0x%x, port %u.\n", 617 be32_to_cpu(daddr->sin_addr.s_addr), 618 be16_to_cpu(daddr->sin_port)); 619 err = -ENETUNREACH; 620 goto err_out; 621 } 622 dst = &rt->dst; 623 n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr); 624 if (!n) { 625 err = -ENODEV; 626 goto rel_rt; 627 } 628 ndev = n->dev; 629 630 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) { 631 pr_info("multi-cast route %pI4, port %u, dev %s.\n", 632 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 633 ndev->name); 634 err = -ENETUNREACH; 635 goto rel_neigh; 636 } 637 638 if (ndev->flags & IFF_LOOPBACK) { 639 ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr); 640 mtu = ndev->mtu; 641 pr_info("rt dev %s, loopback -> %s, mtu %u.\n", 642 n->dev->name, ndev->name, mtu); 643 } 644 645 cdev = cxgbi_device_find_by_netdev(ndev, &port); 646 if (!cdev) { 647 pr_info("dst %pI4, %s, NOT cxgbi device.\n", 648 &daddr->sin_addr.s_addr, ndev->name); 649 err = -ENETUNREACH; 650 goto rel_neigh; 651 } 652 log_debug(1 << CXGBI_DBG_SOCK, 653 "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n", 654 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 655 port, ndev->name, cdev); 656 657 csk = cxgbi_sock_create(cdev); 658 if (!csk) { 659 err = -ENOMEM; 660 goto rel_neigh; 661 } 662 csk->cdev = cdev; 663 csk->port_id = port; 664 csk->mtu = mtu; 665 csk->dst = dst; 666 667 csk->csk_family = AF_INET; 668 csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr; 669 csk->daddr.sin_port = daddr->sin_port; 670 csk->daddr.sin_family = daddr->sin_family; 671 csk->saddr.sin_family = daddr->sin_family; 672 csk->saddr.sin_addr.s_addr = fl4.saddr; 673 neigh_release(n); 674 675 return csk; 676 677 rel_neigh: 678 neigh_release(n); 679 680 rel_rt: 681 ip_rt_put(rt); 682 if (csk) 683 cxgbi_sock_closed(csk); 684 err_out: 685 return ERR_PTR(err); 686 } 687 688 #if IS_ENABLED(CONFIG_IPV6) 689 static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr, 690 const struct in6_addr *daddr) 691 { 692 struct flowi6 fl; 693 694 memset(&fl, 0, sizeof(fl)); 695 if (saddr) 696 memcpy(&fl.saddr, saddr, sizeof(struct in6_addr)); 697 if (daddr) 698 memcpy(&fl.daddr, daddr, sizeof(struct in6_addr)); 699 return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl); 700 } 701 702 static struct cxgbi_sock *cxgbi_check_route6(struct sockaddr *dst_addr) 703 { 704 struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr; 705 struct dst_entry *dst; 706 struct net_device *ndev; 707 struct cxgbi_device *cdev; 708 struct rt6_info *rt = NULL; 709 struct neighbour *n; 710 struct in6_addr pref_saddr; 711 struct cxgbi_sock *csk = NULL; 712 unsigned int mtu = 0; 713 int port = 0xFFFF; 714 int err = 0; 715 716 rt = find_route_ipv6(NULL, &daddr6->sin6_addr); 717 718 if (!rt) { 719 pr_info("no route to ipv6 %pI6 port %u\n", 720 daddr6->sin6_addr.s6_addr, 721 be16_to_cpu(daddr6->sin6_port)); 722 err = -ENETUNREACH; 723 goto err_out; 724 } 725 726 dst = &rt->dst; 727 728 n = dst_neigh_lookup(dst, &daddr6->sin6_addr); 729 730 if (!n) { 731 pr_info("%pI6, port %u, dst no neighbour.\n", 732 daddr6->sin6_addr.s6_addr, 733 be16_to_cpu(daddr6->sin6_port)); 734 err = -ENETUNREACH; 735 goto rel_rt; 736 } 737 ndev = n->dev; 738 739 if (ipv6_addr_is_multicast(&daddr6->sin6_addr)) { 740 pr_info("multi-cast route %pI6 port %u, dev %s.\n", 741 daddr6->sin6_addr.s6_addr, 742 ntohs(daddr6->sin6_port), ndev->name); 743 err = -ENETUNREACH; 744 goto rel_rt; 745 } 746 747 cdev = cxgbi_device_find_by_netdev(ndev, &port); 748 if (!cdev) 749 cdev = cxgbi_device_find_by_mac(ndev, &port); 750 if (!cdev) { 751 pr_info("dst %pI6 %s, NOT cxgbi device.\n", 752 daddr6->sin6_addr.s6_addr, ndev->name); 753 err = -ENETUNREACH; 754 goto rel_rt; 755 } 756 log_debug(1 << CXGBI_DBG_SOCK, 757 "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n", 758 daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port, 759 ndev->name, cdev); 760 761 csk = cxgbi_sock_create(cdev); 762 if (!csk) { 763 err = -ENOMEM; 764 goto rel_rt; 765 } 766 csk->cdev = cdev; 767 csk->port_id = port; 768 csk->mtu = mtu; 769 csk->dst = dst; 770 771 if (ipv6_addr_any(&rt->rt6i_prefsrc.addr)) { 772 struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt); 773 774 err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL, 775 &daddr6->sin6_addr, 0, &pref_saddr); 776 if (err) { 777 pr_info("failed to get source address to reach %pI6\n", 778 &daddr6->sin6_addr); 779 goto rel_rt; 780 } 781 } else { 782 pref_saddr = rt->rt6i_prefsrc.addr; 783 } 784 785 csk->csk_family = AF_INET6; 786 csk->daddr6.sin6_addr = daddr6->sin6_addr; 787 csk->daddr6.sin6_port = daddr6->sin6_port; 788 csk->daddr6.sin6_family = daddr6->sin6_family; 789 csk->saddr6.sin6_family = daddr6->sin6_family; 790 csk->saddr6.sin6_addr = pref_saddr; 791 792 neigh_release(n); 793 return csk; 794 795 rel_rt: 796 if (n) 797 neigh_release(n); 798 799 ip6_rt_put(rt); 800 if (csk) 801 cxgbi_sock_closed(csk); 802 err_out: 803 return ERR_PTR(err); 804 } 805 #endif /* IS_ENABLED(CONFIG_IPV6) */ 806 807 void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn, 808 unsigned int opt) 809 { 810 csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn; 811 dst_confirm(csk->dst); 812 smp_mb(); 813 cxgbi_sock_set_state(csk, CTP_ESTABLISHED); 814 } 815 EXPORT_SYMBOL_GPL(cxgbi_sock_established); 816 817 static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk) 818 { 819 log_debug(1 << CXGBI_DBG_SOCK, 820 "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n", 821 csk, csk->state, csk->flags, csk->user_data); 822 823 if (csk->state != CTP_ESTABLISHED) { 824 read_lock_bh(&csk->callback_lock); 825 if (csk->user_data) 826 iscsi_conn_failure(csk->user_data, 827 ISCSI_ERR_TCP_CONN_CLOSE); 828 read_unlock_bh(&csk->callback_lock); 829 } 830 } 831 832 void cxgbi_sock_closed(struct cxgbi_sock *csk) 833 { 834 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 835 csk, (csk)->state, (csk)->flags, (csk)->tid); 836 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 837 if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED) 838 return; 839 if (csk->saddr.sin_port) 840 sock_put_port(csk); 841 if (csk->dst) 842 dst_release(csk->dst); 843 csk->cdev->csk_release_offload_resources(csk); 844 cxgbi_sock_set_state(csk, CTP_CLOSED); 845 cxgbi_inform_iscsi_conn_closing(csk); 846 cxgbi_sock_put(csk); 847 } 848 EXPORT_SYMBOL_GPL(cxgbi_sock_closed); 849 850 static void need_active_close(struct cxgbi_sock *csk) 851 { 852 int data_lost; 853 int close_req = 0; 854 855 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 856 csk, (csk)->state, (csk)->flags, (csk)->tid); 857 spin_lock_bh(&csk->lock); 858 dst_confirm(csk->dst); 859 data_lost = skb_queue_len(&csk->receive_queue); 860 __skb_queue_purge(&csk->receive_queue); 861 862 if (csk->state == CTP_ACTIVE_OPEN) 863 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 864 else if (csk->state == CTP_ESTABLISHED) { 865 close_req = 1; 866 cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE); 867 } else if (csk->state == CTP_PASSIVE_CLOSE) { 868 close_req = 1; 869 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 870 } 871 872 if (close_req) { 873 if (data_lost) 874 csk->cdev->csk_send_abort_req(csk); 875 else 876 csk->cdev->csk_send_close_req(csk); 877 } 878 879 spin_unlock_bh(&csk->lock); 880 } 881 882 void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno) 883 { 884 pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n", 885 csk, csk->state, csk->flags, 886 &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port, 887 &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port, 888 errno); 889 890 cxgbi_sock_set_state(csk, CTP_CONNECTING); 891 csk->err = errno; 892 cxgbi_sock_closed(csk); 893 } 894 EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open); 895 896 void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb) 897 { 898 struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk; 899 900 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 901 csk, (csk)->state, (csk)->flags, (csk)->tid); 902 cxgbi_sock_get(csk); 903 spin_lock_bh(&csk->lock); 904 if (csk->state == CTP_ACTIVE_OPEN) 905 cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH); 906 spin_unlock_bh(&csk->lock); 907 cxgbi_sock_put(csk); 908 __kfree_skb(skb); 909 } 910 EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure); 911 912 void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk) 913 { 914 cxgbi_sock_get(csk); 915 spin_lock_bh(&csk->lock); 916 917 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD); 918 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 919 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING); 920 if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) 921 pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n", 922 csk, csk->state, csk->flags, csk->tid); 923 cxgbi_sock_closed(csk); 924 } 925 926 spin_unlock_bh(&csk->lock); 927 cxgbi_sock_put(csk); 928 } 929 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl); 930 931 void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk) 932 { 933 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 934 csk, (csk)->state, (csk)->flags, (csk)->tid); 935 cxgbi_sock_get(csk); 936 spin_lock_bh(&csk->lock); 937 938 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 939 goto done; 940 941 switch (csk->state) { 942 case CTP_ESTABLISHED: 943 cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE); 944 break; 945 case CTP_ACTIVE_CLOSE: 946 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 947 break; 948 case CTP_CLOSE_WAIT_1: 949 cxgbi_sock_closed(csk); 950 break; 951 case CTP_ABORTING: 952 break; 953 default: 954 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 955 csk, csk->state, csk->flags, csk->tid); 956 } 957 cxgbi_inform_iscsi_conn_closing(csk); 958 done: 959 spin_unlock_bh(&csk->lock); 960 cxgbi_sock_put(csk); 961 } 962 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close); 963 964 void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt) 965 { 966 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 967 csk, (csk)->state, (csk)->flags, (csk)->tid); 968 cxgbi_sock_get(csk); 969 spin_lock_bh(&csk->lock); 970 971 csk->snd_una = snd_nxt - 1; 972 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 973 goto done; 974 975 switch (csk->state) { 976 case CTP_ACTIVE_CLOSE: 977 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1); 978 break; 979 case CTP_CLOSE_WAIT_1: 980 case CTP_CLOSE_WAIT_2: 981 cxgbi_sock_closed(csk); 982 break; 983 case CTP_ABORTING: 984 break; 985 default: 986 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 987 csk, csk->state, csk->flags, csk->tid); 988 } 989 done: 990 spin_unlock_bh(&csk->lock); 991 cxgbi_sock_put(csk); 992 } 993 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl); 994 995 void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits, 996 unsigned int snd_una, int seq_chk) 997 { 998 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 999 "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n", 1000 csk, csk->state, csk->flags, csk->tid, credits, 1001 csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk); 1002 1003 spin_lock_bh(&csk->lock); 1004 1005 csk->wr_cred += credits; 1006 if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred) 1007 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred; 1008 1009 while (credits) { 1010 struct sk_buff *p = cxgbi_sock_peek_wr(csk); 1011 1012 if (unlikely(!p)) { 1013 pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n", 1014 csk, csk->state, csk->flags, csk->tid, credits, 1015 csk->wr_cred, csk->wr_una_cred); 1016 break; 1017 } 1018 1019 if (unlikely(credits < p->csum)) { 1020 pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n", 1021 csk, csk->state, csk->flags, csk->tid, 1022 credits, csk->wr_cred, csk->wr_una_cred, 1023 p->csum); 1024 p->csum -= credits; 1025 break; 1026 } else { 1027 cxgbi_sock_dequeue_wr(csk); 1028 credits -= p->csum; 1029 kfree_skb(p); 1030 } 1031 } 1032 1033 cxgbi_sock_check_wr_invariants(csk); 1034 1035 if (seq_chk) { 1036 if (unlikely(before(snd_una, csk->snd_una))) { 1037 pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.", 1038 csk, csk->state, csk->flags, csk->tid, snd_una, 1039 csk->snd_una); 1040 goto done; 1041 } 1042 1043 if (csk->snd_una != snd_una) { 1044 csk->snd_una = snd_una; 1045 dst_confirm(csk->dst); 1046 } 1047 } 1048 1049 if (skb_queue_len(&csk->write_queue)) { 1050 if (csk->cdev->csk_push_tx_frames(csk, 0)) 1051 cxgbi_conn_tx_open(csk); 1052 } else 1053 cxgbi_conn_tx_open(csk); 1054 done: 1055 spin_unlock_bh(&csk->lock); 1056 } 1057 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack); 1058 1059 static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk, 1060 unsigned short mtu) 1061 { 1062 int i = 0; 1063 1064 while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu) 1065 ++i; 1066 1067 return i; 1068 } 1069 1070 unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu) 1071 { 1072 unsigned int idx; 1073 struct dst_entry *dst = csk->dst; 1074 1075 csk->advmss = dst_metric_advmss(dst); 1076 1077 if (csk->advmss > pmtu - 40) 1078 csk->advmss = pmtu - 40; 1079 if (csk->advmss < csk->cdev->mtus[0] - 40) 1080 csk->advmss = csk->cdev->mtus[0] - 40; 1081 idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40); 1082 1083 return idx; 1084 } 1085 EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss); 1086 1087 void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb) 1088 { 1089 cxgbi_skcb_tcp_seq(skb) = csk->write_seq; 1090 __skb_queue_tail(&csk->write_queue, skb); 1091 } 1092 EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail); 1093 1094 void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk) 1095 { 1096 struct sk_buff *skb; 1097 1098 while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL) 1099 kfree_skb(skb); 1100 } 1101 EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue); 1102 1103 void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk) 1104 { 1105 int pending = cxgbi_sock_count_pending_wrs(csk); 1106 1107 if (unlikely(csk->wr_cred + pending != csk->wr_max_cred)) 1108 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n", 1109 csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred); 1110 } 1111 EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants); 1112 1113 static int cxgbi_sock_send_pdus(struct cxgbi_sock *csk, struct sk_buff *skb) 1114 { 1115 struct cxgbi_device *cdev = csk->cdev; 1116 struct sk_buff *next; 1117 int err, copied = 0; 1118 1119 spin_lock_bh(&csk->lock); 1120 1121 if (csk->state != CTP_ESTABLISHED) { 1122 log_debug(1 << CXGBI_DBG_PDU_TX, 1123 "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n", 1124 csk, csk->state, csk->flags, csk->tid); 1125 err = -EAGAIN; 1126 goto out_err; 1127 } 1128 1129 if (csk->err) { 1130 log_debug(1 << CXGBI_DBG_PDU_TX, 1131 "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n", 1132 csk, csk->state, csk->flags, csk->tid, csk->err); 1133 err = -EPIPE; 1134 goto out_err; 1135 } 1136 1137 if (csk->write_seq - csk->snd_una >= csk->snd_win) { 1138 log_debug(1 << CXGBI_DBG_PDU_TX, 1139 "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n", 1140 csk, csk->state, csk->flags, csk->tid, csk->write_seq, 1141 csk->snd_una, csk->snd_win); 1142 err = -ENOBUFS; 1143 goto out_err; 1144 } 1145 1146 while (skb) { 1147 int frags = skb_shinfo(skb)->nr_frags + 1148 (skb->len != skb->data_len); 1149 1150 if (unlikely(skb_headroom(skb) < cdev->skb_tx_rsvd)) { 1151 pr_err("csk 0x%p, skb head %u < %u.\n", 1152 csk, skb_headroom(skb), cdev->skb_tx_rsvd); 1153 err = -EINVAL; 1154 goto out_err; 1155 } 1156 1157 if (frags >= SKB_WR_LIST_SIZE) { 1158 pr_err("csk 0x%p, frags %d, %u,%u >%u.\n", 1159 csk, skb_shinfo(skb)->nr_frags, skb->len, 1160 skb->data_len, (uint)(SKB_WR_LIST_SIZE)); 1161 err = -EINVAL; 1162 goto out_err; 1163 } 1164 1165 next = skb->next; 1166 skb->next = NULL; 1167 cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR); 1168 cxgbi_sock_skb_entail(csk, skb); 1169 copied += skb->len; 1170 csk->write_seq += skb->len + 1171 cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 1172 skb = next; 1173 } 1174 done: 1175 if (likely(skb_queue_len(&csk->write_queue))) 1176 cdev->csk_push_tx_frames(csk, 1); 1177 spin_unlock_bh(&csk->lock); 1178 return copied; 1179 1180 out_err: 1181 if (copied == 0 && err == -EPIPE) 1182 copied = csk->err ? csk->err : -EPIPE; 1183 else 1184 copied = err; 1185 goto done; 1186 } 1187 1188 static inline void 1189 scmd_get_params(struct scsi_cmnd *sc, struct scatterlist **sgl, 1190 unsigned int *sgcnt, unsigned int *dlen, 1191 unsigned int prot) 1192 { 1193 struct scsi_data_buffer *sdb = prot ? scsi_prot(sc) : scsi_out(sc); 1194 1195 *sgl = sdb->table.sgl; 1196 *sgcnt = sdb->table.nents; 1197 *dlen = sdb->length; 1198 /* Caution: for protection sdb, sdb->length is invalid */ 1199 } 1200 1201 void cxgbi_ddp_set_one_ppod(struct cxgbi_pagepod *ppod, 1202 struct cxgbi_task_tag_info *ttinfo, 1203 struct scatterlist **sg_pp, unsigned int *sg_off) 1204 { 1205 struct scatterlist *sg = sg_pp ? *sg_pp : NULL; 1206 unsigned int offset = sg_off ? *sg_off : 0; 1207 dma_addr_t addr = 0UL; 1208 unsigned int len = 0; 1209 int i; 1210 1211 memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr)); 1212 1213 if (sg) { 1214 addr = sg_dma_address(sg); 1215 len = sg_dma_len(sg); 1216 } 1217 1218 for (i = 0; i < PPOD_PAGES_MAX; i++) { 1219 if (sg) { 1220 ppod->addr[i] = cpu_to_be64(addr + offset); 1221 offset += PAGE_SIZE; 1222 if (offset == (len + sg->offset)) { 1223 offset = 0; 1224 sg = sg_next(sg); 1225 if (sg) { 1226 addr = sg_dma_address(sg); 1227 len = sg_dma_len(sg); 1228 } 1229 } 1230 } else { 1231 ppod->addr[i] = 0ULL; 1232 } 1233 } 1234 1235 /* 1236 * the fifth address needs to be repeated in the next ppod, so do 1237 * not move sg 1238 */ 1239 if (sg_pp) { 1240 *sg_pp = sg; 1241 *sg_off = offset; 1242 } 1243 1244 if (offset == len) { 1245 offset = 0; 1246 sg = sg_next(sg); 1247 if (sg) { 1248 addr = sg_dma_address(sg); 1249 len = sg_dma_len(sg); 1250 } 1251 } 1252 ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL; 1253 } 1254 EXPORT_SYMBOL_GPL(cxgbi_ddp_set_one_ppod); 1255 1256 /* 1257 * APIs interacting with open-iscsi libraries 1258 */ 1259 1260 static unsigned char padding[4]; 1261 1262 void cxgbi_ddp_ppm_setup(void **ppm_pp, struct cxgbi_device *cdev, 1263 struct cxgbi_tag_format *tformat, unsigned int ppmax, 1264 unsigned int llimit, unsigned int start, 1265 unsigned int rsvd_factor) 1266 { 1267 int err = cxgbi_ppm_init(ppm_pp, cdev->ports[0], cdev->pdev, 1268 cdev->lldev, tformat, ppmax, llimit, start, 1269 rsvd_factor); 1270 1271 if (err >= 0) { 1272 struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*ppm_pp); 1273 1274 if (ppm->ppmax < 1024 || 1275 ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX) 1276 cdev->flags |= CXGBI_FLAG_DDP_OFF; 1277 err = 0; 1278 } else { 1279 cdev->flags |= CXGBI_FLAG_DDP_OFF; 1280 } 1281 } 1282 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppm_setup); 1283 1284 static int cxgbi_ddp_sgl_check(struct scatterlist *sgl, int nents) 1285 { 1286 int i; 1287 int last_sgidx = nents - 1; 1288 struct scatterlist *sg = sgl; 1289 1290 for (i = 0; i < nents; i++, sg = sg_next(sg)) { 1291 unsigned int len = sg->length + sg->offset; 1292 1293 if ((sg->offset & 0x3) || (i && sg->offset) || 1294 ((i != last_sgidx) && len != PAGE_SIZE)) { 1295 log_debug(1 << CXGBI_DBG_DDP, 1296 "sg %u/%u, %u,%u, not aligned.\n", 1297 i, nents, sg->offset, sg->length); 1298 goto err_out; 1299 } 1300 } 1301 return 0; 1302 err_out: 1303 return -EINVAL; 1304 } 1305 1306 static int cxgbi_ddp_reserve(struct cxgbi_conn *cconn, 1307 struct cxgbi_task_data *tdata, u32 sw_tag, 1308 unsigned int xferlen) 1309 { 1310 struct cxgbi_sock *csk = cconn->cep->csk; 1311 struct cxgbi_device *cdev = csk->cdev; 1312 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1313 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 1314 struct scatterlist *sgl = ttinfo->sgl; 1315 unsigned int sgcnt = ttinfo->nents; 1316 unsigned int sg_offset = sgl->offset; 1317 int err; 1318 1319 if (cdev->flags & CXGBI_FLAG_DDP_OFF) { 1320 log_debug(1 << CXGBI_DBG_DDP, 1321 "cdev 0x%p DDP off.\n", cdev); 1322 return -EINVAL; 1323 } 1324 1325 if (!ppm || xferlen < DDP_THRESHOLD || !sgcnt || 1326 ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX) { 1327 log_debug(1 << CXGBI_DBG_DDP, 1328 "ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n", 1329 ppm, ppm ? ppm->tformat.pgsz_idx_dflt : DDP_PGIDX_MAX, 1330 xferlen, ttinfo->nents); 1331 return -EINVAL; 1332 } 1333 1334 /* make sure the buffer is suitable for ddp */ 1335 if (cxgbi_ddp_sgl_check(sgl, sgcnt) < 0) 1336 return -EINVAL; 1337 1338 ttinfo->nr_pages = (xferlen + sgl->offset + (1 << PAGE_SHIFT) - 1) >> 1339 PAGE_SHIFT; 1340 1341 /* 1342 * the ddp tag will be used for the itt in the outgoing pdu, 1343 * the itt genrated by libiscsi is saved in the ppm and can be 1344 * retrieved via the ddp tag 1345 */ 1346 err = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx, 1347 &ttinfo->tag, (unsigned long)sw_tag); 1348 if (err < 0) { 1349 cconn->ddp_full++; 1350 return err; 1351 } 1352 ttinfo->npods = err; 1353 1354 /* setup dma from scsi command sgl */ 1355 sgl->offset = 0; 1356 err = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 1357 sgl->offset = sg_offset; 1358 if (err == 0) { 1359 pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n", 1360 __func__, sw_tag, xferlen, sgcnt); 1361 goto rel_ppods; 1362 } 1363 if (err != ttinfo->nr_pages) { 1364 log_debug(1 << CXGBI_DBG_DDP, 1365 "%s: sw tag 0x%x, xfer %u, sgl %u, dma count %d.\n", 1366 __func__, sw_tag, xferlen, sgcnt, err); 1367 } 1368 1369 ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_MAPPED; 1370 ttinfo->cid = csk->port_id; 1371 1372 cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset, 1373 xferlen, &ttinfo->hdr); 1374 1375 if (cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ) { 1376 /* write ppod from xmit_pdu (of iscsi_scsi_command pdu) */ 1377 ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_VALID; 1378 } else { 1379 /* write ppod from control queue now */ 1380 err = cdev->csk_ddp_set_map(ppm, csk, ttinfo); 1381 if (err < 0) 1382 goto rel_ppods; 1383 } 1384 1385 return 0; 1386 1387 rel_ppods: 1388 cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 1389 1390 if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_MAPPED) { 1391 ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_MAPPED; 1392 dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 1393 } 1394 return -EINVAL; 1395 } 1396 1397 static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt) 1398 { 1399 struct scsi_cmnd *sc = task->sc; 1400 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 1401 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1402 struct cxgbi_device *cdev = cconn->chba->cdev; 1403 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1404 u32 tag = ntohl((__force u32)hdr_itt); 1405 1406 log_debug(1 << CXGBI_DBG_DDP, 1407 "cdev 0x%p, task 0x%p, release tag 0x%x.\n", 1408 cdev, task, tag); 1409 if (sc && 1410 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) && 1411 cxgbi_ppm_is_ddp_tag(ppm, tag)) { 1412 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1413 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 1414 1415 if (!(cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ)) 1416 cdev->csk_ddp_clear_map(cdev, ppm, ttinfo); 1417 cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 1418 dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents, 1419 DMA_FROM_DEVICE); 1420 } 1421 } 1422 1423 static inline u32 cxgbi_build_sw_tag(u32 idx, u32 age) 1424 { 1425 /* assume idx and age both are < 0x7FFF (32767) */ 1426 return (idx << 16) | age; 1427 } 1428 1429 static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt) 1430 { 1431 struct scsi_cmnd *sc = task->sc; 1432 struct iscsi_conn *conn = task->conn; 1433 struct iscsi_session *sess = conn->session; 1434 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1435 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1436 struct cxgbi_device *cdev = cconn->chba->cdev; 1437 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1438 u32 sw_tag = cxgbi_build_sw_tag(task->itt, sess->age); 1439 u32 tag = 0; 1440 int err = -EINVAL; 1441 1442 if (sc && 1443 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) 1444 ) { 1445 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1446 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 1447 1448 scmd_get_params(sc, &ttinfo->sgl, &ttinfo->nents, 1449 &tdata->dlen, 0); 1450 err = cxgbi_ddp_reserve(cconn, tdata, sw_tag, tdata->dlen); 1451 if (!err) 1452 tag = ttinfo->tag; 1453 else 1454 log_debug(1 << CXGBI_DBG_DDP, 1455 "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n", 1456 cconn->cep->csk, task, tdata->dlen, 1457 ttinfo->nents); 1458 } 1459 1460 if (err < 0) { 1461 err = cxgbi_ppm_make_non_ddp_tag(ppm, sw_tag, &tag); 1462 if (err < 0) 1463 return err; 1464 } 1465 /* the itt need to sent in big-endian order */ 1466 *hdr_itt = (__force itt_t)htonl(tag); 1467 1468 log_debug(1 << CXGBI_DBG_DDP, 1469 "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n", 1470 cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt); 1471 return 0; 1472 } 1473 1474 void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age) 1475 { 1476 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1477 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1478 struct cxgbi_device *cdev = cconn->chba->cdev; 1479 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1480 u32 tag = ntohl((__force u32)itt); 1481 u32 sw_bits; 1482 1483 if (ppm) { 1484 if (cxgbi_ppm_is_ddp_tag(ppm, tag)) 1485 sw_bits = cxgbi_ppm_get_tag_caller_data(ppm, tag); 1486 else 1487 sw_bits = cxgbi_ppm_decode_non_ddp_tag(ppm, tag); 1488 } else { 1489 sw_bits = tag; 1490 } 1491 1492 cxgbi_decode_sw_tag(sw_bits, idx, age); 1493 log_debug(1 << CXGBI_DBG_DDP, 1494 "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n", 1495 cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF, 1496 age ? *age : 0xFF); 1497 } 1498 EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt); 1499 1500 void cxgbi_conn_tx_open(struct cxgbi_sock *csk) 1501 { 1502 struct iscsi_conn *conn = csk->user_data; 1503 1504 if (conn) { 1505 log_debug(1 << CXGBI_DBG_SOCK, 1506 "csk 0x%p, cid %d.\n", csk, conn->id); 1507 iscsi_conn_queue_work(conn); 1508 } 1509 } 1510 EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open); 1511 1512 /* 1513 * pdu receive, interact with libiscsi_tcp 1514 */ 1515 static inline int read_pdu_skb(struct iscsi_conn *conn, 1516 struct sk_buff *skb, 1517 unsigned int offset, 1518 int offloaded) 1519 { 1520 int status = 0; 1521 int bytes_read; 1522 1523 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status); 1524 switch (status) { 1525 case ISCSI_TCP_CONN_ERR: 1526 pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n", 1527 skb, offset, offloaded); 1528 return -EIO; 1529 case ISCSI_TCP_SUSPENDED: 1530 log_debug(1 << CXGBI_DBG_PDU_RX, 1531 "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n", 1532 skb, offset, offloaded, bytes_read); 1533 /* no transfer - just have caller flush queue */ 1534 return bytes_read; 1535 case ISCSI_TCP_SKB_DONE: 1536 pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n", 1537 skb, offset, offloaded); 1538 /* 1539 * pdus should always fit in the skb and we should get 1540 * segment done notifcation. 1541 */ 1542 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb."); 1543 return -EFAULT; 1544 case ISCSI_TCP_SEGMENT_DONE: 1545 log_debug(1 << CXGBI_DBG_PDU_RX, 1546 "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n", 1547 skb, offset, offloaded, bytes_read); 1548 return bytes_read; 1549 default: 1550 pr_info("skb 0x%p, off %u, %d, invalid status %d.\n", 1551 skb, offset, offloaded, status); 1552 return -EINVAL; 1553 } 1554 } 1555 1556 static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb) 1557 { 1558 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1559 1560 log_debug(1 << CXGBI_DBG_PDU_RX, 1561 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1562 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1563 1564 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) { 1565 pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb); 1566 iscsi_conn_failure(conn, ISCSI_ERR_PROTO); 1567 return -EIO; 1568 } 1569 1570 if (conn->hdrdgst_en && 1571 cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) { 1572 pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb); 1573 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST); 1574 return -EIO; 1575 } 1576 1577 return read_pdu_skb(conn, skb, 0, 0); 1578 } 1579 1580 static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb, 1581 struct sk_buff *skb, unsigned int offset) 1582 { 1583 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1584 bool offloaded = 0; 1585 int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK; 1586 1587 log_debug(1 << CXGBI_DBG_PDU_RX, 1588 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1589 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1590 1591 if (conn->datadgst_en && 1592 cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) { 1593 pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n", 1594 conn, lskb, cxgbi_skcb_flags(lskb)); 1595 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 1596 return -EIO; 1597 } 1598 1599 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn)) 1600 return 0; 1601 1602 /* coalesced, add header digest length */ 1603 if (lskb == skb && conn->hdrdgst_en) 1604 offset += ISCSI_DIGEST_SIZE; 1605 1606 if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD)) 1607 offloaded = 1; 1608 1609 if (opcode == ISCSI_OP_SCSI_DATA_IN) 1610 log_debug(1 << CXGBI_DBG_PDU_RX, 1611 "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n", 1612 skb, opcode, ntohl(tcp_conn->in.hdr->itt), 1613 tcp_conn->in.datalen, offloaded ? "is" : "not"); 1614 1615 return read_pdu_skb(conn, skb, offset, offloaded); 1616 } 1617 1618 static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied) 1619 { 1620 struct cxgbi_device *cdev = csk->cdev; 1621 int must_send; 1622 u32 credits; 1623 1624 log_debug(1 << CXGBI_DBG_PDU_RX, 1625 "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n", 1626 csk, csk->state, csk->flags, csk->tid, csk->copied_seq, 1627 csk->rcv_wup, cdev->rx_credit_thres, 1628 csk->rcv_win); 1629 1630 if (!cdev->rx_credit_thres) 1631 return; 1632 1633 if (csk->state != CTP_ESTABLISHED) 1634 return; 1635 1636 credits = csk->copied_seq - csk->rcv_wup; 1637 if (unlikely(!credits)) 1638 return; 1639 must_send = credits + 16384 >= csk->rcv_win; 1640 if (must_send || credits >= cdev->rx_credit_thres) 1641 csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits); 1642 } 1643 1644 void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk) 1645 { 1646 struct cxgbi_device *cdev = csk->cdev; 1647 struct iscsi_conn *conn = csk->user_data; 1648 struct sk_buff *skb; 1649 unsigned int read = 0; 1650 int err = 0; 1651 1652 log_debug(1 << CXGBI_DBG_PDU_RX, 1653 "csk 0x%p, conn 0x%p.\n", csk, conn); 1654 1655 if (unlikely(!conn || conn->suspend_rx)) { 1656 log_debug(1 << CXGBI_DBG_PDU_RX, 1657 "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n", 1658 csk, conn, conn ? conn->id : 0xFF, 1659 conn ? conn->suspend_rx : 0xFF); 1660 return; 1661 } 1662 1663 while (!err) { 1664 skb = skb_peek(&csk->receive_queue); 1665 if (!skb || 1666 !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) { 1667 if (skb) 1668 log_debug(1 << CXGBI_DBG_PDU_RX, 1669 "skb 0x%p, NOT ready 0x%lx.\n", 1670 skb, cxgbi_skcb_flags(skb)); 1671 break; 1672 } 1673 __skb_unlink(skb, &csk->receive_queue); 1674 1675 read += cxgbi_skcb_rx_pdulen(skb); 1676 log_debug(1 << CXGBI_DBG_PDU_RX, 1677 "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n", 1678 csk, skb, skb->len, cxgbi_skcb_flags(skb), 1679 cxgbi_skcb_rx_pdulen(skb)); 1680 1681 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) { 1682 err = skb_read_pdu_bhs(conn, skb); 1683 if (err < 0) { 1684 pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, " 1685 "f 0x%lx, plen %u.\n", 1686 csk, skb, skb->len, 1687 cxgbi_skcb_flags(skb), 1688 cxgbi_skcb_rx_pdulen(skb)); 1689 goto skb_done; 1690 } 1691 err = skb_read_pdu_data(conn, skb, skb, 1692 err + cdev->skb_rx_extra); 1693 if (err < 0) 1694 pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, " 1695 "f 0x%lx, plen %u.\n", 1696 csk, skb, skb->len, 1697 cxgbi_skcb_flags(skb), 1698 cxgbi_skcb_rx_pdulen(skb)); 1699 } else { 1700 err = skb_read_pdu_bhs(conn, skb); 1701 if (err < 0) { 1702 pr_err("bhs, csk 0x%p, skb 0x%p,%u, " 1703 "f 0x%lx, plen %u.\n", 1704 csk, skb, skb->len, 1705 cxgbi_skcb_flags(skb), 1706 cxgbi_skcb_rx_pdulen(skb)); 1707 goto skb_done; 1708 } 1709 1710 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) { 1711 struct sk_buff *dskb; 1712 1713 dskb = skb_peek(&csk->receive_queue); 1714 if (!dskb) { 1715 pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx," 1716 " plen %u, NO data.\n", 1717 csk, skb, skb->len, 1718 cxgbi_skcb_flags(skb), 1719 cxgbi_skcb_rx_pdulen(skb)); 1720 err = -EIO; 1721 goto skb_done; 1722 } 1723 __skb_unlink(dskb, &csk->receive_queue); 1724 1725 err = skb_read_pdu_data(conn, skb, dskb, 0); 1726 if (err < 0) 1727 pr_err("data, csk 0x%p, skb 0x%p,%u, " 1728 "f 0x%lx, plen %u, dskb 0x%p," 1729 "%u.\n", 1730 csk, skb, skb->len, 1731 cxgbi_skcb_flags(skb), 1732 cxgbi_skcb_rx_pdulen(skb), 1733 dskb, dskb->len); 1734 __kfree_skb(dskb); 1735 } else 1736 err = skb_read_pdu_data(conn, skb, skb, 0); 1737 } 1738 skb_done: 1739 __kfree_skb(skb); 1740 1741 if (err < 0) 1742 break; 1743 } 1744 1745 log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read); 1746 if (read) { 1747 csk->copied_seq += read; 1748 csk_return_rx_credits(csk, read); 1749 conn->rxdata_octets += read; 1750 } 1751 1752 if (err < 0) { 1753 pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n", 1754 csk, conn, err, read); 1755 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1756 } 1757 } 1758 EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready); 1759 1760 static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt, 1761 unsigned int offset, unsigned int *off, 1762 struct scatterlist **sgp) 1763 { 1764 int i; 1765 struct scatterlist *sg; 1766 1767 for_each_sg(sgl, sg, sgcnt, i) { 1768 if (offset < sg->length) { 1769 *off = offset; 1770 *sgp = sg; 1771 return 0; 1772 } 1773 offset -= sg->length; 1774 } 1775 return -EFAULT; 1776 } 1777 1778 static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset, 1779 unsigned int dlen, struct page_frag *frags, 1780 int frag_max) 1781 { 1782 unsigned int datalen = dlen; 1783 unsigned int sglen = sg->length - sgoffset; 1784 struct page *page = sg_page(sg); 1785 int i; 1786 1787 i = 0; 1788 do { 1789 unsigned int copy; 1790 1791 if (!sglen) { 1792 sg = sg_next(sg); 1793 if (!sg) { 1794 pr_warn("sg %d NULL, len %u/%u.\n", 1795 i, datalen, dlen); 1796 return -EINVAL; 1797 } 1798 sgoffset = 0; 1799 sglen = sg->length; 1800 page = sg_page(sg); 1801 1802 } 1803 copy = min(datalen, sglen); 1804 if (i && page == frags[i - 1].page && 1805 sgoffset + sg->offset == 1806 frags[i - 1].offset + frags[i - 1].size) { 1807 frags[i - 1].size += copy; 1808 } else { 1809 if (i >= frag_max) { 1810 pr_warn("too many pages %u, dlen %u.\n", 1811 frag_max, dlen); 1812 return -EINVAL; 1813 } 1814 1815 frags[i].page = page; 1816 frags[i].offset = sg->offset + sgoffset; 1817 frags[i].size = copy; 1818 i++; 1819 } 1820 datalen -= copy; 1821 sgoffset += copy; 1822 sglen -= copy; 1823 } while (datalen); 1824 1825 return i; 1826 } 1827 1828 int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 opcode) 1829 { 1830 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 1831 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1832 struct cxgbi_device *cdev = cconn->chba->cdev; 1833 struct iscsi_conn *conn = task->conn; 1834 struct iscsi_tcp_task *tcp_task = task->dd_data; 1835 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1836 struct scsi_cmnd *sc = task->sc; 1837 int headroom = SKB_TX_ISCSI_PDU_HEADER_MAX; 1838 1839 tcp_task->dd_data = tdata; 1840 task->hdr = NULL; 1841 1842 if (SKB_MAX_HEAD(cdev->skb_tx_rsvd) > (512 * MAX_SKB_FRAGS) && 1843 (opcode == ISCSI_OP_SCSI_DATA_OUT || 1844 (opcode == ISCSI_OP_SCSI_CMD && 1845 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE)))) 1846 /* data could goes into skb head */ 1847 headroom += min_t(unsigned int, 1848 SKB_MAX_HEAD(cdev->skb_tx_rsvd), 1849 conn->max_xmit_dlength); 1850 1851 tdata->skb = alloc_skb(cdev->skb_tx_rsvd + headroom, GFP_ATOMIC); 1852 if (!tdata->skb) { 1853 struct cxgbi_sock *csk = cconn->cep->csk; 1854 struct net_device *ndev = cdev->ports[csk->port_id]; 1855 ndev->stats.tx_dropped++; 1856 return -ENOMEM; 1857 } 1858 1859 skb_reserve(tdata->skb, cdev->skb_tx_rsvd); 1860 task->hdr = (struct iscsi_hdr *)tdata->skb->data; 1861 task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX; /* BHS + AHS */ 1862 1863 /* data_out uses scsi_cmd's itt */ 1864 if (opcode != ISCSI_OP_SCSI_DATA_OUT) 1865 task_reserve_itt(task, &task->hdr->itt); 1866 1867 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 1868 "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n", 1869 task, opcode, tdata->skb, cdev->skb_tx_rsvd, headroom, 1870 conn->max_xmit_dlength, ntohl(task->hdr->itt)); 1871 1872 return 0; 1873 } 1874 EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu); 1875 1876 static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc) 1877 { 1878 if (hcrc || dcrc) { 1879 u8 submode = 0; 1880 1881 if (hcrc) 1882 submode |= 1; 1883 if (dcrc) 1884 submode |= 2; 1885 cxgbi_skcb_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode; 1886 } else 1887 cxgbi_skcb_ulp_mode(skb) = 0; 1888 } 1889 1890 int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset, 1891 unsigned int count) 1892 { 1893 struct iscsi_conn *conn = task->conn; 1894 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1895 struct sk_buff *skb = tdata->skb; 1896 unsigned int datalen = count; 1897 int i, padlen = iscsi_padding(count); 1898 struct page *pg; 1899 1900 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 1901 "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n", 1902 task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK, 1903 ntohl(task->cmdsn), ntohl(task->hdr->itt), offset, count); 1904 1905 skb_put(skb, task->hdr_len); 1906 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0); 1907 if (!count) 1908 return 0; 1909 1910 if (task->sc) { 1911 struct scsi_data_buffer *sdb = scsi_out(task->sc); 1912 struct scatterlist *sg = NULL; 1913 int err; 1914 1915 tdata->offset = offset; 1916 tdata->count = count; 1917 err = sgl_seek_offset( 1918 sdb->table.sgl, sdb->table.nents, 1919 tdata->offset, &tdata->sgoffset, &sg); 1920 if (err < 0) { 1921 pr_warn("tpdu, sgl %u, bad offset %u/%u.\n", 1922 sdb->table.nents, tdata->offset, sdb->length); 1923 return err; 1924 } 1925 err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count, 1926 tdata->frags, MAX_PDU_FRAGS); 1927 if (err < 0) { 1928 pr_warn("tpdu, sgl %u, bad offset %u + %u.\n", 1929 sdb->table.nents, tdata->offset, tdata->count); 1930 return err; 1931 } 1932 tdata->nr_frags = err; 1933 1934 if (tdata->nr_frags > MAX_SKB_FRAGS || 1935 (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) { 1936 char *dst = skb->data + task->hdr_len; 1937 struct page_frag *frag = tdata->frags; 1938 1939 /* data fits in the skb's headroom */ 1940 for (i = 0; i < tdata->nr_frags; i++, frag++) { 1941 char *src = kmap_atomic(frag->page); 1942 1943 memcpy(dst, src+frag->offset, frag->size); 1944 dst += frag->size; 1945 kunmap_atomic(src); 1946 } 1947 if (padlen) { 1948 memset(dst, 0, padlen); 1949 padlen = 0; 1950 } 1951 skb_put(skb, count + padlen); 1952 } else { 1953 /* data fit into frag_list */ 1954 for (i = 0; i < tdata->nr_frags; i++) { 1955 __skb_fill_page_desc(skb, i, 1956 tdata->frags[i].page, 1957 tdata->frags[i].offset, 1958 tdata->frags[i].size); 1959 skb_frag_ref(skb, i); 1960 } 1961 skb_shinfo(skb)->nr_frags = tdata->nr_frags; 1962 skb->len += count; 1963 skb->data_len += count; 1964 skb->truesize += count; 1965 } 1966 1967 } else { 1968 pg = virt_to_page(task->data); 1969 1970 get_page(pg); 1971 skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data), 1972 count); 1973 skb->len += count; 1974 skb->data_len += count; 1975 skb->truesize += count; 1976 } 1977 1978 if (padlen) { 1979 i = skb_shinfo(skb)->nr_frags; 1980 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 1981 virt_to_page(padding), offset_in_page(padding), 1982 padlen); 1983 1984 skb->data_len += padlen; 1985 skb->truesize += padlen; 1986 skb->len += padlen; 1987 } 1988 1989 return 0; 1990 } 1991 EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu); 1992 1993 int cxgbi_conn_xmit_pdu(struct iscsi_task *task) 1994 { 1995 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 1996 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1997 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1998 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 1999 struct sk_buff *skb = tdata->skb; 2000 struct cxgbi_sock *csk = NULL; 2001 unsigned int datalen; 2002 int err; 2003 2004 if (!skb) { 2005 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2006 "task 0x%p, skb NULL.\n", task); 2007 return 0; 2008 } 2009 2010 if (cconn && cconn->cep) 2011 csk = cconn->cep->csk; 2012 if (!csk) { 2013 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2014 "task 0x%p, csk gone.\n", task); 2015 return -EPIPE; 2016 } 2017 2018 datalen = skb->data_len; 2019 tdata->skb = NULL; 2020 2021 /* write ppod first if using ofldq to write ppod */ 2022 if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_VALID) { 2023 struct cxgbi_ppm *ppm = csk->cdev->cdev2ppm(csk->cdev); 2024 2025 ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_VALID; 2026 if (csk->cdev->csk_ddp_set_map(ppm, csk, ttinfo) < 0) 2027 pr_err("task 0x%p, ppod writing using ofldq failed.\n", 2028 task); 2029 /* continue. Let fl get the data */ 2030 } 2031 2032 err = cxgbi_sock_send_pdus(cconn->cep->csk, skb); 2033 if (err > 0) { 2034 int pdulen = err; 2035 2036 log_debug(1 << CXGBI_DBG_PDU_TX, 2037 "task 0x%p,0x%p, skb 0x%p, len %u/%u, rv %d.\n", 2038 task, task->sc, skb, skb->len, skb->data_len, err); 2039 2040 if (task->conn->hdrdgst_en) 2041 pdulen += ISCSI_DIGEST_SIZE; 2042 2043 if (datalen && task->conn->datadgst_en) 2044 pdulen += ISCSI_DIGEST_SIZE; 2045 2046 task->conn->txdata_octets += pdulen; 2047 return 0; 2048 } 2049 2050 if (err == -EAGAIN || err == -ENOBUFS) { 2051 log_debug(1 << CXGBI_DBG_PDU_TX, 2052 "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n", 2053 task, skb, skb->len, skb->data_len, err); 2054 /* reset skb to send when we are called again */ 2055 tdata->skb = skb; 2056 return err; 2057 } 2058 2059 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2060 "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n", 2061 task->itt, skb, skb->len, skb->data_len, err); 2062 2063 kfree_skb(skb); 2064 2065 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err); 2066 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED); 2067 return err; 2068 } 2069 EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu); 2070 2071 void cxgbi_cleanup_task(struct iscsi_task *task) 2072 { 2073 struct iscsi_tcp_task *tcp_task = task->dd_data; 2074 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2075 2076 log_debug(1 << CXGBI_DBG_ISCSI, 2077 "task 0x%p, skb 0x%p, itt 0x%x.\n", 2078 task, tdata->skb, task->hdr_itt); 2079 2080 tcp_task->dd_data = NULL; 2081 /* never reached the xmit task callout */ 2082 if (tdata->skb) 2083 __kfree_skb(tdata->skb); 2084 2085 task_release_itt(task, task->hdr_itt); 2086 memset(tdata, 0, sizeof(*tdata)); 2087 2088 iscsi_tcp_cleanup_task(task); 2089 } 2090 EXPORT_SYMBOL_GPL(cxgbi_cleanup_task); 2091 2092 void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn, 2093 struct iscsi_stats *stats) 2094 { 2095 struct iscsi_conn *conn = cls_conn->dd_data; 2096 2097 stats->txdata_octets = conn->txdata_octets; 2098 stats->rxdata_octets = conn->rxdata_octets; 2099 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 2100 stats->dataout_pdus = conn->dataout_pdus_cnt; 2101 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 2102 stats->datain_pdus = conn->datain_pdus_cnt; 2103 stats->r2t_pdus = conn->r2t_pdus_cnt; 2104 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 2105 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 2106 stats->digest_err = 0; 2107 stats->timeout_err = 0; 2108 stats->custom_length = 1; 2109 strcpy(stats->custom[0].desc, "eh_abort_cnt"); 2110 stats->custom[0].value = conn->eh_abort_cnt; 2111 } 2112 EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats); 2113 2114 static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn) 2115 { 2116 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2117 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2118 struct cxgbi_device *cdev = cconn->chba->cdev; 2119 unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd); 2120 unsigned int max_def = 512 * MAX_SKB_FRAGS; 2121 unsigned int max = max(max_def, headroom); 2122 2123 max = min(cconn->chba->cdev->tx_max_size, max); 2124 if (conn->max_xmit_dlength) 2125 conn->max_xmit_dlength = min(conn->max_xmit_dlength, max); 2126 else 2127 conn->max_xmit_dlength = max; 2128 cxgbi_align_pdu_size(conn->max_xmit_dlength); 2129 2130 return 0; 2131 } 2132 2133 static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn) 2134 { 2135 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2136 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2137 unsigned int max = cconn->chba->cdev->rx_max_size; 2138 2139 cxgbi_align_pdu_size(max); 2140 2141 if (conn->max_recv_dlength) { 2142 if (conn->max_recv_dlength > max) { 2143 pr_err("MaxRecvDataSegmentLength %u > %u.\n", 2144 conn->max_recv_dlength, max); 2145 return -EINVAL; 2146 } 2147 conn->max_recv_dlength = min(conn->max_recv_dlength, max); 2148 cxgbi_align_pdu_size(conn->max_recv_dlength); 2149 } else 2150 conn->max_recv_dlength = max; 2151 2152 return 0; 2153 } 2154 2155 int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn, 2156 enum iscsi_param param, char *buf, int buflen) 2157 { 2158 struct iscsi_conn *conn = cls_conn->dd_data; 2159 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2160 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2161 struct cxgbi_sock *csk = cconn->cep->csk; 2162 int err; 2163 2164 log_debug(1 << CXGBI_DBG_ISCSI, 2165 "cls_conn 0x%p, param %d, buf(%d) %s.\n", 2166 cls_conn, param, buflen, buf); 2167 2168 switch (param) { 2169 case ISCSI_PARAM_HDRDGST_EN: 2170 err = iscsi_set_param(cls_conn, param, buf, buflen); 2171 if (!err && conn->hdrdgst_en) 2172 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2173 conn->hdrdgst_en, 2174 conn->datadgst_en, 0); 2175 break; 2176 case ISCSI_PARAM_DATADGST_EN: 2177 err = iscsi_set_param(cls_conn, param, buf, buflen); 2178 if (!err && conn->datadgst_en) 2179 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2180 conn->hdrdgst_en, 2181 conn->datadgst_en, 0); 2182 break; 2183 case ISCSI_PARAM_MAX_R2T: 2184 return iscsi_tcp_set_max_r2t(conn, buf); 2185 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2186 err = iscsi_set_param(cls_conn, param, buf, buflen); 2187 if (!err) 2188 err = cxgbi_conn_max_recv_dlength(conn); 2189 break; 2190 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2191 err = iscsi_set_param(cls_conn, param, buf, buflen); 2192 if (!err) 2193 err = cxgbi_conn_max_xmit_dlength(conn); 2194 break; 2195 default: 2196 return iscsi_set_param(cls_conn, param, buf, buflen); 2197 } 2198 return err; 2199 } 2200 EXPORT_SYMBOL_GPL(cxgbi_set_conn_param); 2201 2202 static inline int csk_print_port(struct cxgbi_sock *csk, char *buf) 2203 { 2204 int len; 2205 2206 cxgbi_sock_get(csk); 2207 len = sprintf(buf, "%hu\n", ntohs(csk->daddr.sin_port)); 2208 cxgbi_sock_put(csk); 2209 2210 return len; 2211 } 2212 2213 static inline int csk_print_ip(struct cxgbi_sock *csk, char *buf) 2214 { 2215 int len; 2216 2217 cxgbi_sock_get(csk); 2218 if (csk->csk_family == AF_INET) 2219 len = sprintf(buf, "%pI4", 2220 &csk->daddr.sin_addr.s_addr); 2221 else 2222 len = sprintf(buf, "%pI6", 2223 &csk->daddr6.sin6_addr); 2224 2225 cxgbi_sock_put(csk); 2226 2227 return len; 2228 } 2229 2230 int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param, 2231 char *buf) 2232 { 2233 struct cxgbi_endpoint *cep = ep->dd_data; 2234 struct cxgbi_sock *csk; 2235 int len; 2236 2237 log_debug(1 << CXGBI_DBG_ISCSI, 2238 "cls_conn 0x%p, param %d.\n", ep, param); 2239 2240 switch (param) { 2241 case ISCSI_PARAM_CONN_PORT: 2242 case ISCSI_PARAM_CONN_ADDRESS: 2243 if (!cep) 2244 return -ENOTCONN; 2245 2246 csk = cep->csk; 2247 if (!csk) 2248 return -ENOTCONN; 2249 2250 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 2251 &csk->daddr, param, buf); 2252 default: 2253 return -ENOSYS; 2254 } 2255 return len; 2256 } 2257 EXPORT_SYMBOL_GPL(cxgbi_get_ep_param); 2258 2259 struct iscsi_cls_conn * 2260 cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid) 2261 { 2262 struct iscsi_cls_conn *cls_conn; 2263 struct iscsi_conn *conn; 2264 struct iscsi_tcp_conn *tcp_conn; 2265 struct cxgbi_conn *cconn; 2266 2267 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid); 2268 if (!cls_conn) 2269 return NULL; 2270 2271 conn = cls_conn->dd_data; 2272 tcp_conn = conn->dd_data; 2273 cconn = tcp_conn->dd_data; 2274 cconn->iconn = conn; 2275 2276 log_debug(1 << CXGBI_DBG_ISCSI, 2277 "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n", 2278 cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn); 2279 2280 return cls_conn; 2281 } 2282 EXPORT_SYMBOL_GPL(cxgbi_create_conn); 2283 2284 int cxgbi_bind_conn(struct iscsi_cls_session *cls_session, 2285 struct iscsi_cls_conn *cls_conn, 2286 u64 transport_eph, int is_leading) 2287 { 2288 struct iscsi_conn *conn = cls_conn->dd_data; 2289 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2290 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2291 struct cxgbi_ppm *ppm; 2292 struct iscsi_endpoint *ep; 2293 struct cxgbi_endpoint *cep; 2294 struct cxgbi_sock *csk; 2295 int err; 2296 2297 ep = iscsi_lookup_endpoint(transport_eph); 2298 if (!ep) 2299 return -EINVAL; 2300 2301 /* setup ddp pagesize */ 2302 cep = ep->dd_data; 2303 csk = cep->csk; 2304 2305 ppm = csk->cdev->cdev2ppm(csk->cdev); 2306 err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid, 2307 ppm->tformat.pgsz_idx_dflt, 0); 2308 if (err < 0) 2309 return err; 2310 2311 err = iscsi_conn_bind(cls_session, cls_conn, is_leading); 2312 if (err) 2313 return -EINVAL; 2314 2315 /* calculate the tag idx bits needed for this conn based on cmds_max */ 2316 cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1; 2317 2318 write_lock_bh(&csk->callback_lock); 2319 csk->user_data = conn; 2320 cconn->chba = cep->chba; 2321 cconn->cep = cep; 2322 cep->cconn = cconn; 2323 write_unlock_bh(&csk->callback_lock); 2324 2325 cxgbi_conn_max_xmit_dlength(conn); 2326 cxgbi_conn_max_recv_dlength(conn); 2327 2328 log_debug(1 << CXGBI_DBG_ISCSI, 2329 "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n", 2330 cls_session, cls_conn, ep, cconn, csk); 2331 /* init recv engine */ 2332 iscsi_tcp_hdr_recv_prep(tcp_conn); 2333 2334 return 0; 2335 } 2336 EXPORT_SYMBOL_GPL(cxgbi_bind_conn); 2337 2338 struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep, 2339 u16 cmds_max, u16 qdepth, 2340 u32 initial_cmdsn) 2341 { 2342 struct cxgbi_endpoint *cep; 2343 struct cxgbi_hba *chba; 2344 struct Scsi_Host *shost; 2345 struct iscsi_cls_session *cls_session; 2346 struct iscsi_session *session; 2347 2348 if (!ep) { 2349 pr_err("missing endpoint.\n"); 2350 return NULL; 2351 } 2352 2353 cep = ep->dd_data; 2354 chba = cep->chba; 2355 shost = chba->shost; 2356 2357 BUG_ON(chba != iscsi_host_priv(shost)); 2358 2359 cls_session = iscsi_session_setup(chba->cdev->itp, shost, 2360 cmds_max, 0, 2361 sizeof(struct iscsi_tcp_task) + 2362 sizeof(struct cxgbi_task_data), 2363 initial_cmdsn, ISCSI_MAX_TARGET); 2364 if (!cls_session) 2365 return NULL; 2366 2367 session = cls_session->dd_data; 2368 if (iscsi_tcp_r2tpool_alloc(session)) 2369 goto remove_session; 2370 2371 log_debug(1 << CXGBI_DBG_ISCSI, 2372 "ep 0x%p, cls sess 0x%p.\n", ep, cls_session); 2373 return cls_session; 2374 2375 remove_session: 2376 iscsi_session_teardown(cls_session); 2377 return NULL; 2378 } 2379 EXPORT_SYMBOL_GPL(cxgbi_create_session); 2380 2381 void cxgbi_destroy_session(struct iscsi_cls_session *cls_session) 2382 { 2383 log_debug(1 << CXGBI_DBG_ISCSI, 2384 "cls sess 0x%p.\n", cls_session); 2385 2386 iscsi_tcp_r2tpool_free(cls_session->dd_data); 2387 iscsi_session_teardown(cls_session); 2388 } 2389 EXPORT_SYMBOL_GPL(cxgbi_destroy_session); 2390 2391 int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2392 char *buf, int buflen) 2393 { 2394 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2395 2396 if (!chba->ndev) { 2397 shost_printk(KERN_ERR, shost, "Could not get host param. " 2398 "netdev for host not set.\n"); 2399 return -ENODEV; 2400 } 2401 2402 log_debug(1 << CXGBI_DBG_ISCSI, 2403 "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n", 2404 shost, chba, chba->ndev->name, param, buflen, buf); 2405 2406 switch (param) { 2407 case ISCSI_HOST_PARAM_IPADDRESS: 2408 { 2409 __be32 addr = in_aton(buf); 2410 log_debug(1 << CXGBI_DBG_ISCSI, 2411 "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr); 2412 cxgbi_set_iscsi_ipv4(chba, addr); 2413 return 0; 2414 } 2415 case ISCSI_HOST_PARAM_HWADDRESS: 2416 case ISCSI_HOST_PARAM_NETDEV_NAME: 2417 return 0; 2418 default: 2419 return iscsi_host_set_param(shost, param, buf, buflen); 2420 } 2421 } 2422 EXPORT_SYMBOL_GPL(cxgbi_set_host_param); 2423 2424 int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2425 char *buf) 2426 { 2427 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2428 int len = 0; 2429 2430 if (!chba->ndev) { 2431 shost_printk(KERN_ERR, shost, "Could not get host param. " 2432 "netdev for host not set.\n"); 2433 return -ENODEV; 2434 } 2435 2436 log_debug(1 << CXGBI_DBG_ISCSI, 2437 "shost 0x%p, hba 0x%p,%s, param %d.\n", 2438 shost, chba, chba->ndev->name, param); 2439 2440 switch (param) { 2441 case ISCSI_HOST_PARAM_HWADDRESS: 2442 len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6); 2443 break; 2444 case ISCSI_HOST_PARAM_NETDEV_NAME: 2445 len = sprintf(buf, "%s\n", chba->ndev->name); 2446 break; 2447 case ISCSI_HOST_PARAM_IPADDRESS: 2448 { 2449 struct cxgbi_sock *csk = find_sock_on_port(chba->cdev, 2450 chba->port_id); 2451 if (csk) { 2452 len = sprintf(buf, "%pIS", 2453 (struct sockaddr *)&csk->saddr); 2454 } 2455 log_debug(1 << CXGBI_DBG_ISCSI, 2456 "hba %s, addr %s.\n", chba->ndev->name, buf); 2457 break; 2458 } 2459 default: 2460 return iscsi_host_get_param(shost, param, buf); 2461 } 2462 2463 return len; 2464 } 2465 EXPORT_SYMBOL_GPL(cxgbi_get_host_param); 2466 2467 struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost, 2468 struct sockaddr *dst_addr, 2469 int non_blocking) 2470 { 2471 struct iscsi_endpoint *ep; 2472 struct cxgbi_endpoint *cep; 2473 struct cxgbi_hba *hba = NULL; 2474 struct cxgbi_sock *csk; 2475 int err = -EINVAL; 2476 2477 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2478 "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n", 2479 shost, non_blocking, dst_addr); 2480 2481 if (shost) { 2482 hba = iscsi_host_priv(shost); 2483 if (!hba) { 2484 pr_info("shost 0x%p, priv NULL.\n", shost); 2485 goto err_out; 2486 } 2487 } 2488 2489 if (dst_addr->sa_family == AF_INET) { 2490 csk = cxgbi_check_route(dst_addr); 2491 #if IS_ENABLED(CONFIG_IPV6) 2492 } else if (dst_addr->sa_family == AF_INET6) { 2493 csk = cxgbi_check_route6(dst_addr); 2494 #endif 2495 } else { 2496 pr_info("address family 0x%x NOT supported.\n", 2497 dst_addr->sa_family); 2498 err = -EAFNOSUPPORT; 2499 return (struct iscsi_endpoint *)ERR_PTR(err); 2500 } 2501 2502 if (IS_ERR(csk)) 2503 return (struct iscsi_endpoint *)csk; 2504 cxgbi_sock_get(csk); 2505 2506 if (!hba) 2507 hba = csk->cdev->hbas[csk->port_id]; 2508 else if (hba != csk->cdev->hbas[csk->port_id]) { 2509 pr_info("Could not connect through requested host %u" 2510 "hba 0x%p != 0x%p (%u).\n", 2511 shost->host_no, hba, 2512 csk->cdev->hbas[csk->port_id], csk->port_id); 2513 err = -ENOSPC; 2514 goto release_conn; 2515 } 2516 2517 err = sock_get_port(csk); 2518 if (err) 2519 goto release_conn; 2520 2521 cxgbi_sock_set_state(csk, CTP_CONNECTING); 2522 err = csk->cdev->csk_init_act_open(csk); 2523 if (err) 2524 goto release_conn; 2525 2526 if (cxgbi_sock_is_closing(csk)) { 2527 err = -ENOSPC; 2528 pr_info("csk 0x%p is closing.\n", csk); 2529 goto release_conn; 2530 } 2531 2532 ep = iscsi_create_endpoint(sizeof(*cep)); 2533 if (!ep) { 2534 err = -ENOMEM; 2535 pr_info("iscsi alloc ep, OOM.\n"); 2536 goto release_conn; 2537 } 2538 2539 cep = ep->dd_data; 2540 cep->csk = csk; 2541 cep->chba = hba; 2542 2543 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2544 "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n", 2545 ep, cep, csk, hba, hba->ndev->name); 2546 return ep; 2547 2548 release_conn: 2549 cxgbi_sock_put(csk); 2550 cxgbi_sock_closed(csk); 2551 err_out: 2552 return ERR_PTR(err); 2553 } 2554 EXPORT_SYMBOL_GPL(cxgbi_ep_connect); 2555 2556 int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) 2557 { 2558 struct cxgbi_endpoint *cep = ep->dd_data; 2559 struct cxgbi_sock *csk = cep->csk; 2560 2561 if (!cxgbi_sock_is_established(csk)) 2562 return 0; 2563 return 1; 2564 } 2565 EXPORT_SYMBOL_GPL(cxgbi_ep_poll); 2566 2567 void cxgbi_ep_disconnect(struct iscsi_endpoint *ep) 2568 { 2569 struct cxgbi_endpoint *cep = ep->dd_data; 2570 struct cxgbi_conn *cconn = cep->cconn; 2571 struct cxgbi_sock *csk = cep->csk; 2572 2573 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2574 "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n", 2575 ep, cep, cconn, csk, csk->state, csk->flags); 2576 2577 if (cconn && cconn->iconn) { 2578 iscsi_suspend_tx(cconn->iconn); 2579 write_lock_bh(&csk->callback_lock); 2580 cep->csk->user_data = NULL; 2581 cconn->cep = NULL; 2582 write_unlock_bh(&csk->callback_lock); 2583 } 2584 iscsi_destroy_endpoint(ep); 2585 2586 if (likely(csk->state >= CTP_ESTABLISHED)) 2587 need_active_close(csk); 2588 else 2589 cxgbi_sock_closed(csk); 2590 2591 cxgbi_sock_put(csk); 2592 } 2593 EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect); 2594 2595 int cxgbi_iscsi_init(struct iscsi_transport *itp, 2596 struct scsi_transport_template **stt) 2597 { 2598 *stt = iscsi_register_transport(itp); 2599 if (*stt == NULL) { 2600 pr_err("unable to register %s transport 0x%p.\n", 2601 itp->name, itp); 2602 return -ENODEV; 2603 } 2604 log_debug(1 << CXGBI_DBG_ISCSI, 2605 "%s, registered iscsi transport 0x%p.\n", 2606 itp->name, stt); 2607 return 0; 2608 } 2609 EXPORT_SYMBOL_GPL(cxgbi_iscsi_init); 2610 2611 void cxgbi_iscsi_cleanup(struct iscsi_transport *itp, 2612 struct scsi_transport_template **stt) 2613 { 2614 if (*stt) { 2615 log_debug(1 << CXGBI_DBG_ISCSI, 2616 "de-register transport 0x%p, %s, stt 0x%p.\n", 2617 itp, itp->name, *stt); 2618 *stt = NULL; 2619 iscsi_unregister_transport(itp); 2620 } 2621 } 2622 EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup); 2623 2624 umode_t cxgbi_attr_is_visible(int param_type, int param) 2625 { 2626 switch (param_type) { 2627 case ISCSI_HOST_PARAM: 2628 switch (param) { 2629 case ISCSI_HOST_PARAM_NETDEV_NAME: 2630 case ISCSI_HOST_PARAM_HWADDRESS: 2631 case ISCSI_HOST_PARAM_IPADDRESS: 2632 case ISCSI_HOST_PARAM_INITIATOR_NAME: 2633 return S_IRUGO; 2634 default: 2635 return 0; 2636 } 2637 case ISCSI_PARAM: 2638 switch (param) { 2639 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2640 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2641 case ISCSI_PARAM_HDRDGST_EN: 2642 case ISCSI_PARAM_DATADGST_EN: 2643 case ISCSI_PARAM_CONN_ADDRESS: 2644 case ISCSI_PARAM_CONN_PORT: 2645 case ISCSI_PARAM_EXP_STATSN: 2646 case ISCSI_PARAM_PERSISTENT_ADDRESS: 2647 case ISCSI_PARAM_PERSISTENT_PORT: 2648 case ISCSI_PARAM_PING_TMO: 2649 case ISCSI_PARAM_RECV_TMO: 2650 case ISCSI_PARAM_INITIAL_R2T_EN: 2651 case ISCSI_PARAM_MAX_R2T: 2652 case ISCSI_PARAM_IMM_DATA_EN: 2653 case ISCSI_PARAM_FIRST_BURST: 2654 case ISCSI_PARAM_MAX_BURST: 2655 case ISCSI_PARAM_PDU_INORDER_EN: 2656 case ISCSI_PARAM_DATASEQ_INORDER_EN: 2657 case ISCSI_PARAM_ERL: 2658 case ISCSI_PARAM_TARGET_NAME: 2659 case ISCSI_PARAM_TPGT: 2660 case ISCSI_PARAM_USERNAME: 2661 case ISCSI_PARAM_PASSWORD: 2662 case ISCSI_PARAM_USERNAME_IN: 2663 case ISCSI_PARAM_PASSWORD_IN: 2664 case ISCSI_PARAM_FAST_ABORT: 2665 case ISCSI_PARAM_ABORT_TMO: 2666 case ISCSI_PARAM_LU_RESET_TMO: 2667 case ISCSI_PARAM_TGT_RESET_TMO: 2668 case ISCSI_PARAM_IFACE_NAME: 2669 case ISCSI_PARAM_INITIATOR_NAME: 2670 return S_IRUGO; 2671 default: 2672 return 0; 2673 } 2674 } 2675 2676 return 0; 2677 } 2678 EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible); 2679 2680 static int __init libcxgbi_init_module(void) 2681 { 2682 pr_info("%s", version); 2683 return 0; 2684 } 2685 2686 static void __exit libcxgbi_exit_module(void) 2687 { 2688 cxgbi_device_unregister_all(0xFF); 2689 return; 2690 } 2691 2692 module_init(libcxgbi_init_module); 2693 module_exit(libcxgbi_exit_module); 2694