1 /* 2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver. 3 * 4 * Copyright (c) 2010 Chelsio Communications, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Written by: Karen Xie (kxie@chelsio.com) 11 * Written by: Rakesh Ranjan (rranjan@chelsio.com) 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 15 16 #include <linux/skbuff.h> 17 #include <linux/crypto.h> 18 #include <linux/scatterlist.h> 19 #include <linux/pci.h> 20 #include <scsi/scsi.h> 21 #include <scsi/scsi_cmnd.h> 22 #include <scsi/scsi_host.h> 23 #include <linux/if_vlan.h> 24 #include <linux/inet.h> 25 #include <net/dst.h> 26 #include <net/route.h> 27 #include <linux/inetdevice.h> /* ip_dev_find */ 28 #include <linux/module.h> 29 #include <net/tcp.h> 30 31 static unsigned int dbg_level; 32 33 #include "libcxgbi.h" 34 35 #define DRV_MODULE_NAME "libcxgbi" 36 #define DRV_MODULE_DESC "Chelsio iSCSI driver library" 37 #define DRV_MODULE_VERSION "0.9.0" 38 #define DRV_MODULE_RELDATE "Jun. 2010" 39 40 MODULE_AUTHOR("Chelsio Communications, Inc."); 41 MODULE_DESCRIPTION(DRV_MODULE_DESC); 42 MODULE_VERSION(DRV_MODULE_VERSION); 43 MODULE_LICENSE("GPL"); 44 45 module_param(dbg_level, uint, 0644); 46 MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)"); 47 48 49 /* 50 * cxgbi device management 51 * maintains a list of the cxgbi devices 52 */ 53 static LIST_HEAD(cdev_list); 54 static DEFINE_MUTEX(cdev_mutex); 55 56 int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base, 57 unsigned int max_conn) 58 { 59 struct cxgbi_ports_map *pmap = &cdev->pmap; 60 61 pmap->port_csk = cxgbi_alloc_big_mem(max_conn * 62 sizeof(struct cxgbi_sock *), 63 GFP_KERNEL); 64 if (!pmap->port_csk) { 65 pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn); 66 return -ENOMEM; 67 } 68 69 pmap->max_connect = max_conn; 70 pmap->sport_base = base; 71 spin_lock_init(&pmap->lock); 72 return 0; 73 } 74 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create); 75 76 void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev) 77 { 78 struct cxgbi_ports_map *pmap = &cdev->pmap; 79 struct cxgbi_sock *csk; 80 int i; 81 82 for (i = 0; i < pmap->max_connect; i++) { 83 if (pmap->port_csk[i]) { 84 csk = pmap->port_csk[i]; 85 pmap->port_csk[i] = NULL; 86 log_debug(1 << CXGBI_DBG_SOCK, 87 "csk 0x%p, cdev 0x%p, offload down.\n", 88 csk, cdev); 89 spin_lock_bh(&csk->lock); 90 cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN); 91 cxgbi_sock_closed(csk); 92 spin_unlock_bh(&csk->lock); 93 cxgbi_sock_put(csk); 94 } 95 } 96 } 97 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup); 98 99 static inline void cxgbi_device_destroy(struct cxgbi_device *cdev) 100 { 101 log_debug(1 << CXGBI_DBG_DEV, 102 "cdev 0x%p, p# %u.\n", cdev, cdev->nports); 103 cxgbi_hbas_remove(cdev); 104 cxgbi_device_portmap_cleanup(cdev); 105 if (cdev->dev_ddp_cleanup) 106 cdev->dev_ddp_cleanup(cdev); 107 else 108 cxgbi_ddp_cleanup(cdev); 109 if (cdev->ddp) 110 cxgbi_ddp_cleanup(cdev); 111 if (cdev->pmap.max_connect) 112 cxgbi_free_big_mem(cdev->pmap.port_csk); 113 kfree(cdev); 114 } 115 116 struct cxgbi_device *cxgbi_device_register(unsigned int extra, 117 unsigned int nports) 118 { 119 struct cxgbi_device *cdev; 120 121 cdev = kzalloc(sizeof(*cdev) + extra + nports * 122 (sizeof(struct cxgbi_hba *) + 123 sizeof(struct net_device *)), 124 GFP_KERNEL); 125 if (!cdev) { 126 pr_warn("nport %d, OOM.\n", nports); 127 return NULL; 128 } 129 cdev->ports = (struct net_device **)(cdev + 1); 130 cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports * 131 sizeof(struct net_device *)); 132 if (extra) 133 cdev->dd_data = ((char *)cdev->hbas) + 134 nports * sizeof(struct cxgbi_hba *); 135 spin_lock_init(&cdev->pmap.lock); 136 137 mutex_lock(&cdev_mutex); 138 list_add_tail(&cdev->list_head, &cdev_list); 139 mutex_unlock(&cdev_mutex); 140 141 log_debug(1 << CXGBI_DBG_DEV, 142 "cdev 0x%p, p# %u.\n", cdev, nports); 143 return cdev; 144 } 145 EXPORT_SYMBOL_GPL(cxgbi_device_register); 146 147 void cxgbi_device_unregister(struct cxgbi_device *cdev) 148 { 149 log_debug(1 << CXGBI_DBG_DEV, 150 "cdev 0x%p, p# %u,%s.\n", 151 cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : ""); 152 mutex_lock(&cdev_mutex); 153 list_del(&cdev->list_head); 154 mutex_unlock(&cdev_mutex); 155 cxgbi_device_destroy(cdev); 156 } 157 EXPORT_SYMBOL_GPL(cxgbi_device_unregister); 158 159 void cxgbi_device_unregister_all(unsigned int flag) 160 { 161 struct cxgbi_device *cdev, *tmp; 162 163 mutex_lock(&cdev_mutex); 164 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 165 if ((cdev->flags & flag) == flag) { 166 log_debug(1 << CXGBI_DBG_DEV, 167 "cdev 0x%p, p# %u,%s.\n", 168 cdev, cdev->nports, cdev->nports ? 169 cdev->ports[0]->name : ""); 170 list_del(&cdev->list_head); 171 cxgbi_device_destroy(cdev); 172 } 173 } 174 mutex_unlock(&cdev_mutex); 175 } 176 EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all); 177 178 struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev) 179 { 180 struct cxgbi_device *cdev, *tmp; 181 182 mutex_lock(&cdev_mutex); 183 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 184 if (cdev->lldev == lldev) { 185 mutex_unlock(&cdev_mutex); 186 return cdev; 187 } 188 } 189 mutex_unlock(&cdev_mutex); 190 log_debug(1 << CXGBI_DBG_DEV, 191 "lldev 0x%p, NO match found.\n", lldev); 192 return NULL; 193 } 194 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev); 195 196 static struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev, 197 int *port) 198 { 199 struct net_device *vdev = NULL; 200 struct cxgbi_device *cdev, *tmp; 201 int i; 202 203 if (ndev->priv_flags & IFF_802_1Q_VLAN) { 204 vdev = ndev; 205 ndev = vlan_dev_real_dev(ndev); 206 log_debug(1 << CXGBI_DBG_DEV, 207 "vlan dev %s -> %s.\n", vdev->name, ndev->name); 208 } 209 210 mutex_lock(&cdev_mutex); 211 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 212 for (i = 0; i < cdev->nports; i++) { 213 if (ndev == cdev->ports[i]) { 214 cdev->hbas[i]->vdev = vdev; 215 mutex_unlock(&cdev_mutex); 216 if (port) 217 *port = i; 218 return cdev; 219 } 220 } 221 } 222 mutex_unlock(&cdev_mutex); 223 log_debug(1 << CXGBI_DBG_DEV, 224 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name); 225 return NULL; 226 } 227 228 void cxgbi_hbas_remove(struct cxgbi_device *cdev) 229 { 230 int i; 231 struct cxgbi_hba *chba; 232 233 log_debug(1 << CXGBI_DBG_DEV, 234 "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 235 236 for (i = 0; i < cdev->nports; i++) { 237 chba = cdev->hbas[i]; 238 if (chba) { 239 cdev->hbas[i] = NULL; 240 iscsi_host_remove(chba->shost); 241 pci_dev_put(cdev->pdev); 242 iscsi_host_free(chba->shost); 243 } 244 } 245 } 246 EXPORT_SYMBOL_GPL(cxgbi_hbas_remove); 247 248 int cxgbi_hbas_add(struct cxgbi_device *cdev, unsigned int max_lun, 249 unsigned int max_id, struct scsi_host_template *sht, 250 struct scsi_transport_template *stt) 251 { 252 struct cxgbi_hba *chba; 253 struct Scsi_Host *shost; 254 int i, err; 255 256 log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 257 258 for (i = 0; i < cdev->nports; i++) { 259 shost = iscsi_host_alloc(sht, sizeof(*chba), 1); 260 if (!shost) { 261 pr_info("0x%p, p%d, %s, host alloc failed.\n", 262 cdev, i, cdev->ports[i]->name); 263 err = -ENOMEM; 264 goto err_out; 265 } 266 267 shost->transportt = stt; 268 shost->max_lun = max_lun; 269 shost->max_id = max_id; 270 shost->max_channel = 0; 271 shost->max_cmd_len = 16; 272 273 chba = iscsi_host_priv(shost); 274 chba->cdev = cdev; 275 chba->ndev = cdev->ports[i]; 276 chba->shost = shost; 277 278 log_debug(1 << CXGBI_DBG_DEV, 279 "cdev 0x%p, p#%d %s: chba 0x%p.\n", 280 cdev, i, cdev->ports[i]->name, chba); 281 282 pci_dev_get(cdev->pdev); 283 err = iscsi_host_add(shost, &cdev->pdev->dev); 284 if (err) { 285 pr_info("cdev 0x%p, p#%d %s, host add failed.\n", 286 cdev, i, cdev->ports[i]->name); 287 pci_dev_put(cdev->pdev); 288 scsi_host_put(shost); 289 goto err_out; 290 } 291 292 cdev->hbas[i] = chba; 293 } 294 295 return 0; 296 297 err_out: 298 cxgbi_hbas_remove(cdev); 299 return err; 300 } 301 EXPORT_SYMBOL_GPL(cxgbi_hbas_add); 302 303 /* 304 * iSCSI offload 305 * 306 * - source port management 307 * To find a free source port in the port allocation map we use a very simple 308 * rotor scheme to look for the next free port. 309 * 310 * If a source port has been specified make sure that it doesn't collide with 311 * our normal source port allocation map. If it's outside the range of our 312 * allocation/deallocation scheme just let them use it. 313 * 314 * If the source port is outside our allocation range, the caller is 315 * responsible for keeping track of their port usage. 316 */ 317 static int sock_get_port(struct cxgbi_sock *csk) 318 { 319 struct cxgbi_device *cdev = csk->cdev; 320 struct cxgbi_ports_map *pmap = &cdev->pmap; 321 unsigned int start; 322 int idx; 323 324 if (!pmap->max_connect) { 325 pr_err("cdev 0x%p, p#%u %s, NO port map.\n", 326 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 327 return -EADDRNOTAVAIL; 328 } 329 330 if (csk->saddr.sin_port) { 331 pr_err("source port NON-ZERO %u.\n", 332 ntohs(csk->saddr.sin_port)); 333 return -EADDRINUSE; 334 } 335 336 spin_lock_bh(&pmap->lock); 337 if (pmap->used >= pmap->max_connect) { 338 spin_unlock_bh(&pmap->lock); 339 pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n", 340 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 341 return -EADDRNOTAVAIL; 342 } 343 344 start = idx = pmap->next; 345 do { 346 if (++idx >= pmap->max_connect) 347 idx = 0; 348 if (!pmap->port_csk[idx]) { 349 pmap->used++; 350 csk->saddr.sin_port = 351 htons(pmap->sport_base + idx); 352 pmap->next = idx; 353 pmap->port_csk[idx] = csk; 354 spin_unlock_bh(&pmap->lock); 355 cxgbi_sock_get(csk); 356 log_debug(1 << CXGBI_DBG_SOCK, 357 "cdev 0x%p, p#%u %s, p %u, %u.\n", 358 cdev, csk->port_id, 359 cdev->ports[csk->port_id]->name, 360 pmap->sport_base + idx, pmap->next); 361 return 0; 362 } 363 } while (idx != start); 364 spin_unlock_bh(&pmap->lock); 365 366 /* should not happen */ 367 pr_warn("cdev 0x%p, p#%u %s, next %u?\n", 368 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 369 pmap->next); 370 return -EADDRNOTAVAIL; 371 } 372 373 static void sock_put_port(struct cxgbi_sock *csk) 374 { 375 struct cxgbi_device *cdev = csk->cdev; 376 struct cxgbi_ports_map *pmap = &cdev->pmap; 377 378 if (csk->saddr.sin_port) { 379 int idx = ntohs(csk->saddr.sin_port) - pmap->sport_base; 380 381 csk->saddr.sin_port = 0; 382 if (idx < 0 || idx >= pmap->max_connect) { 383 pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n", 384 cdev, csk->port_id, 385 cdev->ports[csk->port_id]->name, 386 ntohs(csk->saddr.sin_port)); 387 return; 388 } 389 390 spin_lock_bh(&pmap->lock); 391 pmap->port_csk[idx] = NULL; 392 pmap->used--; 393 spin_unlock_bh(&pmap->lock); 394 395 log_debug(1 << CXGBI_DBG_SOCK, 396 "cdev 0x%p, p#%u %s, release %u.\n", 397 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 398 pmap->sport_base + idx); 399 400 cxgbi_sock_put(csk); 401 } 402 } 403 404 /* 405 * iscsi tcp connection 406 */ 407 void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk) 408 { 409 if (csk->cpl_close) { 410 kfree_skb(csk->cpl_close); 411 csk->cpl_close = NULL; 412 } 413 if (csk->cpl_abort_req) { 414 kfree_skb(csk->cpl_abort_req); 415 csk->cpl_abort_req = NULL; 416 } 417 if (csk->cpl_abort_rpl) { 418 kfree_skb(csk->cpl_abort_rpl); 419 csk->cpl_abort_rpl = NULL; 420 } 421 } 422 EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs); 423 424 static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev) 425 { 426 struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO); 427 428 if (!csk) { 429 pr_info("alloc csk %zu failed.\n", sizeof(*csk)); 430 return NULL; 431 } 432 433 if (cdev->csk_alloc_cpls(csk) < 0) { 434 pr_info("csk 0x%p, alloc cpls failed.\n", csk); 435 kfree(csk); 436 return NULL; 437 } 438 439 spin_lock_init(&csk->lock); 440 kref_init(&csk->refcnt); 441 skb_queue_head_init(&csk->receive_queue); 442 skb_queue_head_init(&csk->write_queue); 443 setup_timer(&csk->retry_timer, NULL, (unsigned long)csk); 444 rwlock_init(&csk->callback_lock); 445 csk->cdev = cdev; 446 csk->flags = 0; 447 cxgbi_sock_set_state(csk, CTP_CLOSED); 448 449 log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk); 450 451 return csk; 452 } 453 454 static struct rtable *find_route_ipv4(struct flowi4 *fl4, 455 __be32 saddr, __be32 daddr, 456 __be16 sport, __be16 dport, u8 tos) 457 { 458 struct rtable *rt; 459 460 rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr, 461 dport, sport, IPPROTO_TCP, tos, 0); 462 if (IS_ERR(rt)) 463 return NULL; 464 465 return rt; 466 } 467 468 static struct cxgbi_sock *cxgbi_check_route(struct sockaddr *dst_addr) 469 { 470 struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr; 471 struct dst_entry *dst; 472 struct net_device *ndev; 473 struct cxgbi_device *cdev; 474 struct rtable *rt = NULL; 475 struct neighbour *n; 476 struct flowi4 fl4; 477 struct cxgbi_sock *csk = NULL; 478 unsigned int mtu = 0; 479 int port = 0xFFFF; 480 int err = 0; 481 482 if (daddr->sin_family != AF_INET) { 483 pr_info("address family 0x%x NOT supported.\n", 484 daddr->sin_family); 485 err = -EAFNOSUPPORT; 486 goto err_out; 487 } 488 489 rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0, daddr->sin_port, 0); 490 if (!rt) { 491 pr_info("no route to ipv4 0x%x, port %u.\n", 492 daddr->sin_addr.s_addr, daddr->sin_port); 493 err = -ENETUNREACH; 494 goto err_out; 495 } 496 dst = &rt->dst; 497 n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr); 498 if (!n) { 499 err = -ENODEV; 500 goto rel_rt; 501 } 502 ndev = n->dev; 503 504 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) { 505 pr_info("multi-cast route %pI4, port %u, dev %s.\n", 506 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 507 ndev->name); 508 err = -ENETUNREACH; 509 goto rel_neigh; 510 } 511 512 if (ndev->flags & IFF_LOOPBACK) { 513 ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr); 514 mtu = ndev->mtu; 515 pr_info("rt dev %s, loopback -> %s, mtu %u.\n", 516 n->dev->name, ndev->name, mtu); 517 } 518 519 cdev = cxgbi_device_find_by_netdev(ndev, &port); 520 if (!cdev) { 521 pr_info("dst %pI4, %s, NOT cxgbi device.\n", 522 &daddr->sin_addr.s_addr, ndev->name); 523 err = -ENETUNREACH; 524 goto rel_neigh; 525 } 526 log_debug(1 << CXGBI_DBG_SOCK, 527 "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n", 528 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 529 port, ndev->name, cdev); 530 531 csk = cxgbi_sock_create(cdev); 532 if (!csk) { 533 err = -ENOMEM; 534 goto rel_neigh; 535 } 536 csk->cdev = cdev; 537 csk->port_id = port; 538 csk->mtu = mtu; 539 csk->dst = dst; 540 csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr; 541 csk->daddr.sin_port = daddr->sin_port; 542 csk->daddr.sin_family = daddr->sin_family; 543 csk->saddr.sin_addr.s_addr = fl4.saddr; 544 neigh_release(n); 545 546 return csk; 547 548 rel_neigh: 549 neigh_release(n); 550 551 rel_rt: 552 ip_rt_put(rt); 553 if (csk) 554 cxgbi_sock_closed(csk); 555 err_out: 556 return ERR_PTR(err); 557 } 558 559 void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn, 560 unsigned int opt) 561 { 562 csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn; 563 dst_confirm(csk->dst); 564 smp_mb(); 565 cxgbi_sock_set_state(csk, CTP_ESTABLISHED); 566 } 567 EXPORT_SYMBOL_GPL(cxgbi_sock_established); 568 569 static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk) 570 { 571 log_debug(1 << CXGBI_DBG_SOCK, 572 "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n", 573 csk, csk->state, csk->flags, csk->user_data); 574 575 if (csk->state != CTP_ESTABLISHED) { 576 read_lock_bh(&csk->callback_lock); 577 if (csk->user_data) 578 iscsi_conn_failure(csk->user_data, 579 ISCSI_ERR_CONN_FAILED); 580 read_unlock_bh(&csk->callback_lock); 581 } 582 } 583 584 void cxgbi_sock_closed(struct cxgbi_sock *csk) 585 { 586 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 587 csk, (csk)->state, (csk)->flags, (csk)->tid); 588 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 589 if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED) 590 return; 591 if (csk->saddr.sin_port) 592 sock_put_port(csk); 593 if (csk->dst) 594 dst_release(csk->dst); 595 csk->cdev->csk_release_offload_resources(csk); 596 cxgbi_sock_set_state(csk, CTP_CLOSED); 597 cxgbi_inform_iscsi_conn_closing(csk); 598 cxgbi_sock_put(csk); 599 } 600 EXPORT_SYMBOL_GPL(cxgbi_sock_closed); 601 602 static void need_active_close(struct cxgbi_sock *csk) 603 { 604 int data_lost; 605 int close_req = 0; 606 607 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 608 csk, (csk)->state, (csk)->flags, (csk)->tid); 609 spin_lock_bh(&csk->lock); 610 dst_confirm(csk->dst); 611 data_lost = skb_queue_len(&csk->receive_queue); 612 __skb_queue_purge(&csk->receive_queue); 613 614 if (csk->state == CTP_ACTIVE_OPEN) 615 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 616 else if (csk->state == CTP_ESTABLISHED) { 617 close_req = 1; 618 cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE); 619 } else if (csk->state == CTP_PASSIVE_CLOSE) { 620 close_req = 1; 621 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 622 } 623 624 if (close_req) { 625 if (data_lost) 626 csk->cdev->csk_send_abort_req(csk); 627 else 628 csk->cdev->csk_send_close_req(csk); 629 } 630 631 spin_unlock_bh(&csk->lock); 632 } 633 634 void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno) 635 { 636 pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n", 637 csk, csk->state, csk->flags, 638 &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port, 639 &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port, 640 errno); 641 642 cxgbi_sock_set_state(csk, CTP_CONNECTING); 643 csk->err = errno; 644 cxgbi_sock_closed(csk); 645 } 646 EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open); 647 648 void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb) 649 { 650 struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk; 651 652 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 653 csk, (csk)->state, (csk)->flags, (csk)->tid); 654 cxgbi_sock_get(csk); 655 spin_lock_bh(&csk->lock); 656 if (csk->state == CTP_ACTIVE_OPEN) 657 cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH); 658 spin_unlock_bh(&csk->lock); 659 cxgbi_sock_put(csk); 660 __kfree_skb(skb); 661 } 662 EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure); 663 664 void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk) 665 { 666 cxgbi_sock_get(csk); 667 spin_lock_bh(&csk->lock); 668 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 669 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_RCVD)) 670 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD); 671 else { 672 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_RCVD); 673 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING); 674 if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) 675 pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n", 676 csk, csk->state, csk->flags, csk->tid); 677 cxgbi_sock_closed(csk); 678 } 679 } 680 spin_unlock_bh(&csk->lock); 681 cxgbi_sock_put(csk); 682 } 683 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl); 684 685 void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk) 686 { 687 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 688 csk, (csk)->state, (csk)->flags, (csk)->tid); 689 cxgbi_sock_get(csk); 690 spin_lock_bh(&csk->lock); 691 692 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 693 goto done; 694 695 switch (csk->state) { 696 case CTP_ESTABLISHED: 697 cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE); 698 break; 699 case CTP_ACTIVE_CLOSE: 700 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 701 break; 702 case CTP_CLOSE_WAIT_1: 703 cxgbi_sock_closed(csk); 704 break; 705 case CTP_ABORTING: 706 break; 707 default: 708 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 709 csk, csk->state, csk->flags, csk->tid); 710 } 711 cxgbi_inform_iscsi_conn_closing(csk); 712 done: 713 spin_unlock_bh(&csk->lock); 714 cxgbi_sock_put(csk); 715 } 716 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close); 717 718 void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt) 719 { 720 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 721 csk, (csk)->state, (csk)->flags, (csk)->tid); 722 cxgbi_sock_get(csk); 723 spin_lock_bh(&csk->lock); 724 725 csk->snd_una = snd_nxt - 1; 726 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 727 goto done; 728 729 switch (csk->state) { 730 case CTP_ACTIVE_CLOSE: 731 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1); 732 break; 733 case CTP_CLOSE_WAIT_1: 734 case CTP_CLOSE_WAIT_2: 735 cxgbi_sock_closed(csk); 736 break; 737 case CTP_ABORTING: 738 break; 739 default: 740 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 741 csk, csk->state, csk->flags, csk->tid); 742 } 743 done: 744 spin_unlock_bh(&csk->lock); 745 cxgbi_sock_put(csk); 746 } 747 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl); 748 749 void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits, 750 unsigned int snd_una, int seq_chk) 751 { 752 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 753 "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n", 754 csk, csk->state, csk->flags, csk->tid, credits, 755 csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk); 756 757 spin_lock_bh(&csk->lock); 758 759 csk->wr_cred += credits; 760 if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred) 761 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred; 762 763 while (credits) { 764 struct sk_buff *p = cxgbi_sock_peek_wr(csk); 765 766 if (unlikely(!p)) { 767 pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n", 768 csk, csk->state, csk->flags, csk->tid, credits, 769 csk->wr_cred, csk->wr_una_cred); 770 break; 771 } 772 773 if (unlikely(credits < p->csum)) { 774 pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n", 775 csk, csk->state, csk->flags, csk->tid, 776 credits, csk->wr_cred, csk->wr_una_cred, 777 p->csum); 778 p->csum -= credits; 779 break; 780 } else { 781 cxgbi_sock_dequeue_wr(csk); 782 credits -= p->csum; 783 kfree_skb(p); 784 } 785 } 786 787 cxgbi_sock_check_wr_invariants(csk); 788 789 if (seq_chk) { 790 if (unlikely(before(snd_una, csk->snd_una))) { 791 pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.", 792 csk, csk->state, csk->flags, csk->tid, snd_una, 793 csk->snd_una); 794 goto done; 795 } 796 797 if (csk->snd_una != snd_una) { 798 csk->snd_una = snd_una; 799 dst_confirm(csk->dst); 800 } 801 } 802 803 if (skb_queue_len(&csk->write_queue)) { 804 if (csk->cdev->csk_push_tx_frames(csk, 0)) 805 cxgbi_conn_tx_open(csk); 806 } else 807 cxgbi_conn_tx_open(csk); 808 done: 809 spin_unlock_bh(&csk->lock); 810 } 811 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack); 812 813 static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk, 814 unsigned short mtu) 815 { 816 int i = 0; 817 818 while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu) 819 ++i; 820 821 return i; 822 } 823 824 unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu) 825 { 826 unsigned int idx; 827 struct dst_entry *dst = csk->dst; 828 829 csk->advmss = dst_metric_advmss(dst); 830 831 if (csk->advmss > pmtu - 40) 832 csk->advmss = pmtu - 40; 833 if (csk->advmss < csk->cdev->mtus[0] - 40) 834 csk->advmss = csk->cdev->mtus[0] - 40; 835 idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40); 836 837 return idx; 838 } 839 EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss); 840 841 void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb) 842 { 843 cxgbi_skcb_tcp_seq(skb) = csk->write_seq; 844 __skb_queue_tail(&csk->write_queue, skb); 845 } 846 EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail); 847 848 void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk) 849 { 850 struct sk_buff *skb; 851 852 while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL) 853 kfree_skb(skb); 854 } 855 EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue); 856 857 void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk) 858 { 859 int pending = cxgbi_sock_count_pending_wrs(csk); 860 861 if (unlikely(csk->wr_cred + pending != csk->wr_max_cred)) 862 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n", 863 csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred); 864 } 865 EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants); 866 867 static int cxgbi_sock_send_pdus(struct cxgbi_sock *csk, struct sk_buff *skb) 868 { 869 struct cxgbi_device *cdev = csk->cdev; 870 struct sk_buff *next; 871 int err, copied = 0; 872 873 spin_lock_bh(&csk->lock); 874 875 if (csk->state != CTP_ESTABLISHED) { 876 log_debug(1 << CXGBI_DBG_PDU_TX, 877 "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n", 878 csk, csk->state, csk->flags, csk->tid); 879 err = -EAGAIN; 880 goto out_err; 881 } 882 883 if (csk->err) { 884 log_debug(1 << CXGBI_DBG_PDU_TX, 885 "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n", 886 csk, csk->state, csk->flags, csk->tid, csk->err); 887 err = -EPIPE; 888 goto out_err; 889 } 890 891 if (csk->write_seq - csk->snd_una >= cdev->snd_win) { 892 log_debug(1 << CXGBI_DBG_PDU_TX, 893 "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n", 894 csk, csk->state, csk->flags, csk->tid, csk->write_seq, 895 csk->snd_una, cdev->snd_win); 896 err = -ENOBUFS; 897 goto out_err; 898 } 899 900 while (skb) { 901 int frags = skb_shinfo(skb)->nr_frags + 902 (skb->len != skb->data_len); 903 904 if (unlikely(skb_headroom(skb) < cdev->skb_tx_rsvd)) { 905 pr_err("csk 0x%p, skb head %u < %u.\n", 906 csk, skb_headroom(skb), cdev->skb_tx_rsvd); 907 err = -EINVAL; 908 goto out_err; 909 } 910 911 if (frags >= SKB_WR_LIST_SIZE) { 912 pr_err("csk 0x%p, frags %d, %u,%u >%u.\n", 913 csk, skb_shinfo(skb)->nr_frags, skb->len, 914 skb->data_len, (uint)(SKB_WR_LIST_SIZE)); 915 err = -EINVAL; 916 goto out_err; 917 } 918 919 next = skb->next; 920 skb->next = NULL; 921 cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR); 922 cxgbi_sock_skb_entail(csk, skb); 923 copied += skb->len; 924 csk->write_seq += skb->len + 925 cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 926 skb = next; 927 } 928 done: 929 if (likely(skb_queue_len(&csk->write_queue))) 930 cdev->csk_push_tx_frames(csk, 1); 931 spin_unlock_bh(&csk->lock); 932 return copied; 933 934 out_err: 935 if (copied == 0 && err == -EPIPE) 936 copied = csk->err ? csk->err : -EPIPE; 937 else 938 copied = err; 939 goto done; 940 } 941 942 /* 943 * Direct Data Placement - 944 * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted 945 * final destination host-memory buffers based on the Initiator Task Tag (ITT) 946 * in Data-In or Target Task Tag (TTT) in Data-Out PDUs. 947 * The host memory address is programmed into h/w in the format of pagepod 948 * entries. 949 * The location of the pagepod entry is encoded into ddp tag which is used as 950 * the base for ITT/TTT. 951 */ 952 953 static unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4}; 954 static unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16}; 955 static unsigned char page_idx = DDP_PGIDX_MAX; 956 957 static unsigned char sw_tag_idx_bits; 958 static unsigned char sw_tag_age_bits; 959 960 /* 961 * Direct-Data Placement page size adjustment 962 */ 963 static int ddp_adjust_page_table(void) 964 { 965 int i; 966 unsigned int base_order, order; 967 968 if (PAGE_SIZE < (1UL << ddp_page_shift[0])) { 969 pr_info("PAGE_SIZE 0x%lx too small, min 0x%lx\n", 970 PAGE_SIZE, 1UL << ddp_page_shift[0]); 971 return -EINVAL; 972 } 973 974 base_order = get_order(1UL << ddp_page_shift[0]); 975 order = get_order(1UL << PAGE_SHIFT); 976 977 for (i = 0; i < DDP_PGIDX_MAX; i++) { 978 /* first is the kernel page size, then just doubling */ 979 ddp_page_order[i] = order - base_order + i; 980 ddp_page_shift[i] = PAGE_SHIFT + i; 981 } 982 return 0; 983 } 984 985 static int ddp_find_page_index(unsigned long pgsz) 986 { 987 int i; 988 989 for (i = 0; i < DDP_PGIDX_MAX; i++) { 990 if (pgsz == (1UL << ddp_page_shift[i])) 991 return i; 992 } 993 pr_info("ddp page size %lu not supported.\n", pgsz); 994 return DDP_PGIDX_MAX; 995 } 996 997 static void ddp_setup_host_page_size(void) 998 { 999 if (page_idx == DDP_PGIDX_MAX) { 1000 page_idx = ddp_find_page_index(PAGE_SIZE); 1001 1002 if (page_idx == DDP_PGIDX_MAX) { 1003 pr_info("system PAGE %lu, update hw.\n", PAGE_SIZE); 1004 if (ddp_adjust_page_table() < 0) { 1005 pr_info("PAGE %lu, disable ddp.\n", PAGE_SIZE); 1006 return; 1007 } 1008 page_idx = ddp_find_page_index(PAGE_SIZE); 1009 } 1010 pr_info("system PAGE %lu, ddp idx %u.\n", PAGE_SIZE, page_idx); 1011 } 1012 } 1013 1014 void cxgbi_ddp_page_size_factor(int *pgsz_factor) 1015 { 1016 int i; 1017 1018 for (i = 0; i < DDP_PGIDX_MAX; i++) 1019 pgsz_factor[i] = ddp_page_order[i]; 1020 } 1021 EXPORT_SYMBOL_GPL(cxgbi_ddp_page_size_factor); 1022 1023 /* 1024 * DDP setup & teardown 1025 */ 1026 1027 void cxgbi_ddp_ppod_set(struct cxgbi_pagepod *ppod, 1028 struct cxgbi_pagepod_hdr *hdr, 1029 struct cxgbi_gather_list *gl, unsigned int gidx) 1030 { 1031 int i; 1032 1033 memcpy(ppod, hdr, sizeof(*hdr)); 1034 for (i = 0; i < (PPOD_PAGES_MAX + 1); i++, gidx++) { 1035 ppod->addr[i] = gidx < gl->nelem ? 1036 cpu_to_be64(gl->phys_addr[gidx]) : 0ULL; 1037 } 1038 } 1039 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_set); 1040 1041 void cxgbi_ddp_ppod_clear(struct cxgbi_pagepod *ppod) 1042 { 1043 memset(ppod, 0, sizeof(*ppod)); 1044 } 1045 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_clear); 1046 1047 static inline int ddp_find_unused_entries(struct cxgbi_ddp_info *ddp, 1048 unsigned int start, unsigned int max, 1049 unsigned int count, 1050 struct cxgbi_gather_list *gl) 1051 { 1052 unsigned int i, j, k; 1053 1054 /* not enough entries */ 1055 if ((max - start) < count) { 1056 log_debug(1 << CXGBI_DBG_DDP, 1057 "NOT enough entries %u+%u < %u.\n", start, count, max); 1058 return -EBUSY; 1059 } 1060 1061 max -= count; 1062 spin_lock(&ddp->map_lock); 1063 for (i = start; i < max;) { 1064 for (j = 0, k = i; j < count; j++, k++) { 1065 if (ddp->gl_map[k]) 1066 break; 1067 } 1068 if (j == count) { 1069 for (j = 0, k = i; j < count; j++, k++) 1070 ddp->gl_map[k] = gl; 1071 spin_unlock(&ddp->map_lock); 1072 return i; 1073 } 1074 i += j + 1; 1075 } 1076 spin_unlock(&ddp->map_lock); 1077 log_debug(1 << CXGBI_DBG_DDP, 1078 "NO suitable entries %u available.\n", count); 1079 return -EBUSY; 1080 } 1081 1082 static inline void ddp_unmark_entries(struct cxgbi_ddp_info *ddp, 1083 int start, int count) 1084 { 1085 spin_lock(&ddp->map_lock); 1086 memset(&ddp->gl_map[start], 0, 1087 count * sizeof(struct cxgbi_gather_list *)); 1088 spin_unlock(&ddp->map_lock); 1089 } 1090 1091 static inline void ddp_gl_unmap(struct pci_dev *pdev, 1092 struct cxgbi_gather_list *gl) 1093 { 1094 int i; 1095 1096 for (i = 0; i < gl->nelem; i++) 1097 dma_unmap_page(&pdev->dev, gl->phys_addr[i], PAGE_SIZE, 1098 PCI_DMA_FROMDEVICE); 1099 } 1100 1101 static inline int ddp_gl_map(struct pci_dev *pdev, 1102 struct cxgbi_gather_list *gl) 1103 { 1104 int i; 1105 1106 for (i = 0; i < gl->nelem; i++) { 1107 gl->phys_addr[i] = dma_map_page(&pdev->dev, gl->pages[i], 0, 1108 PAGE_SIZE, 1109 PCI_DMA_FROMDEVICE); 1110 if (unlikely(dma_mapping_error(&pdev->dev, gl->phys_addr[i]))) { 1111 log_debug(1 << CXGBI_DBG_DDP, 1112 "page %d 0x%p, 0x%p dma mapping err.\n", 1113 i, gl->pages[i], pdev); 1114 goto unmap; 1115 } 1116 } 1117 return i; 1118 unmap: 1119 if (i) { 1120 unsigned int nelem = gl->nelem; 1121 1122 gl->nelem = i; 1123 ddp_gl_unmap(pdev, gl); 1124 gl->nelem = nelem; 1125 } 1126 return -EINVAL; 1127 } 1128 1129 static void ddp_release_gl(struct cxgbi_gather_list *gl, 1130 struct pci_dev *pdev) 1131 { 1132 ddp_gl_unmap(pdev, gl); 1133 kfree(gl); 1134 } 1135 1136 static struct cxgbi_gather_list *ddp_make_gl(unsigned int xferlen, 1137 struct scatterlist *sgl, 1138 unsigned int sgcnt, 1139 struct pci_dev *pdev, 1140 gfp_t gfp) 1141 { 1142 struct cxgbi_gather_list *gl; 1143 struct scatterlist *sg = sgl; 1144 struct page *sgpage = sg_page(sg); 1145 unsigned int sglen = sg->length; 1146 unsigned int sgoffset = sg->offset; 1147 unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >> 1148 PAGE_SHIFT; 1149 int i = 1, j = 0; 1150 1151 if (xferlen < DDP_THRESHOLD) { 1152 log_debug(1 << CXGBI_DBG_DDP, 1153 "xfer %u < threshold %u, no ddp.\n", 1154 xferlen, DDP_THRESHOLD); 1155 return NULL; 1156 } 1157 1158 gl = kzalloc(sizeof(struct cxgbi_gather_list) + 1159 npages * (sizeof(dma_addr_t) + 1160 sizeof(struct page *)), gfp); 1161 if (!gl) { 1162 log_debug(1 << CXGBI_DBG_DDP, 1163 "xfer %u, %u pages, OOM.\n", xferlen, npages); 1164 return NULL; 1165 } 1166 1167 log_debug(1 << CXGBI_DBG_DDP, 1168 "xfer %u, sgl %u, gl max %u.\n", xferlen, sgcnt, npages); 1169 1170 gl->pages = (struct page **)&gl->phys_addr[npages]; 1171 gl->nelem = npages; 1172 gl->length = xferlen; 1173 gl->offset = sgoffset; 1174 gl->pages[0] = sgpage; 1175 1176 for (i = 1, sg = sg_next(sgl), j = 0; i < sgcnt; 1177 i++, sg = sg_next(sg)) { 1178 struct page *page = sg_page(sg); 1179 1180 if (sgpage == page && sg->offset == sgoffset + sglen) 1181 sglen += sg->length; 1182 else { 1183 /* make sure the sgl is fit for ddp: 1184 * each has the same page size, and 1185 * all of the middle pages are used completely 1186 */ 1187 if ((j && sgoffset) || ((i != sgcnt - 1) && 1188 ((sglen + sgoffset) & ~PAGE_MASK))) { 1189 log_debug(1 << CXGBI_DBG_DDP, 1190 "page %d/%u, %u + %u.\n", 1191 i, sgcnt, sgoffset, sglen); 1192 goto error_out; 1193 } 1194 1195 j++; 1196 if (j == gl->nelem || sg->offset) { 1197 log_debug(1 << CXGBI_DBG_DDP, 1198 "page %d/%u, offset %u.\n", 1199 j, gl->nelem, sg->offset); 1200 goto error_out; 1201 } 1202 gl->pages[j] = page; 1203 sglen = sg->length; 1204 sgoffset = sg->offset; 1205 sgpage = page; 1206 } 1207 } 1208 gl->nelem = ++j; 1209 1210 if (ddp_gl_map(pdev, gl) < 0) 1211 goto error_out; 1212 1213 return gl; 1214 1215 error_out: 1216 kfree(gl); 1217 return NULL; 1218 } 1219 1220 static void ddp_tag_release(struct cxgbi_hba *chba, u32 tag) 1221 { 1222 struct cxgbi_device *cdev = chba->cdev; 1223 struct cxgbi_ddp_info *ddp = cdev->ddp; 1224 u32 idx; 1225 1226 idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask; 1227 if (idx < ddp->nppods) { 1228 struct cxgbi_gather_list *gl = ddp->gl_map[idx]; 1229 unsigned int npods; 1230 1231 if (!gl || !gl->nelem) { 1232 pr_warn("tag 0x%x, idx %u, gl 0x%p, %u.\n", 1233 tag, idx, gl, gl ? gl->nelem : 0); 1234 return; 1235 } 1236 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT; 1237 log_debug(1 << CXGBI_DBG_DDP, 1238 "tag 0x%x, release idx %u, npods %u.\n", 1239 tag, idx, npods); 1240 cdev->csk_ddp_clear(chba, tag, idx, npods); 1241 ddp_unmark_entries(ddp, idx, npods); 1242 ddp_release_gl(gl, ddp->pdev); 1243 } else 1244 pr_warn("tag 0x%x, idx %u > max %u.\n", tag, idx, ddp->nppods); 1245 } 1246 1247 static int ddp_tag_reserve(struct cxgbi_sock *csk, unsigned int tid, 1248 u32 sw_tag, u32 *tagp, struct cxgbi_gather_list *gl, 1249 gfp_t gfp) 1250 { 1251 struct cxgbi_device *cdev = csk->cdev; 1252 struct cxgbi_ddp_info *ddp = cdev->ddp; 1253 struct cxgbi_tag_format *tformat = &cdev->tag_format; 1254 struct cxgbi_pagepod_hdr hdr; 1255 unsigned int npods; 1256 int idx = -1; 1257 int err = -ENOMEM; 1258 u32 tag; 1259 1260 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT; 1261 if (ddp->idx_last == ddp->nppods) 1262 idx = ddp_find_unused_entries(ddp, 0, ddp->nppods, 1263 npods, gl); 1264 else { 1265 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1, 1266 ddp->nppods, npods, 1267 gl); 1268 if (idx < 0 && ddp->idx_last >= npods) { 1269 idx = ddp_find_unused_entries(ddp, 0, 1270 min(ddp->idx_last + npods, ddp->nppods), 1271 npods, gl); 1272 } 1273 } 1274 if (idx < 0) { 1275 log_debug(1 << CXGBI_DBG_DDP, 1276 "xferlen %u, gl %u, npods %u NO DDP.\n", 1277 gl->length, gl->nelem, npods); 1278 return idx; 1279 } 1280 1281 tag = cxgbi_ddp_tag_base(tformat, sw_tag); 1282 tag |= idx << PPOD_IDX_SHIFT; 1283 1284 hdr.rsvd = 0; 1285 hdr.vld_tid = htonl(PPOD_VALID_FLAG | PPOD_TID(tid)); 1286 hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask); 1287 hdr.max_offset = htonl(gl->length); 1288 hdr.page_offset = htonl(gl->offset); 1289 1290 err = cdev->csk_ddp_set(csk, &hdr, idx, npods, gl); 1291 if (err < 0) 1292 goto unmark_entries; 1293 1294 ddp->idx_last = idx; 1295 log_debug(1 << CXGBI_DBG_DDP, 1296 "xfer %u, gl %u,%u, tid 0x%x, tag 0x%x->0x%x(%u,%u).\n", 1297 gl->length, gl->nelem, gl->offset, tid, sw_tag, tag, idx, 1298 npods); 1299 *tagp = tag; 1300 return 0; 1301 1302 unmark_entries: 1303 ddp_unmark_entries(ddp, idx, npods); 1304 return err; 1305 } 1306 1307 int cxgbi_ddp_reserve(struct cxgbi_sock *csk, unsigned int *tagp, 1308 unsigned int sw_tag, unsigned int xferlen, 1309 struct scatterlist *sgl, unsigned int sgcnt, gfp_t gfp) 1310 { 1311 struct cxgbi_device *cdev = csk->cdev; 1312 struct cxgbi_tag_format *tformat = &cdev->tag_format; 1313 struct cxgbi_gather_list *gl; 1314 int err; 1315 1316 if (page_idx >= DDP_PGIDX_MAX || !cdev->ddp || 1317 xferlen < DDP_THRESHOLD) { 1318 log_debug(1 << CXGBI_DBG_DDP, 1319 "pgidx %u, xfer %u, NO ddp.\n", page_idx, xferlen); 1320 return -EINVAL; 1321 } 1322 1323 if (!cxgbi_sw_tag_usable(tformat, sw_tag)) { 1324 log_debug(1 << CXGBI_DBG_DDP, 1325 "sw_tag 0x%x NOT usable.\n", sw_tag); 1326 return -EINVAL; 1327 } 1328 1329 gl = ddp_make_gl(xferlen, sgl, sgcnt, cdev->pdev, gfp); 1330 if (!gl) 1331 return -ENOMEM; 1332 1333 err = ddp_tag_reserve(csk, csk->tid, sw_tag, tagp, gl, gfp); 1334 if (err < 0) 1335 ddp_release_gl(gl, cdev->pdev); 1336 1337 return err; 1338 } 1339 1340 static void ddp_destroy(struct kref *kref) 1341 { 1342 struct cxgbi_ddp_info *ddp = container_of(kref, 1343 struct cxgbi_ddp_info, 1344 refcnt); 1345 struct cxgbi_device *cdev = ddp->cdev; 1346 int i = 0; 1347 1348 pr_info("kref 0, destroy ddp 0x%p, cdev 0x%p.\n", ddp, cdev); 1349 1350 while (i < ddp->nppods) { 1351 struct cxgbi_gather_list *gl = ddp->gl_map[i]; 1352 1353 if (gl) { 1354 int npods = (gl->nelem + PPOD_PAGES_MAX - 1) 1355 >> PPOD_PAGES_SHIFT; 1356 pr_info("cdev 0x%p, ddp %d + %d.\n", cdev, i, npods); 1357 kfree(gl); 1358 i += npods; 1359 } else 1360 i++; 1361 } 1362 cxgbi_free_big_mem(ddp); 1363 } 1364 1365 int cxgbi_ddp_cleanup(struct cxgbi_device *cdev) 1366 { 1367 struct cxgbi_ddp_info *ddp = cdev->ddp; 1368 1369 log_debug(1 << CXGBI_DBG_DDP, 1370 "cdev 0x%p, release ddp 0x%p.\n", cdev, ddp); 1371 cdev->ddp = NULL; 1372 if (ddp) 1373 return kref_put(&ddp->refcnt, ddp_destroy); 1374 return 0; 1375 } 1376 EXPORT_SYMBOL_GPL(cxgbi_ddp_cleanup); 1377 1378 int cxgbi_ddp_init(struct cxgbi_device *cdev, 1379 unsigned int llimit, unsigned int ulimit, 1380 unsigned int max_txsz, unsigned int max_rxsz) 1381 { 1382 struct cxgbi_ddp_info *ddp; 1383 unsigned int ppmax, bits; 1384 1385 ppmax = (ulimit - llimit + 1) >> PPOD_SIZE_SHIFT; 1386 bits = __ilog2_u32(ppmax) + 1; 1387 if (bits > PPOD_IDX_MAX_SIZE) 1388 bits = PPOD_IDX_MAX_SIZE; 1389 ppmax = (1 << (bits - 1)) - 1; 1390 1391 ddp = cxgbi_alloc_big_mem(sizeof(struct cxgbi_ddp_info) + 1392 ppmax * (sizeof(struct cxgbi_gather_list *) + 1393 sizeof(struct sk_buff *)), 1394 GFP_KERNEL); 1395 if (!ddp) { 1396 pr_warn("cdev 0x%p, ddp ppmax %u OOM.\n", cdev, ppmax); 1397 return -ENOMEM; 1398 } 1399 ddp->gl_map = (struct cxgbi_gather_list **)(ddp + 1); 1400 cdev->ddp = ddp; 1401 1402 spin_lock_init(&ddp->map_lock); 1403 kref_init(&ddp->refcnt); 1404 1405 ddp->cdev = cdev; 1406 ddp->pdev = cdev->pdev; 1407 ddp->llimit = llimit; 1408 ddp->ulimit = ulimit; 1409 ddp->max_txsz = min_t(unsigned int, max_txsz, ULP2_MAX_PKT_SIZE); 1410 ddp->max_rxsz = min_t(unsigned int, max_rxsz, ULP2_MAX_PKT_SIZE); 1411 ddp->nppods = ppmax; 1412 ddp->idx_last = ppmax; 1413 ddp->idx_bits = bits; 1414 ddp->idx_mask = (1 << bits) - 1; 1415 ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1; 1416 1417 cdev->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits; 1418 cdev->tag_format.rsvd_bits = ddp->idx_bits; 1419 cdev->tag_format.rsvd_shift = PPOD_IDX_SHIFT; 1420 cdev->tag_format.rsvd_mask = (1 << cdev->tag_format.rsvd_bits) - 1; 1421 1422 pr_info("%s tag format, sw %u, rsvd %u,%u, mask 0x%x.\n", 1423 cdev->ports[0]->name, cdev->tag_format.sw_bits, 1424 cdev->tag_format.rsvd_bits, cdev->tag_format.rsvd_shift, 1425 cdev->tag_format.rsvd_mask); 1426 1427 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1428 ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN); 1429 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1430 ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN); 1431 1432 log_debug(1 << CXGBI_DBG_DDP, 1433 "%s max payload size: %u/%u, %u/%u.\n", 1434 cdev->ports[0]->name, cdev->tx_max_size, ddp->max_txsz, 1435 cdev->rx_max_size, ddp->max_rxsz); 1436 return 0; 1437 } 1438 EXPORT_SYMBOL_GPL(cxgbi_ddp_init); 1439 1440 /* 1441 * APIs interacting with open-iscsi libraries 1442 */ 1443 1444 static unsigned char padding[4]; 1445 1446 static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt) 1447 { 1448 struct scsi_cmnd *sc = task->sc; 1449 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 1450 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1451 struct cxgbi_hba *chba = cconn->chba; 1452 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format; 1453 u32 tag = ntohl((__force u32)hdr_itt); 1454 1455 log_debug(1 << CXGBI_DBG_DDP, 1456 "cdev 0x%p, release tag 0x%x.\n", chba->cdev, tag); 1457 if (sc && 1458 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) && 1459 cxgbi_is_ddp_tag(tformat, tag)) 1460 ddp_tag_release(chba, tag); 1461 } 1462 1463 static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt) 1464 { 1465 struct scsi_cmnd *sc = task->sc; 1466 struct iscsi_conn *conn = task->conn; 1467 struct iscsi_session *sess = conn->session; 1468 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1469 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1470 struct cxgbi_hba *chba = cconn->chba; 1471 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format; 1472 u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt; 1473 u32 tag = 0; 1474 int err = -EINVAL; 1475 1476 if (sc && 1477 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE)) { 1478 err = cxgbi_ddp_reserve(cconn->cep->csk, &tag, sw_tag, 1479 scsi_in(sc)->length, 1480 scsi_in(sc)->table.sgl, 1481 scsi_in(sc)->table.nents, 1482 GFP_ATOMIC); 1483 if (err < 0) 1484 log_debug(1 << CXGBI_DBG_DDP, 1485 "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n", 1486 cconn->cep->csk, task, scsi_in(sc)->length, 1487 scsi_in(sc)->table.nents); 1488 } 1489 1490 if (err < 0) 1491 tag = cxgbi_set_non_ddp_tag(tformat, sw_tag); 1492 /* the itt need to sent in big-endian order */ 1493 *hdr_itt = (__force itt_t)htonl(tag); 1494 1495 log_debug(1 << CXGBI_DBG_DDP, 1496 "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n", 1497 chba->cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt); 1498 return 0; 1499 } 1500 1501 void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age) 1502 { 1503 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1504 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1505 struct cxgbi_device *cdev = cconn->chba->cdev; 1506 u32 tag = ntohl((__force u32) itt); 1507 u32 sw_bits; 1508 1509 sw_bits = cxgbi_tag_nonrsvd_bits(&cdev->tag_format, tag); 1510 if (idx) 1511 *idx = sw_bits & ((1 << cconn->task_idx_bits) - 1); 1512 if (age) 1513 *age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK; 1514 1515 log_debug(1 << CXGBI_DBG_DDP, 1516 "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n", 1517 cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF, 1518 age ? *age : 0xFF); 1519 } 1520 EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt); 1521 1522 void cxgbi_conn_tx_open(struct cxgbi_sock *csk) 1523 { 1524 struct iscsi_conn *conn = csk->user_data; 1525 1526 if (conn) { 1527 log_debug(1 << CXGBI_DBG_SOCK, 1528 "csk 0x%p, cid %d.\n", csk, conn->id); 1529 iscsi_conn_queue_work(conn); 1530 } 1531 } 1532 EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open); 1533 1534 /* 1535 * pdu receive, interact with libiscsi_tcp 1536 */ 1537 static inline int read_pdu_skb(struct iscsi_conn *conn, 1538 struct sk_buff *skb, 1539 unsigned int offset, 1540 int offloaded) 1541 { 1542 int status = 0; 1543 int bytes_read; 1544 1545 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status); 1546 switch (status) { 1547 case ISCSI_TCP_CONN_ERR: 1548 pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n", 1549 skb, offset, offloaded); 1550 return -EIO; 1551 case ISCSI_TCP_SUSPENDED: 1552 log_debug(1 << CXGBI_DBG_PDU_RX, 1553 "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n", 1554 skb, offset, offloaded, bytes_read); 1555 /* no transfer - just have caller flush queue */ 1556 return bytes_read; 1557 case ISCSI_TCP_SKB_DONE: 1558 pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n", 1559 skb, offset, offloaded); 1560 /* 1561 * pdus should always fit in the skb and we should get 1562 * segment done notifcation. 1563 */ 1564 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb."); 1565 return -EFAULT; 1566 case ISCSI_TCP_SEGMENT_DONE: 1567 log_debug(1 << CXGBI_DBG_PDU_RX, 1568 "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n", 1569 skb, offset, offloaded, bytes_read); 1570 return bytes_read; 1571 default: 1572 pr_info("skb 0x%p, off %u, %d, invalid status %d.\n", 1573 skb, offset, offloaded, status); 1574 return -EINVAL; 1575 } 1576 } 1577 1578 static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb) 1579 { 1580 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1581 1582 log_debug(1 << CXGBI_DBG_PDU_RX, 1583 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1584 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1585 1586 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) { 1587 pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb); 1588 iscsi_conn_failure(conn, ISCSI_ERR_PROTO); 1589 return -EIO; 1590 } 1591 1592 if (conn->hdrdgst_en && 1593 cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) { 1594 pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb); 1595 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST); 1596 return -EIO; 1597 } 1598 1599 return read_pdu_skb(conn, skb, 0, 0); 1600 } 1601 1602 static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb, 1603 struct sk_buff *skb, unsigned int offset) 1604 { 1605 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1606 bool offloaded = 0; 1607 int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK; 1608 1609 log_debug(1 << CXGBI_DBG_PDU_RX, 1610 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1611 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1612 1613 if (conn->datadgst_en && 1614 cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) { 1615 pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n", 1616 conn, lskb, cxgbi_skcb_flags(lskb)); 1617 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 1618 return -EIO; 1619 } 1620 1621 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn)) 1622 return 0; 1623 1624 /* coalesced, add header digest length */ 1625 if (lskb == skb && conn->hdrdgst_en) 1626 offset += ISCSI_DIGEST_SIZE; 1627 1628 if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD)) 1629 offloaded = 1; 1630 1631 if (opcode == ISCSI_OP_SCSI_DATA_IN) 1632 log_debug(1 << CXGBI_DBG_PDU_RX, 1633 "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n", 1634 skb, opcode, ntohl(tcp_conn->in.hdr->itt), 1635 tcp_conn->in.datalen, offloaded ? "is" : "not"); 1636 1637 return read_pdu_skb(conn, skb, offset, offloaded); 1638 } 1639 1640 static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied) 1641 { 1642 struct cxgbi_device *cdev = csk->cdev; 1643 int must_send; 1644 u32 credits; 1645 1646 log_debug(1 << CXGBI_DBG_PDU_RX, 1647 "csk 0x%p,%u,0x%lu,%u, seq %u, wup %u, thre %u, %u.\n", 1648 csk, csk->state, csk->flags, csk->tid, csk->copied_seq, 1649 csk->rcv_wup, cdev->rx_credit_thres, 1650 cdev->rcv_win); 1651 1652 if (csk->state != CTP_ESTABLISHED) 1653 return; 1654 1655 credits = csk->copied_seq - csk->rcv_wup; 1656 if (unlikely(!credits)) 1657 return; 1658 if (unlikely(cdev->rx_credit_thres == 0)) 1659 return; 1660 1661 must_send = credits + 16384 >= cdev->rcv_win; 1662 if (must_send || credits >= cdev->rx_credit_thres) 1663 csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits); 1664 } 1665 1666 void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk) 1667 { 1668 struct cxgbi_device *cdev = csk->cdev; 1669 struct iscsi_conn *conn = csk->user_data; 1670 struct sk_buff *skb; 1671 unsigned int read = 0; 1672 int err = 0; 1673 1674 log_debug(1 << CXGBI_DBG_PDU_RX, 1675 "csk 0x%p, conn 0x%p.\n", csk, conn); 1676 1677 if (unlikely(!conn || conn->suspend_rx)) { 1678 log_debug(1 << CXGBI_DBG_PDU_RX, 1679 "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n", 1680 csk, conn, conn ? conn->id : 0xFF, 1681 conn ? conn->suspend_rx : 0xFF); 1682 return; 1683 } 1684 1685 while (!err) { 1686 skb = skb_peek(&csk->receive_queue); 1687 if (!skb || 1688 !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) { 1689 if (skb) 1690 log_debug(1 << CXGBI_DBG_PDU_RX, 1691 "skb 0x%p, NOT ready 0x%lx.\n", 1692 skb, cxgbi_skcb_flags(skb)); 1693 break; 1694 } 1695 __skb_unlink(skb, &csk->receive_queue); 1696 1697 read += cxgbi_skcb_rx_pdulen(skb); 1698 log_debug(1 << CXGBI_DBG_PDU_RX, 1699 "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n", 1700 csk, skb, skb->len, cxgbi_skcb_flags(skb), 1701 cxgbi_skcb_rx_pdulen(skb)); 1702 1703 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) { 1704 err = skb_read_pdu_bhs(conn, skb); 1705 if (err < 0) { 1706 pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, " 1707 "f 0x%lx, plen %u.\n", 1708 csk, skb, skb->len, 1709 cxgbi_skcb_flags(skb), 1710 cxgbi_skcb_rx_pdulen(skb)); 1711 goto skb_done; 1712 } 1713 err = skb_read_pdu_data(conn, skb, skb, 1714 err + cdev->skb_rx_extra); 1715 if (err < 0) 1716 pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, " 1717 "f 0x%lx, plen %u.\n", 1718 csk, skb, skb->len, 1719 cxgbi_skcb_flags(skb), 1720 cxgbi_skcb_rx_pdulen(skb)); 1721 } else { 1722 err = skb_read_pdu_bhs(conn, skb); 1723 if (err < 0) { 1724 pr_err("bhs, csk 0x%p, skb 0x%p,%u, " 1725 "f 0x%lx, plen %u.\n", 1726 csk, skb, skb->len, 1727 cxgbi_skcb_flags(skb), 1728 cxgbi_skcb_rx_pdulen(skb)); 1729 goto skb_done; 1730 } 1731 1732 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) { 1733 struct sk_buff *dskb; 1734 1735 dskb = skb_peek(&csk->receive_queue); 1736 if (!dskb) { 1737 pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx," 1738 " plen %u, NO data.\n", 1739 csk, skb, skb->len, 1740 cxgbi_skcb_flags(skb), 1741 cxgbi_skcb_rx_pdulen(skb)); 1742 err = -EIO; 1743 goto skb_done; 1744 } 1745 __skb_unlink(dskb, &csk->receive_queue); 1746 1747 err = skb_read_pdu_data(conn, skb, dskb, 0); 1748 if (err < 0) 1749 pr_err("data, csk 0x%p, skb 0x%p,%u, " 1750 "f 0x%lx, plen %u, dskb 0x%p," 1751 "%u.\n", 1752 csk, skb, skb->len, 1753 cxgbi_skcb_flags(skb), 1754 cxgbi_skcb_rx_pdulen(skb), 1755 dskb, dskb->len); 1756 __kfree_skb(dskb); 1757 } else 1758 err = skb_read_pdu_data(conn, skb, skb, 0); 1759 } 1760 skb_done: 1761 __kfree_skb(skb); 1762 1763 if (err < 0) 1764 break; 1765 } 1766 1767 log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read); 1768 if (read) { 1769 csk->copied_seq += read; 1770 csk_return_rx_credits(csk, read); 1771 conn->rxdata_octets += read; 1772 } 1773 1774 if (err < 0) { 1775 pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n", 1776 csk, conn, err, read); 1777 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1778 } 1779 } 1780 EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready); 1781 1782 static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt, 1783 unsigned int offset, unsigned int *off, 1784 struct scatterlist **sgp) 1785 { 1786 int i; 1787 struct scatterlist *sg; 1788 1789 for_each_sg(sgl, sg, sgcnt, i) { 1790 if (offset < sg->length) { 1791 *off = offset; 1792 *sgp = sg; 1793 return 0; 1794 } 1795 offset -= sg->length; 1796 } 1797 return -EFAULT; 1798 } 1799 1800 static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset, 1801 unsigned int dlen, struct page_frag *frags, 1802 int frag_max) 1803 { 1804 unsigned int datalen = dlen; 1805 unsigned int sglen = sg->length - sgoffset; 1806 struct page *page = sg_page(sg); 1807 int i; 1808 1809 i = 0; 1810 do { 1811 unsigned int copy; 1812 1813 if (!sglen) { 1814 sg = sg_next(sg); 1815 if (!sg) { 1816 pr_warn("sg %d NULL, len %u/%u.\n", 1817 i, datalen, dlen); 1818 return -EINVAL; 1819 } 1820 sgoffset = 0; 1821 sglen = sg->length; 1822 page = sg_page(sg); 1823 1824 } 1825 copy = min(datalen, sglen); 1826 if (i && page == frags[i - 1].page && 1827 sgoffset + sg->offset == 1828 frags[i - 1].offset + frags[i - 1].size) { 1829 frags[i - 1].size += copy; 1830 } else { 1831 if (i >= frag_max) { 1832 pr_warn("too many pages %u, dlen %u.\n", 1833 frag_max, dlen); 1834 return -EINVAL; 1835 } 1836 1837 frags[i].page = page; 1838 frags[i].offset = sg->offset + sgoffset; 1839 frags[i].size = copy; 1840 i++; 1841 } 1842 datalen -= copy; 1843 sgoffset += copy; 1844 sglen -= copy; 1845 } while (datalen); 1846 1847 return i; 1848 } 1849 1850 int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 opcode) 1851 { 1852 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 1853 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1854 struct cxgbi_device *cdev = cconn->chba->cdev; 1855 struct iscsi_conn *conn = task->conn; 1856 struct iscsi_tcp_task *tcp_task = task->dd_data; 1857 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1858 struct scsi_cmnd *sc = task->sc; 1859 int headroom = SKB_TX_ISCSI_PDU_HEADER_MAX; 1860 1861 tcp_task->dd_data = tdata; 1862 task->hdr = NULL; 1863 1864 if (SKB_MAX_HEAD(cdev->skb_tx_rsvd) > (512 * MAX_SKB_FRAGS) && 1865 (opcode == ISCSI_OP_SCSI_DATA_OUT || 1866 (opcode == ISCSI_OP_SCSI_CMD && 1867 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE)))) 1868 /* data could goes into skb head */ 1869 headroom += min_t(unsigned int, 1870 SKB_MAX_HEAD(cdev->skb_tx_rsvd), 1871 conn->max_xmit_dlength); 1872 1873 tdata->skb = alloc_skb(cdev->skb_tx_rsvd + headroom, GFP_ATOMIC); 1874 if (!tdata->skb) { 1875 struct cxgbi_sock *csk = cconn->cep->csk; 1876 struct net_device *ndev = cdev->ports[csk->port_id]; 1877 ndev->stats.tx_dropped++; 1878 return -ENOMEM; 1879 } 1880 1881 skb_reserve(tdata->skb, cdev->skb_tx_rsvd); 1882 task->hdr = (struct iscsi_hdr *)tdata->skb->data; 1883 task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX; /* BHS + AHS */ 1884 1885 /* data_out uses scsi_cmd's itt */ 1886 if (opcode != ISCSI_OP_SCSI_DATA_OUT) 1887 task_reserve_itt(task, &task->hdr->itt); 1888 1889 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 1890 "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n", 1891 task, opcode, tdata->skb, cdev->skb_tx_rsvd, headroom, 1892 conn->max_xmit_dlength, ntohl(task->hdr->itt)); 1893 1894 return 0; 1895 } 1896 EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu); 1897 1898 static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc) 1899 { 1900 if (hcrc || dcrc) { 1901 u8 submode = 0; 1902 1903 if (hcrc) 1904 submode |= 1; 1905 if (dcrc) 1906 submode |= 2; 1907 cxgbi_skcb_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode; 1908 } else 1909 cxgbi_skcb_ulp_mode(skb) = 0; 1910 } 1911 1912 int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset, 1913 unsigned int count) 1914 { 1915 struct iscsi_conn *conn = task->conn; 1916 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1917 struct sk_buff *skb = tdata->skb; 1918 unsigned int datalen = count; 1919 int i, padlen = iscsi_padding(count); 1920 struct page *pg; 1921 1922 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 1923 "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n", 1924 task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK, 1925 ntohl(task->cmdsn), ntohl(task->hdr->itt), offset, count); 1926 1927 skb_put(skb, task->hdr_len); 1928 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0); 1929 if (!count) 1930 return 0; 1931 1932 if (task->sc) { 1933 struct scsi_data_buffer *sdb = scsi_out(task->sc); 1934 struct scatterlist *sg = NULL; 1935 int err; 1936 1937 tdata->offset = offset; 1938 tdata->count = count; 1939 err = sgl_seek_offset( 1940 sdb->table.sgl, sdb->table.nents, 1941 tdata->offset, &tdata->sgoffset, &sg); 1942 if (err < 0) { 1943 pr_warn("tpdu, sgl %u, bad offset %u/%u.\n", 1944 sdb->table.nents, tdata->offset, sdb->length); 1945 return err; 1946 } 1947 err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count, 1948 tdata->frags, MAX_PDU_FRAGS); 1949 if (err < 0) { 1950 pr_warn("tpdu, sgl %u, bad offset %u + %u.\n", 1951 sdb->table.nents, tdata->offset, tdata->count); 1952 return err; 1953 } 1954 tdata->nr_frags = err; 1955 1956 if (tdata->nr_frags > MAX_SKB_FRAGS || 1957 (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) { 1958 char *dst = skb->data + task->hdr_len; 1959 struct page_frag *frag = tdata->frags; 1960 1961 /* data fits in the skb's headroom */ 1962 for (i = 0; i < tdata->nr_frags; i++, frag++) { 1963 char *src = kmap_atomic(frag->page); 1964 1965 memcpy(dst, src+frag->offset, frag->size); 1966 dst += frag->size; 1967 kunmap_atomic(src); 1968 } 1969 if (padlen) { 1970 memset(dst, 0, padlen); 1971 padlen = 0; 1972 } 1973 skb_put(skb, count + padlen); 1974 } else { 1975 /* data fit into frag_list */ 1976 for (i = 0; i < tdata->nr_frags; i++) { 1977 __skb_fill_page_desc(skb, i, 1978 tdata->frags[i].page, 1979 tdata->frags[i].offset, 1980 tdata->frags[i].size); 1981 skb_frag_ref(skb, i); 1982 } 1983 skb_shinfo(skb)->nr_frags = tdata->nr_frags; 1984 skb->len += count; 1985 skb->data_len += count; 1986 skb->truesize += count; 1987 } 1988 1989 } else { 1990 pg = virt_to_page(task->data); 1991 1992 get_page(pg); 1993 skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data), 1994 count); 1995 skb->len += count; 1996 skb->data_len += count; 1997 skb->truesize += count; 1998 } 1999 2000 if (padlen) { 2001 i = skb_shinfo(skb)->nr_frags; 2002 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 2003 virt_to_page(padding), offset_in_page(padding), 2004 padlen); 2005 2006 skb->data_len += padlen; 2007 skb->truesize += padlen; 2008 skb->len += padlen; 2009 } 2010 2011 return 0; 2012 } 2013 EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu); 2014 2015 int cxgbi_conn_xmit_pdu(struct iscsi_task *task) 2016 { 2017 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 2018 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2019 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2020 struct sk_buff *skb = tdata->skb; 2021 unsigned int datalen; 2022 int err; 2023 2024 if (!skb) { 2025 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2026 "task 0x%p, skb NULL.\n", task); 2027 return 0; 2028 } 2029 2030 datalen = skb->data_len; 2031 tdata->skb = NULL; 2032 err = cxgbi_sock_send_pdus(cconn->cep->csk, skb); 2033 if (err > 0) { 2034 int pdulen = err; 2035 2036 log_debug(1 << CXGBI_DBG_PDU_TX, 2037 "task 0x%p,0x%p, skb 0x%p, len %u/%u, rv %d.\n", 2038 task, task->sc, skb, skb->len, skb->data_len, err); 2039 2040 if (task->conn->hdrdgst_en) 2041 pdulen += ISCSI_DIGEST_SIZE; 2042 2043 if (datalen && task->conn->datadgst_en) 2044 pdulen += ISCSI_DIGEST_SIZE; 2045 2046 task->conn->txdata_octets += pdulen; 2047 return 0; 2048 } 2049 2050 if (err == -EAGAIN || err == -ENOBUFS) { 2051 log_debug(1 << CXGBI_DBG_PDU_TX, 2052 "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n", 2053 task, skb, skb->len, skb->data_len, err); 2054 /* reset skb to send when we are called again */ 2055 tdata->skb = skb; 2056 return err; 2057 } 2058 2059 kfree_skb(skb); 2060 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2061 "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n", 2062 task->itt, skb, skb->len, skb->data_len, err); 2063 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err); 2064 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED); 2065 return err; 2066 } 2067 EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu); 2068 2069 void cxgbi_cleanup_task(struct iscsi_task *task) 2070 { 2071 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2072 2073 log_debug(1 << CXGBI_DBG_ISCSI, 2074 "task 0x%p, skb 0x%p, itt 0x%x.\n", 2075 task, tdata->skb, task->hdr_itt); 2076 2077 /* never reached the xmit task callout */ 2078 if (tdata->skb) 2079 __kfree_skb(tdata->skb); 2080 memset(tdata, 0, sizeof(*tdata)); 2081 2082 task_release_itt(task, task->hdr_itt); 2083 iscsi_tcp_cleanup_task(task); 2084 } 2085 EXPORT_SYMBOL_GPL(cxgbi_cleanup_task); 2086 2087 void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn, 2088 struct iscsi_stats *stats) 2089 { 2090 struct iscsi_conn *conn = cls_conn->dd_data; 2091 2092 stats->txdata_octets = conn->txdata_octets; 2093 stats->rxdata_octets = conn->rxdata_octets; 2094 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 2095 stats->dataout_pdus = conn->dataout_pdus_cnt; 2096 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 2097 stats->datain_pdus = conn->datain_pdus_cnt; 2098 stats->r2t_pdus = conn->r2t_pdus_cnt; 2099 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 2100 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 2101 stats->digest_err = 0; 2102 stats->timeout_err = 0; 2103 stats->custom_length = 1; 2104 strcpy(stats->custom[0].desc, "eh_abort_cnt"); 2105 stats->custom[0].value = conn->eh_abort_cnt; 2106 } 2107 EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats); 2108 2109 static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn) 2110 { 2111 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2112 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2113 struct cxgbi_device *cdev = cconn->chba->cdev; 2114 unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd); 2115 unsigned int max_def = 512 * MAX_SKB_FRAGS; 2116 unsigned int max = max(max_def, headroom); 2117 2118 max = min(cconn->chba->cdev->tx_max_size, max); 2119 if (conn->max_xmit_dlength) 2120 conn->max_xmit_dlength = min(conn->max_xmit_dlength, max); 2121 else 2122 conn->max_xmit_dlength = max; 2123 cxgbi_align_pdu_size(conn->max_xmit_dlength); 2124 2125 return 0; 2126 } 2127 2128 static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn) 2129 { 2130 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2131 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2132 unsigned int max = cconn->chba->cdev->rx_max_size; 2133 2134 cxgbi_align_pdu_size(max); 2135 2136 if (conn->max_recv_dlength) { 2137 if (conn->max_recv_dlength > max) { 2138 pr_err("MaxRecvDataSegmentLength %u > %u.\n", 2139 conn->max_recv_dlength, max); 2140 return -EINVAL; 2141 } 2142 conn->max_recv_dlength = min(conn->max_recv_dlength, max); 2143 cxgbi_align_pdu_size(conn->max_recv_dlength); 2144 } else 2145 conn->max_recv_dlength = max; 2146 2147 return 0; 2148 } 2149 2150 int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn, 2151 enum iscsi_param param, char *buf, int buflen) 2152 { 2153 struct iscsi_conn *conn = cls_conn->dd_data; 2154 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2155 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2156 struct cxgbi_sock *csk = cconn->cep->csk; 2157 int err; 2158 2159 log_debug(1 << CXGBI_DBG_ISCSI, 2160 "cls_conn 0x%p, param %d, buf(%d) %s.\n", 2161 cls_conn, param, buflen, buf); 2162 2163 switch (param) { 2164 case ISCSI_PARAM_HDRDGST_EN: 2165 err = iscsi_set_param(cls_conn, param, buf, buflen); 2166 if (!err && conn->hdrdgst_en) 2167 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2168 conn->hdrdgst_en, 2169 conn->datadgst_en, 0); 2170 break; 2171 case ISCSI_PARAM_DATADGST_EN: 2172 err = iscsi_set_param(cls_conn, param, buf, buflen); 2173 if (!err && conn->datadgst_en) 2174 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2175 conn->hdrdgst_en, 2176 conn->datadgst_en, 0); 2177 break; 2178 case ISCSI_PARAM_MAX_R2T: 2179 return iscsi_tcp_set_max_r2t(conn, buf); 2180 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2181 err = iscsi_set_param(cls_conn, param, buf, buflen); 2182 if (!err) 2183 err = cxgbi_conn_max_recv_dlength(conn); 2184 break; 2185 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2186 err = iscsi_set_param(cls_conn, param, buf, buflen); 2187 if (!err) 2188 err = cxgbi_conn_max_xmit_dlength(conn); 2189 break; 2190 default: 2191 return iscsi_set_param(cls_conn, param, buf, buflen); 2192 } 2193 return err; 2194 } 2195 EXPORT_SYMBOL_GPL(cxgbi_set_conn_param); 2196 2197 int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param, 2198 char *buf) 2199 { 2200 struct cxgbi_endpoint *cep = ep->dd_data; 2201 struct cxgbi_sock *csk; 2202 int len; 2203 2204 log_debug(1 << CXGBI_DBG_ISCSI, 2205 "cls_conn 0x%p, param %d.\n", ep, param); 2206 2207 switch (param) { 2208 case ISCSI_PARAM_CONN_PORT: 2209 case ISCSI_PARAM_CONN_ADDRESS: 2210 if (!cep) 2211 return -ENOTCONN; 2212 2213 csk = cep->csk; 2214 if (!csk) 2215 return -ENOTCONN; 2216 2217 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 2218 &csk->daddr, param, buf); 2219 default: 2220 return -ENOSYS; 2221 } 2222 return len; 2223 } 2224 EXPORT_SYMBOL_GPL(cxgbi_get_ep_param); 2225 2226 struct iscsi_cls_conn * 2227 cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid) 2228 { 2229 struct iscsi_cls_conn *cls_conn; 2230 struct iscsi_conn *conn; 2231 struct iscsi_tcp_conn *tcp_conn; 2232 struct cxgbi_conn *cconn; 2233 2234 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid); 2235 if (!cls_conn) 2236 return NULL; 2237 2238 conn = cls_conn->dd_data; 2239 tcp_conn = conn->dd_data; 2240 cconn = tcp_conn->dd_data; 2241 cconn->iconn = conn; 2242 2243 log_debug(1 << CXGBI_DBG_ISCSI, 2244 "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n", 2245 cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn); 2246 2247 return cls_conn; 2248 } 2249 EXPORT_SYMBOL_GPL(cxgbi_create_conn); 2250 2251 int cxgbi_bind_conn(struct iscsi_cls_session *cls_session, 2252 struct iscsi_cls_conn *cls_conn, 2253 u64 transport_eph, int is_leading) 2254 { 2255 struct iscsi_conn *conn = cls_conn->dd_data; 2256 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2257 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2258 struct iscsi_endpoint *ep; 2259 struct cxgbi_endpoint *cep; 2260 struct cxgbi_sock *csk; 2261 int err; 2262 2263 ep = iscsi_lookup_endpoint(transport_eph); 2264 if (!ep) 2265 return -EINVAL; 2266 2267 /* setup ddp pagesize */ 2268 cep = ep->dd_data; 2269 csk = cep->csk; 2270 err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid, page_idx, 0); 2271 if (err < 0) 2272 return err; 2273 2274 err = iscsi_conn_bind(cls_session, cls_conn, is_leading); 2275 if (err) 2276 return -EINVAL; 2277 2278 /* calculate the tag idx bits needed for this conn based on cmds_max */ 2279 cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1; 2280 2281 write_lock_bh(&csk->callback_lock); 2282 csk->user_data = conn; 2283 cconn->chba = cep->chba; 2284 cconn->cep = cep; 2285 cep->cconn = cconn; 2286 write_unlock_bh(&csk->callback_lock); 2287 2288 cxgbi_conn_max_xmit_dlength(conn); 2289 cxgbi_conn_max_recv_dlength(conn); 2290 2291 log_debug(1 << CXGBI_DBG_ISCSI, 2292 "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n", 2293 cls_session, cls_conn, ep, cconn, csk); 2294 /* init recv engine */ 2295 iscsi_tcp_hdr_recv_prep(tcp_conn); 2296 2297 return 0; 2298 } 2299 EXPORT_SYMBOL_GPL(cxgbi_bind_conn); 2300 2301 struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep, 2302 u16 cmds_max, u16 qdepth, 2303 u32 initial_cmdsn) 2304 { 2305 struct cxgbi_endpoint *cep; 2306 struct cxgbi_hba *chba; 2307 struct Scsi_Host *shost; 2308 struct iscsi_cls_session *cls_session; 2309 struct iscsi_session *session; 2310 2311 if (!ep) { 2312 pr_err("missing endpoint.\n"); 2313 return NULL; 2314 } 2315 2316 cep = ep->dd_data; 2317 chba = cep->chba; 2318 shost = chba->shost; 2319 2320 BUG_ON(chba != iscsi_host_priv(shost)); 2321 2322 cls_session = iscsi_session_setup(chba->cdev->itp, shost, 2323 cmds_max, 0, 2324 sizeof(struct iscsi_tcp_task) + 2325 sizeof(struct cxgbi_task_data), 2326 initial_cmdsn, ISCSI_MAX_TARGET); 2327 if (!cls_session) 2328 return NULL; 2329 2330 session = cls_session->dd_data; 2331 if (iscsi_tcp_r2tpool_alloc(session)) 2332 goto remove_session; 2333 2334 log_debug(1 << CXGBI_DBG_ISCSI, 2335 "ep 0x%p, cls sess 0x%p.\n", ep, cls_session); 2336 return cls_session; 2337 2338 remove_session: 2339 iscsi_session_teardown(cls_session); 2340 return NULL; 2341 } 2342 EXPORT_SYMBOL_GPL(cxgbi_create_session); 2343 2344 void cxgbi_destroy_session(struct iscsi_cls_session *cls_session) 2345 { 2346 log_debug(1 << CXGBI_DBG_ISCSI, 2347 "cls sess 0x%p.\n", cls_session); 2348 2349 iscsi_tcp_r2tpool_free(cls_session->dd_data); 2350 iscsi_session_teardown(cls_session); 2351 } 2352 EXPORT_SYMBOL_GPL(cxgbi_destroy_session); 2353 2354 int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2355 char *buf, int buflen) 2356 { 2357 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2358 2359 if (!chba->ndev) { 2360 shost_printk(KERN_ERR, shost, "Could not get host param. " 2361 "netdev for host not set.\n"); 2362 return -ENODEV; 2363 } 2364 2365 log_debug(1 << CXGBI_DBG_ISCSI, 2366 "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n", 2367 shost, chba, chba->ndev->name, param, buflen, buf); 2368 2369 switch (param) { 2370 case ISCSI_HOST_PARAM_IPADDRESS: 2371 { 2372 __be32 addr = in_aton(buf); 2373 log_debug(1 << CXGBI_DBG_ISCSI, 2374 "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr); 2375 cxgbi_set_iscsi_ipv4(chba, addr); 2376 return 0; 2377 } 2378 case ISCSI_HOST_PARAM_HWADDRESS: 2379 case ISCSI_HOST_PARAM_NETDEV_NAME: 2380 return 0; 2381 default: 2382 return iscsi_host_set_param(shost, param, buf, buflen); 2383 } 2384 } 2385 EXPORT_SYMBOL_GPL(cxgbi_set_host_param); 2386 2387 int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2388 char *buf) 2389 { 2390 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2391 int len = 0; 2392 2393 if (!chba->ndev) { 2394 shost_printk(KERN_ERR, shost, "Could not get host param. " 2395 "netdev for host not set.\n"); 2396 return -ENODEV; 2397 } 2398 2399 log_debug(1 << CXGBI_DBG_ISCSI, 2400 "shost 0x%p, hba 0x%p,%s, param %d.\n", 2401 shost, chba, chba->ndev->name, param); 2402 2403 switch (param) { 2404 case ISCSI_HOST_PARAM_HWADDRESS: 2405 len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6); 2406 break; 2407 case ISCSI_HOST_PARAM_NETDEV_NAME: 2408 len = sprintf(buf, "%s\n", chba->ndev->name); 2409 break; 2410 case ISCSI_HOST_PARAM_IPADDRESS: 2411 { 2412 __be32 addr; 2413 2414 addr = cxgbi_get_iscsi_ipv4(chba); 2415 len = sprintf(buf, "%pI4", &addr); 2416 log_debug(1 << CXGBI_DBG_ISCSI, 2417 "hba %s, ipv4 %pI4.\n", chba->ndev->name, &addr); 2418 break; 2419 } 2420 default: 2421 return iscsi_host_get_param(shost, param, buf); 2422 } 2423 2424 return len; 2425 } 2426 EXPORT_SYMBOL_GPL(cxgbi_get_host_param); 2427 2428 struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost, 2429 struct sockaddr *dst_addr, 2430 int non_blocking) 2431 { 2432 struct iscsi_endpoint *ep; 2433 struct cxgbi_endpoint *cep; 2434 struct cxgbi_hba *hba = NULL; 2435 struct cxgbi_sock *csk; 2436 int err = -EINVAL; 2437 2438 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2439 "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n", 2440 shost, non_blocking, dst_addr); 2441 2442 if (shost) { 2443 hba = iscsi_host_priv(shost); 2444 if (!hba) { 2445 pr_info("shost 0x%p, priv NULL.\n", shost); 2446 goto err_out; 2447 } 2448 } 2449 2450 csk = cxgbi_check_route(dst_addr); 2451 if (IS_ERR(csk)) 2452 return (struct iscsi_endpoint *)csk; 2453 cxgbi_sock_get(csk); 2454 2455 if (!hba) 2456 hba = csk->cdev->hbas[csk->port_id]; 2457 else if (hba != csk->cdev->hbas[csk->port_id]) { 2458 pr_info("Could not connect through requested host %u" 2459 "hba 0x%p != 0x%p (%u).\n", 2460 shost->host_no, hba, 2461 csk->cdev->hbas[csk->port_id], csk->port_id); 2462 err = -ENOSPC; 2463 goto release_conn; 2464 } 2465 2466 err = sock_get_port(csk); 2467 if (err) 2468 goto release_conn; 2469 2470 cxgbi_sock_set_state(csk, CTP_CONNECTING); 2471 err = csk->cdev->csk_init_act_open(csk); 2472 if (err) 2473 goto release_conn; 2474 2475 if (cxgbi_sock_is_closing(csk)) { 2476 err = -ENOSPC; 2477 pr_info("csk 0x%p is closing.\n", csk); 2478 goto release_conn; 2479 } 2480 2481 ep = iscsi_create_endpoint(sizeof(*cep)); 2482 if (!ep) { 2483 err = -ENOMEM; 2484 pr_info("iscsi alloc ep, OOM.\n"); 2485 goto release_conn; 2486 } 2487 2488 cep = ep->dd_data; 2489 cep->csk = csk; 2490 cep->chba = hba; 2491 2492 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2493 "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n", 2494 ep, cep, csk, hba, hba->ndev->name); 2495 return ep; 2496 2497 release_conn: 2498 cxgbi_sock_put(csk); 2499 cxgbi_sock_closed(csk); 2500 err_out: 2501 return ERR_PTR(err); 2502 } 2503 EXPORT_SYMBOL_GPL(cxgbi_ep_connect); 2504 2505 int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) 2506 { 2507 struct cxgbi_endpoint *cep = ep->dd_data; 2508 struct cxgbi_sock *csk = cep->csk; 2509 2510 if (!cxgbi_sock_is_established(csk)) 2511 return 0; 2512 return 1; 2513 } 2514 EXPORT_SYMBOL_GPL(cxgbi_ep_poll); 2515 2516 void cxgbi_ep_disconnect(struct iscsi_endpoint *ep) 2517 { 2518 struct cxgbi_endpoint *cep = ep->dd_data; 2519 struct cxgbi_conn *cconn = cep->cconn; 2520 struct cxgbi_sock *csk = cep->csk; 2521 2522 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2523 "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n", 2524 ep, cep, cconn, csk, csk->state, csk->flags); 2525 2526 if (cconn && cconn->iconn) { 2527 iscsi_suspend_tx(cconn->iconn); 2528 write_lock_bh(&csk->callback_lock); 2529 cep->csk->user_data = NULL; 2530 cconn->cep = NULL; 2531 write_unlock_bh(&csk->callback_lock); 2532 } 2533 iscsi_destroy_endpoint(ep); 2534 2535 if (likely(csk->state >= CTP_ESTABLISHED)) 2536 need_active_close(csk); 2537 else 2538 cxgbi_sock_closed(csk); 2539 2540 cxgbi_sock_put(csk); 2541 } 2542 EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect); 2543 2544 int cxgbi_iscsi_init(struct iscsi_transport *itp, 2545 struct scsi_transport_template **stt) 2546 { 2547 *stt = iscsi_register_transport(itp); 2548 if (*stt == NULL) { 2549 pr_err("unable to register %s transport 0x%p.\n", 2550 itp->name, itp); 2551 return -ENODEV; 2552 } 2553 log_debug(1 << CXGBI_DBG_ISCSI, 2554 "%s, registered iscsi transport 0x%p.\n", 2555 itp->name, stt); 2556 return 0; 2557 } 2558 EXPORT_SYMBOL_GPL(cxgbi_iscsi_init); 2559 2560 void cxgbi_iscsi_cleanup(struct iscsi_transport *itp, 2561 struct scsi_transport_template **stt) 2562 { 2563 if (*stt) { 2564 log_debug(1 << CXGBI_DBG_ISCSI, 2565 "de-register transport 0x%p, %s, stt 0x%p.\n", 2566 itp, itp->name, *stt); 2567 *stt = NULL; 2568 iscsi_unregister_transport(itp); 2569 } 2570 } 2571 EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup); 2572 2573 umode_t cxgbi_attr_is_visible(int param_type, int param) 2574 { 2575 switch (param_type) { 2576 case ISCSI_HOST_PARAM: 2577 switch (param) { 2578 case ISCSI_HOST_PARAM_NETDEV_NAME: 2579 case ISCSI_HOST_PARAM_HWADDRESS: 2580 case ISCSI_HOST_PARAM_IPADDRESS: 2581 case ISCSI_HOST_PARAM_INITIATOR_NAME: 2582 return S_IRUGO; 2583 default: 2584 return 0; 2585 } 2586 case ISCSI_PARAM: 2587 switch (param) { 2588 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2589 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2590 case ISCSI_PARAM_HDRDGST_EN: 2591 case ISCSI_PARAM_DATADGST_EN: 2592 case ISCSI_PARAM_CONN_ADDRESS: 2593 case ISCSI_PARAM_CONN_PORT: 2594 case ISCSI_PARAM_EXP_STATSN: 2595 case ISCSI_PARAM_PERSISTENT_ADDRESS: 2596 case ISCSI_PARAM_PERSISTENT_PORT: 2597 case ISCSI_PARAM_PING_TMO: 2598 case ISCSI_PARAM_RECV_TMO: 2599 case ISCSI_PARAM_INITIAL_R2T_EN: 2600 case ISCSI_PARAM_MAX_R2T: 2601 case ISCSI_PARAM_IMM_DATA_EN: 2602 case ISCSI_PARAM_FIRST_BURST: 2603 case ISCSI_PARAM_MAX_BURST: 2604 case ISCSI_PARAM_PDU_INORDER_EN: 2605 case ISCSI_PARAM_DATASEQ_INORDER_EN: 2606 case ISCSI_PARAM_ERL: 2607 case ISCSI_PARAM_TARGET_NAME: 2608 case ISCSI_PARAM_TPGT: 2609 case ISCSI_PARAM_USERNAME: 2610 case ISCSI_PARAM_PASSWORD: 2611 case ISCSI_PARAM_USERNAME_IN: 2612 case ISCSI_PARAM_PASSWORD_IN: 2613 case ISCSI_PARAM_FAST_ABORT: 2614 case ISCSI_PARAM_ABORT_TMO: 2615 case ISCSI_PARAM_LU_RESET_TMO: 2616 case ISCSI_PARAM_TGT_RESET_TMO: 2617 case ISCSI_PARAM_IFACE_NAME: 2618 case ISCSI_PARAM_INITIATOR_NAME: 2619 return S_IRUGO; 2620 default: 2621 return 0; 2622 } 2623 } 2624 2625 return 0; 2626 } 2627 EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible); 2628 2629 static int __init libcxgbi_init_module(void) 2630 { 2631 sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1; 2632 sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1; 2633 2634 pr_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n", 2635 ISCSI_ITT_MASK, sw_tag_idx_bits, 2636 ISCSI_AGE_MASK, sw_tag_age_bits); 2637 2638 ddp_setup_host_page_size(); 2639 return 0; 2640 } 2641 2642 static void __exit libcxgbi_exit_module(void) 2643 { 2644 cxgbi_device_unregister_all(0xFF); 2645 return; 2646 } 2647 2648 module_init(libcxgbi_init_module); 2649 module_exit(libcxgbi_exit_module); 2650