1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2016 Chelsio Communications, Inc. 4 */ 5 6 #define DRV_NAME "cxgbit" 7 #define DRV_VERSION "1.0.0-ko" 8 #define pr_fmt(fmt) DRV_NAME ": " fmt 9 10 #include "cxgbit.h" 11 12 #ifdef CONFIG_CHELSIO_T4_DCB 13 #include <net/dcbevent.h> 14 #include "cxgb4_dcb.h" 15 #endif 16 17 LIST_HEAD(cdev_list_head); 18 /* cdev list lock */ 19 DEFINE_MUTEX(cdev_list_lock); 20 21 void _cxgbit_free_cdev(struct kref *kref) 22 { 23 struct cxgbit_device *cdev; 24 25 cdev = container_of(kref, struct cxgbit_device, kref); 26 27 cxgbi_ppm_release(cdev2ppm(cdev)); 28 kfree(cdev); 29 } 30 31 static void cxgbit_set_mdsl(struct cxgbit_device *cdev) 32 { 33 struct cxgb4_lld_info *lldi = &cdev->lldi; 34 u32 mdsl; 35 36 #define CXGBIT_T5_MAX_PDU_LEN 16224 37 #define CXGBIT_PDU_NONPAYLOAD_LEN 312 /* 48(BHS) + 256(AHS) + 8(Digest) */ 38 if (is_t5(lldi->adapter_type)) { 39 mdsl = min_t(u32, lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN, 40 CXGBIT_T5_MAX_PDU_LEN - CXGBIT_PDU_NONPAYLOAD_LEN); 41 } else { 42 mdsl = lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN; 43 mdsl = min(mdsl, 16384U); 44 } 45 46 mdsl = round_down(mdsl, 4); 47 mdsl = min_t(u32, mdsl, 4 * PAGE_SIZE); 48 mdsl = min_t(u32, mdsl, (MAX_SKB_FRAGS - 1) * PAGE_SIZE); 49 50 cdev->mdsl = mdsl; 51 } 52 53 static void *cxgbit_uld_add(const struct cxgb4_lld_info *lldi) 54 { 55 struct cxgbit_device *cdev; 56 57 if (is_t4(lldi->adapter_type)) 58 return ERR_PTR(-ENODEV); 59 60 cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); 61 if (!cdev) 62 return ERR_PTR(-ENOMEM); 63 64 kref_init(&cdev->kref); 65 spin_lock_init(&cdev->np_lock); 66 67 cdev->lldi = *lldi; 68 69 cxgbit_set_mdsl(cdev); 70 71 if (cxgbit_ddp_init(cdev) < 0) { 72 kfree(cdev); 73 return ERR_PTR(-EINVAL); 74 } 75 76 if (!test_bit(CDEV_DDP_ENABLE, &cdev->flags)) 77 pr_info("cdev %s ddp init failed\n", 78 pci_name(lldi->pdev)); 79 80 if (lldi->fw_vers >= 0x10d2b00) 81 set_bit(CDEV_ISO_ENABLE, &cdev->flags); 82 83 spin_lock_init(&cdev->cskq.lock); 84 INIT_LIST_HEAD(&cdev->cskq.list); 85 86 mutex_lock(&cdev_list_lock); 87 list_add_tail(&cdev->list, &cdev_list_head); 88 mutex_unlock(&cdev_list_lock); 89 90 pr_info("cdev %s added for iSCSI target transport\n", 91 pci_name(lldi->pdev)); 92 93 return cdev; 94 } 95 96 static void cxgbit_close_conn(struct cxgbit_device *cdev) 97 { 98 struct cxgbit_sock *csk; 99 struct sk_buff *skb; 100 bool wakeup_thread = false; 101 102 spin_lock_bh(&cdev->cskq.lock); 103 list_for_each_entry(csk, &cdev->cskq.list, list) { 104 skb = alloc_skb(0, GFP_ATOMIC); 105 if (!skb) 106 continue; 107 108 spin_lock_bh(&csk->rxq.lock); 109 __skb_queue_tail(&csk->rxq, skb); 110 if (skb_queue_len(&csk->rxq) == 1) 111 wakeup_thread = true; 112 spin_unlock_bh(&csk->rxq.lock); 113 114 if (wakeup_thread) { 115 wake_up(&csk->waitq); 116 wakeup_thread = false; 117 } 118 } 119 spin_unlock_bh(&cdev->cskq.lock); 120 } 121 122 static void cxgbit_detach_cdev(struct cxgbit_device *cdev) 123 { 124 bool free_cdev = false; 125 126 spin_lock_bh(&cdev->cskq.lock); 127 if (list_empty(&cdev->cskq.list)) 128 free_cdev = true; 129 spin_unlock_bh(&cdev->cskq.lock); 130 131 if (free_cdev) { 132 mutex_lock(&cdev_list_lock); 133 list_del(&cdev->list); 134 mutex_unlock(&cdev_list_lock); 135 136 cxgbit_put_cdev(cdev); 137 } else { 138 cxgbit_close_conn(cdev); 139 } 140 } 141 142 static int cxgbit_uld_state_change(void *handle, enum cxgb4_state state) 143 { 144 struct cxgbit_device *cdev = handle; 145 146 switch (state) { 147 case CXGB4_STATE_UP: 148 set_bit(CDEV_STATE_UP, &cdev->flags); 149 pr_info("cdev %s state UP.\n", pci_name(cdev->lldi.pdev)); 150 break; 151 case CXGB4_STATE_START_RECOVERY: 152 clear_bit(CDEV_STATE_UP, &cdev->flags); 153 cxgbit_close_conn(cdev); 154 pr_info("cdev %s state RECOVERY.\n", pci_name(cdev->lldi.pdev)); 155 break; 156 case CXGB4_STATE_DOWN: 157 pr_info("cdev %s state DOWN.\n", pci_name(cdev->lldi.pdev)); 158 break; 159 case CXGB4_STATE_DETACH: 160 clear_bit(CDEV_STATE_UP, &cdev->flags); 161 pr_info("cdev %s state DETACH.\n", pci_name(cdev->lldi.pdev)); 162 cxgbit_detach_cdev(cdev); 163 break; 164 default: 165 pr_info("cdev %s unknown state %d.\n", 166 pci_name(cdev->lldi.pdev), state); 167 break; 168 } 169 return 0; 170 } 171 172 static void 173 cxgbit_process_ddpvld(struct cxgbit_sock *csk, struct cxgbit_lro_pdu_cb *pdu_cb, 174 u32 ddpvld) 175 { 176 177 if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) { 178 pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", csk->tid, ddpvld); 179 pdu_cb->flags |= PDUCBF_RX_HCRC_ERR; 180 } 181 182 if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) { 183 pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", csk->tid, ddpvld); 184 pdu_cb->flags |= PDUCBF_RX_DCRC_ERR; 185 } 186 187 if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT)) 188 pr_info("tid 0x%x, status 0x%x, pad bad.\n", csk->tid, ddpvld); 189 190 if ((ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) && 191 (!(pdu_cb->flags & PDUCBF_RX_DATA))) { 192 pdu_cb->flags |= PDUCBF_RX_DATA_DDPD; 193 } 194 } 195 196 static void 197 cxgbit_lro_add_packet_rsp(struct sk_buff *skb, u8 op, const __be64 *rsp) 198 { 199 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 200 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 201 lro_cb->pdu_idx); 202 struct cpl_rx_iscsi_ddp *cpl = (struct cpl_rx_iscsi_ddp *)(rsp + 1); 203 204 cxgbit_process_ddpvld(lro_cb->csk, pdu_cb, be32_to_cpu(cpl->ddpvld)); 205 206 pdu_cb->flags |= PDUCBF_RX_STATUS; 207 pdu_cb->ddigest = ntohl(cpl->ulp_crc); 208 pdu_cb->pdulen = ntohs(cpl->len); 209 210 if (pdu_cb->flags & PDUCBF_RX_HDR) 211 pdu_cb->complete = true; 212 213 lro_cb->pdu_totallen += pdu_cb->pdulen; 214 lro_cb->complete = true; 215 lro_cb->pdu_idx++; 216 } 217 218 static void 219 cxgbit_copy_frags(struct sk_buff *skb, const struct pkt_gl *gl, 220 unsigned int offset) 221 { 222 u8 skb_frag_idx = skb_shinfo(skb)->nr_frags; 223 u8 i; 224 225 /* usually there's just one frag */ 226 __skb_fill_page_desc(skb, skb_frag_idx, gl->frags[0].page, 227 gl->frags[0].offset + offset, 228 gl->frags[0].size - offset); 229 for (i = 1; i < gl->nfrags; i++) 230 __skb_fill_page_desc(skb, skb_frag_idx + i, 231 gl->frags[i].page, 232 gl->frags[i].offset, 233 gl->frags[i].size); 234 235 skb_shinfo(skb)->nr_frags += gl->nfrags; 236 237 /* get a reference to the last page, we don't own it */ 238 get_page(gl->frags[gl->nfrags - 1].page); 239 } 240 241 static void 242 cxgbit_lro_add_packet_gl(struct sk_buff *skb, u8 op, const struct pkt_gl *gl) 243 { 244 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 245 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 246 lro_cb->pdu_idx); 247 u32 len, offset; 248 249 if (op == CPL_ISCSI_HDR) { 250 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)gl->va; 251 252 offset = sizeof(struct cpl_iscsi_hdr); 253 pdu_cb->flags |= PDUCBF_RX_HDR; 254 pdu_cb->seq = ntohl(cpl->seq); 255 len = ntohs(cpl->len); 256 pdu_cb->hdr = gl->va + offset; 257 pdu_cb->hlen = len; 258 pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags; 259 260 if (unlikely(gl->nfrags > 1)) 261 cxgbit_skcb_flags(skb) = 0; 262 263 lro_cb->complete = false; 264 } else if (op == CPL_ISCSI_DATA) { 265 struct cpl_iscsi_data *cpl = (struct cpl_iscsi_data *)gl->va; 266 267 offset = sizeof(struct cpl_iscsi_data); 268 pdu_cb->flags |= PDUCBF_RX_DATA; 269 len = ntohs(cpl->len); 270 pdu_cb->dlen = len; 271 pdu_cb->doffset = lro_cb->offset; 272 pdu_cb->nr_dfrags = gl->nfrags; 273 pdu_cb->dfrag_idx = skb_shinfo(skb)->nr_frags; 274 lro_cb->complete = false; 275 } else { 276 struct cpl_rx_iscsi_cmp *cpl; 277 278 cpl = (struct cpl_rx_iscsi_cmp *)gl->va; 279 offset = sizeof(struct cpl_rx_iscsi_cmp); 280 pdu_cb->flags |= (PDUCBF_RX_HDR | PDUCBF_RX_STATUS); 281 len = be16_to_cpu(cpl->len); 282 pdu_cb->hdr = gl->va + offset; 283 pdu_cb->hlen = len; 284 pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags; 285 pdu_cb->ddigest = be32_to_cpu(cpl->ulp_crc); 286 pdu_cb->pdulen = ntohs(cpl->len); 287 288 if (unlikely(gl->nfrags > 1)) 289 cxgbit_skcb_flags(skb) = 0; 290 291 cxgbit_process_ddpvld(lro_cb->csk, pdu_cb, 292 be32_to_cpu(cpl->ddpvld)); 293 294 if (pdu_cb->flags & PDUCBF_RX_DATA_DDPD) { 295 pdu_cb->flags |= PDUCBF_RX_DDP_CMP; 296 pdu_cb->complete = true; 297 } else if (pdu_cb->flags & PDUCBF_RX_DATA) { 298 pdu_cb->complete = true; 299 } 300 301 lro_cb->pdu_totallen += pdu_cb->hlen + pdu_cb->dlen; 302 lro_cb->complete = true; 303 lro_cb->pdu_idx++; 304 } 305 306 cxgbit_copy_frags(skb, gl, offset); 307 308 pdu_cb->frags += gl->nfrags; 309 lro_cb->offset += len; 310 skb->len += len; 311 skb->data_len += len; 312 skb->truesize += len; 313 } 314 315 static struct sk_buff * 316 cxgbit_lro_init_skb(struct cxgbit_sock *csk, u8 op, const struct pkt_gl *gl, 317 const __be64 *rsp, struct napi_struct *napi) 318 { 319 struct sk_buff *skb; 320 struct cxgbit_lro_cb *lro_cb; 321 322 skb = napi_alloc_skb(napi, LRO_SKB_MAX_HEADROOM); 323 324 if (unlikely(!skb)) 325 return NULL; 326 327 memset(skb->data, 0, LRO_SKB_MAX_HEADROOM); 328 329 cxgbit_skcb_flags(skb) |= SKCBF_RX_LRO; 330 331 lro_cb = cxgbit_skb_lro_cb(skb); 332 333 cxgbit_get_csk(csk); 334 335 lro_cb->csk = csk; 336 337 return skb; 338 } 339 340 static void cxgbit_queue_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 341 { 342 bool wakeup_thread = false; 343 344 spin_lock(&csk->rxq.lock); 345 __skb_queue_tail(&csk->rxq, skb); 346 if (skb_queue_len(&csk->rxq) == 1) 347 wakeup_thread = true; 348 spin_unlock(&csk->rxq.lock); 349 350 if (wakeup_thread) 351 wake_up(&csk->waitq); 352 } 353 354 static void cxgbit_lro_flush(struct t4_lro_mgr *lro_mgr, struct sk_buff *skb) 355 { 356 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 357 struct cxgbit_sock *csk = lro_cb->csk; 358 359 csk->lro_skb = NULL; 360 361 __skb_unlink(skb, &lro_mgr->lroq); 362 cxgbit_queue_lro_skb(csk, skb); 363 364 cxgbit_put_csk(csk); 365 366 lro_mgr->lro_pkts++; 367 lro_mgr->lro_session_cnt--; 368 } 369 370 static void cxgbit_uld_lro_flush(struct t4_lro_mgr *lro_mgr) 371 { 372 struct sk_buff *skb; 373 374 while ((skb = skb_peek(&lro_mgr->lroq))) 375 cxgbit_lro_flush(lro_mgr, skb); 376 } 377 378 static int 379 cxgbit_lro_receive(struct cxgbit_sock *csk, u8 op, const __be64 *rsp, 380 const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr, 381 struct napi_struct *napi) 382 { 383 struct sk_buff *skb; 384 struct cxgbit_lro_cb *lro_cb; 385 386 if (!csk) { 387 pr_err("%s: csk NULL, op 0x%x.\n", __func__, op); 388 goto out; 389 } 390 391 if (csk->lro_skb) 392 goto add_packet; 393 394 start_lro: 395 if (lro_mgr->lro_session_cnt >= MAX_LRO_SESSIONS) { 396 cxgbit_uld_lro_flush(lro_mgr); 397 goto start_lro; 398 } 399 400 skb = cxgbit_lro_init_skb(csk, op, gl, rsp, napi); 401 if (unlikely(!skb)) 402 goto out; 403 404 csk->lro_skb = skb; 405 406 __skb_queue_tail(&lro_mgr->lroq, skb); 407 lro_mgr->lro_session_cnt++; 408 409 add_packet: 410 skb = csk->lro_skb; 411 lro_cb = cxgbit_skb_lro_cb(skb); 412 413 if ((gl && (((skb_shinfo(skb)->nr_frags + gl->nfrags) > 414 MAX_SKB_FRAGS) || (lro_cb->pdu_totallen >= LRO_FLUSH_LEN_MAX))) || 415 (lro_cb->pdu_idx >= MAX_SKB_FRAGS)) { 416 cxgbit_lro_flush(lro_mgr, skb); 417 goto start_lro; 418 } 419 420 if (gl) 421 cxgbit_lro_add_packet_gl(skb, op, gl); 422 else 423 cxgbit_lro_add_packet_rsp(skb, op, rsp); 424 425 lro_mgr->lro_merged++; 426 427 return 0; 428 429 out: 430 return -1; 431 } 432 433 static int 434 cxgbit_uld_lro_rx_handler(void *hndl, const __be64 *rsp, 435 const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr, 436 struct napi_struct *napi) 437 { 438 struct cxgbit_device *cdev = hndl; 439 struct cxgb4_lld_info *lldi = &cdev->lldi; 440 struct cpl_tx_data *rpl = NULL; 441 struct cxgbit_sock *csk = NULL; 442 unsigned int tid = 0; 443 struct sk_buff *skb; 444 unsigned int op = *(u8 *)rsp; 445 bool lro_flush = true; 446 447 switch (op) { 448 case CPL_ISCSI_HDR: 449 case CPL_ISCSI_DATA: 450 case CPL_RX_ISCSI_CMP: 451 case CPL_RX_ISCSI_DDP: 452 case CPL_FW4_ACK: 453 lro_flush = false; 454 fallthrough; 455 case CPL_ABORT_RPL_RSS: 456 case CPL_PASS_ESTABLISH: 457 case CPL_PEER_CLOSE: 458 case CPL_CLOSE_CON_RPL: 459 case CPL_ABORT_REQ_RSS: 460 case CPL_SET_TCB_RPL: 461 case CPL_RX_DATA: 462 rpl = gl ? (struct cpl_tx_data *)gl->va : 463 (struct cpl_tx_data *)(rsp + 1); 464 tid = GET_TID(rpl); 465 csk = lookup_tid(lldi->tids, tid); 466 break; 467 default: 468 break; 469 } 470 471 if (csk && csk->lro_skb && lro_flush) 472 cxgbit_lro_flush(lro_mgr, csk->lro_skb); 473 474 if (!gl) { 475 unsigned int len; 476 477 if (op == CPL_RX_ISCSI_DDP) { 478 if (!cxgbit_lro_receive(csk, op, rsp, NULL, lro_mgr, 479 napi)) 480 return 0; 481 } 482 483 len = 64 - sizeof(struct rsp_ctrl) - 8; 484 skb = napi_alloc_skb(napi, len); 485 if (!skb) 486 goto nomem; 487 __skb_put(skb, len); 488 skb_copy_to_linear_data(skb, &rsp[1], len); 489 } else { 490 if (unlikely(op != *(u8 *)gl->va)) { 491 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n", 492 gl->va, be64_to_cpu(*rsp), 493 get_unaligned_be64(gl->va), 494 gl->tot_len); 495 return 0; 496 } 497 498 if ((op == CPL_ISCSI_HDR) || (op == CPL_ISCSI_DATA) || 499 (op == CPL_RX_ISCSI_CMP)) { 500 if (!cxgbit_lro_receive(csk, op, rsp, gl, lro_mgr, 501 napi)) 502 return 0; 503 } 504 505 #define RX_PULL_LEN 128 506 skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); 507 if (unlikely(!skb)) 508 goto nomem; 509 } 510 511 rpl = (struct cpl_tx_data *)skb->data; 512 op = rpl->ot.opcode; 513 cxgbit_skcb_rx_opcode(skb) = op; 514 515 pr_debug("cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n", 516 cdev, op, rpl->ot.opcode_tid, 517 ntohl(rpl->ot.opcode_tid), skb); 518 519 if (op < NUM_CPL_CMDS && cxgbit_cplhandlers[op]) { 520 cxgbit_cplhandlers[op](cdev, skb); 521 } else { 522 pr_err("No handler for opcode 0x%x.\n", op); 523 __kfree_skb(skb); 524 } 525 return 0; 526 nomem: 527 pr_err("%s OOM bailing out.\n", __func__); 528 return 1; 529 } 530 531 #ifdef CONFIG_CHELSIO_T4_DCB 532 struct cxgbit_dcb_work { 533 struct dcb_app_type dcb_app; 534 struct work_struct work; 535 }; 536 537 static void 538 cxgbit_update_dcb_priority(struct cxgbit_device *cdev, u8 port_id, 539 u8 dcb_priority, u16 port_num) 540 { 541 struct cxgbit_sock *csk; 542 struct sk_buff *skb; 543 u16 local_port; 544 bool wakeup_thread = false; 545 546 spin_lock_bh(&cdev->cskq.lock); 547 list_for_each_entry(csk, &cdev->cskq.list, list) { 548 if (csk->port_id != port_id) 549 continue; 550 551 if (csk->com.local_addr.ss_family == AF_INET6) { 552 struct sockaddr_in6 *sock_in6; 553 554 sock_in6 = (struct sockaddr_in6 *)&csk->com.local_addr; 555 local_port = ntohs(sock_in6->sin6_port); 556 } else { 557 struct sockaddr_in *sock_in; 558 559 sock_in = (struct sockaddr_in *)&csk->com.local_addr; 560 local_port = ntohs(sock_in->sin_port); 561 } 562 563 if (local_port != port_num) 564 continue; 565 566 if (csk->dcb_priority == dcb_priority) 567 continue; 568 569 skb = alloc_skb(0, GFP_ATOMIC); 570 if (!skb) 571 continue; 572 573 spin_lock(&csk->rxq.lock); 574 __skb_queue_tail(&csk->rxq, skb); 575 if (skb_queue_len(&csk->rxq) == 1) 576 wakeup_thread = true; 577 spin_unlock(&csk->rxq.lock); 578 579 if (wakeup_thread) { 580 wake_up(&csk->waitq); 581 wakeup_thread = false; 582 } 583 } 584 spin_unlock_bh(&cdev->cskq.lock); 585 } 586 587 static void cxgbit_dcb_workfn(struct work_struct *work) 588 { 589 struct cxgbit_dcb_work *dcb_work; 590 struct net_device *ndev; 591 struct cxgbit_device *cdev = NULL; 592 struct dcb_app_type *iscsi_app; 593 u8 priority, port_id = 0xff; 594 595 dcb_work = container_of(work, struct cxgbit_dcb_work, work); 596 iscsi_app = &dcb_work->dcb_app; 597 598 if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) { 599 if ((iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_STREAM) && 600 (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY)) 601 goto out; 602 603 priority = iscsi_app->app.priority; 604 605 } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) { 606 if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM) 607 goto out; 608 609 if (!iscsi_app->app.priority) 610 goto out; 611 612 priority = ffs(iscsi_app->app.priority) - 1; 613 } else { 614 goto out; 615 } 616 617 pr_debug("priority for ifid %d is %u\n", 618 iscsi_app->ifindex, priority); 619 620 ndev = dev_get_by_index(&init_net, iscsi_app->ifindex); 621 622 if (!ndev) 623 goto out; 624 625 mutex_lock(&cdev_list_lock); 626 cdev = cxgbit_find_device(ndev, &port_id); 627 628 dev_put(ndev); 629 630 if (!cdev) { 631 mutex_unlock(&cdev_list_lock); 632 goto out; 633 } 634 635 cxgbit_update_dcb_priority(cdev, port_id, priority, 636 iscsi_app->app.protocol); 637 mutex_unlock(&cdev_list_lock); 638 out: 639 kfree(dcb_work); 640 } 641 642 static int 643 cxgbit_dcbevent_notify(struct notifier_block *nb, unsigned long action, 644 void *data) 645 { 646 struct cxgbit_dcb_work *dcb_work; 647 struct dcb_app_type *dcb_app = data; 648 649 dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); 650 if (!dcb_work) 651 return NOTIFY_DONE; 652 653 dcb_work->dcb_app = *dcb_app; 654 INIT_WORK(&dcb_work->work, cxgbit_dcb_workfn); 655 schedule_work(&dcb_work->work); 656 return NOTIFY_OK; 657 } 658 #endif 659 660 static enum target_prot_op cxgbit_get_sup_prot_ops(struct iscsi_conn *conn) 661 { 662 return TARGET_PROT_NORMAL; 663 } 664 665 static struct iscsit_transport cxgbit_transport = { 666 .name = DRV_NAME, 667 .transport_type = ISCSI_CXGBIT, 668 .rdma_shutdown = false, 669 .priv_size = sizeof(struct cxgbit_cmd), 670 .owner = THIS_MODULE, 671 .iscsit_setup_np = cxgbit_setup_np, 672 .iscsit_accept_np = cxgbit_accept_np, 673 .iscsit_free_np = cxgbit_free_np, 674 .iscsit_free_conn = cxgbit_free_conn, 675 .iscsit_get_login_rx = cxgbit_get_login_rx, 676 .iscsit_put_login_tx = cxgbit_put_login_tx, 677 .iscsit_immediate_queue = iscsit_immediate_queue, 678 .iscsit_response_queue = iscsit_response_queue, 679 .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, 680 .iscsit_queue_data_in = iscsit_queue_rsp, 681 .iscsit_queue_status = iscsit_queue_rsp, 682 .iscsit_xmit_pdu = cxgbit_xmit_pdu, 683 .iscsit_get_r2t_ttt = cxgbit_get_r2t_ttt, 684 .iscsit_get_rx_pdu = cxgbit_get_rx_pdu, 685 .iscsit_validate_params = cxgbit_validate_params, 686 .iscsit_unmap_cmd = cxgbit_unmap_cmd, 687 .iscsit_aborted_task = iscsit_aborted_task, 688 .iscsit_get_sup_prot_ops = cxgbit_get_sup_prot_ops, 689 }; 690 691 static struct cxgb4_uld_info cxgbit_uld_info = { 692 .name = DRV_NAME, 693 .nrxq = MAX_ULD_QSETS, 694 .ntxq = MAX_ULD_QSETS, 695 .rxq_size = 1024, 696 .lro = true, 697 .add = cxgbit_uld_add, 698 .state_change = cxgbit_uld_state_change, 699 .lro_rx_handler = cxgbit_uld_lro_rx_handler, 700 .lro_flush = cxgbit_uld_lro_flush, 701 }; 702 703 #ifdef CONFIG_CHELSIO_T4_DCB 704 static struct notifier_block cxgbit_dcbevent_nb = { 705 .notifier_call = cxgbit_dcbevent_notify, 706 }; 707 #endif 708 709 static int __init cxgbit_init(void) 710 { 711 cxgb4_register_uld(CXGB4_ULD_ISCSIT, &cxgbit_uld_info); 712 iscsit_register_transport(&cxgbit_transport); 713 714 #ifdef CONFIG_CHELSIO_T4_DCB 715 pr_info("%s dcb enabled.\n", DRV_NAME); 716 register_dcbevent_notifier(&cxgbit_dcbevent_nb); 717 #endif 718 BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) < 719 sizeof(union cxgbit_skb_cb)); 720 return 0; 721 } 722 723 static void __exit cxgbit_exit(void) 724 { 725 struct cxgbit_device *cdev, *tmp; 726 727 #ifdef CONFIG_CHELSIO_T4_DCB 728 unregister_dcbevent_notifier(&cxgbit_dcbevent_nb); 729 #endif 730 mutex_lock(&cdev_list_lock); 731 list_for_each_entry_safe(cdev, tmp, &cdev_list_head, list) { 732 list_del(&cdev->list); 733 cxgbit_put_cdev(cdev); 734 } 735 mutex_unlock(&cdev_list_lock); 736 iscsit_unregister_transport(&cxgbit_transport); 737 cxgb4_unregister_uld(CXGB4_ULD_ISCSIT); 738 } 739 740 module_init(cxgbit_init); 741 module_exit(cxgbit_exit); 742 743 MODULE_DESCRIPTION("Chelsio iSCSI target offload driver"); 744 MODULE_AUTHOR("Chelsio Communications"); 745 MODULE_VERSION(DRV_VERSION); 746 MODULE_LICENSE("GPL"); 747