1 /** 2 * Copyright (C) 2005 - 2013 Emulex 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com) 11 * 12 * Contact Information: 13 * linux-drivers@emulex.com 14 * 15 * Emulex 16 * 3333 Susan Street 17 * Costa Mesa, CA 92626 18 */ 19 20 #include <linux/reboot.h> 21 #include <linux/delay.h> 22 #include <linux/slab.h> 23 #include <linux/interrupt.h> 24 #include <linux/blkdev.h> 25 #include <linux/pci.h> 26 #include <linux/string.h> 27 #include <linux/kernel.h> 28 #include <linux/semaphore.h> 29 #include <linux/iscsi_boot_sysfs.h> 30 #include <linux/module.h> 31 #include <linux/bsg-lib.h> 32 33 #include <scsi/libiscsi.h> 34 #include <scsi/scsi_bsg_iscsi.h> 35 #include <scsi/scsi_netlink.h> 36 #include <scsi/scsi_transport_iscsi.h> 37 #include <scsi/scsi_transport.h> 38 #include <scsi/scsi_cmnd.h> 39 #include <scsi/scsi_device.h> 40 #include <scsi/scsi_host.h> 41 #include <scsi/scsi.h> 42 #include "be_main.h" 43 #include "be_iscsi.h" 44 #include "be_mgmt.h" 45 #include "be_cmds.h" 46 47 static unsigned int be_iopoll_budget = 10; 48 static unsigned int be_max_phys_size = 64; 49 static unsigned int enable_msix = 1; 50 51 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 52 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR); 53 MODULE_VERSION(BUILD_STR); 54 MODULE_AUTHOR("Emulex Corporation"); 55 MODULE_LICENSE("GPL"); 56 module_param(be_iopoll_budget, int, 0); 57 module_param(enable_msix, int, 0); 58 module_param(be_max_phys_size, uint, S_IRUGO); 59 MODULE_PARM_DESC(be_max_phys_size, 60 "Maximum Size (In Kilobytes) of physically contiguous " 61 "memory that can be allocated. Range is 16 - 128"); 62 63 #define beiscsi_disp_param(_name)\ 64 ssize_t \ 65 beiscsi_##_name##_disp(struct device *dev,\ 66 struct device_attribute *attrib, char *buf) \ 67 { \ 68 struct Scsi_Host *shost = class_to_shost(dev);\ 69 struct beiscsi_hba *phba = iscsi_host_priv(shost); \ 70 uint32_t param_val = 0; \ 71 param_val = phba->attr_##_name;\ 72 return snprintf(buf, PAGE_SIZE, "%d\n",\ 73 phba->attr_##_name);\ 74 } 75 76 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\ 77 int \ 78 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\ 79 {\ 80 if (val >= _minval && val <= _maxval) {\ 81 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\ 82 "BA_%d : beiscsi_"#_name" updated "\ 83 "from 0x%x ==> 0x%x\n",\ 84 phba->attr_##_name, val); \ 85 phba->attr_##_name = val;\ 86 return 0;\ 87 } \ 88 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \ 89 "BA_%d beiscsi_"#_name" attribute "\ 90 "cannot be updated to 0x%x, "\ 91 "range allowed is ["#_minval" - "#_maxval"]\n", val);\ 92 return -EINVAL;\ 93 } 94 95 #define beiscsi_store_param(_name) \ 96 ssize_t \ 97 beiscsi_##_name##_store(struct device *dev,\ 98 struct device_attribute *attr, const char *buf,\ 99 size_t count) \ 100 { \ 101 struct Scsi_Host *shost = class_to_shost(dev);\ 102 struct beiscsi_hba *phba = iscsi_host_priv(shost);\ 103 uint32_t param_val = 0;\ 104 if (!isdigit(buf[0]))\ 105 return -EINVAL;\ 106 if (sscanf(buf, "%i", ¶m_val) != 1)\ 107 return -EINVAL;\ 108 if (beiscsi_##_name##_change(phba, param_val) == 0) \ 109 return strlen(buf);\ 110 else \ 111 return -EINVAL;\ 112 } 113 114 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \ 115 int \ 116 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \ 117 { \ 118 if (val >= _minval && val <= _maxval) {\ 119 phba->attr_##_name = val;\ 120 return 0;\ 121 } \ 122 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\ 123 "BA_%d beiscsi_"#_name" attribute " \ 124 "cannot be updated to 0x%x, "\ 125 "range allowed is ["#_minval" - "#_maxval"]\n", val);\ 126 phba->attr_##_name = _defval;\ 127 return -EINVAL;\ 128 } 129 130 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \ 131 static uint beiscsi_##_name = _defval;\ 132 module_param(beiscsi_##_name, uint, S_IRUGO);\ 133 MODULE_PARM_DESC(beiscsi_##_name, _descp);\ 134 beiscsi_disp_param(_name)\ 135 beiscsi_change_param(_name, _minval, _maxval, _defval)\ 136 beiscsi_store_param(_name)\ 137 beiscsi_init_param(_name, _minval, _maxval, _defval)\ 138 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\ 139 beiscsi_##_name##_disp, beiscsi_##_name##_store) 140 141 /* 142 * When new log level added update the 143 * the MAX allowed value for log_enable 144 */ 145 BEISCSI_RW_ATTR(log_enable, 0x00, 146 0xFF, 0x00, "Enable logging Bit Mask\n" 147 "\t\t\t\tInitialization Events : 0x01\n" 148 "\t\t\t\tMailbox Events : 0x02\n" 149 "\t\t\t\tMiscellaneous Events : 0x04\n" 150 "\t\t\t\tError Handling : 0x08\n" 151 "\t\t\t\tIO Path Events : 0x10\n" 152 "\t\t\t\tConfiguration Path : 0x20\n" 153 "\t\t\t\tiSCSI Protocol : 0x40\n"); 154 155 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL); 156 DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL); 157 DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL); 158 DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL); 159 DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO, 160 beiscsi_active_session_disp, NULL); 161 DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO, 162 beiscsi_free_session_disp, NULL); 163 struct device_attribute *beiscsi_attrs[] = { 164 &dev_attr_beiscsi_log_enable, 165 &dev_attr_beiscsi_drvr_ver, 166 &dev_attr_beiscsi_adapter_family, 167 &dev_attr_beiscsi_fw_ver, 168 &dev_attr_beiscsi_active_session_count, 169 &dev_attr_beiscsi_free_session_count, 170 &dev_attr_beiscsi_phys_port, 171 NULL, 172 }; 173 174 static char const *cqe_desc[] = { 175 "RESERVED_DESC", 176 "SOL_CMD_COMPLETE", 177 "SOL_CMD_KILLED_DATA_DIGEST_ERR", 178 "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL", 179 "CXN_KILLED_BURST_LEN_MISMATCH", 180 "CXN_KILLED_AHS_RCVD", 181 "CXN_KILLED_HDR_DIGEST_ERR", 182 "CXN_KILLED_UNKNOWN_HDR", 183 "CXN_KILLED_STALE_ITT_TTT_RCVD", 184 "CXN_KILLED_INVALID_ITT_TTT_RCVD", 185 "CXN_KILLED_RST_RCVD", 186 "CXN_KILLED_TIMED_OUT", 187 "CXN_KILLED_RST_SENT", 188 "CXN_KILLED_FIN_RCVD", 189 "CXN_KILLED_BAD_UNSOL_PDU_RCVD", 190 "CXN_KILLED_BAD_WRB_INDEX_ERROR", 191 "CXN_KILLED_OVER_RUN_RESIDUAL", 192 "CXN_KILLED_UNDER_RUN_RESIDUAL", 193 "CMD_KILLED_INVALID_STATSN_RCVD", 194 "CMD_KILLED_INVALID_R2T_RCVD", 195 "CMD_CXN_KILLED_LUN_INVALID", 196 "CMD_CXN_KILLED_ICD_INVALID", 197 "CMD_CXN_KILLED_ITT_INVALID", 198 "CMD_CXN_KILLED_SEQ_OUTOFORDER", 199 "CMD_CXN_KILLED_INVALID_DATASN_RCVD", 200 "CXN_INVALIDATE_NOTIFY", 201 "CXN_INVALIDATE_INDEX_NOTIFY", 202 "CMD_INVALIDATED_NOTIFY", 203 "UNSOL_HDR_NOTIFY", 204 "UNSOL_DATA_NOTIFY", 205 "UNSOL_DATA_DIGEST_ERROR_NOTIFY", 206 "DRIVERMSG_NOTIFY", 207 "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN", 208 "SOL_CMD_KILLED_DIF_ERR", 209 "CXN_KILLED_SYN_RCVD", 210 "CXN_KILLED_IMM_DATA_RCVD" 211 }; 212 213 static int beiscsi_slave_configure(struct scsi_device *sdev) 214 { 215 blk_queue_max_segment_size(sdev->request_queue, 65536); 216 return 0; 217 } 218 219 static int beiscsi_eh_abort(struct scsi_cmnd *sc) 220 { 221 struct iscsi_cls_session *cls_session; 222 struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr; 223 struct beiscsi_io_task *aborted_io_task; 224 struct iscsi_conn *conn; 225 struct beiscsi_conn *beiscsi_conn; 226 struct beiscsi_hba *phba; 227 struct iscsi_session *session; 228 struct invalidate_command_table *inv_tbl; 229 struct be_dma_mem nonemb_cmd; 230 unsigned int cid, tag, num_invalidate; 231 int rc; 232 233 cls_session = starget_to_session(scsi_target(sc->device)); 234 session = cls_session->dd_data; 235 236 spin_lock_bh(&session->frwd_lock); 237 if (!aborted_task || !aborted_task->sc) { 238 /* we raced */ 239 spin_unlock_bh(&session->frwd_lock); 240 return SUCCESS; 241 } 242 243 aborted_io_task = aborted_task->dd_data; 244 if (!aborted_io_task->scsi_cmnd) { 245 /* raced or invalid command */ 246 spin_unlock_bh(&session->frwd_lock); 247 return SUCCESS; 248 } 249 spin_unlock_bh(&session->frwd_lock); 250 /* Invalidate WRB Posted for this Task */ 251 AMAP_SET_BITS(struct amap_iscsi_wrb, invld, 252 aborted_io_task->pwrb_handle->pwrb, 253 1); 254 255 conn = aborted_task->conn; 256 beiscsi_conn = conn->dd_data; 257 phba = beiscsi_conn->phba; 258 259 /* invalidate iocb */ 260 cid = beiscsi_conn->beiscsi_conn_cid; 261 inv_tbl = phba->inv_tbl; 262 memset(inv_tbl, 0x0, sizeof(*inv_tbl)); 263 inv_tbl->cid = cid; 264 inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index; 265 num_invalidate = 1; 266 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 267 sizeof(struct invalidate_commands_params_in), 268 &nonemb_cmd.dma); 269 if (nonemb_cmd.va == NULL) { 270 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH, 271 "BM_%d : Failed to allocate memory for" 272 "mgmt_invalidate_icds\n"); 273 return FAILED; 274 } 275 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in); 276 277 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, 278 cid, &nonemb_cmd); 279 if (!tag) { 280 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH, 281 "BM_%d : mgmt_invalidate_icds could not be" 282 "submitted\n"); 283 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 284 nonemb_cmd.va, nonemb_cmd.dma); 285 286 return FAILED; 287 } 288 289 rc = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd); 290 if (rc != -EBUSY) 291 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 292 nonemb_cmd.va, nonemb_cmd.dma); 293 294 return iscsi_eh_abort(sc); 295 } 296 297 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc) 298 { 299 struct iscsi_task *abrt_task; 300 struct beiscsi_io_task *abrt_io_task; 301 struct iscsi_conn *conn; 302 struct beiscsi_conn *beiscsi_conn; 303 struct beiscsi_hba *phba; 304 struct iscsi_session *session; 305 struct iscsi_cls_session *cls_session; 306 struct invalidate_command_table *inv_tbl; 307 struct be_dma_mem nonemb_cmd; 308 unsigned int cid, tag, i, num_invalidate; 309 int rc; 310 311 /* invalidate iocbs */ 312 cls_session = starget_to_session(scsi_target(sc->device)); 313 session = cls_session->dd_data; 314 spin_lock_bh(&session->frwd_lock); 315 if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) { 316 spin_unlock_bh(&session->frwd_lock); 317 return FAILED; 318 } 319 conn = session->leadconn; 320 beiscsi_conn = conn->dd_data; 321 phba = beiscsi_conn->phba; 322 cid = beiscsi_conn->beiscsi_conn_cid; 323 inv_tbl = phba->inv_tbl; 324 memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN); 325 num_invalidate = 0; 326 for (i = 0; i < conn->session->cmds_max; i++) { 327 abrt_task = conn->session->cmds[i]; 328 abrt_io_task = abrt_task->dd_data; 329 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE) 330 continue; 331 332 if (sc->device->lun != abrt_task->sc->device->lun) 333 continue; 334 335 /* Invalidate WRB Posted for this Task */ 336 AMAP_SET_BITS(struct amap_iscsi_wrb, invld, 337 abrt_io_task->pwrb_handle->pwrb, 338 1); 339 340 inv_tbl->cid = cid; 341 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index; 342 num_invalidate++; 343 inv_tbl++; 344 } 345 spin_unlock_bh(&session->frwd_lock); 346 inv_tbl = phba->inv_tbl; 347 348 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 349 sizeof(struct invalidate_commands_params_in), 350 &nonemb_cmd.dma); 351 if (nonemb_cmd.va == NULL) { 352 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH, 353 "BM_%d : Failed to allocate memory for" 354 "mgmt_invalidate_icds\n"); 355 return FAILED; 356 } 357 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in); 358 memset(nonemb_cmd.va, 0, nonemb_cmd.size); 359 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, 360 cid, &nonemb_cmd); 361 if (!tag) { 362 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH, 363 "BM_%d : mgmt_invalidate_icds could not be" 364 " submitted\n"); 365 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 366 nonemb_cmd.va, nonemb_cmd.dma); 367 return FAILED; 368 } 369 370 rc = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd); 371 if (rc != -EBUSY) 372 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 373 nonemb_cmd.va, nonemb_cmd.dma); 374 return iscsi_eh_device_reset(sc); 375 } 376 377 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf) 378 { 379 struct beiscsi_hba *phba = data; 380 struct mgmt_session_info *boot_sess = &phba->boot_sess; 381 struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0]; 382 char *str = buf; 383 int rc; 384 385 switch (type) { 386 case ISCSI_BOOT_TGT_NAME: 387 rc = sprintf(buf, "%.*s\n", 388 (int)strlen(boot_sess->target_name), 389 (char *)&boot_sess->target_name); 390 break; 391 case ISCSI_BOOT_TGT_IP_ADDR: 392 if (boot_conn->dest_ipaddr.ip_type == 0x1) 393 rc = sprintf(buf, "%pI4\n", 394 (char *)&boot_conn->dest_ipaddr.addr); 395 else 396 rc = sprintf(str, "%pI6\n", 397 (char *)&boot_conn->dest_ipaddr.addr); 398 break; 399 case ISCSI_BOOT_TGT_PORT: 400 rc = sprintf(str, "%d\n", boot_conn->dest_port); 401 break; 402 403 case ISCSI_BOOT_TGT_CHAP_NAME: 404 rc = sprintf(str, "%.*s\n", 405 boot_conn->negotiated_login_options.auth_data.chap. 406 target_chap_name_length, 407 (char *)&boot_conn->negotiated_login_options. 408 auth_data.chap.target_chap_name); 409 break; 410 case ISCSI_BOOT_TGT_CHAP_SECRET: 411 rc = sprintf(str, "%.*s\n", 412 boot_conn->negotiated_login_options.auth_data.chap. 413 target_secret_length, 414 (char *)&boot_conn->negotiated_login_options. 415 auth_data.chap.target_secret); 416 break; 417 case ISCSI_BOOT_TGT_REV_CHAP_NAME: 418 rc = sprintf(str, "%.*s\n", 419 boot_conn->negotiated_login_options.auth_data.chap. 420 intr_chap_name_length, 421 (char *)&boot_conn->negotiated_login_options. 422 auth_data.chap.intr_chap_name); 423 break; 424 case ISCSI_BOOT_TGT_REV_CHAP_SECRET: 425 rc = sprintf(str, "%.*s\n", 426 boot_conn->negotiated_login_options.auth_data.chap. 427 intr_secret_length, 428 (char *)&boot_conn->negotiated_login_options. 429 auth_data.chap.intr_secret); 430 break; 431 case ISCSI_BOOT_TGT_FLAGS: 432 rc = sprintf(str, "2\n"); 433 break; 434 case ISCSI_BOOT_TGT_NIC_ASSOC: 435 rc = sprintf(str, "0\n"); 436 break; 437 default: 438 rc = -ENOSYS; 439 break; 440 } 441 return rc; 442 } 443 444 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf) 445 { 446 struct beiscsi_hba *phba = data; 447 char *str = buf; 448 int rc; 449 450 switch (type) { 451 case ISCSI_BOOT_INI_INITIATOR_NAME: 452 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname); 453 break; 454 default: 455 rc = -ENOSYS; 456 break; 457 } 458 return rc; 459 } 460 461 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf) 462 { 463 struct beiscsi_hba *phba = data; 464 char *str = buf; 465 int rc; 466 467 switch (type) { 468 case ISCSI_BOOT_ETH_FLAGS: 469 rc = sprintf(str, "2\n"); 470 break; 471 case ISCSI_BOOT_ETH_INDEX: 472 rc = sprintf(str, "0\n"); 473 break; 474 case ISCSI_BOOT_ETH_MAC: 475 rc = beiscsi_get_macaddr(str, phba); 476 break; 477 default: 478 rc = -ENOSYS; 479 break; 480 } 481 return rc; 482 } 483 484 485 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type) 486 { 487 umode_t rc; 488 489 switch (type) { 490 case ISCSI_BOOT_TGT_NAME: 491 case ISCSI_BOOT_TGT_IP_ADDR: 492 case ISCSI_BOOT_TGT_PORT: 493 case ISCSI_BOOT_TGT_CHAP_NAME: 494 case ISCSI_BOOT_TGT_CHAP_SECRET: 495 case ISCSI_BOOT_TGT_REV_CHAP_NAME: 496 case ISCSI_BOOT_TGT_REV_CHAP_SECRET: 497 case ISCSI_BOOT_TGT_NIC_ASSOC: 498 case ISCSI_BOOT_TGT_FLAGS: 499 rc = S_IRUGO; 500 break; 501 default: 502 rc = 0; 503 break; 504 } 505 return rc; 506 } 507 508 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type) 509 { 510 umode_t rc; 511 512 switch (type) { 513 case ISCSI_BOOT_INI_INITIATOR_NAME: 514 rc = S_IRUGO; 515 break; 516 default: 517 rc = 0; 518 break; 519 } 520 return rc; 521 } 522 523 524 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type) 525 { 526 umode_t rc; 527 528 switch (type) { 529 case ISCSI_BOOT_ETH_FLAGS: 530 case ISCSI_BOOT_ETH_MAC: 531 case ISCSI_BOOT_ETH_INDEX: 532 rc = S_IRUGO; 533 break; 534 default: 535 rc = 0; 536 break; 537 } 538 return rc; 539 } 540 541 /*------------------- PCI Driver operations and data ----------------- */ 542 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = { 543 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 544 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 545 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 546 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 547 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) }, 548 { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) }, 549 { 0 } 550 }; 551 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); 552 553 554 static struct scsi_host_template beiscsi_sht = { 555 .module = THIS_MODULE, 556 .name = "Emulex 10Gbe open-iscsi Initiator Driver", 557 .proc_name = DRV_NAME, 558 .queuecommand = iscsi_queuecommand, 559 .change_queue_depth = iscsi_change_queue_depth, 560 .slave_configure = beiscsi_slave_configure, 561 .target_alloc = iscsi_target_alloc, 562 .eh_abort_handler = beiscsi_eh_abort, 563 .eh_device_reset_handler = beiscsi_eh_device_reset, 564 .eh_target_reset_handler = iscsi_eh_session_reset, 565 .shost_attrs = beiscsi_attrs, 566 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS, 567 .can_queue = BE2_IO_DEPTH, 568 .this_id = -1, 569 .max_sectors = BEISCSI_MAX_SECTORS, 570 .cmd_per_lun = BEISCSI_CMD_PER_LUN, 571 .use_clustering = ENABLE_CLUSTERING, 572 .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID, 573 574 }; 575 576 static struct scsi_transport_template *beiscsi_scsi_transport; 577 578 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev) 579 { 580 struct beiscsi_hba *phba; 581 struct Scsi_Host *shost; 582 583 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0); 584 if (!shost) { 585 dev_err(&pcidev->dev, 586 "beiscsi_hba_alloc - iscsi_host_alloc failed\n"); 587 return NULL; 588 } 589 shost->dma_boundary = pcidev->dma_mask; 590 shost->max_id = BE2_MAX_SESSIONS; 591 shost->max_channel = 0; 592 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN; 593 shost->max_lun = BEISCSI_NUM_MAX_LUN; 594 shost->transportt = beiscsi_scsi_transport; 595 phba = iscsi_host_priv(shost); 596 memset(phba, 0, sizeof(*phba)); 597 phba->shost = shost; 598 phba->pcidev = pci_dev_get(pcidev); 599 pci_set_drvdata(pcidev, phba); 600 phba->interface_handle = 0xFFFFFFFF; 601 602 if (iscsi_host_add(shost, &phba->pcidev->dev)) 603 goto free_devices; 604 605 return phba; 606 607 free_devices: 608 pci_dev_put(phba->pcidev); 609 iscsi_host_free(phba->shost); 610 return NULL; 611 } 612 613 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba) 614 { 615 if (phba->csr_va) { 616 iounmap(phba->csr_va); 617 phba->csr_va = NULL; 618 } 619 if (phba->db_va) { 620 iounmap(phba->db_va); 621 phba->db_va = NULL; 622 } 623 if (phba->pci_va) { 624 iounmap(phba->pci_va); 625 phba->pci_va = NULL; 626 } 627 } 628 629 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, 630 struct pci_dev *pcidev) 631 { 632 u8 __iomem *addr; 633 int pcicfg_reg; 634 635 addr = ioremap_nocache(pci_resource_start(pcidev, 2), 636 pci_resource_len(pcidev, 2)); 637 if (addr == NULL) 638 return -ENOMEM; 639 phba->ctrl.csr = addr; 640 phba->csr_va = addr; 641 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2); 642 643 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024); 644 if (addr == NULL) 645 goto pci_map_err; 646 phba->ctrl.db = addr; 647 phba->db_va = addr; 648 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4); 649 650 if (phba->generation == BE_GEN2) 651 pcicfg_reg = 1; 652 else 653 pcicfg_reg = 0; 654 655 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg), 656 pci_resource_len(pcidev, pcicfg_reg)); 657 658 if (addr == NULL) 659 goto pci_map_err; 660 phba->ctrl.pcicfg = addr; 661 phba->pci_va = addr; 662 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg); 663 return 0; 664 665 pci_map_err: 666 beiscsi_unmap_pci_function(phba); 667 return -ENOMEM; 668 } 669 670 static int beiscsi_enable_pci(struct pci_dev *pcidev) 671 { 672 int ret; 673 674 ret = pci_enable_device(pcidev); 675 if (ret) { 676 dev_err(&pcidev->dev, 677 "beiscsi_enable_pci - enable device failed\n"); 678 return ret; 679 } 680 681 pci_set_master(pcidev); 682 ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64)); 683 if (ret) { 684 ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32)); 685 if (ret) { 686 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n"); 687 pci_disable_device(pcidev); 688 return ret; 689 } else { 690 ret = pci_set_consistent_dma_mask(pcidev, 691 DMA_BIT_MASK(32)); 692 } 693 } else { 694 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64)); 695 if (ret) { 696 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n"); 697 pci_disable_device(pcidev); 698 return ret; 699 } 700 } 701 return 0; 702 } 703 704 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev) 705 { 706 struct be_ctrl_info *ctrl = &phba->ctrl; 707 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced; 708 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem; 709 int status = 0; 710 711 ctrl->pdev = pdev; 712 status = beiscsi_map_pci_bars(phba, pdev); 713 if (status) 714 return status; 715 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 716 mbox_mem_alloc->va = pci_alloc_consistent(pdev, 717 mbox_mem_alloc->size, 718 &mbox_mem_alloc->dma); 719 if (!mbox_mem_alloc->va) { 720 beiscsi_unmap_pci_function(phba); 721 return -ENOMEM; 722 } 723 724 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 725 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 726 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 727 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 728 spin_lock_init(&ctrl->mbox_lock); 729 spin_lock_init(&phba->ctrl.mcc_lock); 730 spin_lock_init(&phba->ctrl.mcc_cq_lock); 731 732 return status; 733 } 734 735 /** 736 * beiscsi_get_params()- Set the config paramters 737 * @phba: ptr device priv structure 738 **/ 739 static void beiscsi_get_params(struct beiscsi_hba *phba) 740 { 741 uint32_t total_cid_count = 0; 742 uint32_t total_icd_count = 0; 743 uint8_t ulp_num = 0; 744 745 total_cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) + 746 BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1); 747 748 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 749 uint32_t align_mask = 0; 750 uint32_t icd_post_per_page = 0; 751 uint32_t icd_count_unavailable = 0; 752 uint32_t icd_start = 0, icd_count = 0; 753 uint32_t icd_start_align = 0, icd_count_align = 0; 754 755 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 756 icd_start = phba->fw_config.iscsi_icd_start[ulp_num]; 757 icd_count = phba->fw_config.iscsi_icd_count[ulp_num]; 758 759 /* Get ICD count that can be posted on each page */ 760 icd_post_per_page = (PAGE_SIZE / (BE2_SGE * 761 sizeof(struct iscsi_sge))); 762 align_mask = (icd_post_per_page - 1); 763 764 /* Check if icd_start is aligned ICD per page posting */ 765 if (icd_start % icd_post_per_page) { 766 icd_start_align = ((icd_start + 767 icd_post_per_page) & 768 ~(align_mask)); 769 phba->fw_config. 770 iscsi_icd_start[ulp_num] = 771 icd_start_align; 772 } 773 774 icd_count_align = (icd_count & ~align_mask); 775 776 /* ICD discarded in the process of alignment */ 777 if (icd_start_align) 778 icd_count_unavailable = ((icd_start_align - 779 icd_start) + 780 (icd_count - 781 icd_count_align)); 782 783 /* Updated ICD count available */ 784 phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count - 785 icd_count_unavailable); 786 787 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 788 "BM_%d : Aligned ICD values\n" 789 "\t ICD Start : %d\n" 790 "\t ICD Count : %d\n" 791 "\t ICD Discarded : %d\n", 792 phba->fw_config. 793 iscsi_icd_start[ulp_num], 794 phba->fw_config. 795 iscsi_icd_count[ulp_num], 796 icd_count_unavailable); 797 break; 798 } 799 } 800 801 total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num]; 802 phba->params.ios_per_ctrl = (total_icd_count - 803 (total_cid_count + 804 BE2_TMFS + BE2_NOPOUT_REQ)); 805 phba->params.cxns_per_ctrl = total_cid_count; 806 phba->params.asyncpdus_per_ctrl = total_cid_count; 807 phba->params.icds_per_ctrl = total_icd_count; 808 phba->params.num_sge_per_io = BE2_SGE; 809 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ; 810 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ; 811 phba->params.eq_timer = 64; 812 phba->params.num_eq_entries = 1024; 813 phba->params.num_cq_entries = 1024; 814 phba->params.wrbs_per_cxn = 256; 815 } 816 817 static void hwi_ring_eq_db(struct beiscsi_hba *phba, 818 unsigned int id, unsigned int clr_interrupt, 819 unsigned int num_processed, 820 unsigned char rearm, unsigned char event) 821 { 822 u32 val = 0; 823 824 if (rearm) 825 val |= 1 << DB_EQ_REARM_SHIFT; 826 if (clr_interrupt) 827 val |= 1 << DB_EQ_CLR_SHIFT; 828 if (event) 829 val |= 1 << DB_EQ_EVNT_SHIFT; 830 831 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT; 832 /* Setting lower order EQ_ID Bits */ 833 val |= (id & DB_EQ_RING_ID_LOW_MASK); 834 835 /* Setting Higher order EQ_ID Bits */ 836 val |= (((id >> DB_EQ_HIGH_FEILD_SHIFT) & 837 DB_EQ_RING_ID_HIGH_MASK) 838 << DB_EQ_HIGH_SET_SHIFT); 839 840 iowrite32(val, phba->db_va + DB_EQ_OFFSET); 841 } 842 843 /** 844 * be_isr_mcc - The isr routine of the driver. 845 * @irq: Not used 846 * @dev_id: Pointer to host adapter structure 847 */ 848 static irqreturn_t be_isr_mcc(int irq, void *dev_id) 849 { 850 struct beiscsi_hba *phba; 851 struct be_eq_entry *eqe = NULL; 852 struct be_queue_info *eq; 853 struct be_queue_info *mcc; 854 unsigned int num_eq_processed; 855 struct be_eq_obj *pbe_eq; 856 unsigned long flags; 857 858 pbe_eq = dev_id; 859 eq = &pbe_eq->q; 860 phba = pbe_eq->phba; 861 mcc = &phba->ctrl.mcc_obj.cq; 862 eqe = queue_tail_node(eq); 863 864 num_eq_processed = 0; 865 866 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 867 & EQE_VALID_MASK) { 868 if (((eqe->dw[offsetof(struct amap_eq_entry, 869 resource_id) / 32] & 870 EQE_RESID_MASK) >> 16) == mcc->id) { 871 spin_lock_irqsave(&phba->isr_lock, flags); 872 pbe_eq->todo_mcc_cq = true; 873 spin_unlock_irqrestore(&phba->isr_lock, flags); 874 } 875 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 876 queue_tail_inc(eq); 877 eqe = queue_tail_node(eq); 878 num_eq_processed++; 879 } 880 if (pbe_eq->todo_mcc_cq) 881 queue_work(phba->wq, &pbe_eq->work_cqs); 882 if (num_eq_processed) 883 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1); 884 885 return IRQ_HANDLED; 886 } 887 888 /** 889 * be_isr_msix - The isr routine of the driver. 890 * @irq: Not used 891 * @dev_id: Pointer to host adapter structure 892 */ 893 static irqreturn_t be_isr_msix(int irq, void *dev_id) 894 { 895 struct beiscsi_hba *phba; 896 struct be_eq_entry *eqe = NULL; 897 struct be_queue_info *eq; 898 struct be_queue_info *cq; 899 unsigned int num_eq_processed; 900 struct be_eq_obj *pbe_eq; 901 902 pbe_eq = dev_id; 903 eq = &pbe_eq->q; 904 cq = pbe_eq->cq; 905 eqe = queue_tail_node(eq); 906 907 phba = pbe_eq->phba; 908 num_eq_processed = 0; 909 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 910 & EQE_VALID_MASK) { 911 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 912 blk_iopoll_sched(&pbe_eq->iopoll); 913 914 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 915 queue_tail_inc(eq); 916 eqe = queue_tail_node(eq); 917 num_eq_processed++; 918 } 919 920 if (num_eq_processed) 921 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1); 922 923 return IRQ_HANDLED; 924 } 925 926 /** 927 * be_isr - The isr routine of the driver. 928 * @irq: Not used 929 * @dev_id: Pointer to host adapter structure 930 */ 931 static irqreturn_t be_isr(int irq, void *dev_id) 932 { 933 struct beiscsi_hba *phba; 934 struct hwi_controller *phwi_ctrlr; 935 struct hwi_context_memory *phwi_context; 936 struct be_eq_entry *eqe = NULL; 937 struct be_queue_info *eq; 938 struct be_queue_info *mcc; 939 unsigned long flags, index; 940 unsigned int num_mcceq_processed, num_ioeq_processed; 941 struct be_ctrl_info *ctrl; 942 struct be_eq_obj *pbe_eq; 943 int isr; 944 945 phba = dev_id; 946 ctrl = &phba->ctrl; 947 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET + 948 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE)); 949 if (!isr) 950 return IRQ_NONE; 951 952 phwi_ctrlr = phba->phwi_ctrlr; 953 phwi_context = phwi_ctrlr->phwi_ctxt; 954 pbe_eq = &phwi_context->be_eq[0]; 955 956 eq = &phwi_context->be_eq[0].q; 957 mcc = &phba->ctrl.mcc_obj.cq; 958 index = 0; 959 eqe = queue_tail_node(eq); 960 961 num_ioeq_processed = 0; 962 num_mcceq_processed = 0; 963 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 964 & EQE_VALID_MASK) { 965 if (((eqe->dw[offsetof(struct amap_eq_entry, 966 resource_id) / 32] & 967 EQE_RESID_MASK) >> 16) == mcc->id) { 968 spin_lock_irqsave(&phba->isr_lock, flags); 969 pbe_eq->todo_mcc_cq = true; 970 spin_unlock_irqrestore(&phba->isr_lock, flags); 971 num_mcceq_processed++; 972 } else { 973 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 974 blk_iopoll_sched(&pbe_eq->iopoll); 975 num_ioeq_processed++; 976 } 977 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 978 queue_tail_inc(eq); 979 eqe = queue_tail_node(eq); 980 } 981 if (num_ioeq_processed || num_mcceq_processed) { 982 if (pbe_eq->todo_mcc_cq) 983 queue_work(phba->wq, &pbe_eq->work_cqs); 984 985 if ((num_mcceq_processed) && (!num_ioeq_processed)) 986 hwi_ring_eq_db(phba, eq->id, 0, 987 (num_ioeq_processed + 988 num_mcceq_processed) , 1, 1); 989 else 990 hwi_ring_eq_db(phba, eq->id, 0, 991 (num_ioeq_processed + 992 num_mcceq_processed), 0, 1); 993 994 return IRQ_HANDLED; 995 } else 996 return IRQ_NONE; 997 } 998 999 static int beiscsi_init_irqs(struct beiscsi_hba *phba) 1000 { 1001 struct pci_dev *pcidev = phba->pcidev; 1002 struct hwi_controller *phwi_ctrlr; 1003 struct hwi_context_memory *phwi_context; 1004 int ret, msix_vec, i, j; 1005 1006 phwi_ctrlr = phba->phwi_ctrlr; 1007 phwi_context = phwi_ctrlr->phwi_ctxt; 1008 1009 if (phba->msix_enabled) { 1010 for (i = 0; i < phba->num_cpus; i++) { 1011 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, 1012 GFP_KERNEL); 1013 if (!phba->msi_name[i]) { 1014 ret = -ENOMEM; 1015 goto free_msix_irqs; 1016 } 1017 1018 sprintf(phba->msi_name[i], "beiscsi_%02x_%02x", 1019 phba->shost->host_no, i); 1020 msix_vec = phba->msix_entries[i].vector; 1021 ret = request_irq(msix_vec, be_isr_msix, 0, 1022 phba->msi_name[i], 1023 &phwi_context->be_eq[i]); 1024 if (ret) { 1025 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 1026 "BM_%d : beiscsi_init_irqs-Failed to" 1027 "register msix for i = %d\n", 1028 i); 1029 kfree(phba->msi_name[i]); 1030 goto free_msix_irqs; 1031 } 1032 } 1033 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL); 1034 if (!phba->msi_name[i]) { 1035 ret = -ENOMEM; 1036 goto free_msix_irqs; 1037 } 1038 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x", 1039 phba->shost->host_no); 1040 msix_vec = phba->msix_entries[i].vector; 1041 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i], 1042 &phwi_context->be_eq[i]); 1043 if (ret) { 1044 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT , 1045 "BM_%d : beiscsi_init_irqs-" 1046 "Failed to register beiscsi_msix_mcc\n"); 1047 kfree(phba->msi_name[i]); 1048 goto free_msix_irqs; 1049 } 1050 1051 } else { 1052 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, 1053 "beiscsi", phba); 1054 if (ret) { 1055 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 1056 "BM_%d : beiscsi_init_irqs-" 1057 "Failed to register irq\\n"); 1058 return ret; 1059 } 1060 } 1061 return 0; 1062 free_msix_irqs: 1063 for (j = i - 1; j >= 0; j--) { 1064 kfree(phba->msi_name[j]); 1065 msix_vec = phba->msix_entries[j].vector; 1066 free_irq(msix_vec, &phwi_context->be_eq[j]); 1067 } 1068 return ret; 1069 } 1070 1071 void hwi_ring_cq_db(struct beiscsi_hba *phba, 1072 unsigned int id, unsigned int num_processed, 1073 unsigned char rearm, unsigned char event) 1074 { 1075 u32 val = 0; 1076 1077 if (rearm) 1078 val |= 1 << DB_CQ_REARM_SHIFT; 1079 1080 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT; 1081 1082 /* Setting lower order CQ_ID Bits */ 1083 val |= (id & DB_CQ_RING_ID_LOW_MASK); 1084 1085 /* Setting Higher order CQ_ID Bits */ 1086 val |= (((id >> DB_CQ_HIGH_FEILD_SHIFT) & 1087 DB_CQ_RING_ID_HIGH_MASK) 1088 << DB_CQ_HIGH_SET_SHIFT); 1089 1090 iowrite32(val, phba->db_va + DB_CQ_OFFSET); 1091 } 1092 1093 static unsigned int 1094 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn, 1095 struct beiscsi_hba *phba, 1096 struct pdu_base *ppdu, 1097 unsigned long pdu_len, 1098 void *pbuffer, unsigned long buf_len) 1099 { 1100 struct iscsi_conn *conn = beiscsi_conn->conn; 1101 struct iscsi_session *session = conn->session; 1102 struct iscsi_task *task; 1103 struct beiscsi_io_task *io_task; 1104 struct iscsi_hdr *login_hdr; 1105 1106 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] & 1107 PDUBASE_OPCODE_MASK) { 1108 case ISCSI_OP_NOOP_IN: 1109 pbuffer = NULL; 1110 buf_len = 0; 1111 break; 1112 case ISCSI_OP_ASYNC_EVENT: 1113 break; 1114 case ISCSI_OP_REJECT: 1115 WARN_ON(!pbuffer); 1116 WARN_ON(!(buf_len == 48)); 1117 beiscsi_log(phba, KERN_ERR, 1118 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 1119 "BM_%d : In ISCSI_OP_REJECT\n"); 1120 break; 1121 case ISCSI_OP_LOGIN_RSP: 1122 case ISCSI_OP_TEXT_RSP: 1123 task = conn->login_task; 1124 io_task = task->dd_data; 1125 login_hdr = (struct iscsi_hdr *)ppdu; 1126 login_hdr->itt = io_task->libiscsi_itt; 1127 break; 1128 default: 1129 beiscsi_log(phba, KERN_WARNING, 1130 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 1131 "BM_%d : Unrecognized opcode 0x%x in async msg\n", 1132 (ppdu-> 1133 dw[offsetof(struct amap_pdu_base, opcode) / 32] 1134 & PDUBASE_OPCODE_MASK)); 1135 return 1; 1136 } 1137 1138 spin_lock_bh(&session->back_lock); 1139 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len); 1140 spin_unlock_bh(&session->back_lock); 1141 return 0; 1142 } 1143 1144 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba) 1145 { 1146 struct sgl_handle *psgl_handle; 1147 1148 if (phba->io_sgl_hndl_avbl) { 1149 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 1150 "BM_%d : In alloc_io_sgl_handle," 1151 " io_sgl_alloc_index=%d\n", 1152 phba->io_sgl_alloc_index); 1153 1154 psgl_handle = phba->io_sgl_hndl_base[phba-> 1155 io_sgl_alloc_index]; 1156 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL; 1157 phba->io_sgl_hndl_avbl--; 1158 if (phba->io_sgl_alloc_index == (phba->params. 1159 ios_per_ctrl - 1)) 1160 phba->io_sgl_alloc_index = 0; 1161 else 1162 phba->io_sgl_alloc_index++; 1163 } else 1164 psgl_handle = NULL; 1165 return psgl_handle; 1166 } 1167 1168 static void 1169 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 1170 { 1171 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 1172 "BM_%d : In free_,io_sgl_free_index=%d\n", 1173 phba->io_sgl_free_index); 1174 1175 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) { 1176 /* 1177 * this can happen if clean_task is called on a task that 1178 * failed in xmit_task or alloc_pdu. 1179 */ 1180 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, 1181 "BM_%d : Double Free in IO SGL io_sgl_free_index=%d," 1182 "value there=%p\n", phba->io_sgl_free_index, 1183 phba->io_sgl_hndl_base 1184 [phba->io_sgl_free_index]); 1185 return; 1186 } 1187 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle; 1188 phba->io_sgl_hndl_avbl++; 1189 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1)) 1190 phba->io_sgl_free_index = 0; 1191 else 1192 phba->io_sgl_free_index++; 1193 } 1194 1195 /** 1196 * alloc_wrb_handle - To allocate a wrb handle 1197 * @phba: The hba pointer 1198 * @cid: The cid to use for allocation 1199 * 1200 * This happens under session_lock until submission to chip 1201 */ 1202 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid) 1203 { 1204 struct hwi_wrb_context *pwrb_context; 1205 struct hwi_controller *phwi_ctrlr; 1206 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp; 1207 uint16_t cri_index = BE_GET_CRI_FROM_CID(cid); 1208 1209 phwi_ctrlr = phba->phwi_ctrlr; 1210 pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 1211 if (pwrb_context->wrb_handles_available >= 2) { 1212 pwrb_handle = pwrb_context->pwrb_handle_base[ 1213 pwrb_context->alloc_index]; 1214 pwrb_context->wrb_handles_available--; 1215 if (pwrb_context->alloc_index == 1216 (phba->params.wrbs_per_cxn - 1)) 1217 pwrb_context->alloc_index = 0; 1218 else 1219 pwrb_context->alloc_index++; 1220 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[ 1221 pwrb_context->alloc_index]; 1222 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index; 1223 } else 1224 pwrb_handle = NULL; 1225 return pwrb_handle; 1226 } 1227 1228 /** 1229 * free_wrb_handle - To free the wrb handle back to pool 1230 * @phba: The hba pointer 1231 * @pwrb_context: The context to free from 1232 * @pwrb_handle: The wrb_handle to free 1233 * 1234 * This happens under session_lock until submission to chip 1235 */ 1236 static void 1237 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context, 1238 struct wrb_handle *pwrb_handle) 1239 { 1240 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle; 1241 pwrb_context->wrb_handles_available++; 1242 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1)) 1243 pwrb_context->free_index = 0; 1244 else 1245 pwrb_context->free_index++; 1246 1247 beiscsi_log(phba, KERN_INFO, 1248 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 1249 "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x" 1250 "wrb_handles_available=%d\n", 1251 pwrb_handle, pwrb_context->free_index, 1252 pwrb_context->wrb_handles_available); 1253 } 1254 1255 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) 1256 { 1257 struct sgl_handle *psgl_handle; 1258 1259 if (phba->eh_sgl_hndl_avbl) { 1260 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index]; 1261 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL; 1262 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, 1263 "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n", 1264 phba->eh_sgl_alloc_index, 1265 phba->eh_sgl_alloc_index); 1266 1267 phba->eh_sgl_hndl_avbl--; 1268 if (phba->eh_sgl_alloc_index == 1269 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1270 1)) 1271 phba->eh_sgl_alloc_index = 0; 1272 else 1273 phba->eh_sgl_alloc_index++; 1274 } else 1275 psgl_handle = NULL; 1276 return psgl_handle; 1277 } 1278 1279 void 1280 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) 1281 { 1282 1283 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, 1284 "BM_%d : In free_mgmt_sgl_handle," 1285 "eh_sgl_free_index=%d\n", 1286 phba->eh_sgl_free_index); 1287 1288 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) { 1289 /* 1290 * this can happen if clean_task is called on a task that 1291 * failed in xmit_task or alloc_pdu. 1292 */ 1293 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG, 1294 "BM_%d : Double Free in eh SGL ," 1295 "eh_sgl_free_index=%d\n", 1296 phba->eh_sgl_free_index); 1297 return; 1298 } 1299 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle; 1300 phba->eh_sgl_hndl_avbl++; 1301 if (phba->eh_sgl_free_index == 1302 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1)) 1303 phba->eh_sgl_free_index = 0; 1304 else 1305 phba->eh_sgl_free_index++; 1306 } 1307 1308 static void 1309 be_complete_io(struct beiscsi_conn *beiscsi_conn, 1310 struct iscsi_task *task, 1311 struct common_sol_cqe *csol_cqe) 1312 { 1313 struct beiscsi_io_task *io_task = task->dd_data; 1314 struct be_status_bhs *sts_bhs = 1315 (struct be_status_bhs *)io_task->cmd_bhs; 1316 struct iscsi_conn *conn = beiscsi_conn->conn; 1317 unsigned char *sense; 1318 u32 resid = 0, exp_cmdsn, max_cmdsn; 1319 u8 rsp, status, flags; 1320 1321 exp_cmdsn = csol_cqe->exp_cmdsn; 1322 max_cmdsn = (csol_cqe->exp_cmdsn + 1323 csol_cqe->cmd_wnd - 1); 1324 rsp = csol_cqe->i_resp; 1325 status = csol_cqe->i_sts; 1326 flags = csol_cqe->i_flags; 1327 resid = csol_cqe->res_cnt; 1328 1329 if (!task->sc) { 1330 if (io_task->scsi_cmnd) { 1331 scsi_dma_unmap(io_task->scsi_cmnd); 1332 io_task->scsi_cmnd = NULL; 1333 } 1334 1335 return; 1336 } 1337 task->sc->result = (DID_OK << 16) | status; 1338 if (rsp != ISCSI_STATUS_CMD_COMPLETED) { 1339 task->sc->result = DID_ERROR << 16; 1340 goto unmap; 1341 } 1342 1343 /* bidi not initially supported */ 1344 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) { 1345 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW)) 1346 task->sc->result = DID_ERROR << 16; 1347 1348 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) { 1349 scsi_set_resid(task->sc, resid); 1350 if (!status && (scsi_bufflen(task->sc) - resid < 1351 task->sc->underflow)) 1352 task->sc->result = DID_ERROR << 16; 1353 } 1354 } 1355 1356 if (status == SAM_STAT_CHECK_CONDITION) { 1357 u16 sense_len; 1358 unsigned short *slen = (unsigned short *)sts_bhs->sense_info; 1359 1360 sense = sts_bhs->sense_info + sizeof(unsigned short); 1361 sense_len = be16_to_cpu(*slen); 1362 memcpy(task->sc->sense_buffer, sense, 1363 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE)); 1364 } 1365 1366 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) 1367 conn->rxdata_octets += resid; 1368 unmap: 1369 scsi_dma_unmap(io_task->scsi_cmnd); 1370 io_task->scsi_cmnd = NULL; 1371 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn); 1372 } 1373 1374 static void 1375 be_complete_logout(struct beiscsi_conn *beiscsi_conn, 1376 struct iscsi_task *task, 1377 struct common_sol_cqe *csol_cqe) 1378 { 1379 struct iscsi_logout_rsp *hdr; 1380 struct beiscsi_io_task *io_task = task->dd_data; 1381 struct iscsi_conn *conn = beiscsi_conn->conn; 1382 1383 hdr = (struct iscsi_logout_rsp *)task->hdr; 1384 hdr->opcode = ISCSI_OP_LOGOUT_RSP; 1385 hdr->t2wait = 5; 1386 hdr->t2retain = 0; 1387 hdr->flags = csol_cqe->i_flags; 1388 hdr->response = csol_cqe->i_resp; 1389 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn); 1390 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn + 1391 csol_cqe->cmd_wnd - 1); 1392 1393 hdr->dlength[0] = 0; 1394 hdr->dlength[1] = 0; 1395 hdr->dlength[2] = 0; 1396 hdr->hlength = 0; 1397 hdr->itt = io_task->libiscsi_itt; 1398 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 1399 } 1400 1401 static void 1402 be_complete_tmf(struct beiscsi_conn *beiscsi_conn, 1403 struct iscsi_task *task, 1404 struct common_sol_cqe *csol_cqe) 1405 { 1406 struct iscsi_tm_rsp *hdr; 1407 struct iscsi_conn *conn = beiscsi_conn->conn; 1408 struct beiscsi_io_task *io_task = task->dd_data; 1409 1410 hdr = (struct iscsi_tm_rsp *)task->hdr; 1411 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 1412 hdr->flags = csol_cqe->i_flags; 1413 hdr->response = csol_cqe->i_resp; 1414 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn); 1415 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn + 1416 csol_cqe->cmd_wnd - 1); 1417 1418 hdr->itt = io_task->libiscsi_itt; 1419 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 1420 } 1421 1422 static void 1423 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn, 1424 struct beiscsi_hba *phba, struct sol_cqe *psol) 1425 { 1426 struct hwi_wrb_context *pwrb_context; 1427 struct wrb_handle *pwrb_handle = NULL; 1428 struct hwi_controller *phwi_ctrlr; 1429 struct iscsi_task *task; 1430 struct beiscsi_io_task *io_task; 1431 uint16_t wrb_index, cid, cri_index; 1432 1433 phwi_ctrlr = phba->phwi_ctrlr; 1434 if (is_chip_be2_be3r(phba)) { 1435 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe, 1436 wrb_idx, psol); 1437 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe, 1438 cid, psol); 1439 } else { 1440 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2, 1441 wrb_idx, psol); 1442 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2, 1443 cid, psol); 1444 } 1445 1446 cri_index = BE_GET_CRI_FROM_CID(cid); 1447 pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 1448 pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index]; 1449 task = pwrb_handle->pio_handle; 1450 1451 io_task = task->dd_data; 1452 memset(io_task->pwrb_handle->pwrb, 0, sizeof(struct iscsi_wrb)); 1453 iscsi_put_task(task); 1454 } 1455 1456 static void 1457 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn, 1458 struct iscsi_task *task, 1459 struct common_sol_cqe *csol_cqe) 1460 { 1461 struct iscsi_nopin *hdr; 1462 struct iscsi_conn *conn = beiscsi_conn->conn; 1463 struct beiscsi_io_task *io_task = task->dd_data; 1464 1465 hdr = (struct iscsi_nopin *)task->hdr; 1466 hdr->flags = csol_cqe->i_flags; 1467 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn); 1468 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn + 1469 csol_cqe->cmd_wnd - 1); 1470 1471 hdr->opcode = ISCSI_OP_NOOP_IN; 1472 hdr->itt = io_task->libiscsi_itt; 1473 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); 1474 } 1475 1476 static void adapter_get_sol_cqe(struct beiscsi_hba *phba, 1477 struct sol_cqe *psol, 1478 struct common_sol_cqe *csol_cqe) 1479 { 1480 if (is_chip_be2_be3r(phba)) { 1481 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe, 1482 i_exp_cmd_sn, psol); 1483 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe, 1484 i_res_cnt, psol); 1485 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe, 1486 i_cmd_wnd, psol); 1487 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe, 1488 wrb_index, psol); 1489 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe, 1490 cid, psol); 1491 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe, 1492 hw_sts, psol); 1493 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe, 1494 i_resp, psol); 1495 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe, 1496 i_sts, psol); 1497 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe, 1498 i_flags, psol); 1499 } else { 1500 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1501 i_exp_cmd_sn, psol); 1502 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1503 i_res_cnt, psol); 1504 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1505 wrb_index, psol); 1506 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1507 cid, psol); 1508 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1509 hw_sts, psol); 1510 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1511 i_cmd_wnd, psol); 1512 if (AMAP_GET_BITS(struct amap_sol_cqe_v2, 1513 cmd_cmpl, psol)) 1514 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1515 i_sts, psol); 1516 else 1517 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2, 1518 i_sts, psol); 1519 if (AMAP_GET_BITS(struct amap_sol_cqe_v2, 1520 u, psol)) 1521 csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW; 1522 1523 if (AMAP_GET_BITS(struct amap_sol_cqe_v2, 1524 o, psol)) 1525 csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW; 1526 } 1527 } 1528 1529 1530 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn, 1531 struct beiscsi_hba *phba, struct sol_cqe *psol) 1532 { 1533 struct hwi_wrb_context *pwrb_context; 1534 struct wrb_handle *pwrb_handle; 1535 struct iscsi_wrb *pwrb = NULL; 1536 struct hwi_controller *phwi_ctrlr; 1537 struct iscsi_task *task; 1538 unsigned int type; 1539 struct iscsi_conn *conn = beiscsi_conn->conn; 1540 struct iscsi_session *session = conn->session; 1541 struct common_sol_cqe csol_cqe = {0}; 1542 uint16_t cri_index = 0; 1543 1544 phwi_ctrlr = phba->phwi_ctrlr; 1545 1546 /* Copy the elements to a common structure */ 1547 adapter_get_sol_cqe(phba, psol, &csol_cqe); 1548 1549 cri_index = BE_GET_CRI_FROM_CID(csol_cqe.cid); 1550 pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 1551 1552 pwrb_handle = pwrb_context->pwrb_handle_basestd[ 1553 csol_cqe.wrb_index]; 1554 1555 task = pwrb_handle->pio_handle; 1556 pwrb = pwrb_handle->pwrb; 1557 type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type; 1558 1559 spin_lock_bh(&session->back_lock); 1560 switch (type) { 1561 case HWH_TYPE_IO: 1562 case HWH_TYPE_IO_RD: 1563 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == 1564 ISCSI_OP_NOOP_OUT) 1565 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe); 1566 else 1567 be_complete_io(beiscsi_conn, task, &csol_cqe); 1568 break; 1569 1570 case HWH_TYPE_LOGOUT: 1571 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT) 1572 be_complete_logout(beiscsi_conn, task, &csol_cqe); 1573 else 1574 be_complete_tmf(beiscsi_conn, task, &csol_cqe); 1575 break; 1576 1577 case HWH_TYPE_LOGIN: 1578 beiscsi_log(phba, KERN_ERR, 1579 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 1580 "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in" 1581 " hwi_complete_cmd- Solicited path\n"); 1582 break; 1583 1584 case HWH_TYPE_NOP: 1585 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe); 1586 break; 1587 1588 default: 1589 beiscsi_log(phba, KERN_WARNING, 1590 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 1591 "BM_%d : In hwi_complete_cmd, unknown type = %d" 1592 "wrb_index 0x%x CID 0x%x\n", type, 1593 csol_cqe.wrb_index, 1594 csol_cqe.cid); 1595 break; 1596 } 1597 1598 spin_unlock_bh(&session->back_lock); 1599 } 1600 1601 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context 1602 *pasync_ctx, unsigned int is_header, 1603 unsigned int host_write_ptr) 1604 { 1605 if (is_header) 1606 return &pasync_ctx->async_entry[host_write_ptr]. 1607 header_busy_list; 1608 else 1609 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list; 1610 } 1611 1612 static struct async_pdu_handle * 1613 hwi_get_async_handle(struct beiscsi_hba *phba, 1614 struct beiscsi_conn *beiscsi_conn, 1615 struct hwi_async_pdu_context *pasync_ctx, 1616 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index) 1617 { 1618 struct be_bus_address phys_addr; 1619 struct list_head *pbusy_list; 1620 struct async_pdu_handle *pasync_handle = NULL; 1621 unsigned char is_header = 0; 1622 unsigned int index, dpl; 1623 1624 if (is_chip_be2_be3r(phba)) { 1625 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 1626 dpl, pdpdu_cqe); 1627 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, 1628 index, pdpdu_cqe); 1629 } else { 1630 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2, 1631 dpl, pdpdu_cqe); 1632 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2, 1633 index, pdpdu_cqe); 1634 } 1635 1636 phys_addr.u.a32.address_lo = 1637 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 1638 db_addr_lo) / 32] - dpl); 1639 phys_addr.u.a32.address_hi = 1640 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 1641 db_addr_hi) / 32]; 1642 1643 phys_addr.u.a64.address = 1644 *((unsigned long long *)(&phys_addr.u.a64.address)); 1645 1646 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32] 1647 & PDUCQE_CODE_MASK) { 1648 case UNSOL_HDR_NOTIFY: 1649 is_header = 1; 1650 1651 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1652 is_header, index); 1653 break; 1654 case UNSOL_DATA_NOTIFY: 1655 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1656 is_header, index); 1657 break; 1658 default: 1659 pbusy_list = NULL; 1660 beiscsi_log(phba, KERN_WARNING, 1661 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 1662 "BM_%d : Unexpected code=%d\n", 1663 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, 1664 code) / 32] & PDUCQE_CODE_MASK); 1665 return NULL; 1666 } 1667 1668 WARN_ON(list_empty(pbusy_list)); 1669 list_for_each_entry(pasync_handle, pbusy_list, link) { 1670 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address) 1671 break; 1672 } 1673 1674 WARN_ON(!pasync_handle); 1675 1676 pasync_handle->cri = BE_GET_ASYNC_CRI_FROM_CID( 1677 beiscsi_conn->beiscsi_conn_cid); 1678 pasync_handle->is_header = is_header; 1679 pasync_handle->buffer_len = dpl; 1680 *pcq_index = index; 1681 1682 return pasync_handle; 1683 } 1684 1685 static unsigned int 1686 hwi_update_async_writables(struct beiscsi_hba *phba, 1687 struct hwi_async_pdu_context *pasync_ctx, 1688 unsigned int is_header, unsigned int cq_index) 1689 { 1690 struct list_head *pbusy_list; 1691 struct async_pdu_handle *pasync_handle; 1692 unsigned int num_entries, writables = 0; 1693 unsigned int *pep_read_ptr, *pwritables; 1694 1695 num_entries = pasync_ctx->num_entries; 1696 if (is_header) { 1697 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr; 1698 pwritables = &pasync_ctx->async_header.writables; 1699 } else { 1700 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr; 1701 pwritables = &pasync_ctx->async_data.writables; 1702 } 1703 1704 while ((*pep_read_ptr) != cq_index) { 1705 (*pep_read_ptr)++; 1706 *pep_read_ptr = (*pep_read_ptr) % num_entries; 1707 1708 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header, 1709 *pep_read_ptr); 1710 if (writables == 0) 1711 WARN_ON(list_empty(pbusy_list)); 1712 1713 if (!list_empty(pbusy_list)) { 1714 pasync_handle = list_entry(pbusy_list->next, 1715 struct async_pdu_handle, 1716 link); 1717 WARN_ON(!pasync_handle); 1718 pasync_handle->consumed = 1; 1719 } 1720 1721 writables++; 1722 } 1723 1724 if (!writables) { 1725 beiscsi_log(phba, KERN_ERR, 1726 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 1727 "BM_%d : Duplicate notification received - index 0x%x!!\n", 1728 cq_index); 1729 WARN_ON(1); 1730 } 1731 1732 *pwritables = *pwritables + writables; 1733 return 0; 1734 } 1735 1736 static void hwi_free_async_msg(struct beiscsi_hba *phba, 1737 struct hwi_async_pdu_context *pasync_ctx, 1738 unsigned int cri) 1739 { 1740 struct async_pdu_handle *pasync_handle, *tmp_handle; 1741 struct list_head *plist; 1742 1743 plist = &pasync_ctx->async_entry[cri].wait_queue.list; 1744 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) { 1745 list_del(&pasync_handle->link); 1746 1747 if (pasync_handle->is_header) { 1748 list_add_tail(&pasync_handle->link, 1749 &pasync_ctx->async_header.free_list); 1750 pasync_ctx->async_header.free_entries++; 1751 } else { 1752 list_add_tail(&pasync_handle->link, 1753 &pasync_ctx->async_data.free_list); 1754 pasync_ctx->async_data.free_entries++; 1755 } 1756 } 1757 1758 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list); 1759 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0; 1760 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; 1761 } 1762 1763 static struct phys_addr * 1764 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx, 1765 unsigned int is_header, unsigned int host_write_ptr) 1766 { 1767 struct phys_addr *pasync_sge = NULL; 1768 1769 if (is_header) 1770 pasync_sge = pasync_ctx->async_header.ring_base; 1771 else 1772 pasync_sge = pasync_ctx->async_data.ring_base; 1773 1774 return pasync_sge + host_write_ptr; 1775 } 1776 1777 static void hwi_post_async_buffers(struct beiscsi_hba *phba, 1778 unsigned int is_header, uint8_t ulp_num) 1779 { 1780 struct hwi_controller *phwi_ctrlr; 1781 struct hwi_async_pdu_context *pasync_ctx; 1782 struct async_pdu_handle *pasync_handle; 1783 struct list_head *pfree_link, *pbusy_list; 1784 struct phys_addr *pasync_sge; 1785 unsigned int ring_id, num_entries; 1786 unsigned int host_write_num, doorbell_offset; 1787 unsigned int writables; 1788 unsigned int i = 0; 1789 u32 doorbell = 0; 1790 1791 phwi_ctrlr = phba->phwi_ctrlr; 1792 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num); 1793 num_entries = pasync_ctx->num_entries; 1794 1795 if (is_header) { 1796 writables = min(pasync_ctx->async_header.writables, 1797 pasync_ctx->async_header.free_entries); 1798 pfree_link = pasync_ctx->async_header.free_list.next; 1799 host_write_num = pasync_ctx->async_header.host_write_ptr; 1800 ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id; 1801 doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num]. 1802 doorbell_offset; 1803 } else { 1804 writables = min(pasync_ctx->async_data.writables, 1805 pasync_ctx->async_data.free_entries); 1806 pfree_link = pasync_ctx->async_data.free_list.next; 1807 host_write_num = pasync_ctx->async_data.host_write_ptr; 1808 ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id; 1809 doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num]. 1810 doorbell_offset; 1811 } 1812 1813 writables = (writables / 8) * 8; 1814 if (writables) { 1815 for (i = 0; i < writables; i++) { 1816 pbusy_list = 1817 hwi_get_async_busy_list(pasync_ctx, is_header, 1818 host_write_num); 1819 pasync_handle = 1820 list_entry(pfree_link, struct async_pdu_handle, 1821 link); 1822 WARN_ON(!pasync_handle); 1823 pasync_handle->consumed = 0; 1824 1825 pfree_link = pfree_link->next; 1826 1827 pasync_sge = hwi_get_ring_address(pasync_ctx, 1828 is_header, host_write_num); 1829 1830 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo; 1831 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi; 1832 1833 list_move(&pasync_handle->link, pbusy_list); 1834 1835 host_write_num++; 1836 host_write_num = host_write_num % num_entries; 1837 } 1838 1839 if (is_header) { 1840 pasync_ctx->async_header.host_write_ptr = 1841 host_write_num; 1842 pasync_ctx->async_header.free_entries -= writables; 1843 pasync_ctx->async_header.writables -= writables; 1844 pasync_ctx->async_header.busy_entries += writables; 1845 } else { 1846 pasync_ctx->async_data.host_write_ptr = host_write_num; 1847 pasync_ctx->async_data.free_entries -= writables; 1848 pasync_ctx->async_data.writables -= writables; 1849 pasync_ctx->async_data.busy_entries += writables; 1850 } 1851 1852 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK; 1853 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT; 1854 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT; 1855 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK) 1856 << DB_DEF_PDU_CQPROC_SHIFT; 1857 1858 iowrite32(doorbell, phba->db_va + doorbell_offset); 1859 } 1860 } 1861 1862 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba, 1863 struct beiscsi_conn *beiscsi_conn, 1864 struct i_t_dpdu_cqe *pdpdu_cqe) 1865 { 1866 struct hwi_controller *phwi_ctrlr; 1867 struct hwi_async_pdu_context *pasync_ctx; 1868 struct async_pdu_handle *pasync_handle = NULL; 1869 unsigned int cq_index = -1; 1870 uint16_t cri_index = BE_GET_CRI_FROM_CID( 1871 beiscsi_conn->beiscsi_conn_cid); 1872 1873 phwi_ctrlr = phba->phwi_ctrlr; 1874 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, 1875 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, 1876 cri_index)); 1877 1878 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, 1879 pdpdu_cqe, &cq_index); 1880 BUG_ON(pasync_handle->is_header != 0); 1881 if (pasync_handle->consumed == 0) 1882 hwi_update_async_writables(phba, pasync_ctx, 1883 pasync_handle->is_header, cq_index); 1884 1885 hwi_free_async_msg(phba, pasync_ctx, pasync_handle->cri); 1886 hwi_post_async_buffers(phba, pasync_handle->is_header, 1887 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, 1888 cri_index)); 1889 } 1890 1891 static unsigned int 1892 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn, 1893 struct beiscsi_hba *phba, 1894 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri) 1895 { 1896 struct list_head *plist; 1897 struct async_pdu_handle *pasync_handle; 1898 void *phdr = NULL; 1899 unsigned int hdr_len = 0, buf_len = 0; 1900 unsigned int status, index = 0, offset = 0; 1901 void *pfirst_buffer = NULL; 1902 unsigned int num_buf = 0; 1903 1904 plist = &pasync_ctx->async_entry[cri].wait_queue.list; 1905 1906 list_for_each_entry(pasync_handle, plist, link) { 1907 if (index == 0) { 1908 phdr = pasync_handle->pbuffer; 1909 hdr_len = pasync_handle->buffer_len; 1910 } else { 1911 buf_len = pasync_handle->buffer_len; 1912 if (!num_buf) { 1913 pfirst_buffer = pasync_handle->pbuffer; 1914 num_buf++; 1915 } 1916 memcpy(pfirst_buffer + offset, 1917 pasync_handle->pbuffer, buf_len); 1918 offset += buf_len; 1919 } 1920 index++; 1921 } 1922 1923 status = beiscsi_process_async_pdu(beiscsi_conn, phba, 1924 phdr, hdr_len, pfirst_buffer, 1925 offset); 1926 1927 hwi_free_async_msg(phba, pasync_ctx, cri); 1928 return 0; 1929 } 1930 1931 static unsigned int 1932 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn, 1933 struct beiscsi_hba *phba, 1934 struct async_pdu_handle *pasync_handle) 1935 { 1936 struct hwi_async_pdu_context *pasync_ctx; 1937 struct hwi_controller *phwi_ctrlr; 1938 unsigned int bytes_needed = 0, status = 0; 1939 unsigned short cri = pasync_handle->cri; 1940 struct pdu_base *ppdu; 1941 1942 phwi_ctrlr = phba->phwi_ctrlr; 1943 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, 1944 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, 1945 BE_GET_CRI_FROM_CID(beiscsi_conn-> 1946 beiscsi_conn_cid))); 1947 1948 list_del(&pasync_handle->link); 1949 if (pasync_handle->is_header) { 1950 pasync_ctx->async_header.busy_entries--; 1951 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { 1952 hwi_free_async_msg(phba, pasync_ctx, cri); 1953 BUG(); 1954 } 1955 1956 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; 1957 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1; 1958 pasync_ctx->async_entry[cri].wait_queue.hdr_len = 1959 (unsigned short)pasync_handle->buffer_len; 1960 list_add_tail(&pasync_handle->link, 1961 &pasync_ctx->async_entry[cri].wait_queue.list); 1962 1963 ppdu = pasync_handle->pbuffer; 1964 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base, 1965 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) & 1966 0xFFFF0000) | ((be16_to_cpu((ppdu-> 1967 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32] 1968 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF)); 1969 1970 if (status == 0) { 1971 pasync_ctx->async_entry[cri].wait_queue.bytes_needed = 1972 bytes_needed; 1973 1974 if (bytes_needed == 0) 1975 status = hwi_fwd_async_msg(beiscsi_conn, phba, 1976 pasync_ctx, cri); 1977 } 1978 } else { 1979 pasync_ctx->async_data.busy_entries--; 1980 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { 1981 list_add_tail(&pasync_handle->link, 1982 &pasync_ctx->async_entry[cri].wait_queue. 1983 list); 1984 pasync_ctx->async_entry[cri].wait_queue. 1985 bytes_received += 1986 (unsigned short)pasync_handle->buffer_len; 1987 1988 if (pasync_ctx->async_entry[cri].wait_queue. 1989 bytes_received >= 1990 pasync_ctx->async_entry[cri].wait_queue. 1991 bytes_needed) 1992 status = hwi_fwd_async_msg(beiscsi_conn, phba, 1993 pasync_ctx, cri); 1994 } 1995 } 1996 return status; 1997 } 1998 1999 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn, 2000 struct beiscsi_hba *phba, 2001 struct i_t_dpdu_cqe *pdpdu_cqe) 2002 { 2003 struct hwi_controller *phwi_ctrlr; 2004 struct hwi_async_pdu_context *pasync_ctx; 2005 struct async_pdu_handle *pasync_handle = NULL; 2006 unsigned int cq_index = -1; 2007 uint16_t cri_index = BE_GET_CRI_FROM_CID( 2008 beiscsi_conn->beiscsi_conn_cid); 2009 2010 phwi_ctrlr = phba->phwi_ctrlr; 2011 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, 2012 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, 2013 cri_index)); 2014 2015 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, 2016 pdpdu_cqe, &cq_index); 2017 2018 if (pasync_handle->consumed == 0) 2019 hwi_update_async_writables(phba, pasync_ctx, 2020 pasync_handle->is_header, cq_index); 2021 2022 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle); 2023 hwi_post_async_buffers(phba, pasync_handle->is_header, 2024 BEISCSI_GET_ULP_FROM_CRI( 2025 phwi_ctrlr, cri_index)); 2026 } 2027 2028 static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba) 2029 { 2030 struct be_queue_info *mcc_cq; 2031 struct be_mcc_compl *mcc_compl; 2032 unsigned int num_processed = 0; 2033 2034 mcc_cq = &phba->ctrl.mcc_obj.cq; 2035 mcc_compl = queue_tail_node(mcc_cq); 2036 mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 2037 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) { 2038 2039 if (num_processed >= 32) { 2040 hwi_ring_cq_db(phba, mcc_cq->id, 2041 num_processed, 0, 0); 2042 num_processed = 0; 2043 } 2044 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) { 2045 /* Interpret flags as an async trailer */ 2046 if (is_link_state_evt(mcc_compl->flags)) 2047 /* Interpret compl as a async link evt */ 2048 beiscsi_async_link_state_process(phba, 2049 (struct be_async_event_link_state *) mcc_compl); 2050 else 2051 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX, 2052 "BM_%d : Unsupported Async Event, flags" 2053 " = 0x%08x\n", 2054 mcc_compl->flags); 2055 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) { 2056 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl); 2057 atomic_dec(&phba->ctrl.mcc_obj.q.used); 2058 } 2059 2060 mcc_compl->flags = 0; 2061 queue_tail_inc(mcc_cq); 2062 mcc_compl = queue_tail_node(mcc_cq); 2063 mcc_compl->flags = le32_to_cpu(mcc_compl->flags); 2064 num_processed++; 2065 } 2066 2067 if (num_processed > 0) 2068 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0); 2069 2070 } 2071 2072 /** 2073 * beiscsi_process_cq()- Process the Completion Queue 2074 * @pbe_eq: Event Q on which the Completion has come 2075 * 2076 * return 2077 * Number of Completion Entries processed. 2078 **/ 2079 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq) 2080 { 2081 struct be_queue_info *cq; 2082 struct sol_cqe *sol; 2083 struct dmsg_cqe *dmsg; 2084 unsigned int num_processed = 0; 2085 unsigned int tot_nump = 0; 2086 unsigned short code = 0, cid = 0; 2087 uint16_t cri_index = 0; 2088 struct beiscsi_conn *beiscsi_conn; 2089 struct beiscsi_endpoint *beiscsi_ep; 2090 struct iscsi_endpoint *ep; 2091 struct beiscsi_hba *phba; 2092 2093 cq = pbe_eq->cq; 2094 sol = queue_tail_node(cq); 2095 phba = pbe_eq->phba; 2096 2097 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] & 2098 CQE_VALID_MASK) { 2099 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe)); 2100 2101 code = (sol->dw[offsetof(struct amap_sol_cqe, code) / 2102 32] & CQE_CODE_MASK); 2103 2104 /* Get the CID */ 2105 if (is_chip_be2_be3r(phba)) { 2106 cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol); 2107 } else { 2108 if ((code == DRIVERMSG_NOTIFY) || 2109 (code == UNSOL_HDR_NOTIFY) || 2110 (code == UNSOL_DATA_NOTIFY)) 2111 cid = AMAP_GET_BITS( 2112 struct amap_i_t_dpdu_cqe_v2, 2113 cid, sol); 2114 else 2115 cid = AMAP_GET_BITS(struct amap_sol_cqe_v2, 2116 cid, sol); 2117 } 2118 2119 cri_index = BE_GET_CRI_FROM_CID(cid); 2120 ep = phba->ep_array[cri_index]; 2121 beiscsi_ep = ep->dd_data; 2122 beiscsi_conn = beiscsi_ep->conn; 2123 2124 if (num_processed >= 32) { 2125 hwi_ring_cq_db(phba, cq->id, 2126 num_processed, 0, 0); 2127 tot_nump += num_processed; 2128 num_processed = 0; 2129 } 2130 2131 switch (code) { 2132 case SOL_CMD_COMPLETE: 2133 hwi_complete_cmd(beiscsi_conn, phba, sol); 2134 break; 2135 case DRIVERMSG_NOTIFY: 2136 beiscsi_log(phba, KERN_INFO, 2137 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 2138 "BM_%d : Received %s[%d] on CID : %d\n", 2139 cqe_desc[code], code, cid); 2140 2141 dmsg = (struct dmsg_cqe *)sol; 2142 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol); 2143 break; 2144 case UNSOL_HDR_NOTIFY: 2145 beiscsi_log(phba, KERN_INFO, 2146 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 2147 "BM_%d : Received %s[%d] on CID : %d\n", 2148 cqe_desc[code], code, cid); 2149 2150 spin_lock_bh(&phba->async_pdu_lock); 2151 hwi_process_default_pdu_ring(beiscsi_conn, phba, 2152 (struct i_t_dpdu_cqe *)sol); 2153 spin_unlock_bh(&phba->async_pdu_lock); 2154 break; 2155 case UNSOL_DATA_NOTIFY: 2156 beiscsi_log(phba, KERN_INFO, 2157 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 2158 "BM_%d : Received %s[%d] on CID : %d\n", 2159 cqe_desc[code], code, cid); 2160 2161 spin_lock_bh(&phba->async_pdu_lock); 2162 hwi_process_default_pdu_ring(beiscsi_conn, phba, 2163 (struct i_t_dpdu_cqe *)sol); 2164 spin_unlock_bh(&phba->async_pdu_lock); 2165 break; 2166 case CXN_INVALIDATE_INDEX_NOTIFY: 2167 case CMD_INVALIDATED_NOTIFY: 2168 case CXN_INVALIDATE_NOTIFY: 2169 beiscsi_log(phba, KERN_ERR, 2170 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 2171 "BM_%d : Ignoring %s[%d] on CID : %d\n", 2172 cqe_desc[code], code, cid); 2173 break; 2174 case SOL_CMD_KILLED_DATA_DIGEST_ERR: 2175 case CMD_KILLED_INVALID_STATSN_RCVD: 2176 case CMD_KILLED_INVALID_R2T_RCVD: 2177 case CMD_CXN_KILLED_LUN_INVALID: 2178 case CMD_CXN_KILLED_ICD_INVALID: 2179 case CMD_CXN_KILLED_ITT_INVALID: 2180 case CMD_CXN_KILLED_SEQ_OUTOFORDER: 2181 case CMD_CXN_KILLED_INVALID_DATASN_RCVD: 2182 beiscsi_log(phba, KERN_ERR, 2183 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 2184 "BM_%d : Cmd Notification %s[%d] on CID : %d\n", 2185 cqe_desc[code], code, cid); 2186 break; 2187 case UNSOL_DATA_DIGEST_ERROR_NOTIFY: 2188 beiscsi_log(phba, KERN_ERR, 2189 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 2190 "BM_%d : Dropping %s[%d] on DPDU ring on CID : %d\n", 2191 cqe_desc[code], code, cid); 2192 spin_lock_bh(&phba->async_pdu_lock); 2193 hwi_flush_default_pdu_buffer(phba, beiscsi_conn, 2194 (struct i_t_dpdu_cqe *) sol); 2195 spin_unlock_bh(&phba->async_pdu_lock); 2196 break; 2197 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL: 2198 case CXN_KILLED_BURST_LEN_MISMATCH: 2199 case CXN_KILLED_AHS_RCVD: 2200 case CXN_KILLED_HDR_DIGEST_ERR: 2201 case CXN_KILLED_UNKNOWN_HDR: 2202 case CXN_KILLED_STALE_ITT_TTT_RCVD: 2203 case CXN_KILLED_INVALID_ITT_TTT_RCVD: 2204 case CXN_KILLED_TIMED_OUT: 2205 case CXN_KILLED_FIN_RCVD: 2206 case CXN_KILLED_RST_SENT: 2207 case CXN_KILLED_RST_RCVD: 2208 case CXN_KILLED_BAD_UNSOL_PDU_RCVD: 2209 case CXN_KILLED_BAD_WRB_INDEX_ERROR: 2210 case CXN_KILLED_OVER_RUN_RESIDUAL: 2211 case CXN_KILLED_UNDER_RUN_RESIDUAL: 2212 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN: 2213 beiscsi_log(phba, KERN_ERR, 2214 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 2215 "BM_%d : Event %s[%d] received on CID : %d\n", 2216 cqe_desc[code], code, cid); 2217 if (beiscsi_conn) 2218 iscsi_conn_failure(beiscsi_conn->conn, 2219 ISCSI_ERR_CONN_FAILED); 2220 break; 2221 default: 2222 beiscsi_log(phba, KERN_ERR, 2223 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 2224 "BM_%d : Invalid CQE Event Received Code : %d" 2225 "CID 0x%x...\n", 2226 code, cid); 2227 break; 2228 } 2229 2230 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0); 2231 queue_tail_inc(cq); 2232 sol = queue_tail_node(cq); 2233 num_processed++; 2234 } 2235 2236 if (num_processed > 0) { 2237 tot_nump += num_processed; 2238 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0); 2239 } 2240 return tot_nump; 2241 } 2242 2243 void beiscsi_process_all_cqs(struct work_struct *work) 2244 { 2245 unsigned long flags; 2246 struct hwi_controller *phwi_ctrlr; 2247 struct hwi_context_memory *phwi_context; 2248 struct beiscsi_hba *phba; 2249 struct be_eq_obj *pbe_eq = 2250 container_of(work, struct be_eq_obj, work_cqs); 2251 2252 phba = pbe_eq->phba; 2253 phwi_ctrlr = phba->phwi_ctrlr; 2254 phwi_context = phwi_ctrlr->phwi_ctxt; 2255 2256 if (pbe_eq->todo_mcc_cq) { 2257 spin_lock_irqsave(&phba->isr_lock, flags); 2258 pbe_eq->todo_mcc_cq = false; 2259 spin_unlock_irqrestore(&phba->isr_lock, flags); 2260 beiscsi_process_mcc_isr(phba); 2261 } 2262 2263 if (pbe_eq->todo_cq) { 2264 spin_lock_irqsave(&phba->isr_lock, flags); 2265 pbe_eq->todo_cq = false; 2266 spin_unlock_irqrestore(&phba->isr_lock, flags); 2267 beiscsi_process_cq(pbe_eq); 2268 } 2269 2270 /* rearm EQ for further interrupts */ 2271 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); 2272 } 2273 2274 static int be_iopoll(struct blk_iopoll *iop, int budget) 2275 { 2276 unsigned int ret; 2277 struct beiscsi_hba *phba; 2278 struct be_eq_obj *pbe_eq; 2279 2280 pbe_eq = container_of(iop, struct be_eq_obj, iopoll); 2281 ret = beiscsi_process_cq(pbe_eq); 2282 if (ret < budget) { 2283 phba = pbe_eq->phba; 2284 blk_iopoll_complete(iop); 2285 beiscsi_log(phba, KERN_INFO, 2286 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 2287 "BM_%d : rearm pbe_eq->q.id =%d\n", 2288 pbe_eq->q.id); 2289 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); 2290 } 2291 return ret; 2292 } 2293 2294 static void 2295 hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg, 2296 unsigned int num_sg, struct beiscsi_io_task *io_task) 2297 { 2298 struct iscsi_sge *psgl; 2299 unsigned int sg_len, index; 2300 unsigned int sge_len = 0; 2301 unsigned long long addr; 2302 struct scatterlist *l_sg; 2303 unsigned int offset; 2304 2305 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb, 2306 io_task->bhs_pa.u.a32.address_lo); 2307 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb, 2308 io_task->bhs_pa.u.a32.address_hi); 2309 2310 l_sg = sg; 2311 for (index = 0; (index < num_sg) && (index < 2); index++, 2312 sg = sg_next(sg)) { 2313 if (index == 0) { 2314 sg_len = sg_dma_len(sg); 2315 addr = (u64) sg_dma_address(sg); 2316 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 2317 sge0_addr_lo, pwrb, 2318 lower_32_bits(addr)); 2319 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 2320 sge0_addr_hi, pwrb, 2321 upper_32_bits(addr)); 2322 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 2323 sge0_len, pwrb, 2324 sg_len); 2325 sge_len = sg_len; 2326 } else { 2327 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset, 2328 pwrb, sge_len); 2329 sg_len = sg_dma_len(sg); 2330 addr = (u64) sg_dma_address(sg); 2331 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 2332 sge1_addr_lo, pwrb, 2333 lower_32_bits(addr)); 2334 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 2335 sge1_addr_hi, pwrb, 2336 upper_32_bits(addr)); 2337 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 2338 sge1_len, pwrb, 2339 sg_len); 2340 } 2341 } 2342 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 2343 memset(psgl, 0, sizeof(*psgl) * BE2_SGE); 2344 2345 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2); 2346 2347 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2348 io_task->bhs_pa.u.a32.address_hi); 2349 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2350 io_task->bhs_pa.u.a32.address_lo); 2351 2352 if (num_sg == 1) { 2353 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb, 2354 1); 2355 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb, 2356 0); 2357 } else if (num_sg == 2) { 2358 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb, 2359 0); 2360 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb, 2361 1); 2362 } else { 2363 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb, 2364 0); 2365 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb, 2366 0); 2367 } 2368 2369 sg = l_sg; 2370 psgl++; 2371 psgl++; 2372 offset = 0; 2373 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) { 2374 sg_len = sg_dma_len(sg); 2375 addr = (u64) sg_dma_address(sg); 2376 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2377 lower_32_bits(addr)); 2378 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2379 upper_32_bits(addr)); 2380 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len); 2381 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset); 2382 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 2383 offset += sg_len; 2384 } 2385 psgl--; 2386 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 2387 } 2388 2389 static void 2390 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg, 2391 unsigned int num_sg, struct beiscsi_io_task *io_task) 2392 { 2393 struct iscsi_sge *psgl; 2394 unsigned int sg_len, index; 2395 unsigned int sge_len = 0; 2396 unsigned long long addr; 2397 struct scatterlist *l_sg; 2398 unsigned int offset; 2399 2400 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 2401 io_task->bhs_pa.u.a32.address_lo); 2402 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 2403 io_task->bhs_pa.u.a32.address_hi); 2404 2405 l_sg = sg; 2406 for (index = 0; (index < num_sg) && (index < 2); index++, 2407 sg = sg_next(sg)) { 2408 if (index == 0) { 2409 sg_len = sg_dma_len(sg); 2410 addr = (u64) sg_dma_address(sg); 2411 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 2412 ((u32)(addr & 0xFFFFFFFF))); 2413 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 2414 ((u32)(addr >> 32))); 2415 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 2416 sg_len); 2417 sge_len = sg_len; 2418 } else { 2419 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset, 2420 pwrb, sge_len); 2421 sg_len = sg_dma_len(sg); 2422 addr = (u64) sg_dma_address(sg); 2423 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb, 2424 ((u32)(addr & 0xFFFFFFFF))); 2425 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb, 2426 ((u32)(addr >> 32))); 2427 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb, 2428 sg_len); 2429 } 2430 } 2431 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 2432 memset(psgl, 0, sizeof(*psgl) * BE2_SGE); 2433 2434 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2); 2435 2436 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2437 io_task->bhs_pa.u.a32.address_hi); 2438 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2439 io_task->bhs_pa.u.a32.address_lo); 2440 2441 if (num_sg == 1) { 2442 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2443 1); 2444 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2445 0); 2446 } else if (num_sg == 2) { 2447 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2448 0); 2449 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2450 1); 2451 } else { 2452 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 2453 0); 2454 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 2455 0); 2456 } 2457 sg = l_sg; 2458 psgl++; 2459 psgl++; 2460 offset = 0; 2461 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) { 2462 sg_len = sg_dma_len(sg); 2463 addr = (u64) sg_dma_address(sg); 2464 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2465 (addr & 0xFFFFFFFF)); 2466 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2467 (addr >> 32)); 2468 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len); 2469 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset); 2470 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 2471 offset += sg_len; 2472 } 2473 psgl--; 2474 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 2475 } 2476 2477 /** 2478 * hwi_write_buffer()- Populate the WRB with task info 2479 * @pwrb: ptr to the WRB entry 2480 * @task: iscsi task which is to be executed 2481 **/ 2482 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task) 2483 { 2484 struct iscsi_sge *psgl; 2485 struct beiscsi_io_task *io_task = task->dd_data; 2486 struct beiscsi_conn *beiscsi_conn = io_task->conn; 2487 struct beiscsi_hba *phba = beiscsi_conn->phba; 2488 uint8_t dsp_value = 0; 2489 2490 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2; 2491 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, 2492 io_task->bhs_pa.u.a32.address_lo); 2493 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, 2494 io_task->bhs_pa.u.a32.address_hi); 2495 2496 if (task->data) { 2497 2498 /* Check for the data_count */ 2499 dsp_value = (task->data_count) ? 1 : 0; 2500 2501 if (is_chip_be2_be3r(phba)) 2502 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, 2503 pwrb, dsp_value); 2504 else 2505 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, 2506 pwrb, dsp_value); 2507 2508 /* Map addr only if there is data_count */ 2509 if (dsp_value) { 2510 io_task->mtask_addr = pci_map_single(phba->pcidev, 2511 task->data, 2512 task->data_count, 2513 PCI_DMA_TODEVICE); 2514 io_task->mtask_data_count = task->data_count; 2515 } else 2516 io_task->mtask_addr = 0; 2517 2518 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, 2519 lower_32_bits(io_task->mtask_addr)); 2520 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, 2521 upper_32_bits(io_task->mtask_addr)); 2522 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, 2523 task->data_count); 2524 2525 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1); 2526 } else { 2527 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 2528 io_task->mtask_addr = 0; 2529 } 2530 2531 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; 2532 2533 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len); 2534 2535 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2536 io_task->bhs_pa.u.a32.address_hi); 2537 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2538 io_task->bhs_pa.u.a32.address_lo); 2539 if (task->data) { 2540 psgl++; 2541 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0); 2542 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0); 2543 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0); 2544 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0); 2545 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0); 2546 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); 2547 2548 psgl++; 2549 if (task->data) { 2550 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 2551 lower_32_bits(io_task->mtask_addr)); 2552 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 2553 upper_32_bits(io_task->mtask_addr)); 2554 } 2555 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106); 2556 } 2557 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); 2558 } 2559 2560 /** 2561 * beiscsi_find_mem_req()- Find mem needed 2562 * @phba: ptr to HBA struct 2563 **/ 2564 static void beiscsi_find_mem_req(struct beiscsi_hba *phba) 2565 { 2566 uint8_t mem_descr_index, ulp_num; 2567 unsigned int num_cq_pages, num_async_pdu_buf_pages; 2568 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn; 2569 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages; 2570 2571 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 2572 sizeof(struct sol_cqe)); 2573 2574 phba->params.hwi_ws_sz = sizeof(struct hwi_controller); 2575 2576 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 * 2577 BE_ISCSI_PDU_HEADER_SIZE; 2578 phba->mem_req[HWI_MEM_ADDN_CONTEXT] = 2579 sizeof(struct hwi_context_memory); 2580 2581 2582 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb) 2583 * (phba->params.wrbs_per_cxn) 2584 * phba->params.cxns_per_ctrl; 2585 wrb_sz_per_cxn = sizeof(struct wrb_handle) * 2586 (phba->params.wrbs_per_cxn); 2587 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) * 2588 phba->params.cxns_per_ctrl); 2589 2590 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) * 2591 phba->params.icds_per_ctrl; 2592 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) * 2593 phba->params.num_sge_per_io * phba->params.icds_per_ctrl; 2594 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 2595 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 2596 2597 num_async_pdu_buf_sgl_pages = 2598 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT( 2599 phba, ulp_num) * 2600 sizeof(struct phys_addr)); 2601 2602 num_async_pdu_buf_pages = 2603 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT( 2604 phba, ulp_num) * 2605 phba->params.defpdu_hdr_sz); 2606 2607 num_async_pdu_data_pages = 2608 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT( 2609 phba, ulp_num) * 2610 phba->params.defpdu_data_sz); 2611 2612 num_async_pdu_data_sgl_pages = 2613 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT( 2614 phba, ulp_num) * 2615 sizeof(struct phys_addr)); 2616 2617 mem_descr_index = (HWI_MEM_TEMPLATE_HDR_ULP0 + 2618 (ulp_num * MEM_DESCR_OFFSET)); 2619 phba->mem_req[mem_descr_index] = 2620 BEISCSI_GET_CID_COUNT(phba, ulp_num) * 2621 BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE; 2622 2623 mem_descr_index = (HWI_MEM_ASYNC_HEADER_BUF_ULP0 + 2624 (ulp_num * MEM_DESCR_OFFSET)); 2625 phba->mem_req[mem_descr_index] = 2626 num_async_pdu_buf_pages * 2627 PAGE_SIZE; 2628 2629 mem_descr_index = (HWI_MEM_ASYNC_DATA_BUF_ULP0 + 2630 (ulp_num * MEM_DESCR_OFFSET)); 2631 phba->mem_req[mem_descr_index] = 2632 num_async_pdu_data_pages * 2633 PAGE_SIZE; 2634 2635 mem_descr_index = (HWI_MEM_ASYNC_HEADER_RING_ULP0 + 2636 (ulp_num * MEM_DESCR_OFFSET)); 2637 phba->mem_req[mem_descr_index] = 2638 num_async_pdu_buf_sgl_pages * 2639 PAGE_SIZE; 2640 2641 mem_descr_index = (HWI_MEM_ASYNC_DATA_RING_ULP0 + 2642 (ulp_num * MEM_DESCR_OFFSET)); 2643 phba->mem_req[mem_descr_index] = 2644 num_async_pdu_data_sgl_pages * 2645 PAGE_SIZE; 2646 2647 mem_descr_index = (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 + 2648 (ulp_num * MEM_DESCR_OFFSET)); 2649 phba->mem_req[mem_descr_index] = 2650 BEISCSI_GET_CID_COUNT(phba, ulp_num) * 2651 sizeof(struct async_pdu_handle); 2652 2653 mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 + 2654 (ulp_num * MEM_DESCR_OFFSET)); 2655 phba->mem_req[mem_descr_index] = 2656 BEISCSI_GET_CID_COUNT(phba, ulp_num) * 2657 sizeof(struct async_pdu_handle); 2658 2659 mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 + 2660 (ulp_num * MEM_DESCR_OFFSET)); 2661 phba->mem_req[mem_descr_index] = 2662 sizeof(struct hwi_async_pdu_context) + 2663 (BEISCSI_GET_CID_COUNT(phba, ulp_num) * 2664 sizeof(struct hwi_async_entry)); 2665 } 2666 } 2667 } 2668 2669 static int beiscsi_alloc_mem(struct beiscsi_hba *phba) 2670 { 2671 dma_addr_t bus_add; 2672 struct hwi_controller *phwi_ctrlr; 2673 struct be_mem_descriptor *mem_descr; 2674 struct mem_array *mem_arr, *mem_arr_orig; 2675 unsigned int i, j, alloc_size, curr_alloc_size; 2676 2677 phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL); 2678 if (!phba->phwi_ctrlr) 2679 return -ENOMEM; 2680 2681 /* Allocate memory for wrb_context */ 2682 phwi_ctrlr = phba->phwi_ctrlr; 2683 phwi_ctrlr->wrb_context = kzalloc(sizeof(struct hwi_wrb_context) * 2684 phba->params.cxns_per_ctrl, 2685 GFP_KERNEL); 2686 if (!phwi_ctrlr->wrb_context) 2687 return -ENOMEM; 2688 2689 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr), 2690 GFP_KERNEL); 2691 if (!phba->init_mem) { 2692 kfree(phwi_ctrlr->wrb_context); 2693 kfree(phba->phwi_ctrlr); 2694 return -ENOMEM; 2695 } 2696 2697 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT, 2698 GFP_KERNEL); 2699 if (!mem_arr_orig) { 2700 kfree(phba->init_mem); 2701 kfree(phwi_ctrlr->wrb_context); 2702 kfree(phba->phwi_ctrlr); 2703 return -ENOMEM; 2704 } 2705 2706 mem_descr = phba->init_mem; 2707 for (i = 0; i < SE_MEM_MAX; i++) { 2708 if (!phba->mem_req[i]) { 2709 mem_descr->mem_array = NULL; 2710 mem_descr++; 2711 continue; 2712 } 2713 2714 j = 0; 2715 mem_arr = mem_arr_orig; 2716 alloc_size = phba->mem_req[i]; 2717 memset(mem_arr, 0, sizeof(struct mem_array) * 2718 BEISCSI_MAX_FRAGS_INIT); 2719 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size); 2720 do { 2721 mem_arr->virtual_address = pci_alloc_consistent( 2722 phba->pcidev, 2723 curr_alloc_size, 2724 &bus_add); 2725 if (!mem_arr->virtual_address) { 2726 if (curr_alloc_size <= BE_MIN_MEM_SIZE) 2727 goto free_mem; 2728 if (curr_alloc_size - 2729 rounddown_pow_of_two(curr_alloc_size)) 2730 curr_alloc_size = rounddown_pow_of_two 2731 (curr_alloc_size); 2732 else 2733 curr_alloc_size = curr_alloc_size / 2; 2734 } else { 2735 mem_arr->bus_address.u. 2736 a64.address = (__u64) bus_add; 2737 mem_arr->size = curr_alloc_size; 2738 alloc_size -= curr_alloc_size; 2739 curr_alloc_size = min(be_max_phys_size * 2740 1024, alloc_size); 2741 j++; 2742 mem_arr++; 2743 } 2744 } while (alloc_size); 2745 mem_descr->num_elements = j; 2746 mem_descr->size_in_bytes = phba->mem_req[i]; 2747 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j, 2748 GFP_KERNEL); 2749 if (!mem_descr->mem_array) 2750 goto free_mem; 2751 2752 memcpy(mem_descr->mem_array, mem_arr_orig, 2753 sizeof(struct mem_array) * j); 2754 mem_descr++; 2755 } 2756 kfree(mem_arr_orig); 2757 return 0; 2758 free_mem: 2759 mem_descr->num_elements = j; 2760 while ((i) || (j)) { 2761 for (j = mem_descr->num_elements; j > 0; j--) { 2762 pci_free_consistent(phba->pcidev, 2763 mem_descr->mem_array[j - 1].size, 2764 mem_descr->mem_array[j - 1]. 2765 virtual_address, 2766 (unsigned long)mem_descr-> 2767 mem_array[j - 1]. 2768 bus_address.u.a64.address); 2769 } 2770 if (i) { 2771 i--; 2772 kfree(mem_descr->mem_array); 2773 mem_descr--; 2774 } 2775 } 2776 kfree(mem_arr_orig); 2777 kfree(phba->init_mem); 2778 kfree(phba->phwi_ctrlr->wrb_context); 2779 kfree(phba->phwi_ctrlr); 2780 return -ENOMEM; 2781 } 2782 2783 static int beiscsi_get_memory(struct beiscsi_hba *phba) 2784 { 2785 beiscsi_find_mem_req(phba); 2786 return beiscsi_alloc_mem(phba); 2787 } 2788 2789 static void iscsi_init_global_templates(struct beiscsi_hba *phba) 2790 { 2791 struct pdu_data_out *pdata_out; 2792 struct pdu_nop_out *pnop_out; 2793 struct be_mem_descriptor *mem_descr; 2794 2795 mem_descr = phba->init_mem; 2796 mem_descr += ISCSI_MEM_GLOBAL_HEADER; 2797 pdata_out = 2798 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address; 2799 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 2800 2801 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out, 2802 IIOC_SCSI_DATA); 2803 2804 pnop_out = 2805 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0]. 2806 virtual_address + BE_ISCSI_PDU_HEADER_SIZE); 2807 2808 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE); 2809 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF); 2810 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1); 2811 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0); 2812 } 2813 2814 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba) 2815 { 2816 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb; 2817 struct hwi_context_memory *phwi_ctxt; 2818 struct wrb_handle *pwrb_handle = NULL; 2819 struct hwi_controller *phwi_ctrlr; 2820 struct hwi_wrb_context *pwrb_context; 2821 struct iscsi_wrb *pwrb = NULL; 2822 unsigned int num_cxn_wrbh = 0; 2823 unsigned int num_cxn_wrb = 0, j, idx = 0, index; 2824 2825 mem_descr_wrbh = phba->init_mem; 2826 mem_descr_wrbh += HWI_MEM_WRBH; 2827 2828 mem_descr_wrb = phba->init_mem; 2829 mem_descr_wrb += HWI_MEM_WRB; 2830 phwi_ctrlr = phba->phwi_ctrlr; 2831 2832 /* Allocate memory for WRBQ */ 2833 phwi_ctxt = phwi_ctrlr->phwi_ctxt; 2834 phwi_ctxt->be_wrbq = kzalloc(sizeof(struct be_queue_info) * 2835 phba->params.cxns_per_ctrl, 2836 GFP_KERNEL); 2837 if (!phwi_ctxt->be_wrbq) { 2838 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 2839 "BM_%d : WRBQ Mem Alloc Failed\n"); 2840 return -ENOMEM; 2841 } 2842 2843 for (index = 0; index < phba->params.cxns_per_ctrl; index++) { 2844 pwrb_context = &phwi_ctrlr->wrb_context[index]; 2845 pwrb_context->pwrb_handle_base = 2846 kzalloc(sizeof(struct wrb_handle *) * 2847 phba->params.wrbs_per_cxn, GFP_KERNEL); 2848 if (!pwrb_context->pwrb_handle_base) { 2849 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 2850 "BM_%d : Mem Alloc Failed. Failing to load\n"); 2851 goto init_wrb_hndl_failed; 2852 } 2853 pwrb_context->pwrb_handle_basestd = 2854 kzalloc(sizeof(struct wrb_handle *) * 2855 phba->params.wrbs_per_cxn, GFP_KERNEL); 2856 if (!pwrb_context->pwrb_handle_basestd) { 2857 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 2858 "BM_%d : Mem Alloc Failed. Failing to load\n"); 2859 goto init_wrb_hndl_failed; 2860 } 2861 if (!num_cxn_wrbh) { 2862 pwrb_handle = 2863 mem_descr_wrbh->mem_array[idx].virtual_address; 2864 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) / 2865 ((sizeof(struct wrb_handle)) * 2866 phba->params.wrbs_per_cxn)); 2867 idx++; 2868 } 2869 pwrb_context->alloc_index = 0; 2870 pwrb_context->wrb_handles_available = 0; 2871 pwrb_context->free_index = 0; 2872 2873 if (num_cxn_wrbh) { 2874 for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 2875 pwrb_context->pwrb_handle_base[j] = pwrb_handle; 2876 pwrb_context->pwrb_handle_basestd[j] = 2877 pwrb_handle; 2878 pwrb_context->wrb_handles_available++; 2879 pwrb_handle->wrb_index = j; 2880 pwrb_handle++; 2881 } 2882 num_cxn_wrbh--; 2883 } 2884 } 2885 idx = 0; 2886 for (index = 0; index < phba->params.cxns_per_ctrl; index++) { 2887 pwrb_context = &phwi_ctrlr->wrb_context[index]; 2888 if (!num_cxn_wrb) { 2889 pwrb = mem_descr_wrb->mem_array[idx].virtual_address; 2890 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) / 2891 ((sizeof(struct iscsi_wrb) * 2892 phba->params.wrbs_per_cxn)); 2893 idx++; 2894 } 2895 2896 if (num_cxn_wrb) { 2897 for (j = 0; j < phba->params.wrbs_per_cxn; j++) { 2898 pwrb_handle = pwrb_context->pwrb_handle_base[j]; 2899 pwrb_handle->pwrb = pwrb; 2900 pwrb++; 2901 } 2902 num_cxn_wrb--; 2903 } 2904 } 2905 return 0; 2906 init_wrb_hndl_failed: 2907 for (j = index; j > 0; j--) { 2908 pwrb_context = &phwi_ctrlr->wrb_context[j]; 2909 kfree(pwrb_context->pwrb_handle_base); 2910 kfree(pwrb_context->pwrb_handle_basestd); 2911 } 2912 return -ENOMEM; 2913 } 2914 2915 static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba) 2916 { 2917 uint8_t ulp_num; 2918 struct hwi_controller *phwi_ctrlr; 2919 struct hba_parameters *p = &phba->params; 2920 struct hwi_async_pdu_context *pasync_ctx; 2921 struct async_pdu_handle *pasync_header_h, *pasync_data_h; 2922 unsigned int index, idx, num_per_mem, num_async_data; 2923 struct be_mem_descriptor *mem_descr; 2924 2925 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 2926 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 2927 2928 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2929 mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 + 2930 (ulp_num * MEM_DESCR_OFFSET)); 2931 2932 phwi_ctrlr = phba->phwi_ctrlr; 2933 phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] = 2934 (struct hwi_async_pdu_context *) 2935 mem_descr->mem_array[0].virtual_address; 2936 2937 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num]; 2938 memset(pasync_ctx, 0, sizeof(*pasync_ctx)); 2939 2940 pasync_ctx->async_entry = 2941 (struct hwi_async_entry *) 2942 ((long unsigned int)pasync_ctx + 2943 sizeof(struct hwi_async_pdu_context)); 2944 2945 pasync_ctx->num_entries = BEISCSI_GET_CID_COUNT(phba, 2946 ulp_num); 2947 pasync_ctx->buffer_size = p->defpdu_hdr_sz; 2948 2949 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2950 mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 + 2951 (ulp_num * MEM_DESCR_OFFSET); 2952 if (mem_descr->mem_array[0].virtual_address) { 2953 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 2954 "BM_%d : hwi_init_async_pdu_ctx" 2955 " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n", 2956 ulp_num, 2957 mem_descr->mem_array[0]. 2958 virtual_address); 2959 } else 2960 beiscsi_log(phba, KERN_WARNING, 2961 BEISCSI_LOG_INIT, 2962 "BM_%d : No Virtual address for ULP : %d\n", 2963 ulp_num); 2964 2965 pasync_ctx->async_header.va_base = 2966 mem_descr->mem_array[0].virtual_address; 2967 2968 pasync_ctx->async_header.pa_base.u.a64.address = 2969 mem_descr->mem_array[0]. 2970 bus_address.u.a64.address; 2971 2972 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2973 mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 + 2974 (ulp_num * MEM_DESCR_OFFSET); 2975 if (mem_descr->mem_array[0].virtual_address) { 2976 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 2977 "BM_%d : hwi_init_async_pdu_ctx" 2978 " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n", 2979 ulp_num, 2980 mem_descr->mem_array[0]. 2981 virtual_address); 2982 } else 2983 beiscsi_log(phba, KERN_WARNING, 2984 BEISCSI_LOG_INIT, 2985 "BM_%d : No Virtual address for ULP : %d\n", 2986 ulp_num); 2987 2988 pasync_ctx->async_header.ring_base = 2989 mem_descr->mem_array[0].virtual_address; 2990 2991 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 2992 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 + 2993 (ulp_num * MEM_DESCR_OFFSET); 2994 if (mem_descr->mem_array[0].virtual_address) { 2995 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 2996 "BM_%d : hwi_init_async_pdu_ctx" 2997 " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n", 2998 ulp_num, 2999 mem_descr->mem_array[0]. 3000 virtual_address); 3001 } else 3002 beiscsi_log(phba, KERN_WARNING, 3003 BEISCSI_LOG_INIT, 3004 "BM_%d : No Virtual address for ULP : %d\n", 3005 ulp_num); 3006 3007 pasync_ctx->async_header.handle_base = 3008 mem_descr->mem_array[0].virtual_address; 3009 pasync_ctx->async_header.writables = 0; 3010 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list); 3011 3012 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 3013 mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 + 3014 (ulp_num * MEM_DESCR_OFFSET); 3015 if (mem_descr->mem_array[0].virtual_address) { 3016 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3017 "BM_%d : hwi_init_async_pdu_ctx" 3018 " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n", 3019 ulp_num, 3020 mem_descr->mem_array[0]. 3021 virtual_address); 3022 } else 3023 beiscsi_log(phba, KERN_WARNING, 3024 BEISCSI_LOG_INIT, 3025 "BM_%d : No Virtual address for ULP : %d\n", 3026 ulp_num); 3027 3028 pasync_ctx->async_data.ring_base = 3029 mem_descr->mem_array[0].virtual_address; 3030 3031 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 3032 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 + 3033 (ulp_num * MEM_DESCR_OFFSET); 3034 if (!mem_descr->mem_array[0].virtual_address) 3035 beiscsi_log(phba, KERN_WARNING, 3036 BEISCSI_LOG_INIT, 3037 "BM_%d : No Virtual address for ULP : %d\n", 3038 ulp_num); 3039 3040 pasync_ctx->async_data.handle_base = 3041 mem_descr->mem_array[0].virtual_address; 3042 pasync_ctx->async_data.writables = 0; 3043 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list); 3044 3045 pasync_header_h = 3046 (struct async_pdu_handle *) 3047 pasync_ctx->async_header.handle_base; 3048 pasync_data_h = 3049 (struct async_pdu_handle *) 3050 pasync_ctx->async_data.handle_base; 3051 3052 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 3053 mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 + 3054 (ulp_num * MEM_DESCR_OFFSET); 3055 if (mem_descr->mem_array[0].virtual_address) { 3056 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3057 "BM_%d : hwi_init_async_pdu_ctx" 3058 " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n", 3059 ulp_num, 3060 mem_descr->mem_array[0]. 3061 virtual_address); 3062 } else 3063 beiscsi_log(phba, KERN_WARNING, 3064 BEISCSI_LOG_INIT, 3065 "BM_%d : No Virtual address for ULP : %d\n", 3066 ulp_num); 3067 3068 idx = 0; 3069 pasync_ctx->async_data.va_base = 3070 mem_descr->mem_array[idx].virtual_address; 3071 pasync_ctx->async_data.pa_base.u.a64.address = 3072 mem_descr->mem_array[idx]. 3073 bus_address.u.a64.address; 3074 3075 num_async_data = ((mem_descr->mem_array[idx].size) / 3076 phba->params.defpdu_data_sz); 3077 num_per_mem = 0; 3078 3079 for (index = 0; index < BEISCSI_GET_CID_COUNT 3080 (phba, ulp_num); index++) { 3081 pasync_header_h->cri = -1; 3082 pasync_header_h->index = (char)index; 3083 INIT_LIST_HEAD(&pasync_header_h->link); 3084 pasync_header_h->pbuffer = 3085 (void *)((unsigned long) 3086 (pasync_ctx-> 3087 async_header.va_base) + 3088 (p->defpdu_hdr_sz * index)); 3089 3090 pasync_header_h->pa.u.a64.address = 3091 pasync_ctx->async_header.pa_base.u.a64. 3092 address + (p->defpdu_hdr_sz * index); 3093 3094 list_add_tail(&pasync_header_h->link, 3095 &pasync_ctx->async_header. 3096 free_list); 3097 pasync_header_h++; 3098 pasync_ctx->async_header.free_entries++; 3099 pasync_ctx->async_header.writables++; 3100 3101 INIT_LIST_HEAD(&pasync_ctx->async_entry[index]. 3102 wait_queue.list); 3103 INIT_LIST_HEAD(&pasync_ctx->async_entry[index]. 3104 header_busy_list); 3105 pasync_data_h->cri = -1; 3106 pasync_data_h->index = (char)index; 3107 INIT_LIST_HEAD(&pasync_data_h->link); 3108 3109 if (!num_async_data) { 3110 num_per_mem = 0; 3111 idx++; 3112 pasync_ctx->async_data.va_base = 3113 mem_descr->mem_array[idx]. 3114 virtual_address; 3115 pasync_ctx->async_data.pa_base.u. 3116 a64.address = 3117 mem_descr->mem_array[idx]. 3118 bus_address.u.a64.address; 3119 num_async_data = 3120 ((mem_descr->mem_array[idx]. 3121 size) / 3122 phba->params.defpdu_data_sz); 3123 } 3124 pasync_data_h->pbuffer = 3125 (void *)((unsigned long) 3126 (pasync_ctx->async_data.va_base) + 3127 (p->defpdu_data_sz * num_per_mem)); 3128 3129 pasync_data_h->pa.u.a64.address = 3130 pasync_ctx->async_data.pa_base.u.a64. 3131 address + (p->defpdu_data_sz * 3132 num_per_mem); 3133 num_per_mem++; 3134 num_async_data--; 3135 3136 list_add_tail(&pasync_data_h->link, 3137 &pasync_ctx->async_data. 3138 free_list); 3139 pasync_data_h++; 3140 pasync_ctx->async_data.free_entries++; 3141 pasync_ctx->async_data.writables++; 3142 3143 INIT_LIST_HEAD(&pasync_ctx->async_entry[index]. 3144 data_busy_list); 3145 } 3146 3147 pasync_ctx->async_header.host_write_ptr = 0; 3148 pasync_ctx->async_header.ep_read_ptr = -1; 3149 pasync_ctx->async_data.host_write_ptr = 0; 3150 pasync_ctx->async_data.ep_read_ptr = -1; 3151 } 3152 } 3153 3154 return 0; 3155 } 3156 3157 static int 3158 be_sgl_create_contiguous(void *virtual_address, 3159 u64 physical_address, u32 length, 3160 struct be_dma_mem *sgl) 3161 { 3162 WARN_ON(!virtual_address); 3163 WARN_ON(!physical_address); 3164 WARN_ON(!length > 0); 3165 WARN_ON(!sgl); 3166 3167 sgl->va = virtual_address; 3168 sgl->dma = (unsigned long)physical_address; 3169 sgl->size = length; 3170 3171 return 0; 3172 } 3173 3174 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl) 3175 { 3176 memset(sgl, 0, sizeof(*sgl)); 3177 } 3178 3179 static void 3180 hwi_build_be_sgl_arr(struct beiscsi_hba *phba, 3181 struct mem_array *pmem, struct be_dma_mem *sgl) 3182 { 3183 if (sgl->va) 3184 be_sgl_destroy_contiguous(sgl); 3185 3186 be_sgl_create_contiguous(pmem->virtual_address, 3187 pmem->bus_address.u.a64.address, 3188 pmem->size, sgl); 3189 } 3190 3191 static void 3192 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba, 3193 struct mem_array *pmem, struct be_dma_mem *sgl) 3194 { 3195 if (sgl->va) 3196 be_sgl_destroy_contiguous(sgl); 3197 3198 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address, 3199 pmem->bus_address.u.a64.address, 3200 pmem->size, sgl); 3201 } 3202 3203 static int be_fill_queue(struct be_queue_info *q, 3204 u16 len, u16 entry_size, void *vaddress) 3205 { 3206 struct be_dma_mem *mem = &q->dma_mem; 3207 3208 memset(q, 0, sizeof(*q)); 3209 q->len = len; 3210 q->entry_size = entry_size; 3211 mem->size = len * entry_size; 3212 mem->va = vaddress; 3213 if (!mem->va) 3214 return -ENOMEM; 3215 memset(mem->va, 0, mem->size); 3216 return 0; 3217 } 3218 3219 static int beiscsi_create_eqs(struct beiscsi_hba *phba, 3220 struct hwi_context_memory *phwi_context) 3221 { 3222 unsigned int i, num_eq_pages; 3223 int ret = 0, eq_for_mcc; 3224 struct be_queue_info *eq; 3225 struct be_dma_mem *mem; 3226 void *eq_vaddress; 3227 dma_addr_t paddr; 3228 3229 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \ 3230 sizeof(struct be_eq_entry)); 3231 3232 if (phba->msix_enabled) 3233 eq_for_mcc = 1; 3234 else 3235 eq_for_mcc = 0; 3236 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 3237 eq = &phwi_context->be_eq[i].q; 3238 mem = &eq->dma_mem; 3239 phwi_context->be_eq[i].phba = phba; 3240 eq_vaddress = pci_alloc_consistent(phba->pcidev, 3241 num_eq_pages * PAGE_SIZE, 3242 &paddr); 3243 if (!eq_vaddress) 3244 goto create_eq_error; 3245 3246 mem->va = eq_vaddress; 3247 ret = be_fill_queue(eq, phba->params.num_eq_entries, 3248 sizeof(struct be_eq_entry), eq_vaddress); 3249 if (ret) { 3250 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3251 "BM_%d : be_fill_queue Failed for EQ\n"); 3252 goto create_eq_error; 3253 } 3254 3255 mem->dma = paddr; 3256 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq, 3257 phwi_context->cur_eqd); 3258 if (ret) { 3259 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3260 "BM_%d : beiscsi_cmd_eq_create" 3261 "Failed for EQ\n"); 3262 goto create_eq_error; 3263 } 3264 3265 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3266 "BM_%d : eqid = %d\n", 3267 phwi_context->be_eq[i].q.id); 3268 } 3269 return 0; 3270 create_eq_error: 3271 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { 3272 eq = &phwi_context->be_eq[i].q; 3273 mem = &eq->dma_mem; 3274 if (mem->va) 3275 pci_free_consistent(phba->pcidev, num_eq_pages 3276 * PAGE_SIZE, 3277 mem->va, mem->dma); 3278 } 3279 return ret; 3280 } 3281 3282 static int beiscsi_create_cqs(struct beiscsi_hba *phba, 3283 struct hwi_context_memory *phwi_context) 3284 { 3285 unsigned int i, num_cq_pages; 3286 int ret = 0; 3287 struct be_queue_info *cq, *eq; 3288 struct be_dma_mem *mem; 3289 struct be_eq_obj *pbe_eq; 3290 void *cq_vaddress; 3291 dma_addr_t paddr; 3292 3293 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ 3294 sizeof(struct sol_cqe)); 3295 3296 for (i = 0; i < phba->num_cpus; i++) { 3297 cq = &phwi_context->be_cq[i]; 3298 eq = &phwi_context->be_eq[i].q; 3299 pbe_eq = &phwi_context->be_eq[i]; 3300 pbe_eq->cq = cq; 3301 pbe_eq->phba = phba; 3302 mem = &cq->dma_mem; 3303 cq_vaddress = pci_alloc_consistent(phba->pcidev, 3304 num_cq_pages * PAGE_SIZE, 3305 &paddr); 3306 if (!cq_vaddress) 3307 goto create_cq_error; 3308 ret = be_fill_queue(cq, phba->params.num_cq_entries, 3309 sizeof(struct sol_cqe), cq_vaddress); 3310 if (ret) { 3311 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3312 "BM_%d : be_fill_queue Failed " 3313 "for ISCSI CQ\n"); 3314 goto create_cq_error; 3315 } 3316 3317 mem->dma = paddr; 3318 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, 3319 false, 0); 3320 if (ret) { 3321 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3322 "BM_%d : beiscsi_cmd_eq_create" 3323 "Failed for ISCSI CQ\n"); 3324 goto create_cq_error; 3325 } 3326 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3327 "BM_%d : iscsi cq_id is %d for eq_id %d\n" 3328 "iSCSI CQ CREATED\n", cq->id, eq->id); 3329 } 3330 return 0; 3331 3332 create_cq_error: 3333 for (i = 0; i < phba->num_cpus; i++) { 3334 cq = &phwi_context->be_cq[i]; 3335 mem = &cq->dma_mem; 3336 if (mem->va) 3337 pci_free_consistent(phba->pcidev, num_cq_pages 3338 * PAGE_SIZE, 3339 mem->va, mem->dma); 3340 } 3341 return ret; 3342 3343 } 3344 3345 static int 3346 beiscsi_create_def_hdr(struct beiscsi_hba *phba, 3347 struct hwi_context_memory *phwi_context, 3348 struct hwi_controller *phwi_ctrlr, 3349 unsigned int def_pdu_ring_sz, uint8_t ulp_num) 3350 { 3351 unsigned int idx; 3352 int ret; 3353 struct be_queue_info *dq, *cq; 3354 struct be_dma_mem *mem; 3355 struct be_mem_descriptor *mem_descr; 3356 void *dq_vaddress; 3357 3358 idx = 0; 3359 dq = &phwi_context->be_def_hdrq[ulp_num]; 3360 cq = &phwi_context->be_cq[0]; 3361 mem = &dq->dma_mem; 3362 mem_descr = phba->init_mem; 3363 mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 + 3364 (ulp_num * MEM_DESCR_OFFSET); 3365 dq_vaddress = mem_descr->mem_array[idx].virtual_address; 3366 ret = be_fill_queue(dq, mem_descr->mem_array[0].size / 3367 sizeof(struct phys_addr), 3368 sizeof(struct phys_addr), dq_vaddress); 3369 if (ret) { 3370 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3371 "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n", 3372 ulp_num); 3373 3374 return ret; 3375 } 3376 mem->dma = (unsigned long)mem_descr->mem_array[idx]. 3377 bus_address.u.a64.address; 3378 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq, 3379 def_pdu_ring_sz, 3380 phba->params.defpdu_hdr_sz, 3381 BEISCSI_DEFQ_HDR, ulp_num); 3382 if (ret) { 3383 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3384 "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n", 3385 ulp_num); 3386 3387 return ret; 3388 } 3389 3390 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3391 "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n", 3392 ulp_num, 3393 phwi_context->be_def_hdrq[ulp_num].id); 3394 hwi_post_async_buffers(phba, BEISCSI_DEFQ_HDR, ulp_num); 3395 return 0; 3396 } 3397 3398 static int 3399 beiscsi_create_def_data(struct beiscsi_hba *phba, 3400 struct hwi_context_memory *phwi_context, 3401 struct hwi_controller *phwi_ctrlr, 3402 unsigned int def_pdu_ring_sz, uint8_t ulp_num) 3403 { 3404 unsigned int idx; 3405 int ret; 3406 struct be_queue_info *dataq, *cq; 3407 struct be_dma_mem *mem; 3408 struct be_mem_descriptor *mem_descr; 3409 void *dq_vaddress; 3410 3411 idx = 0; 3412 dataq = &phwi_context->be_def_dataq[ulp_num]; 3413 cq = &phwi_context->be_cq[0]; 3414 mem = &dataq->dma_mem; 3415 mem_descr = phba->init_mem; 3416 mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 + 3417 (ulp_num * MEM_DESCR_OFFSET); 3418 dq_vaddress = mem_descr->mem_array[idx].virtual_address; 3419 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size / 3420 sizeof(struct phys_addr), 3421 sizeof(struct phys_addr), dq_vaddress); 3422 if (ret) { 3423 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3424 "BM_%d : be_fill_queue Failed for DEF PDU " 3425 "DATA on ULP : %d\n", 3426 ulp_num); 3427 3428 return ret; 3429 } 3430 mem->dma = (unsigned long)mem_descr->mem_array[idx]. 3431 bus_address.u.a64.address; 3432 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq, 3433 def_pdu_ring_sz, 3434 phba->params.defpdu_data_sz, 3435 BEISCSI_DEFQ_DATA, ulp_num); 3436 if (ret) { 3437 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3438 "BM_%d be_cmd_create_default_pdu_queue" 3439 " Failed for DEF PDU DATA on ULP : %d\n", 3440 ulp_num); 3441 return ret; 3442 } 3443 3444 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3445 "BM_%d : iscsi def data id on ULP : %d is %d\n", 3446 ulp_num, 3447 phwi_context->be_def_dataq[ulp_num].id); 3448 3449 hwi_post_async_buffers(phba, BEISCSI_DEFQ_DATA, ulp_num); 3450 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3451 "BM_%d : DEFAULT PDU DATA RING CREATED" 3452 "on ULP : %d\n", ulp_num); 3453 3454 return 0; 3455 } 3456 3457 3458 static int 3459 beiscsi_post_template_hdr(struct beiscsi_hba *phba) 3460 { 3461 struct be_mem_descriptor *mem_descr; 3462 struct mem_array *pm_arr; 3463 struct be_dma_mem sgl; 3464 int status, ulp_num; 3465 3466 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 3467 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 3468 mem_descr = (struct be_mem_descriptor *)phba->init_mem; 3469 mem_descr += HWI_MEM_TEMPLATE_HDR_ULP0 + 3470 (ulp_num * MEM_DESCR_OFFSET); 3471 pm_arr = mem_descr->mem_array; 3472 3473 hwi_build_be_sgl_arr(phba, pm_arr, &sgl); 3474 status = be_cmd_iscsi_post_template_hdr( 3475 &phba->ctrl, &sgl); 3476 3477 if (status != 0) { 3478 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3479 "BM_%d : Post Template HDR Failed for" 3480 "ULP_%d\n", ulp_num); 3481 return status; 3482 } 3483 3484 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3485 "BM_%d : Template HDR Pages Posted for" 3486 "ULP_%d\n", ulp_num); 3487 } 3488 } 3489 return 0; 3490 } 3491 3492 static int 3493 beiscsi_post_pages(struct beiscsi_hba *phba) 3494 { 3495 struct be_mem_descriptor *mem_descr; 3496 struct mem_array *pm_arr; 3497 unsigned int page_offset, i; 3498 struct be_dma_mem sgl; 3499 int status, ulp_num = 0; 3500 3501 mem_descr = phba->init_mem; 3502 mem_descr += HWI_MEM_SGE; 3503 pm_arr = mem_descr->mem_array; 3504 3505 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) 3506 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) 3507 break; 3508 3509 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io * 3510 phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE; 3511 for (i = 0; i < mem_descr->num_elements; i++) { 3512 hwi_build_be_sgl_arr(phba, pm_arr, &sgl); 3513 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl, 3514 page_offset, 3515 (pm_arr->size / PAGE_SIZE)); 3516 page_offset += pm_arr->size / PAGE_SIZE; 3517 if (status != 0) { 3518 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3519 "BM_%d : post sgl failed.\n"); 3520 return status; 3521 } 3522 pm_arr++; 3523 } 3524 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3525 "BM_%d : POSTED PAGES\n"); 3526 return 0; 3527 } 3528 3529 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q) 3530 { 3531 struct be_dma_mem *mem = &q->dma_mem; 3532 if (mem->va) { 3533 pci_free_consistent(phba->pcidev, mem->size, 3534 mem->va, mem->dma); 3535 mem->va = NULL; 3536 } 3537 } 3538 3539 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q, 3540 u16 len, u16 entry_size) 3541 { 3542 struct be_dma_mem *mem = &q->dma_mem; 3543 3544 memset(q, 0, sizeof(*q)); 3545 q->len = len; 3546 q->entry_size = entry_size; 3547 mem->size = len * entry_size; 3548 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma); 3549 if (!mem->va) 3550 return -ENOMEM; 3551 memset(mem->va, 0, mem->size); 3552 return 0; 3553 } 3554 3555 static int 3556 beiscsi_create_wrb_rings(struct beiscsi_hba *phba, 3557 struct hwi_context_memory *phwi_context, 3558 struct hwi_controller *phwi_ctrlr) 3559 { 3560 unsigned int wrb_mem_index, offset, size, num_wrb_rings; 3561 u64 pa_addr_lo; 3562 unsigned int idx, num, i, ulp_num; 3563 struct mem_array *pwrb_arr; 3564 void *wrb_vaddr; 3565 struct be_dma_mem sgl; 3566 struct be_mem_descriptor *mem_descr; 3567 struct hwi_wrb_context *pwrb_context; 3568 int status; 3569 uint8_t ulp_count = 0, ulp_base_num = 0; 3570 uint16_t cid_count_ulp[BEISCSI_ULP_COUNT] = { 0 }; 3571 3572 idx = 0; 3573 mem_descr = phba->init_mem; 3574 mem_descr += HWI_MEM_WRB; 3575 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl, 3576 GFP_KERNEL); 3577 if (!pwrb_arr) { 3578 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3579 "BM_%d : Memory alloc failed in create wrb ring.\n"); 3580 return -ENOMEM; 3581 } 3582 wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 3583 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address; 3584 num_wrb_rings = mem_descr->mem_array[idx].size / 3585 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb)); 3586 3587 for (num = 0; num < phba->params.cxns_per_ctrl; num++) { 3588 if (num_wrb_rings) { 3589 pwrb_arr[num].virtual_address = wrb_vaddr; 3590 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo; 3591 pwrb_arr[num].size = phba->params.wrbs_per_cxn * 3592 sizeof(struct iscsi_wrb); 3593 wrb_vaddr += pwrb_arr[num].size; 3594 pa_addr_lo += pwrb_arr[num].size; 3595 num_wrb_rings--; 3596 } else { 3597 idx++; 3598 wrb_vaddr = mem_descr->mem_array[idx].virtual_address; 3599 pa_addr_lo = mem_descr->mem_array[idx].\ 3600 bus_address.u.a64.address; 3601 num_wrb_rings = mem_descr->mem_array[idx].size / 3602 (phba->params.wrbs_per_cxn * 3603 sizeof(struct iscsi_wrb)); 3604 pwrb_arr[num].virtual_address = wrb_vaddr; 3605 pwrb_arr[num].bus_address.u.a64.address\ 3606 = pa_addr_lo; 3607 pwrb_arr[num].size = phba->params.wrbs_per_cxn * 3608 sizeof(struct iscsi_wrb); 3609 wrb_vaddr += pwrb_arr[num].size; 3610 pa_addr_lo += pwrb_arr[num].size; 3611 num_wrb_rings--; 3612 } 3613 } 3614 3615 /* Get the ULP Count */ 3616 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) 3617 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 3618 ulp_count++; 3619 ulp_base_num = ulp_num; 3620 cid_count_ulp[ulp_num] = 3621 BEISCSI_GET_CID_COUNT(phba, ulp_num); 3622 } 3623 3624 for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 3625 wrb_mem_index = 0; 3626 offset = 0; 3627 size = 0; 3628 3629 if (ulp_count > 1) { 3630 ulp_base_num = (ulp_base_num + 1) % BEISCSI_ULP_COUNT; 3631 3632 if (!cid_count_ulp[ulp_base_num]) 3633 ulp_base_num = (ulp_base_num + 1) % 3634 BEISCSI_ULP_COUNT; 3635 3636 cid_count_ulp[ulp_base_num]--; 3637 } 3638 3639 3640 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl); 3641 status = be_cmd_wrbq_create(&phba->ctrl, &sgl, 3642 &phwi_context->be_wrbq[i], 3643 &phwi_ctrlr->wrb_context[i], 3644 ulp_base_num); 3645 if (status != 0) { 3646 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3647 "BM_%d : wrbq create failed."); 3648 kfree(pwrb_arr); 3649 return status; 3650 } 3651 pwrb_context = &phwi_ctrlr->wrb_context[i]; 3652 BE_SET_CID_TO_CRI(i, pwrb_context->cid); 3653 } 3654 kfree(pwrb_arr); 3655 return 0; 3656 } 3657 3658 static void free_wrb_handles(struct beiscsi_hba *phba) 3659 { 3660 unsigned int index; 3661 struct hwi_controller *phwi_ctrlr; 3662 struct hwi_wrb_context *pwrb_context; 3663 3664 phwi_ctrlr = phba->phwi_ctrlr; 3665 for (index = 0; index < phba->params.cxns_per_ctrl; index++) { 3666 pwrb_context = &phwi_ctrlr->wrb_context[index]; 3667 kfree(pwrb_context->pwrb_handle_base); 3668 kfree(pwrb_context->pwrb_handle_basestd); 3669 } 3670 } 3671 3672 static void be_mcc_queues_destroy(struct beiscsi_hba *phba) 3673 { 3674 struct be_queue_info *q; 3675 struct be_ctrl_info *ctrl = &phba->ctrl; 3676 3677 q = &phba->ctrl.mcc_obj.q; 3678 if (q->created) 3679 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ); 3680 be_queue_free(phba, q); 3681 3682 q = &phba->ctrl.mcc_obj.cq; 3683 if (q->created) 3684 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 3685 be_queue_free(phba, q); 3686 } 3687 3688 static void hwi_cleanup(struct beiscsi_hba *phba) 3689 { 3690 struct be_queue_info *q; 3691 struct be_ctrl_info *ctrl = &phba->ctrl; 3692 struct hwi_controller *phwi_ctrlr; 3693 struct hwi_context_memory *phwi_context; 3694 struct hwi_async_pdu_context *pasync_ctx; 3695 int i, eq_num, ulp_num; 3696 3697 phwi_ctrlr = phba->phwi_ctrlr; 3698 phwi_context = phwi_ctrlr->phwi_ctxt; 3699 3700 be_cmd_iscsi_remove_template_hdr(ctrl); 3701 3702 for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 3703 q = &phwi_context->be_wrbq[i]; 3704 if (q->created) 3705 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ); 3706 } 3707 kfree(phwi_context->be_wrbq); 3708 free_wrb_handles(phba); 3709 3710 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 3711 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 3712 3713 q = &phwi_context->be_def_hdrq[ulp_num]; 3714 if (q->created) 3715 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 3716 3717 q = &phwi_context->be_def_dataq[ulp_num]; 3718 if (q->created) 3719 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); 3720 3721 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num]; 3722 } 3723 } 3724 3725 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL); 3726 3727 for (i = 0; i < (phba->num_cpus); i++) { 3728 q = &phwi_context->be_cq[i]; 3729 if (q->created) 3730 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); 3731 } 3732 if (phba->msix_enabled) 3733 eq_num = 1; 3734 else 3735 eq_num = 0; 3736 for (i = 0; i < (phba->num_cpus + eq_num); i++) { 3737 q = &phwi_context->be_eq[i].q; 3738 if (q->created) 3739 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ); 3740 } 3741 be_mcc_queues_destroy(phba); 3742 be_cmd_fw_uninit(ctrl); 3743 } 3744 3745 static int be_mcc_queues_create(struct beiscsi_hba *phba, 3746 struct hwi_context_memory *phwi_context) 3747 { 3748 struct be_queue_info *q, *cq; 3749 struct be_ctrl_info *ctrl = &phba->ctrl; 3750 3751 /* Alloc MCC compl queue */ 3752 cq = &phba->ctrl.mcc_obj.cq; 3753 if (be_queue_alloc(phba, cq, MCC_CQ_LEN, 3754 sizeof(struct be_mcc_compl))) 3755 goto err; 3756 /* Ask BE to create MCC compl queue; */ 3757 if (phba->msix_enabled) { 3758 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq 3759 [phba->num_cpus].q, false, true, 0)) 3760 goto mcc_cq_free; 3761 } else { 3762 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q, 3763 false, true, 0)) 3764 goto mcc_cq_free; 3765 } 3766 3767 /* Alloc MCC queue */ 3768 q = &phba->ctrl.mcc_obj.q; 3769 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 3770 goto mcc_cq_destroy; 3771 3772 /* Ask BE to create MCC queue */ 3773 if (beiscsi_cmd_mccq_create(phba, q, cq)) 3774 goto mcc_q_free; 3775 3776 return 0; 3777 3778 mcc_q_free: 3779 be_queue_free(phba, q); 3780 mcc_cq_destroy: 3781 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ); 3782 mcc_cq_free: 3783 be_queue_free(phba, cq); 3784 err: 3785 return -ENOMEM; 3786 } 3787 3788 /** 3789 * find_num_cpus()- Get the CPU online count 3790 * @phba: ptr to priv structure 3791 * 3792 * CPU count is used for creating EQ. 3793 **/ 3794 static void find_num_cpus(struct beiscsi_hba *phba) 3795 { 3796 int num_cpus = 0; 3797 3798 num_cpus = num_online_cpus(); 3799 3800 switch (phba->generation) { 3801 case BE_GEN2: 3802 case BE_GEN3: 3803 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ? 3804 BEISCSI_MAX_NUM_CPUS : num_cpus; 3805 break; 3806 case BE_GEN4: 3807 /* 3808 * If eqid_count == 1 fall back to 3809 * INTX mechanism 3810 **/ 3811 if (phba->fw_config.eqid_count == 1) { 3812 enable_msix = 0; 3813 phba->num_cpus = 1; 3814 return; 3815 } 3816 3817 phba->num_cpus = 3818 (num_cpus > (phba->fw_config.eqid_count - 1)) ? 3819 (phba->fw_config.eqid_count - 1) : num_cpus; 3820 break; 3821 default: 3822 phba->num_cpus = 1; 3823 } 3824 } 3825 3826 static int hwi_init_port(struct beiscsi_hba *phba) 3827 { 3828 struct hwi_controller *phwi_ctrlr; 3829 struct hwi_context_memory *phwi_context; 3830 unsigned int def_pdu_ring_sz; 3831 struct be_ctrl_info *ctrl = &phba->ctrl; 3832 int status, ulp_num; 3833 3834 phwi_ctrlr = phba->phwi_ctrlr; 3835 phwi_context = phwi_ctrlr->phwi_ctxt; 3836 phwi_context->max_eqd = 0; 3837 phwi_context->min_eqd = 0; 3838 phwi_context->cur_eqd = 64; 3839 be_cmd_fw_initialize(&phba->ctrl); 3840 3841 status = beiscsi_create_eqs(phba, phwi_context); 3842 if (status != 0) { 3843 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3844 "BM_%d : EQ not created\n"); 3845 goto error; 3846 } 3847 3848 status = be_mcc_queues_create(phba, phwi_context); 3849 if (status != 0) 3850 goto error; 3851 3852 status = mgmt_check_supported_fw(ctrl, phba); 3853 if (status != 0) { 3854 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3855 "BM_%d : Unsupported fw version\n"); 3856 goto error; 3857 } 3858 3859 status = beiscsi_create_cqs(phba, phwi_context); 3860 if (status != 0) { 3861 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3862 "BM_%d : CQ not created\n"); 3863 goto error; 3864 } 3865 3866 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 3867 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 3868 3869 def_pdu_ring_sz = 3870 BEISCSI_GET_CID_COUNT(phba, ulp_num) * 3871 sizeof(struct phys_addr); 3872 3873 status = beiscsi_create_def_hdr(phba, phwi_context, 3874 phwi_ctrlr, 3875 def_pdu_ring_sz, 3876 ulp_num); 3877 if (status != 0) { 3878 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3879 "BM_%d : Default Header not created for ULP : %d\n", 3880 ulp_num); 3881 goto error; 3882 } 3883 3884 status = beiscsi_create_def_data(phba, phwi_context, 3885 phwi_ctrlr, 3886 def_pdu_ring_sz, 3887 ulp_num); 3888 if (status != 0) { 3889 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3890 "BM_%d : Default Data not created for ULP : %d\n", 3891 ulp_num); 3892 goto error; 3893 } 3894 } 3895 } 3896 3897 status = beiscsi_post_pages(phba); 3898 if (status != 0) { 3899 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3900 "BM_%d : Post SGL Pages Failed\n"); 3901 goto error; 3902 } 3903 3904 status = beiscsi_post_template_hdr(phba); 3905 if (status != 0) { 3906 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3907 "BM_%d : Template HDR Posting for CXN Failed\n"); 3908 } 3909 3910 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr); 3911 if (status != 0) { 3912 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3913 "BM_%d : WRB Rings not created\n"); 3914 goto error; 3915 } 3916 3917 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 3918 uint16_t async_arr_idx = 0; 3919 3920 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) { 3921 uint16_t cri = 0; 3922 struct hwi_async_pdu_context *pasync_ctx; 3923 3924 pasync_ctx = HWI_GET_ASYNC_PDU_CTX( 3925 phwi_ctrlr, ulp_num); 3926 for (cri = 0; cri < 3927 phba->params.cxns_per_ctrl; cri++) { 3928 if (ulp_num == BEISCSI_GET_ULP_FROM_CRI 3929 (phwi_ctrlr, cri)) 3930 pasync_ctx->cid_to_async_cri_map[ 3931 phwi_ctrlr->wrb_context[cri].cid] = 3932 async_arr_idx++; 3933 } 3934 } 3935 } 3936 3937 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3938 "BM_%d : hwi_init_port success\n"); 3939 return 0; 3940 3941 error: 3942 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3943 "BM_%d : hwi_init_port failed"); 3944 hwi_cleanup(phba); 3945 return status; 3946 } 3947 3948 static int hwi_init_controller(struct beiscsi_hba *phba) 3949 { 3950 struct hwi_controller *phwi_ctrlr; 3951 3952 phwi_ctrlr = phba->phwi_ctrlr; 3953 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) { 3954 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba-> 3955 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address; 3956 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 3957 "BM_%d : phwi_ctrlr->phwi_ctxt=%p\n", 3958 phwi_ctrlr->phwi_ctxt); 3959 } else { 3960 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3961 "BM_%d : HWI_MEM_ADDN_CONTEXT is more " 3962 "than one element.Failing to load\n"); 3963 return -ENOMEM; 3964 } 3965 3966 iscsi_init_global_templates(phba); 3967 if (beiscsi_init_wrb_handle(phba)) 3968 return -ENOMEM; 3969 3970 if (hwi_init_async_pdu_ctx(phba)) { 3971 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3972 "BM_%d : hwi_init_async_pdu_ctx failed\n"); 3973 return -ENOMEM; 3974 } 3975 3976 if (hwi_init_port(phba) != 0) { 3977 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 3978 "BM_%d : hwi_init_controller failed\n"); 3979 3980 return -ENOMEM; 3981 } 3982 return 0; 3983 } 3984 3985 static void beiscsi_free_mem(struct beiscsi_hba *phba) 3986 { 3987 struct be_mem_descriptor *mem_descr; 3988 int i, j; 3989 3990 mem_descr = phba->init_mem; 3991 i = 0; 3992 j = 0; 3993 for (i = 0; i < SE_MEM_MAX; i++) { 3994 for (j = mem_descr->num_elements; j > 0; j--) { 3995 pci_free_consistent(phba->pcidev, 3996 mem_descr->mem_array[j - 1].size, 3997 mem_descr->mem_array[j - 1].virtual_address, 3998 (unsigned long)mem_descr->mem_array[j - 1]. 3999 bus_address.u.a64.address); 4000 } 4001 4002 kfree(mem_descr->mem_array); 4003 mem_descr++; 4004 } 4005 kfree(phba->init_mem); 4006 kfree(phba->phwi_ctrlr->wrb_context); 4007 kfree(phba->phwi_ctrlr); 4008 } 4009 4010 static int beiscsi_init_controller(struct beiscsi_hba *phba) 4011 { 4012 int ret = -ENOMEM; 4013 4014 ret = beiscsi_get_memory(phba); 4015 if (ret < 0) { 4016 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4017 "BM_%d : beiscsi_dev_probe -" 4018 "Failed in beiscsi_alloc_memory\n"); 4019 return ret; 4020 } 4021 4022 ret = hwi_init_controller(phba); 4023 if (ret) 4024 goto free_init; 4025 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 4026 "BM_%d : Return success from beiscsi_init_controller"); 4027 4028 return 0; 4029 4030 free_init: 4031 beiscsi_free_mem(phba); 4032 return ret; 4033 } 4034 4035 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba) 4036 { 4037 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg; 4038 struct sgl_handle *psgl_handle; 4039 struct iscsi_sge *pfrag; 4040 unsigned int arr_index, i, idx; 4041 unsigned int ulp_icd_start, ulp_num = 0; 4042 4043 phba->io_sgl_hndl_avbl = 0; 4044 phba->eh_sgl_hndl_avbl = 0; 4045 4046 mem_descr_sglh = phba->init_mem; 4047 mem_descr_sglh += HWI_MEM_SGLH; 4048 if (1 == mem_descr_sglh->num_elements) { 4049 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 4050 phba->params.ios_per_ctrl, 4051 GFP_KERNEL); 4052 if (!phba->io_sgl_hndl_base) { 4053 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4054 "BM_%d : Mem Alloc Failed. Failing to load\n"); 4055 return -ENOMEM; 4056 } 4057 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * 4058 (phba->params.icds_per_ctrl - 4059 phba->params.ios_per_ctrl), 4060 GFP_KERNEL); 4061 if (!phba->eh_sgl_hndl_base) { 4062 kfree(phba->io_sgl_hndl_base); 4063 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4064 "BM_%d : Mem Alloc Failed. Failing to load\n"); 4065 return -ENOMEM; 4066 } 4067 } else { 4068 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4069 "BM_%d : HWI_MEM_SGLH is more than one element." 4070 "Failing to load\n"); 4071 return -ENOMEM; 4072 } 4073 4074 arr_index = 0; 4075 idx = 0; 4076 while (idx < mem_descr_sglh->num_elements) { 4077 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address; 4078 4079 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size / 4080 sizeof(struct sgl_handle)); i++) { 4081 if (arr_index < phba->params.ios_per_ctrl) { 4082 phba->io_sgl_hndl_base[arr_index] = psgl_handle; 4083 phba->io_sgl_hndl_avbl++; 4084 arr_index++; 4085 } else { 4086 phba->eh_sgl_hndl_base[arr_index - 4087 phba->params.ios_per_ctrl] = 4088 psgl_handle; 4089 arr_index++; 4090 phba->eh_sgl_hndl_avbl++; 4091 } 4092 psgl_handle++; 4093 } 4094 idx++; 4095 } 4096 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 4097 "BM_%d : phba->io_sgl_hndl_avbl=%d" 4098 "phba->eh_sgl_hndl_avbl=%d\n", 4099 phba->io_sgl_hndl_avbl, 4100 phba->eh_sgl_hndl_avbl); 4101 4102 mem_descr_sg = phba->init_mem; 4103 mem_descr_sg += HWI_MEM_SGE; 4104 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 4105 "\n BM_%d : mem_descr_sg->num_elements=%d\n", 4106 mem_descr_sg->num_elements); 4107 4108 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) 4109 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) 4110 break; 4111 4112 ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num]; 4113 4114 arr_index = 0; 4115 idx = 0; 4116 while (idx < mem_descr_sg->num_elements) { 4117 pfrag = mem_descr_sg->mem_array[idx].virtual_address; 4118 4119 for (i = 0; 4120 i < (mem_descr_sg->mem_array[idx].size) / 4121 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io); 4122 i++) { 4123 if (arr_index < phba->params.ios_per_ctrl) 4124 psgl_handle = phba->io_sgl_hndl_base[arr_index]; 4125 else 4126 psgl_handle = phba->eh_sgl_hndl_base[arr_index - 4127 phba->params.ios_per_ctrl]; 4128 psgl_handle->pfrag = pfrag; 4129 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0); 4130 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0); 4131 pfrag += phba->params.num_sge_per_io; 4132 psgl_handle->sgl_index = ulp_icd_start + arr_index++; 4133 } 4134 idx++; 4135 } 4136 phba->io_sgl_free_index = 0; 4137 phba->io_sgl_alloc_index = 0; 4138 phba->eh_sgl_free_index = 0; 4139 phba->eh_sgl_alloc_index = 0; 4140 return 0; 4141 } 4142 4143 static int hba_setup_cid_tbls(struct beiscsi_hba *phba) 4144 { 4145 int ret; 4146 uint16_t i, ulp_num; 4147 struct ulp_cid_info *ptr_cid_info = NULL; 4148 4149 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 4150 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 4151 ptr_cid_info = kzalloc(sizeof(struct ulp_cid_info), 4152 GFP_KERNEL); 4153 4154 if (!ptr_cid_info) { 4155 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4156 "BM_%d : Failed to allocate memory" 4157 "for ULP_CID_INFO for ULP : %d\n", 4158 ulp_num); 4159 ret = -ENOMEM; 4160 goto free_memory; 4161 4162 } 4163 4164 /* Allocate memory for CID array */ 4165 ptr_cid_info->cid_array = kzalloc(sizeof(void *) * 4166 BEISCSI_GET_CID_COUNT(phba, 4167 ulp_num), GFP_KERNEL); 4168 if (!ptr_cid_info->cid_array) { 4169 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4170 "BM_%d : Failed to allocate memory" 4171 "for CID_ARRAY for ULP : %d\n", 4172 ulp_num); 4173 kfree(ptr_cid_info); 4174 ptr_cid_info = NULL; 4175 ret = -ENOMEM; 4176 4177 goto free_memory; 4178 } 4179 ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT( 4180 phba, ulp_num); 4181 4182 /* Save the cid_info_array ptr */ 4183 phba->cid_array_info[ulp_num] = ptr_cid_info; 4184 } 4185 } 4186 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) * 4187 phba->params.cxns_per_ctrl, GFP_KERNEL); 4188 if (!phba->ep_array) { 4189 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4190 "BM_%d : Failed to allocate memory in " 4191 "hba_setup_cid_tbls\n"); 4192 ret = -ENOMEM; 4193 4194 goto free_memory; 4195 } 4196 4197 phba->conn_table = kzalloc(sizeof(struct beiscsi_conn *) * 4198 phba->params.cxns_per_ctrl, GFP_KERNEL); 4199 if (!phba->conn_table) { 4200 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4201 "BM_%d : Failed to allocate memory in" 4202 "hba_setup_cid_tbls\n"); 4203 4204 kfree(phba->ep_array); 4205 phba->ep_array = NULL; 4206 ret = -ENOMEM; 4207 } 4208 4209 for (i = 0; i < phba->params.cxns_per_ctrl; i++) { 4210 ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num; 4211 4212 ptr_cid_info = phba->cid_array_info[ulp_num]; 4213 ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] = 4214 phba->phwi_ctrlr->wrb_context[i].cid; 4215 4216 } 4217 4218 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 4219 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 4220 ptr_cid_info = phba->cid_array_info[ulp_num]; 4221 4222 ptr_cid_info->cid_alloc = 0; 4223 ptr_cid_info->cid_free = 0; 4224 } 4225 } 4226 return 0; 4227 4228 free_memory: 4229 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 4230 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 4231 ptr_cid_info = phba->cid_array_info[ulp_num]; 4232 4233 if (ptr_cid_info) { 4234 kfree(ptr_cid_info->cid_array); 4235 kfree(ptr_cid_info); 4236 phba->cid_array_info[ulp_num] = NULL; 4237 } 4238 } 4239 } 4240 4241 return ret; 4242 } 4243 4244 static void hwi_enable_intr(struct beiscsi_hba *phba) 4245 { 4246 struct be_ctrl_info *ctrl = &phba->ctrl; 4247 struct hwi_controller *phwi_ctrlr; 4248 struct hwi_context_memory *phwi_context; 4249 struct be_queue_info *eq; 4250 u8 __iomem *addr; 4251 u32 reg, i; 4252 u32 enabled; 4253 4254 phwi_ctrlr = phba->phwi_ctrlr; 4255 phwi_context = phwi_ctrlr->phwi_ctxt; 4256 4257 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg + 4258 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET); 4259 reg = ioread32(addr); 4260 4261 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 4262 if (!enabled) { 4263 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 4264 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 4265 "BM_%d : reg =x%08x addr=%p\n", reg, addr); 4266 iowrite32(reg, addr); 4267 } 4268 4269 if (!phba->msix_enabled) { 4270 eq = &phwi_context->be_eq[0].q; 4271 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 4272 "BM_%d : eq->id=%d\n", eq->id); 4273 4274 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 4275 } else { 4276 for (i = 0; i <= phba->num_cpus; i++) { 4277 eq = &phwi_context->be_eq[i].q; 4278 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 4279 "BM_%d : eq->id=%d\n", eq->id); 4280 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); 4281 } 4282 } 4283 } 4284 4285 static void hwi_disable_intr(struct beiscsi_hba *phba) 4286 { 4287 struct be_ctrl_info *ctrl = &phba->ctrl; 4288 4289 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET; 4290 u32 reg = ioread32(addr); 4291 4292 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 4293 if (enabled) { 4294 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 4295 iowrite32(reg, addr); 4296 } else 4297 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 4298 "BM_%d : In hwi_disable_intr, Already Disabled\n"); 4299 } 4300 4301 /** 4302 * beiscsi_get_boot_info()- Get the boot session info 4303 * @phba: The device priv structure instance 4304 * 4305 * Get the boot target info and store in driver priv structure 4306 * 4307 * return values 4308 * Success: 0 4309 * Failure: Non-Zero Value 4310 **/ 4311 static int beiscsi_get_boot_info(struct beiscsi_hba *phba) 4312 { 4313 struct be_cmd_get_session_resp *session_resp; 4314 struct be_dma_mem nonemb_cmd; 4315 unsigned int tag; 4316 unsigned int s_handle; 4317 int ret = -ENOMEM; 4318 4319 /* Get the session handle of the boot target */ 4320 ret = be_mgmt_get_boot_shandle(phba, &s_handle); 4321 if (ret) { 4322 beiscsi_log(phba, KERN_ERR, 4323 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 4324 "BM_%d : No boot session\n"); 4325 return ret; 4326 } 4327 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 4328 sizeof(*session_resp), 4329 &nonemb_cmd.dma); 4330 if (nonemb_cmd.va == NULL) { 4331 beiscsi_log(phba, KERN_ERR, 4332 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 4333 "BM_%d : Failed to allocate memory for" 4334 "beiscsi_get_session_info\n"); 4335 4336 return -ENOMEM; 4337 } 4338 4339 memset(nonemb_cmd.va, 0, sizeof(*session_resp)); 4340 tag = mgmt_get_session_info(phba, s_handle, 4341 &nonemb_cmd); 4342 if (!tag) { 4343 beiscsi_log(phba, KERN_ERR, 4344 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 4345 "BM_%d : beiscsi_get_session_info" 4346 " Failed\n"); 4347 4348 goto boot_freemem; 4349 } 4350 4351 ret = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd); 4352 if (ret) { 4353 beiscsi_log(phba, KERN_ERR, 4354 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG, 4355 "BM_%d : beiscsi_get_session_info Failed"); 4356 4357 if (ret != -EBUSY) 4358 goto boot_freemem; 4359 else 4360 return ret; 4361 } 4362 4363 session_resp = nonemb_cmd.va ; 4364 4365 memcpy(&phba->boot_sess, &session_resp->session_info, 4366 sizeof(struct mgmt_session_info)); 4367 ret = 0; 4368 4369 boot_freemem: 4370 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 4371 nonemb_cmd.va, nonemb_cmd.dma); 4372 return ret; 4373 } 4374 4375 static void beiscsi_boot_release(void *data) 4376 { 4377 struct beiscsi_hba *phba = data; 4378 4379 scsi_host_put(phba->shost); 4380 } 4381 4382 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba) 4383 { 4384 struct iscsi_boot_kobj *boot_kobj; 4385 4386 /* get boot info using mgmt cmd */ 4387 if (beiscsi_get_boot_info(phba)) 4388 /* Try to see if we can carry on without this */ 4389 return 0; 4390 4391 phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no); 4392 if (!phba->boot_kset) 4393 return -ENOMEM; 4394 4395 /* get a ref because the show function will ref the phba */ 4396 if (!scsi_host_get(phba->shost)) 4397 goto free_kset; 4398 boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba, 4399 beiscsi_show_boot_tgt_info, 4400 beiscsi_tgt_get_attr_visibility, 4401 beiscsi_boot_release); 4402 if (!boot_kobj) 4403 goto put_shost; 4404 4405 if (!scsi_host_get(phba->shost)) 4406 goto free_kset; 4407 boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba, 4408 beiscsi_show_boot_ini_info, 4409 beiscsi_ini_get_attr_visibility, 4410 beiscsi_boot_release); 4411 if (!boot_kobj) 4412 goto put_shost; 4413 4414 if (!scsi_host_get(phba->shost)) 4415 goto free_kset; 4416 boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba, 4417 beiscsi_show_boot_eth_info, 4418 beiscsi_eth_get_attr_visibility, 4419 beiscsi_boot_release); 4420 if (!boot_kobj) 4421 goto put_shost; 4422 return 0; 4423 4424 put_shost: 4425 scsi_host_put(phba->shost); 4426 free_kset: 4427 iscsi_boot_destroy_kset(phba->boot_kset); 4428 return -ENOMEM; 4429 } 4430 4431 static int beiscsi_init_port(struct beiscsi_hba *phba) 4432 { 4433 int ret; 4434 4435 ret = beiscsi_init_controller(phba); 4436 if (ret < 0) { 4437 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4438 "BM_%d : beiscsi_dev_probe - Failed in" 4439 "beiscsi_init_controller\n"); 4440 return ret; 4441 } 4442 ret = beiscsi_init_sgl_handle(phba); 4443 if (ret < 0) { 4444 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4445 "BM_%d : beiscsi_dev_probe - Failed in" 4446 "beiscsi_init_sgl_handle\n"); 4447 goto do_cleanup_ctrlr; 4448 } 4449 4450 if (hba_setup_cid_tbls(phba)) { 4451 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 4452 "BM_%d : Failed in hba_setup_cid_tbls\n"); 4453 kfree(phba->io_sgl_hndl_base); 4454 kfree(phba->eh_sgl_hndl_base); 4455 goto do_cleanup_ctrlr; 4456 } 4457 4458 return ret; 4459 4460 do_cleanup_ctrlr: 4461 hwi_cleanup(phba); 4462 return ret; 4463 } 4464 4465 static void hwi_purge_eq(struct beiscsi_hba *phba) 4466 { 4467 struct hwi_controller *phwi_ctrlr; 4468 struct hwi_context_memory *phwi_context; 4469 struct be_queue_info *eq; 4470 struct be_eq_entry *eqe = NULL; 4471 int i, eq_msix; 4472 unsigned int num_processed; 4473 4474 phwi_ctrlr = phba->phwi_ctrlr; 4475 phwi_context = phwi_ctrlr->phwi_ctxt; 4476 if (phba->msix_enabled) 4477 eq_msix = 1; 4478 else 4479 eq_msix = 0; 4480 4481 for (i = 0; i < (phba->num_cpus + eq_msix); i++) { 4482 eq = &phwi_context->be_eq[i].q; 4483 eqe = queue_tail_node(eq); 4484 num_processed = 0; 4485 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 4486 & EQE_VALID_MASK) { 4487 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 4488 queue_tail_inc(eq); 4489 eqe = queue_tail_node(eq); 4490 num_processed++; 4491 } 4492 4493 if (num_processed) 4494 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1); 4495 } 4496 } 4497 4498 static void beiscsi_clean_port(struct beiscsi_hba *phba) 4499 { 4500 int mgmt_status, ulp_num; 4501 struct ulp_cid_info *ptr_cid_info = NULL; 4502 4503 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 4504 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 4505 mgmt_status = mgmt_epfw_cleanup(phba, ulp_num); 4506 if (mgmt_status) 4507 beiscsi_log(phba, KERN_WARNING, 4508 BEISCSI_LOG_INIT, 4509 "BM_%d : mgmt_epfw_cleanup FAILED" 4510 " for ULP_%d\n", ulp_num); 4511 } 4512 } 4513 4514 hwi_purge_eq(phba); 4515 hwi_cleanup(phba); 4516 kfree(phba->io_sgl_hndl_base); 4517 kfree(phba->eh_sgl_hndl_base); 4518 kfree(phba->ep_array); 4519 kfree(phba->conn_table); 4520 4521 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) { 4522 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) { 4523 ptr_cid_info = phba->cid_array_info[ulp_num]; 4524 4525 if (ptr_cid_info) { 4526 kfree(ptr_cid_info->cid_array); 4527 kfree(ptr_cid_info); 4528 phba->cid_array_info[ulp_num] = NULL; 4529 } 4530 } 4531 } 4532 4533 } 4534 4535 /** 4536 * beiscsi_free_mgmt_task_handles()- Free driver CXN resources 4537 * @beiscsi_conn: ptr to the conn to be cleaned up 4538 * @task: ptr to iscsi_task resource to be freed. 4539 * 4540 * Free driver mgmt resources binded to CXN. 4541 **/ 4542 void 4543 beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn, 4544 struct iscsi_task *task) 4545 { 4546 struct beiscsi_io_task *io_task; 4547 struct beiscsi_hba *phba = beiscsi_conn->phba; 4548 struct hwi_wrb_context *pwrb_context; 4549 struct hwi_controller *phwi_ctrlr; 4550 uint16_t cri_index = BE_GET_CRI_FROM_CID( 4551 beiscsi_conn->beiscsi_conn_cid); 4552 4553 phwi_ctrlr = phba->phwi_ctrlr; 4554 pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 4555 4556 io_task = task->dd_data; 4557 4558 if (io_task->pwrb_handle) { 4559 memset(io_task->pwrb_handle->pwrb, 0, 4560 sizeof(struct iscsi_wrb)); 4561 free_wrb_handle(phba, pwrb_context, 4562 io_task->pwrb_handle); 4563 io_task->pwrb_handle = NULL; 4564 } 4565 4566 if (io_task->psgl_handle) { 4567 spin_lock_bh(&phba->mgmt_sgl_lock); 4568 free_mgmt_sgl_handle(phba, 4569 io_task->psgl_handle); 4570 io_task->psgl_handle = NULL; 4571 spin_unlock_bh(&phba->mgmt_sgl_lock); 4572 } 4573 4574 if (io_task->mtask_addr) 4575 pci_unmap_single(phba->pcidev, 4576 io_task->mtask_addr, 4577 io_task->mtask_data_count, 4578 PCI_DMA_TODEVICE); 4579 } 4580 4581 /** 4582 * beiscsi_cleanup_task()- Free driver resources of the task 4583 * @task: ptr to the iscsi task 4584 * 4585 **/ 4586 static void beiscsi_cleanup_task(struct iscsi_task *task) 4587 { 4588 struct beiscsi_io_task *io_task = task->dd_data; 4589 struct iscsi_conn *conn = task->conn; 4590 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 4591 struct beiscsi_hba *phba = beiscsi_conn->phba; 4592 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 4593 struct hwi_wrb_context *pwrb_context; 4594 struct hwi_controller *phwi_ctrlr; 4595 uint16_t cri_index = BE_GET_CRI_FROM_CID( 4596 beiscsi_conn->beiscsi_conn_cid); 4597 4598 phwi_ctrlr = phba->phwi_ctrlr; 4599 pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 4600 4601 if (io_task->cmd_bhs) { 4602 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 4603 io_task->bhs_pa.u.a64.address); 4604 io_task->cmd_bhs = NULL; 4605 } 4606 4607 if (task->sc) { 4608 if (io_task->pwrb_handle) { 4609 free_wrb_handle(phba, pwrb_context, 4610 io_task->pwrb_handle); 4611 io_task->pwrb_handle = NULL; 4612 } 4613 4614 if (io_task->psgl_handle) { 4615 spin_lock(&phba->io_sgl_lock); 4616 free_io_sgl_handle(phba, io_task->psgl_handle); 4617 spin_unlock(&phba->io_sgl_lock); 4618 io_task->psgl_handle = NULL; 4619 } 4620 4621 if (io_task->scsi_cmnd) { 4622 scsi_dma_unmap(io_task->scsi_cmnd); 4623 io_task->scsi_cmnd = NULL; 4624 } 4625 } else { 4626 if (!beiscsi_conn->login_in_progress) 4627 beiscsi_free_mgmt_task_handles(beiscsi_conn, task); 4628 } 4629 } 4630 4631 void 4632 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn, 4633 struct beiscsi_offload_params *params) 4634 { 4635 struct wrb_handle *pwrb_handle; 4636 struct beiscsi_hba *phba = beiscsi_conn->phba; 4637 struct iscsi_task *task = beiscsi_conn->task; 4638 struct iscsi_session *session = task->conn->session; 4639 u32 doorbell = 0; 4640 4641 /* 4642 * We can always use 0 here because it is reserved by libiscsi for 4643 * login/startup related tasks. 4644 */ 4645 beiscsi_conn->login_in_progress = 0; 4646 spin_lock_bh(&session->back_lock); 4647 beiscsi_cleanup_task(task); 4648 spin_unlock_bh(&session->back_lock); 4649 4650 pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid); 4651 4652 /* Check for the adapter family */ 4653 if (is_chip_be2_be3r(phba)) 4654 beiscsi_offload_cxn_v0(params, pwrb_handle, 4655 phba->init_mem); 4656 else 4657 beiscsi_offload_cxn_v2(params, pwrb_handle); 4658 4659 be_dws_le_to_cpu(pwrb_handle->pwrb, 4660 sizeof(struct iscsi_target_context_update_wrb)); 4661 4662 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 4663 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) 4664 << DB_DEF_PDU_WRB_INDEX_SHIFT; 4665 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 4666 iowrite32(doorbell, phba->db_va + 4667 beiscsi_conn->doorbell_offset); 4668 } 4669 4670 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt, 4671 int *index, int *age) 4672 { 4673 *index = (int)itt; 4674 if (age) 4675 *age = conn->session->age; 4676 } 4677 4678 /** 4679 * beiscsi_alloc_pdu - allocates pdu and related resources 4680 * @task: libiscsi task 4681 * @opcode: opcode of pdu for task 4682 * 4683 * This is called with the session lock held. It will allocate 4684 * the wrb and sgl if needed for the command. And it will prep 4685 * the pdu's itt. beiscsi_parse_pdu will later translate 4686 * the pdu itt to the libiscsi task itt. 4687 */ 4688 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode) 4689 { 4690 struct beiscsi_io_task *io_task = task->dd_data; 4691 struct iscsi_conn *conn = task->conn; 4692 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 4693 struct beiscsi_hba *phba = beiscsi_conn->phba; 4694 struct hwi_wrb_context *pwrb_context; 4695 struct hwi_controller *phwi_ctrlr; 4696 itt_t itt; 4697 uint16_t cri_index = 0; 4698 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; 4699 dma_addr_t paddr; 4700 4701 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool, 4702 GFP_ATOMIC, &paddr); 4703 if (!io_task->cmd_bhs) 4704 return -ENOMEM; 4705 io_task->bhs_pa.u.a64.address = paddr; 4706 io_task->libiscsi_itt = (itt_t)task->itt; 4707 io_task->conn = beiscsi_conn; 4708 4709 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr; 4710 task->hdr_max = sizeof(struct be_cmd_bhs); 4711 io_task->psgl_handle = NULL; 4712 io_task->pwrb_handle = NULL; 4713 4714 if (task->sc) { 4715 spin_lock(&phba->io_sgl_lock); 4716 io_task->psgl_handle = alloc_io_sgl_handle(phba); 4717 spin_unlock(&phba->io_sgl_lock); 4718 if (!io_task->psgl_handle) { 4719 beiscsi_log(phba, KERN_ERR, 4720 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 4721 "BM_%d : Alloc of IO_SGL_ICD Failed" 4722 "for the CID : %d\n", 4723 beiscsi_conn->beiscsi_conn_cid); 4724 goto free_hndls; 4725 } 4726 io_task->pwrb_handle = alloc_wrb_handle(phba, 4727 beiscsi_conn->beiscsi_conn_cid); 4728 if (!io_task->pwrb_handle) { 4729 beiscsi_log(phba, KERN_ERR, 4730 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 4731 "BM_%d : Alloc of WRB_HANDLE Failed" 4732 "for the CID : %d\n", 4733 beiscsi_conn->beiscsi_conn_cid); 4734 goto free_io_hndls; 4735 } 4736 } else { 4737 io_task->scsi_cmnd = NULL; 4738 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) { 4739 beiscsi_conn->task = task; 4740 if (!beiscsi_conn->login_in_progress) { 4741 spin_lock(&phba->mgmt_sgl_lock); 4742 io_task->psgl_handle = (struct sgl_handle *) 4743 alloc_mgmt_sgl_handle(phba); 4744 spin_unlock(&phba->mgmt_sgl_lock); 4745 if (!io_task->psgl_handle) { 4746 beiscsi_log(phba, KERN_ERR, 4747 BEISCSI_LOG_IO | 4748 BEISCSI_LOG_CONFIG, 4749 "BM_%d : Alloc of MGMT_SGL_ICD Failed" 4750 "for the CID : %d\n", 4751 beiscsi_conn-> 4752 beiscsi_conn_cid); 4753 goto free_hndls; 4754 } 4755 4756 beiscsi_conn->login_in_progress = 1; 4757 beiscsi_conn->plogin_sgl_handle = 4758 io_task->psgl_handle; 4759 io_task->pwrb_handle = 4760 alloc_wrb_handle(phba, 4761 beiscsi_conn->beiscsi_conn_cid); 4762 if (!io_task->pwrb_handle) { 4763 beiscsi_log(phba, KERN_ERR, 4764 BEISCSI_LOG_IO | 4765 BEISCSI_LOG_CONFIG, 4766 "BM_%d : Alloc of WRB_HANDLE Failed" 4767 "for the CID : %d\n", 4768 beiscsi_conn-> 4769 beiscsi_conn_cid); 4770 goto free_mgmt_hndls; 4771 } 4772 beiscsi_conn->plogin_wrb_handle = 4773 io_task->pwrb_handle; 4774 4775 } else { 4776 io_task->psgl_handle = 4777 beiscsi_conn->plogin_sgl_handle; 4778 io_task->pwrb_handle = 4779 beiscsi_conn->plogin_wrb_handle; 4780 } 4781 } else { 4782 spin_lock(&phba->mgmt_sgl_lock); 4783 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba); 4784 spin_unlock(&phba->mgmt_sgl_lock); 4785 if (!io_task->psgl_handle) { 4786 beiscsi_log(phba, KERN_ERR, 4787 BEISCSI_LOG_IO | 4788 BEISCSI_LOG_CONFIG, 4789 "BM_%d : Alloc of MGMT_SGL_ICD Failed" 4790 "for the CID : %d\n", 4791 beiscsi_conn-> 4792 beiscsi_conn_cid); 4793 goto free_hndls; 4794 } 4795 io_task->pwrb_handle = 4796 alloc_wrb_handle(phba, 4797 beiscsi_conn->beiscsi_conn_cid); 4798 if (!io_task->pwrb_handle) { 4799 beiscsi_log(phba, KERN_ERR, 4800 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG, 4801 "BM_%d : Alloc of WRB_HANDLE Failed" 4802 "for the CID : %d\n", 4803 beiscsi_conn->beiscsi_conn_cid); 4804 goto free_mgmt_hndls; 4805 } 4806 4807 } 4808 } 4809 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle-> 4810 wrb_index << 16) | (unsigned int) 4811 (io_task->psgl_handle->sgl_index)); 4812 io_task->pwrb_handle->pio_handle = task; 4813 4814 io_task->cmd_bhs->iscsi_hdr.itt = itt; 4815 return 0; 4816 4817 free_io_hndls: 4818 spin_lock(&phba->io_sgl_lock); 4819 free_io_sgl_handle(phba, io_task->psgl_handle); 4820 spin_unlock(&phba->io_sgl_lock); 4821 goto free_hndls; 4822 free_mgmt_hndls: 4823 spin_lock(&phba->mgmt_sgl_lock); 4824 free_mgmt_sgl_handle(phba, io_task->psgl_handle); 4825 io_task->psgl_handle = NULL; 4826 spin_unlock(&phba->mgmt_sgl_lock); 4827 free_hndls: 4828 phwi_ctrlr = phba->phwi_ctrlr; 4829 cri_index = BE_GET_CRI_FROM_CID( 4830 beiscsi_conn->beiscsi_conn_cid); 4831 pwrb_context = &phwi_ctrlr->wrb_context[cri_index]; 4832 if (io_task->pwrb_handle) 4833 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); 4834 io_task->pwrb_handle = NULL; 4835 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, 4836 io_task->bhs_pa.u.a64.address); 4837 io_task->cmd_bhs = NULL; 4838 return -ENOMEM; 4839 } 4840 int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg, 4841 unsigned int num_sg, unsigned int xferlen, 4842 unsigned int writedir) 4843 { 4844 4845 struct beiscsi_io_task *io_task = task->dd_data; 4846 struct iscsi_conn *conn = task->conn; 4847 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 4848 struct beiscsi_hba *phba = beiscsi_conn->phba; 4849 struct iscsi_wrb *pwrb = NULL; 4850 unsigned int doorbell = 0; 4851 4852 pwrb = io_task->pwrb_handle->pwrb; 4853 4854 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0; 4855 io_task->bhs_len = sizeof(struct be_cmd_bhs); 4856 4857 if (writedir) { 4858 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb, 4859 INI_WR_CMD); 4860 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1); 4861 } else { 4862 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb, 4863 INI_RD_CMD); 4864 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0); 4865 } 4866 4867 io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2, 4868 type, pwrb); 4869 4870 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb, 4871 cpu_to_be16(*(unsigned short *) 4872 &io_task->cmd_bhs->iscsi_hdr.lun)); 4873 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen); 4874 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb, 4875 io_task->pwrb_handle->wrb_index); 4876 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb, 4877 be32_to_cpu(task->cmdsn)); 4878 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb, 4879 io_task->psgl_handle->sgl_index); 4880 4881 hwi_write_sgl_v2(pwrb, sg, num_sg, io_task); 4882 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb, 4883 io_task->pwrb_handle->nxt_wrb_index); 4884 4885 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 4886 4887 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 4888 doorbell |= (io_task->pwrb_handle->wrb_index & 4889 DB_DEF_PDU_WRB_INDEX_MASK) << 4890 DB_DEF_PDU_WRB_INDEX_SHIFT; 4891 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 4892 iowrite32(doorbell, phba->db_va + 4893 beiscsi_conn->doorbell_offset); 4894 return 0; 4895 } 4896 4897 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg, 4898 unsigned int num_sg, unsigned int xferlen, 4899 unsigned int writedir) 4900 { 4901 4902 struct beiscsi_io_task *io_task = task->dd_data; 4903 struct iscsi_conn *conn = task->conn; 4904 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 4905 struct beiscsi_hba *phba = beiscsi_conn->phba; 4906 struct iscsi_wrb *pwrb = NULL; 4907 unsigned int doorbell = 0; 4908 4909 pwrb = io_task->pwrb_handle->pwrb; 4910 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0; 4911 io_task->bhs_len = sizeof(struct be_cmd_bhs); 4912 4913 if (writedir) { 4914 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4915 INI_WR_CMD); 4916 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); 4917 } else { 4918 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, 4919 INI_RD_CMD); 4920 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); 4921 } 4922 4923 io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb, 4924 type, pwrb); 4925 4926 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb, 4927 cpu_to_be16(*(unsigned short *) 4928 &io_task->cmd_bhs->iscsi_hdr.lun)); 4929 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen); 4930 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 4931 io_task->pwrb_handle->wrb_index); 4932 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 4933 be32_to_cpu(task->cmdsn)); 4934 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 4935 io_task->psgl_handle->sgl_index); 4936 4937 hwi_write_sgl(pwrb, sg, num_sg, io_task); 4938 4939 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 4940 io_task->pwrb_handle->nxt_wrb_index); 4941 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); 4942 4943 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; 4944 doorbell |= (io_task->pwrb_handle->wrb_index & 4945 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 4946 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 4947 4948 iowrite32(doorbell, phba->db_va + 4949 beiscsi_conn->doorbell_offset); 4950 return 0; 4951 } 4952 4953 static int beiscsi_mtask(struct iscsi_task *task) 4954 { 4955 struct beiscsi_io_task *io_task = task->dd_data; 4956 struct iscsi_conn *conn = task->conn; 4957 struct beiscsi_conn *beiscsi_conn = conn->dd_data; 4958 struct beiscsi_hba *phba = beiscsi_conn->phba; 4959 struct iscsi_wrb *pwrb = NULL; 4960 unsigned int doorbell = 0; 4961 unsigned int cid; 4962 unsigned int pwrb_typeoffset = 0; 4963 4964 cid = beiscsi_conn->beiscsi_conn_cid; 4965 pwrb = io_task->pwrb_handle->pwrb; 4966 memset(pwrb, 0, sizeof(*pwrb)); 4967 4968 if (is_chip_be2_be3r(phba)) { 4969 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 4970 be32_to_cpu(task->cmdsn)); 4971 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, 4972 io_task->pwrb_handle->wrb_index); 4973 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, 4974 io_task->psgl_handle->sgl_index); 4975 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, 4976 task->data_count); 4977 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, 4978 io_task->pwrb_handle->nxt_wrb_index); 4979 pwrb_typeoffset = BE_WRB_TYPE_OFFSET; 4980 } else { 4981 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb, 4982 be32_to_cpu(task->cmdsn)); 4983 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb, 4984 io_task->pwrb_handle->wrb_index); 4985 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb, 4986 io_task->psgl_handle->sgl_index); 4987 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, 4988 task->data_count); 4989 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb, 4990 io_task->pwrb_handle->nxt_wrb_index); 4991 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET; 4992 } 4993 4994 4995 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) { 4996 case ISCSI_OP_LOGIN: 4997 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1); 4998 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset); 4999 hwi_write_buffer(pwrb, task); 5000 break; 5001 case ISCSI_OP_NOOP_OUT: 5002 if (task->hdr->ttt != ISCSI_RESERVED_TAG) { 5003 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset); 5004 if (is_chip_be2_be3r(phba)) 5005 AMAP_SET_BITS(struct amap_iscsi_wrb, 5006 dmsg, pwrb, 1); 5007 else 5008 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 5009 dmsg, pwrb, 1); 5010 } else { 5011 ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset); 5012 if (is_chip_be2_be3r(phba)) 5013 AMAP_SET_BITS(struct amap_iscsi_wrb, 5014 dmsg, pwrb, 0); 5015 else 5016 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, 5017 dmsg, pwrb, 0); 5018 } 5019 hwi_write_buffer(pwrb, task); 5020 break; 5021 case ISCSI_OP_TEXT: 5022 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset); 5023 hwi_write_buffer(pwrb, task); 5024 break; 5025 case ISCSI_OP_SCSI_TMFUNC: 5026 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset); 5027 hwi_write_buffer(pwrb, task); 5028 break; 5029 case ISCSI_OP_LOGOUT: 5030 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset); 5031 hwi_write_buffer(pwrb, task); 5032 break; 5033 5034 default: 5035 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 5036 "BM_%d : opcode =%d Not supported\n", 5037 task->hdr->opcode & ISCSI_OPCODE_MASK); 5038 5039 return -EINVAL; 5040 } 5041 5042 /* Set the task type */ 5043 io_task->wrb_type = (is_chip_be2_be3r(phba)) ? 5044 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb) : 5045 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb); 5046 5047 doorbell |= cid & DB_WRB_POST_CID_MASK; 5048 doorbell |= (io_task->pwrb_handle->wrb_index & 5049 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; 5050 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; 5051 iowrite32(doorbell, phba->db_va + 5052 beiscsi_conn->doorbell_offset); 5053 return 0; 5054 } 5055 5056 static int beiscsi_task_xmit(struct iscsi_task *task) 5057 { 5058 struct beiscsi_io_task *io_task = task->dd_data; 5059 struct scsi_cmnd *sc = task->sc; 5060 struct beiscsi_hba *phba = NULL; 5061 struct scatterlist *sg; 5062 int num_sg; 5063 unsigned int writedir = 0, xferlen = 0; 5064 5065 phba = ((struct beiscsi_conn *)task->conn->dd_data)->phba; 5066 5067 if (!sc) 5068 return beiscsi_mtask(task); 5069 5070 io_task->scsi_cmnd = sc; 5071 num_sg = scsi_dma_map(sc); 5072 if (num_sg < 0) { 5073 struct iscsi_conn *conn = task->conn; 5074 struct beiscsi_hba *phba = NULL; 5075 5076 phba = ((struct beiscsi_conn *)conn->dd_data)->phba; 5077 beiscsi_log(phba, KERN_ERR, 5078 BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI, 5079 "BM_%d : scsi_dma_map Failed " 5080 "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n", 5081 be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt), 5082 io_task->libiscsi_itt, scsi_bufflen(sc)); 5083 5084 return num_sg; 5085 } 5086 xferlen = scsi_bufflen(sc); 5087 sg = scsi_sglist(sc); 5088 if (sc->sc_data_direction == DMA_TO_DEVICE) 5089 writedir = 1; 5090 else 5091 writedir = 0; 5092 5093 return phba->iotask_fn(task, sg, num_sg, xferlen, writedir); 5094 } 5095 5096 /** 5097 * beiscsi_bsg_request - handle bsg request from ISCSI transport 5098 * @job: job to handle 5099 */ 5100 static int beiscsi_bsg_request(struct bsg_job *job) 5101 { 5102 struct Scsi_Host *shost; 5103 struct beiscsi_hba *phba; 5104 struct iscsi_bsg_request *bsg_req = job->request; 5105 int rc = -EINVAL; 5106 unsigned int tag; 5107 struct be_dma_mem nonemb_cmd; 5108 struct be_cmd_resp_hdr *resp; 5109 struct iscsi_bsg_reply *bsg_reply = job->reply; 5110 unsigned short status, extd_status; 5111 5112 shost = iscsi_job_to_shost(job); 5113 phba = iscsi_host_priv(shost); 5114 5115 switch (bsg_req->msgcode) { 5116 case ISCSI_BSG_HST_VENDOR: 5117 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, 5118 job->request_payload.payload_len, 5119 &nonemb_cmd.dma); 5120 if (nonemb_cmd.va == NULL) { 5121 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 5122 "BM_%d : Failed to allocate memory for " 5123 "beiscsi_bsg_request\n"); 5124 return -ENOMEM; 5125 } 5126 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job, 5127 &nonemb_cmd); 5128 if (!tag) { 5129 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 5130 "BM_%d : MBX Tag Allocation Failed\n"); 5131 5132 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 5133 nonemb_cmd.va, nonemb_cmd.dma); 5134 return -EAGAIN; 5135 } 5136 5137 rc = wait_event_interruptible_timeout( 5138 phba->ctrl.mcc_wait[tag], 5139 phba->ctrl.mcc_numtag[tag], 5140 msecs_to_jiffies( 5141 BEISCSI_HOST_MBX_TIMEOUT)); 5142 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8; 5143 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF; 5144 free_mcc_tag(&phba->ctrl, tag); 5145 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va; 5146 sg_copy_from_buffer(job->reply_payload.sg_list, 5147 job->reply_payload.sg_cnt, 5148 nonemb_cmd.va, (resp->response_length 5149 + sizeof(*resp))); 5150 bsg_reply->reply_payload_rcv_len = resp->response_length; 5151 bsg_reply->result = status; 5152 bsg_job_done(job, bsg_reply->result, 5153 bsg_reply->reply_payload_rcv_len); 5154 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, 5155 nonemb_cmd.va, nonemb_cmd.dma); 5156 if (status || extd_status) { 5157 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 5158 "BM_%d : MBX Cmd Failed" 5159 " status = %d extd_status = %d\n", 5160 status, extd_status); 5161 5162 return -EIO; 5163 } else { 5164 rc = 0; 5165 } 5166 break; 5167 5168 default: 5169 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG, 5170 "BM_%d : Unsupported bsg command: 0x%x\n", 5171 bsg_req->msgcode); 5172 break; 5173 } 5174 5175 return rc; 5176 } 5177 5178 void beiscsi_hba_attrs_init(struct beiscsi_hba *phba) 5179 { 5180 /* Set the logging parameter */ 5181 beiscsi_log_enable_init(phba, beiscsi_log_enable); 5182 } 5183 5184 /* 5185 * beiscsi_quiesce()- Cleanup Driver resources 5186 * @phba: Instance Priv structure 5187 * @unload_state:i Clean or EEH unload state 5188 * 5189 * Free the OS and HW resources held by the driver 5190 **/ 5191 static void beiscsi_quiesce(struct beiscsi_hba *phba, 5192 uint32_t unload_state) 5193 { 5194 struct hwi_controller *phwi_ctrlr; 5195 struct hwi_context_memory *phwi_context; 5196 struct be_eq_obj *pbe_eq; 5197 unsigned int i, msix_vec; 5198 5199 phwi_ctrlr = phba->phwi_ctrlr; 5200 phwi_context = phwi_ctrlr->phwi_ctxt; 5201 hwi_disable_intr(phba); 5202 if (phba->msix_enabled) { 5203 for (i = 0; i <= phba->num_cpus; i++) { 5204 msix_vec = phba->msix_entries[i].vector; 5205 synchronize_irq(msix_vec); 5206 free_irq(msix_vec, &phwi_context->be_eq[i]); 5207 kfree(phba->msi_name[i]); 5208 } 5209 } else 5210 if (phba->pcidev->irq) { 5211 synchronize_irq(phba->pcidev->irq); 5212 free_irq(phba->pcidev->irq, phba); 5213 } 5214 pci_disable_msix(phba->pcidev); 5215 5216 for (i = 0; i < phba->num_cpus; i++) { 5217 pbe_eq = &phwi_context->be_eq[i]; 5218 blk_iopoll_disable(&pbe_eq->iopoll); 5219 } 5220 5221 if (unload_state == BEISCSI_CLEAN_UNLOAD) { 5222 destroy_workqueue(phba->wq); 5223 beiscsi_clean_port(phba); 5224 beiscsi_free_mem(phba); 5225 5226 beiscsi_unmap_pci_function(phba); 5227 pci_free_consistent(phba->pcidev, 5228 phba->ctrl.mbox_mem_alloced.size, 5229 phba->ctrl.mbox_mem_alloced.va, 5230 phba->ctrl.mbox_mem_alloced.dma); 5231 } else { 5232 hwi_purge_eq(phba); 5233 hwi_cleanup(phba); 5234 } 5235 5236 cancel_delayed_work_sync(&phba->beiscsi_hw_check_task); 5237 } 5238 5239 static void beiscsi_remove(struct pci_dev *pcidev) 5240 { 5241 5242 struct beiscsi_hba *phba = NULL; 5243 5244 phba = pci_get_drvdata(pcidev); 5245 if (!phba) { 5246 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n"); 5247 return; 5248 } 5249 5250 beiscsi_destroy_def_ifaces(phba); 5251 beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD); 5252 iscsi_boot_destroy_kset(phba->boot_kset); 5253 iscsi_host_remove(phba->shost); 5254 pci_dev_put(phba->pcidev); 5255 iscsi_host_free(phba->shost); 5256 pci_disable_pcie_error_reporting(pcidev); 5257 pci_set_drvdata(pcidev, NULL); 5258 pci_disable_device(pcidev); 5259 } 5260 5261 static void beiscsi_shutdown(struct pci_dev *pcidev) 5262 { 5263 5264 struct beiscsi_hba *phba = NULL; 5265 5266 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev); 5267 if (!phba) { 5268 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n"); 5269 return; 5270 } 5271 5272 phba->state = BE_ADAPTER_STATE_SHUTDOWN; 5273 iscsi_host_for_each_session(phba->shost, be2iscsi_fail_session); 5274 beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD); 5275 pci_disable_device(pcidev); 5276 } 5277 5278 static void beiscsi_msix_enable(struct beiscsi_hba *phba) 5279 { 5280 int i, status; 5281 5282 for (i = 0; i <= phba->num_cpus; i++) 5283 phba->msix_entries[i].entry = i; 5284 5285 status = pci_enable_msix(phba->pcidev, phba->msix_entries, 5286 (phba->num_cpus + 1)); 5287 if (!status) 5288 phba->msix_enabled = true; 5289 5290 return; 5291 } 5292 5293 /* 5294 * beiscsi_hw_health_check()- Check adapter health 5295 * @work: work item to check HW health 5296 * 5297 * Check if adapter in an unrecoverable state or not. 5298 **/ 5299 static void 5300 beiscsi_hw_health_check(struct work_struct *work) 5301 { 5302 struct beiscsi_hba *phba = 5303 container_of(work, struct beiscsi_hba, 5304 beiscsi_hw_check_task.work); 5305 5306 beiscsi_ue_detect(phba); 5307 5308 schedule_delayed_work(&phba->beiscsi_hw_check_task, 5309 msecs_to_jiffies(1000)); 5310 } 5311 5312 5313 static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev, 5314 pci_channel_state_t state) 5315 { 5316 struct beiscsi_hba *phba = NULL; 5317 5318 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev); 5319 phba->state |= BE_ADAPTER_PCI_ERR; 5320 5321 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5322 "BM_%d : EEH error detected\n"); 5323 5324 beiscsi_quiesce(phba, BEISCSI_EEH_UNLOAD); 5325 5326 if (state == pci_channel_io_perm_failure) { 5327 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5328 "BM_%d : EEH : State PERM Failure"); 5329 return PCI_ERS_RESULT_DISCONNECT; 5330 } 5331 5332 pci_disable_device(pdev); 5333 5334 /* The error could cause the FW to trigger a flash debug dump. 5335 * Resetting the card while flash dump is in progress 5336 * can cause it not to recover; wait for it to finish. 5337 * Wait only for first function as it is needed only once per 5338 * adapter. 5339 **/ 5340 if (pdev->devfn == 0) 5341 ssleep(30); 5342 5343 return PCI_ERS_RESULT_NEED_RESET; 5344 } 5345 5346 static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev) 5347 { 5348 struct beiscsi_hba *phba = NULL; 5349 int status = 0; 5350 5351 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev); 5352 5353 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5354 "BM_%d : EEH Reset\n"); 5355 5356 status = pci_enable_device(pdev); 5357 if (status) 5358 return PCI_ERS_RESULT_DISCONNECT; 5359 5360 pci_set_master(pdev); 5361 pci_set_power_state(pdev, PCI_D0); 5362 pci_restore_state(pdev); 5363 5364 /* Wait for the CHIP Reset to complete */ 5365 status = be_chk_reset_complete(phba); 5366 if (!status) { 5367 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 5368 "BM_%d : EEH Reset Completed\n"); 5369 } else { 5370 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 5371 "BM_%d : EEH Reset Completion Failure\n"); 5372 return PCI_ERS_RESULT_DISCONNECT; 5373 } 5374 5375 pci_cleanup_aer_uncorrect_error_status(pdev); 5376 return PCI_ERS_RESULT_RECOVERED; 5377 } 5378 5379 static void beiscsi_eeh_resume(struct pci_dev *pdev) 5380 { 5381 int ret = 0, i; 5382 struct be_eq_obj *pbe_eq; 5383 struct beiscsi_hba *phba = NULL; 5384 struct hwi_controller *phwi_ctrlr; 5385 struct hwi_context_memory *phwi_context; 5386 5387 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev); 5388 pci_save_state(pdev); 5389 5390 if (enable_msix) 5391 find_num_cpus(phba); 5392 else 5393 phba->num_cpus = 1; 5394 5395 if (enable_msix) { 5396 beiscsi_msix_enable(phba); 5397 if (!phba->msix_enabled) 5398 phba->num_cpus = 1; 5399 } 5400 5401 ret = beiscsi_cmd_reset_function(phba); 5402 if (ret) { 5403 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5404 "BM_%d : Reset Failed\n"); 5405 goto ret_err; 5406 } 5407 5408 ret = be_chk_reset_complete(phba); 5409 if (ret) { 5410 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5411 "BM_%d : Failed to get out of reset.\n"); 5412 goto ret_err; 5413 } 5414 5415 beiscsi_get_params(phba); 5416 phba->shost->max_id = phba->params.cxns_per_ctrl; 5417 phba->shost->can_queue = phba->params.ios_per_ctrl; 5418 ret = hwi_init_controller(phba); 5419 5420 for (i = 0; i < MAX_MCC_CMD; i++) { 5421 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]); 5422 phba->ctrl.mcc_tag[i] = i + 1; 5423 phba->ctrl.mcc_numtag[i + 1] = 0; 5424 phba->ctrl.mcc_tag_available++; 5425 } 5426 5427 phwi_ctrlr = phba->phwi_ctrlr; 5428 phwi_context = phwi_ctrlr->phwi_ctxt; 5429 5430 for (i = 0; i < phba->num_cpus; i++) { 5431 pbe_eq = &phwi_context->be_eq[i]; 5432 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget, 5433 be_iopoll); 5434 blk_iopoll_enable(&pbe_eq->iopoll); 5435 } 5436 5437 i = (phba->msix_enabled) ? i : 0; 5438 /* Work item for MCC handling */ 5439 pbe_eq = &phwi_context->be_eq[i]; 5440 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs); 5441 5442 ret = beiscsi_init_irqs(phba); 5443 if (ret < 0) { 5444 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5445 "BM_%d : beiscsi_eeh_resume - " 5446 "Failed to beiscsi_init_irqs\n"); 5447 goto ret_err; 5448 } 5449 5450 hwi_enable_intr(phba); 5451 phba->state &= ~BE_ADAPTER_PCI_ERR; 5452 5453 return; 5454 ret_err: 5455 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5456 "BM_%d : AER EEH Resume Failed\n"); 5457 } 5458 5459 static int beiscsi_dev_probe(struct pci_dev *pcidev, 5460 const struct pci_device_id *id) 5461 { 5462 struct beiscsi_hba *phba = NULL; 5463 struct hwi_controller *phwi_ctrlr; 5464 struct hwi_context_memory *phwi_context; 5465 struct be_eq_obj *pbe_eq; 5466 int ret = 0, i; 5467 5468 ret = beiscsi_enable_pci(pcidev); 5469 if (ret < 0) { 5470 dev_err(&pcidev->dev, 5471 "beiscsi_dev_probe - Failed to enable pci device\n"); 5472 return ret; 5473 } 5474 5475 phba = beiscsi_hba_alloc(pcidev); 5476 if (!phba) { 5477 dev_err(&pcidev->dev, 5478 "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n"); 5479 goto disable_pci; 5480 } 5481 5482 /* Enable EEH reporting */ 5483 ret = pci_enable_pcie_error_reporting(pcidev); 5484 if (ret) 5485 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT, 5486 "BM_%d : PCIe Error Reporting " 5487 "Enabling Failed\n"); 5488 5489 pci_save_state(pcidev); 5490 5491 /* Initialize Driver configuration Paramters */ 5492 beiscsi_hba_attrs_init(phba); 5493 5494 phba->fw_timeout = false; 5495 phba->mac_addr_set = false; 5496 5497 5498 switch (pcidev->device) { 5499 case BE_DEVICE_ID1: 5500 case OC_DEVICE_ID1: 5501 case OC_DEVICE_ID2: 5502 phba->generation = BE_GEN2; 5503 phba->iotask_fn = beiscsi_iotask; 5504 break; 5505 case BE_DEVICE_ID2: 5506 case OC_DEVICE_ID3: 5507 phba->generation = BE_GEN3; 5508 phba->iotask_fn = beiscsi_iotask; 5509 break; 5510 case OC_SKH_ID1: 5511 phba->generation = BE_GEN4; 5512 phba->iotask_fn = beiscsi_iotask_v2; 5513 break; 5514 default: 5515 phba->generation = 0; 5516 } 5517 5518 ret = be_ctrl_init(phba, pcidev); 5519 if (ret) { 5520 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5521 "BM_%d : beiscsi_dev_probe-" 5522 "Failed in be_ctrl_init\n"); 5523 goto hba_free; 5524 } 5525 5526 ret = beiscsi_cmd_reset_function(phba); 5527 if (ret) { 5528 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5529 "BM_%d : Reset Failed\n"); 5530 goto hba_free; 5531 } 5532 ret = be_chk_reset_complete(phba); 5533 if (ret) { 5534 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5535 "BM_%d : Failed to get out of reset.\n"); 5536 goto hba_free; 5537 } 5538 5539 spin_lock_init(&phba->io_sgl_lock); 5540 spin_lock_init(&phba->mgmt_sgl_lock); 5541 spin_lock_init(&phba->isr_lock); 5542 spin_lock_init(&phba->async_pdu_lock); 5543 ret = mgmt_get_fw_config(&phba->ctrl, phba); 5544 if (ret != 0) { 5545 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5546 "BM_%d : Error getting fw config\n"); 5547 goto free_port; 5548 } 5549 5550 if (enable_msix) 5551 find_num_cpus(phba); 5552 else 5553 phba->num_cpus = 1; 5554 5555 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 5556 "BM_%d : num_cpus = %d\n", 5557 phba->num_cpus); 5558 5559 if (enable_msix) { 5560 beiscsi_msix_enable(phba); 5561 if (!phba->msix_enabled) 5562 phba->num_cpus = 1; 5563 } 5564 5565 phba->shost->max_id = phba->params.cxns_per_ctrl; 5566 beiscsi_get_params(phba); 5567 phba->shost->can_queue = phba->params.ios_per_ctrl; 5568 ret = beiscsi_init_port(phba); 5569 if (ret < 0) { 5570 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5571 "BM_%d : beiscsi_dev_probe-" 5572 "Failed in beiscsi_init_port\n"); 5573 goto free_port; 5574 } 5575 5576 for (i = 0; i < MAX_MCC_CMD; i++) { 5577 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]); 5578 phba->ctrl.mcc_tag[i] = i + 1; 5579 phba->ctrl.mcc_numtag[i + 1] = 0; 5580 phba->ctrl.mcc_tag_available++; 5581 memset(&phba->ctrl.ptag_state[i].tag_mem_state, 0, 5582 sizeof(struct beiscsi_mcc_tag_state)); 5583 } 5584 5585 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0; 5586 5587 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq", 5588 phba->shost->host_no); 5589 phba->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 1, phba->wq_name); 5590 if (!phba->wq) { 5591 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5592 "BM_%d : beiscsi_dev_probe-" 5593 "Failed to allocate work queue\n"); 5594 goto free_twq; 5595 } 5596 5597 INIT_DELAYED_WORK(&phba->beiscsi_hw_check_task, 5598 beiscsi_hw_health_check); 5599 5600 phwi_ctrlr = phba->phwi_ctrlr; 5601 phwi_context = phwi_ctrlr->phwi_ctxt; 5602 5603 for (i = 0; i < phba->num_cpus; i++) { 5604 pbe_eq = &phwi_context->be_eq[i]; 5605 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget, 5606 be_iopoll); 5607 blk_iopoll_enable(&pbe_eq->iopoll); 5608 } 5609 5610 i = (phba->msix_enabled) ? i : 0; 5611 /* Work item for MCC handling */ 5612 pbe_eq = &phwi_context->be_eq[i]; 5613 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs); 5614 5615 ret = beiscsi_init_irqs(phba); 5616 if (ret < 0) { 5617 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5618 "BM_%d : beiscsi_dev_probe-" 5619 "Failed to beiscsi_init_irqs\n"); 5620 goto free_blkenbld; 5621 } 5622 hwi_enable_intr(phba); 5623 5624 if (beiscsi_setup_boot_info(phba)) 5625 /* 5626 * log error but continue, because we may not be using 5627 * iscsi boot. 5628 */ 5629 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5630 "BM_%d : Could not set up " 5631 "iSCSI boot info.\n"); 5632 5633 beiscsi_create_def_ifaces(phba); 5634 schedule_delayed_work(&phba->beiscsi_hw_check_task, 5635 msecs_to_jiffies(1000)); 5636 5637 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT, 5638 "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n"); 5639 return 0; 5640 5641 free_blkenbld: 5642 destroy_workqueue(phba->wq); 5643 for (i = 0; i < phba->num_cpus; i++) { 5644 pbe_eq = &phwi_context->be_eq[i]; 5645 blk_iopoll_disable(&pbe_eq->iopoll); 5646 } 5647 free_twq: 5648 beiscsi_clean_port(phba); 5649 beiscsi_free_mem(phba); 5650 free_port: 5651 pci_free_consistent(phba->pcidev, 5652 phba->ctrl.mbox_mem_alloced.size, 5653 phba->ctrl.mbox_mem_alloced.va, 5654 phba->ctrl.mbox_mem_alloced.dma); 5655 beiscsi_unmap_pci_function(phba); 5656 hba_free: 5657 if (phba->msix_enabled) 5658 pci_disable_msix(phba->pcidev); 5659 iscsi_host_remove(phba->shost); 5660 pci_dev_put(phba->pcidev); 5661 iscsi_host_free(phba->shost); 5662 disable_pci: 5663 pci_disable_device(pcidev); 5664 return ret; 5665 } 5666 5667 static struct pci_error_handlers beiscsi_eeh_handlers = { 5668 .error_detected = beiscsi_eeh_err_detected, 5669 .slot_reset = beiscsi_eeh_reset, 5670 .resume = beiscsi_eeh_resume, 5671 }; 5672 5673 struct iscsi_transport beiscsi_iscsi_transport = { 5674 .owner = THIS_MODULE, 5675 .name = DRV_NAME, 5676 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO | 5677 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD, 5678 .create_session = beiscsi_session_create, 5679 .destroy_session = beiscsi_session_destroy, 5680 .create_conn = beiscsi_conn_create, 5681 .bind_conn = beiscsi_conn_bind, 5682 .destroy_conn = iscsi_conn_teardown, 5683 .attr_is_visible = be2iscsi_attr_is_visible, 5684 .set_iface_param = be2iscsi_iface_set_param, 5685 .get_iface_param = be2iscsi_iface_get_param, 5686 .set_param = beiscsi_set_param, 5687 .get_conn_param = iscsi_conn_get_param, 5688 .get_session_param = iscsi_session_get_param, 5689 .get_host_param = beiscsi_get_host_param, 5690 .start_conn = beiscsi_conn_start, 5691 .stop_conn = iscsi_conn_stop, 5692 .send_pdu = iscsi_conn_send_pdu, 5693 .xmit_task = beiscsi_task_xmit, 5694 .cleanup_task = beiscsi_cleanup_task, 5695 .alloc_pdu = beiscsi_alloc_pdu, 5696 .parse_pdu_itt = beiscsi_parse_pdu, 5697 .get_stats = beiscsi_conn_get_stats, 5698 .get_ep_param = beiscsi_ep_get_param, 5699 .ep_connect = beiscsi_ep_connect, 5700 .ep_poll = beiscsi_ep_poll, 5701 .ep_disconnect = beiscsi_ep_disconnect, 5702 .session_recovery_timedout = iscsi_session_recovery_timedout, 5703 .bsg_request = beiscsi_bsg_request, 5704 }; 5705 5706 static struct pci_driver beiscsi_pci_driver = { 5707 .name = DRV_NAME, 5708 .probe = beiscsi_dev_probe, 5709 .remove = beiscsi_remove, 5710 .shutdown = beiscsi_shutdown, 5711 .id_table = beiscsi_pci_id_table, 5712 .err_handler = &beiscsi_eeh_handlers 5713 }; 5714 5715 5716 static int __init beiscsi_module_init(void) 5717 { 5718 int ret; 5719 5720 beiscsi_scsi_transport = 5721 iscsi_register_transport(&beiscsi_iscsi_transport); 5722 if (!beiscsi_scsi_transport) { 5723 printk(KERN_ERR 5724 "beiscsi_module_init - Unable to register beiscsi transport.\n"); 5725 return -ENOMEM; 5726 } 5727 printk(KERN_INFO "In beiscsi_module_init, tt=%p\n", 5728 &beiscsi_iscsi_transport); 5729 5730 ret = pci_register_driver(&beiscsi_pci_driver); 5731 if (ret) { 5732 printk(KERN_ERR 5733 "beiscsi_module_init - Unable to register beiscsi pci driver.\n"); 5734 goto unregister_iscsi_transport; 5735 } 5736 return 0; 5737 5738 unregister_iscsi_transport: 5739 iscsi_unregister_transport(&beiscsi_iscsi_transport); 5740 return ret; 5741 } 5742 5743 static void __exit beiscsi_module_exit(void) 5744 { 5745 pci_unregister_driver(&beiscsi_pci_driver); 5746 iscsi_unregister_transport(&beiscsi_iscsi_transport); 5747 } 5748 5749 module_init(beiscsi_module_init); 5750 module_exit(beiscsi_module_exit); 5751