1 /* 2 * Serial Attached SCSI (SAS) Expander discovery and configuration 3 * 4 * Copyright (C) 2005 Adaptec, Inc. All rights reserved. 5 * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com> 6 * 7 * This file is licensed under GPLv2. 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License as 11 * published by the Free Software Foundation; either version 2 of the 12 * License, or (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, but 15 * WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software 21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 22 * 23 */ 24 25 #include <linux/scatterlist.h> 26 #include <linux/blkdev.h> 27 28 #include "sas_internal.h" 29 30 #include <scsi/scsi_transport.h> 31 #include <scsi/scsi_transport_sas.h> 32 #include "../scsi_sas_internal.h" 33 34 static int sas_discover_expander(struct domain_device *dev); 35 static int sas_configure_routing(struct domain_device *dev, u8 *sas_addr); 36 static int sas_configure_phy(struct domain_device *dev, int phy_id, 37 u8 *sas_addr, int include); 38 static int sas_disable_routing(struct domain_device *dev, u8 *sas_addr); 39 40 /* ---------- SMP task management ---------- */ 41 42 static void smp_task_timedout(unsigned long _task) 43 { 44 struct sas_task *task = (void *) _task; 45 unsigned long flags; 46 47 spin_lock_irqsave(&task->task_state_lock, flags); 48 if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) 49 task->task_state_flags |= SAS_TASK_STATE_ABORTED; 50 spin_unlock_irqrestore(&task->task_state_lock, flags); 51 52 complete(&task->completion); 53 } 54 55 static void smp_task_done(struct sas_task *task) 56 { 57 if (!del_timer(&task->timer)) 58 return; 59 complete(&task->completion); 60 } 61 62 /* Give it some long enough timeout. In seconds. */ 63 #define SMP_TIMEOUT 10 64 65 static int smp_execute_task(struct domain_device *dev, void *req, int req_size, 66 void *resp, int resp_size) 67 { 68 int res, retry; 69 struct sas_task *task = NULL; 70 struct sas_internal *i = 71 to_sas_internal(dev->port->ha->core.shost->transportt); 72 73 for (retry = 0; retry < 3; retry++) { 74 task = sas_alloc_task(GFP_KERNEL); 75 if (!task) 76 return -ENOMEM; 77 78 task->dev = dev; 79 task->task_proto = dev->tproto; 80 sg_init_one(&task->smp_task.smp_req, req, req_size); 81 sg_init_one(&task->smp_task.smp_resp, resp, resp_size); 82 83 task->task_done = smp_task_done; 84 85 task->timer.data = (unsigned long) task; 86 task->timer.function = smp_task_timedout; 87 task->timer.expires = jiffies + SMP_TIMEOUT*HZ; 88 add_timer(&task->timer); 89 90 res = i->dft->lldd_execute_task(task, 1, GFP_KERNEL); 91 92 if (res) { 93 del_timer(&task->timer); 94 SAS_DPRINTK("executing SMP task failed:%d\n", res); 95 goto ex_err; 96 } 97 98 wait_for_completion(&task->completion); 99 res = -ECOMM; 100 if ((task->task_state_flags & SAS_TASK_STATE_ABORTED)) { 101 SAS_DPRINTK("smp task timed out or aborted\n"); 102 i->dft->lldd_abort_task(task); 103 if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) { 104 SAS_DPRINTK("SMP task aborted and not done\n"); 105 goto ex_err; 106 } 107 } 108 if (task->task_status.resp == SAS_TASK_COMPLETE && 109 task->task_status.stat == SAM_GOOD) { 110 res = 0; 111 break; 112 } if (task->task_status.resp == SAS_TASK_COMPLETE && 113 task->task_status.stat == SAS_DATA_UNDERRUN) { 114 /* no error, but return the number of bytes of 115 * underrun */ 116 res = task->task_status.residual; 117 break; 118 } if (task->task_status.resp == SAS_TASK_COMPLETE && 119 task->task_status.stat == SAS_DATA_OVERRUN) { 120 res = -EMSGSIZE; 121 break; 122 } else { 123 SAS_DPRINTK("%s: task to dev %016llx response: 0x%x " 124 "status 0x%x\n", __func__, 125 SAS_ADDR(dev->sas_addr), 126 task->task_status.resp, 127 task->task_status.stat); 128 sas_free_task(task); 129 task = NULL; 130 } 131 } 132 ex_err: 133 BUG_ON(retry == 3 && task != NULL); 134 if (task != NULL) { 135 sas_free_task(task); 136 } 137 return res; 138 } 139 140 /* ---------- Allocations ---------- */ 141 142 static inline void *alloc_smp_req(int size) 143 { 144 u8 *p = kzalloc(size, GFP_KERNEL); 145 if (p) 146 p[0] = SMP_REQUEST; 147 return p; 148 } 149 150 static inline void *alloc_smp_resp(int size) 151 { 152 return kzalloc(size, GFP_KERNEL); 153 } 154 155 /* ---------- Expander configuration ---------- */ 156 157 static void sas_set_ex_phy(struct domain_device *dev, int phy_id, 158 void *disc_resp) 159 { 160 struct expander_device *ex = &dev->ex_dev; 161 struct ex_phy *phy = &ex->ex_phy[phy_id]; 162 struct smp_resp *resp = disc_resp; 163 struct discover_resp *dr = &resp->disc; 164 struct sas_rphy *rphy = dev->rphy; 165 int rediscover = (phy->phy != NULL); 166 167 if (!rediscover) { 168 phy->phy = sas_phy_alloc(&rphy->dev, phy_id); 169 170 /* FIXME: error_handling */ 171 BUG_ON(!phy->phy); 172 } 173 174 switch (resp->result) { 175 case SMP_RESP_PHY_VACANT: 176 phy->phy_state = PHY_VACANT; 177 return; 178 default: 179 phy->phy_state = PHY_NOT_PRESENT; 180 return; 181 case SMP_RESP_FUNC_ACC: 182 phy->phy_state = PHY_EMPTY; /* do not know yet */ 183 break; 184 } 185 186 phy->phy_id = phy_id; 187 phy->attached_dev_type = dr->attached_dev_type; 188 phy->linkrate = dr->linkrate; 189 phy->attached_sata_host = dr->attached_sata_host; 190 phy->attached_sata_dev = dr->attached_sata_dev; 191 phy->attached_sata_ps = dr->attached_sata_ps; 192 phy->attached_iproto = dr->iproto << 1; 193 phy->attached_tproto = dr->tproto << 1; 194 memcpy(phy->attached_sas_addr, dr->attached_sas_addr, SAS_ADDR_SIZE); 195 phy->attached_phy_id = dr->attached_phy_id; 196 phy->phy_change_count = dr->change_count; 197 phy->routing_attr = dr->routing_attr; 198 phy->virtual = dr->virtual; 199 phy->last_da_index = -1; 200 201 phy->phy->identify.initiator_port_protocols = phy->attached_iproto; 202 phy->phy->identify.target_port_protocols = phy->attached_tproto; 203 phy->phy->identify.phy_identifier = phy_id; 204 phy->phy->minimum_linkrate_hw = dr->hmin_linkrate; 205 phy->phy->maximum_linkrate_hw = dr->hmax_linkrate; 206 phy->phy->minimum_linkrate = dr->pmin_linkrate; 207 phy->phy->maximum_linkrate = dr->pmax_linkrate; 208 phy->phy->negotiated_linkrate = phy->linkrate; 209 210 if (!rediscover) 211 sas_phy_add(phy->phy); 212 213 SAS_DPRINTK("ex %016llx phy%02d:%c attached: %016llx\n", 214 SAS_ADDR(dev->sas_addr), phy->phy_id, 215 phy->routing_attr == TABLE_ROUTING ? 'T' : 216 phy->routing_attr == DIRECT_ROUTING ? 'D' : 217 phy->routing_attr == SUBTRACTIVE_ROUTING ? 'S' : '?', 218 SAS_ADDR(phy->attached_sas_addr)); 219 220 return; 221 } 222 223 #define DISCOVER_REQ_SIZE 16 224 #define DISCOVER_RESP_SIZE 56 225 226 static int sas_ex_phy_discover_helper(struct domain_device *dev, u8 *disc_req, 227 u8 *disc_resp, int single) 228 { 229 int i, res; 230 231 disc_req[9] = single; 232 for (i = 1 ; i < 3; i++) { 233 struct discover_resp *dr; 234 235 res = smp_execute_task(dev, disc_req, DISCOVER_REQ_SIZE, 236 disc_resp, DISCOVER_RESP_SIZE); 237 if (res) 238 return res; 239 /* This is detecting a failure to transmit inital 240 * dev to host FIS as described in section G.5 of 241 * sas-2 r 04b */ 242 dr = &((struct smp_resp *)disc_resp)->disc; 243 if (!(dr->attached_dev_type == 0 && 244 dr->attached_sata_dev)) 245 break; 246 /* In order to generate the dev to host FIS, we 247 * send a link reset to the expander port */ 248 sas_smp_phy_control(dev, single, PHY_FUNC_LINK_RESET, NULL); 249 /* Wait for the reset to trigger the negotiation */ 250 msleep(500); 251 } 252 sas_set_ex_phy(dev, single, disc_resp); 253 return 0; 254 } 255 256 static int sas_ex_phy_discover(struct domain_device *dev, int single) 257 { 258 struct expander_device *ex = &dev->ex_dev; 259 int res = 0; 260 u8 *disc_req; 261 u8 *disc_resp; 262 263 disc_req = alloc_smp_req(DISCOVER_REQ_SIZE); 264 if (!disc_req) 265 return -ENOMEM; 266 267 disc_resp = alloc_smp_req(DISCOVER_RESP_SIZE); 268 if (!disc_resp) { 269 kfree(disc_req); 270 return -ENOMEM; 271 } 272 273 disc_req[1] = SMP_DISCOVER; 274 275 if (0 <= single && single < ex->num_phys) { 276 res = sas_ex_phy_discover_helper(dev, disc_req, disc_resp, single); 277 } else { 278 int i; 279 280 for (i = 0; i < ex->num_phys; i++) { 281 res = sas_ex_phy_discover_helper(dev, disc_req, 282 disc_resp, i); 283 if (res) 284 goto out_err; 285 } 286 } 287 out_err: 288 kfree(disc_resp); 289 kfree(disc_req); 290 return res; 291 } 292 293 static int sas_expander_discover(struct domain_device *dev) 294 { 295 struct expander_device *ex = &dev->ex_dev; 296 int res = -ENOMEM; 297 298 ex->ex_phy = kzalloc(sizeof(*ex->ex_phy)*ex->num_phys, GFP_KERNEL); 299 if (!ex->ex_phy) 300 return -ENOMEM; 301 302 res = sas_ex_phy_discover(dev, -1); 303 if (res) 304 goto out_err; 305 306 return 0; 307 out_err: 308 kfree(ex->ex_phy); 309 ex->ex_phy = NULL; 310 return res; 311 } 312 313 #define MAX_EXPANDER_PHYS 128 314 315 static void ex_assign_report_general(struct domain_device *dev, 316 struct smp_resp *resp) 317 { 318 struct report_general_resp *rg = &resp->rg; 319 320 dev->ex_dev.ex_change_count = be16_to_cpu(rg->change_count); 321 dev->ex_dev.max_route_indexes = be16_to_cpu(rg->route_indexes); 322 dev->ex_dev.num_phys = min(rg->num_phys, (u8)MAX_EXPANDER_PHYS); 323 dev->ex_dev.conf_route_table = rg->conf_route_table; 324 dev->ex_dev.configuring = rg->configuring; 325 memcpy(dev->ex_dev.enclosure_logical_id, rg->enclosure_logical_id, 8); 326 } 327 328 #define RG_REQ_SIZE 8 329 #define RG_RESP_SIZE 32 330 331 static int sas_ex_general(struct domain_device *dev) 332 { 333 u8 *rg_req; 334 struct smp_resp *rg_resp; 335 int res; 336 int i; 337 338 rg_req = alloc_smp_req(RG_REQ_SIZE); 339 if (!rg_req) 340 return -ENOMEM; 341 342 rg_resp = alloc_smp_resp(RG_RESP_SIZE); 343 if (!rg_resp) { 344 kfree(rg_req); 345 return -ENOMEM; 346 } 347 348 rg_req[1] = SMP_REPORT_GENERAL; 349 350 for (i = 0; i < 5; i++) { 351 res = smp_execute_task(dev, rg_req, RG_REQ_SIZE, rg_resp, 352 RG_RESP_SIZE); 353 354 if (res) { 355 SAS_DPRINTK("RG to ex %016llx failed:0x%x\n", 356 SAS_ADDR(dev->sas_addr), res); 357 goto out; 358 } else if (rg_resp->result != SMP_RESP_FUNC_ACC) { 359 SAS_DPRINTK("RG:ex %016llx returned SMP result:0x%x\n", 360 SAS_ADDR(dev->sas_addr), rg_resp->result); 361 res = rg_resp->result; 362 goto out; 363 } 364 365 ex_assign_report_general(dev, rg_resp); 366 367 if (dev->ex_dev.configuring) { 368 SAS_DPRINTK("RG: ex %llx self-configuring...\n", 369 SAS_ADDR(dev->sas_addr)); 370 schedule_timeout_interruptible(5*HZ); 371 } else 372 break; 373 } 374 out: 375 kfree(rg_req); 376 kfree(rg_resp); 377 return res; 378 } 379 380 static void ex_assign_manuf_info(struct domain_device *dev, void 381 *_mi_resp) 382 { 383 u8 *mi_resp = _mi_resp; 384 struct sas_rphy *rphy = dev->rphy; 385 struct sas_expander_device *edev = rphy_to_expander_device(rphy); 386 387 memcpy(edev->vendor_id, mi_resp + 12, SAS_EXPANDER_VENDOR_ID_LEN); 388 memcpy(edev->product_id, mi_resp + 20, SAS_EXPANDER_PRODUCT_ID_LEN); 389 memcpy(edev->product_rev, mi_resp + 36, 390 SAS_EXPANDER_PRODUCT_REV_LEN); 391 392 if (mi_resp[8] & 1) { 393 memcpy(edev->component_vendor_id, mi_resp + 40, 394 SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN); 395 edev->component_id = mi_resp[48] << 8 | mi_resp[49]; 396 edev->component_revision_id = mi_resp[50]; 397 } 398 } 399 400 #define MI_REQ_SIZE 8 401 #define MI_RESP_SIZE 64 402 403 static int sas_ex_manuf_info(struct domain_device *dev) 404 { 405 u8 *mi_req; 406 u8 *mi_resp; 407 int res; 408 409 mi_req = alloc_smp_req(MI_REQ_SIZE); 410 if (!mi_req) 411 return -ENOMEM; 412 413 mi_resp = alloc_smp_resp(MI_RESP_SIZE); 414 if (!mi_resp) { 415 kfree(mi_req); 416 return -ENOMEM; 417 } 418 419 mi_req[1] = SMP_REPORT_MANUF_INFO; 420 421 res = smp_execute_task(dev, mi_req, MI_REQ_SIZE, mi_resp,MI_RESP_SIZE); 422 if (res) { 423 SAS_DPRINTK("MI: ex %016llx failed:0x%x\n", 424 SAS_ADDR(dev->sas_addr), res); 425 goto out; 426 } else if (mi_resp[2] != SMP_RESP_FUNC_ACC) { 427 SAS_DPRINTK("MI ex %016llx returned SMP result:0x%x\n", 428 SAS_ADDR(dev->sas_addr), mi_resp[2]); 429 goto out; 430 } 431 432 ex_assign_manuf_info(dev, mi_resp); 433 out: 434 kfree(mi_req); 435 kfree(mi_resp); 436 return res; 437 } 438 439 #define PC_REQ_SIZE 44 440 #define PC_RESP_SIZE 8 441 442 int sas_smp_phy_control(struct domain_device *dev, int phy_id, 443 enum phy_func phy_func, 444 struct sas_phy_linkrates *rates) 445 { 446 u8 *pc_req; 447 u8 *pc_resp; 448 int res; 449 450 pc_req = alloc_smp_req(PC_REQ_SIZE); 451 if (!pc_req) 452 return -ENOMEM; 453 454 pc_resp = alloc_smp_resp(PC_RESP_SIZE); 455 if (!pc_resp) { 456 kfree(pc_req); 457 return -ENOMEM; 458 } 459 460 pc_req[1] = SMP_PHY_CONTROL; 461 pc_req[9] = phy_id; 462 pc_req[10]= phy_func; 463 if (rates) { 464 pc_req[32] = rates->minimum_linkrate << 4; 465 pc_req[33] = rates->maximum_linkrate << 4; 466 } 467 468 res = smp_execute_task(dev, pc_req, PC_REQ_SIZE, pc_resp,PC_RESP_SIZE); 469 470 kfree(pc_resp); 471 kfree(pc_req); 472 return res; 473 } 474 475 static void sas_ex_disable_phy(struct domain_device *dev, int phy_id) 476 { 477 struct expander_device *ex = &dev->ex_dev; 478 struct ex_phy *phy = &ex->ex_phy[phy_id]; 479 480 sas_smp_phy_control(dev, phy_id, PHY_FUNC_DISABLE, NULL); 481 phy->linkrate = SAS_PHY_DISABLED; 482 } 483 484 static void sas_ex_disable_port(struct domain_device *dev, u8 *sas_addr) 485 { 486 struct expander_device *ex = &dev->ex_dev; 487 int i; 488 489 for (i = 0; i < ex->num_phys; i++) { 490 struct ex_phy *phy = &ex->ex_phy[i]; 491 492 if (phy->phy_state == PHY_VACANT || 493 phy->phy_state == PHY_NOT_PRESENT) 494 continue; 495 496 if (SAS_ADDR(phy->attached_sas_addr) == SAS_ADDR(sas_addr)) 497 sas_ex_disable_phy(dev, i); 498 } 499 } 500 501 static int sas_dev_present_in_domain(struct asd_sas_port *port, 502 u8 *sas_addr) 503 { 504 struct domain_device *dev; 505 506 if (SAS_ADDR(port->sas_addr) == SAS_ADDR(sas_addr)) 507 return 1; 508 list_for_each_entry(dev, &port->dev_list, dev_list_node) { 509 if (SAS_ADDR(dev->sas_addr) == SAS_ADDR(sas_addr)) 510 return 1; 511 } 512 return 0; 513 } 514 515 #define RPEL_REQ_SIZE 16 516 #define RPEL_RESP_SIZE 32 517 int sas_smp_get_phy_events(struct sas_phy *phy) 518 { 519 int res; 520 u8 *req; 521 u8 *resp; 522 struct sas_rphy *rphy = dev_to_rphy(phy->dev.parent); 523 struct domain_device *dev = sas_find_dev_by_rphy(rphy); 524 525 req = alloc_smp_req(RPEL_REQ_SIZE); 526 if (!req) 527 return -ENOMEM; 528 529 resp = alloc_smp_resp(RPEL_RESP_SIZE); 530 if (!resp) { 531 kfree(req); 532 return -ENOMEM; 533 } 534 535 req[1] = SMP_REPORT_PHY_ERR_LOG; 536 req[9] = phy->number; 537 538 res = smp_execute_task(dev, req, RPEL_REQ_SIZE, 539 resp, RPEL_RESP_SIZE); 540 541 if (!res) 542 goto out; 543 544 phy->invalid_dword_count = scsi_to_u32(&resp[12]); 545 phy->running_disparity_error_count = scsi_to_u32(&resp[16]); 546 phy->loss_of_dword_sync_count = scsi_to_u32(&resp[20]); 547 phy->phy_reset_problem_count = scsi_to_u32(&resp[24]); 548 549 out: 550 kfree(resp); 551 return res; 552 553 } 554 555 #ifdef CONFIG_SCSI_SAS_ATA 556 557 #define RPS_REQ_SIZE 16 558 #define RPS_RESP_SIZE 60 559 560 static int sas_get_report_phy_sata(struct domain_device *dev, 561 int phy_id, 562 struct smp_resp *rps_resp) 563 { 564 int res; 565 u8 *rps_req = alloc_smp_req(RPS_REQ_SIZE); 566 u8 *resp = (u8 *)rps_resp; 567 568 if (!rps_req) 569 return -ENOMEM; 570 571 rps_req[1] = SMP_REPORT_PHY_SATA; 572 rps_req[9] = phy_id; 573 574 res = smp_execute_task(dev, rps_req, RPS_REQ_SIZE, 575 rps_resp, RPS_RESP_SIZE); 576 577 /* 0x34 is the FIS type for the D2H fis. There's a potential 578 * standards cockup here. sas-2 explicitly specifies the FIS 579 * should be encoded so that FIS type is in resp[24]. 580 * However, some expanders endian reverse this. Undo the 581 * reversal here */ 582 if (!res && resp[27] == 0x34 && resp[24] != 0x34) { 583 int i; 584 585 for (i = 0; i < 5; i++) { 586 int j = 24 + (i*4); 587 u8 a, b; 588 a = resp[j + 0]; 589 b = resp[j + 1]; 590 resp[j + 0] = resp[j + 3]; 591 resp[j + 1] = resp[j + 2]; 592 resp[j + 2] = b; 593 resp[j + 3] = a; 594 } 595 } 596 597 kfree(rps_req); 598 return res; 599 } 600 #endif 601 602 static void sas_ex_get_linkrate(struct domain_device *parent, 603 struct domain_device *child, 604 struct ex_phy *parent_phy) 605 { 606 struct expander_device *parent_ex = &parent->ex_dev; 607 struct sas_port *port; 608 int i; 609 610 child->pathways = 0; 611 612 port = parent_phy->port; 613 614 for (i = 0; i < parent_ex->num_phys; i++) { 615 struct ex_phy *phy = &parent_ex->ex_phy[i]; 616 617 if (phy->phy_state == PHY_VACANT || 618 phy->phy_state == PHY_NOT_PRESENT) 619 continue; 620 621 if (SAS_ADDR(phy->attached_sas_addr) == 622 SAS_ADDR(child->sas_addr)) { 623 624 child->min_linkrate = min(parent->min_linkrate, 625 phy->linkrate); 626 child->max_linkrate = max(parent->max_linkrate, 627 phy->linkrate); 628 child->pathways++; 629 sas_port_add_phy(port, phy->phy); 630 } 631 } 632 child->linkrate = min(parent_phy->linkrate, child->max_linkrate); 633 child->pathways = min(child->pathways, parent->pathways); 634 } 635 636 static struct domain_device *sas_ex_discover_end_dev( 637 struct domain_device *parent, int phy_id) 638 { 639 struct expander_device *parent_ex = &parent->ex_dev; 640 struct ex_phy *phy = &parent_ex->ex_phy[phy_id]; 641 struct domain_device *child = NULL; 642 struct sas_rphy *rphy; 643 int res; 644 645 if (phy->attached_sata_host || phy->attached_sata_ps) 646 return NULL; 647 648 child = kzalloc(sizeof(*child), GFP_KERNEL); 649 if (!child) 650 return NULL; 651 652 child->parent = parent; 653 child->port = parent->port; 654 child->iproto = phy->attached_iproto; 655 memcpy(child->sas_addr, phy->attached_sas_addr, SAS_ADDR_SIZE); 656 sas_hash_addr(child->hashed_sas_addr, child->sas_addr); 657 if (!phy->port) { 658 phy->port = sas_port_alloc(&parent->rphy->dev, phy_id); 659 if (unlikely(!phy->port)) 660 goto out_err; 661 if (unlikely(sas_port_add(phy->port) != 0)) { 662 sas_port_free(phy->port); 663 goto out_err; 664 } 665 } 666 sas_ex_get_linkrate(parent, child, phy); 667 668 #ifdef CONFIG_SCSI_SAS_ATA 669 if ((phy->attached_tproto & SAS_PROTOCOL_STP) || phy->attached_sata_dev) { 670 child->dev_type = SATA_DEV; 671 if (phy->attached_tproto & SAS_PROTOCOL_STP) 672 child->tproto = phy->attached_tproto; 673 if (phy->attached_sata_dev) 674 child->tproto |= SATA_DEV; 675 res = sas_get_report_phy_sata(parent, phy_id, 676 &child->sata_dev.rps_resp); 677 if (res) { 678 SAS_DPRINTK("report phy sata to %016llx:0x%x returned " 679 "0x%x\n", SAS_ADDR(parent->sas_addr), 680 phy_id, res); 681 goto out_free; 682 } 683 memcpy(child->frame_rcvd, &child->sata_dev.rps_resp.rps.fis, 684 sizeof(struct dev_to_host_fis)); 685 686 rphy = sas_end_device_alloc(phy->port); 687 if (unlikely(!rphy)) 688 goto out_free; 689 690 sas_init_dev(child); 691 692 child->rphy = rphy; 693 694 spin_lock_irq(&parent->port->dev_list_lock); 695 list_add_tail(&child->dev_list_node, &parent->port->dev_list); 696 spin_unlock_irq(&parent->port->dev_list_lock); 697 698 res = sas_discover_sata(child); 699 if (res) { 700 SAS_DPRINTK("sas_discover_sata() for device %16llx at " 701 "%016llx:0x%x returned 0x%x\n", 702 SAS_ADDR(child->sas_addr), 703 SAS_ADDR(parent->sas_addr), phy_id, res); 704 goto out_list_del; 705 } 706 } else 707 #endif 708 if (phy->attached_tproto & SAS_PROTOCOL_SSP) { 709 child->dev_type = SAS_END_DEV; 710 rphy = sas_end_device_alloc(phy->port); 711 /* FIXME: error handling */ 712 if (unlikely(!rphy)) 713 goto out_free; 714 child->tproto = phy->attached_tproto; 715 sas_init_dev(child); 716 717 child->rphy = rphy; 718 sas_fill_in_rphy(child, rphy); 719 720 spin_lock_irq(&parent->port->dev_list_lock); 721 list_add_tail(&child->dev_list_node, &parent->port->dev_list); 722 spin_unlock_irq(&parent->port->dev_list_lock); 723 724 res = sas_discover_end_dev(child); 725 if (res) { 726 SAS_DPRINTK("sas_discover_end_dev() for device %16llx " 727 "at %016llx:0x%x returned 0x%x\n", 728 SAS_ADDR(child->sas_addr), 729 SAS_ADDR(parent->sas_addr), phy_id, res); 730 goto out_list_del; 731 } 732 } else { 733 SAS_DPRINTK("target proto 0x%x at %016llx:0x%x not handled\n", 734 phy->attached_tproto, SAS_ADDR(parent->sas_addr), 735 phy_id); 736 goto out_free; 737 } 738 739 list_add_tail(&child->siblings, &parent_ex->children); 740 return child; 741 742 out_list_del: 743 sas_rphy_free(child->rphy); 744 child->rphy = NULL; 745 list_del(&child->dev_list_node); 746 out_free: 747 sas_port_delete(phy->port); 748 out_err: 749 phy->port = NULL; 750 kfree(child); 751 return NULL; 752 } 753 754 /* See if this phy is part of a wide port */ 755 static int sas_ex_join_wide_port(struct domain_device *parent, int phy_id) 756 { 757 struct ex_phy *phy = &parent->ex_dev.ex_phy[phy_id]; 758 int i; 759 760 for (i = 0; i < parent->ex_dev.num_phys; i++) { 761 struct ex_phy *ephy = &parent->ex_dev.ex_phy[i]; 762 763 if (ephy == phy) 764 continue; 765 766 if (!memcmp(phy->attached_sas_addr, ephy->attached_sas_addr, 767 SAS_ADDR_SIZE) && ephy->port) { 768 sas_port_add_phy(ephy->port, phy->phy); 769 phy->port = ephy->port; 770 phy->phy_state = PHY_DEVICE_DISCOVERED; 771 return 0; 772 } 773 } 774 775 return -ENODEV; 776 } 777 778 static struct domain_device *sas_ex_discover_expander( 779 struct domain_device *parent, int phy_id) 780 { 781 struct sas_expander_device *parent_ex = rphy_to_expander_device(parent->rphy); 782 struct ex_phy *phy = &parent->ex_dev.ex_phy[phy_id]; 783 struct domain_device *child = NULL; 784 struct sas_rphy *rphy; 785 struct sas_expander_device *edev; 786 struct asd_sas_port *port; 787 int res; 788 789 if (phy->routing_attr == DIRECT_ROUTING) { 790 SAS_DPRINTK("ex %016llx:0x%x:D <--> ex %016llx:0x%x is not " 791 "allowed\n", 792 SAS_ADDR(parent->sas_addr), phy_id, 793 SAS_ADDR(phy->attached_sas_addr), 794 phy->attached_phy_id); 795 return NULL; 796 } 797 child = kzalloc(sizeof(*child), GFP_KERNEL); 798 if (!child) 799 return NULL; 800 801 phy->port = sas_port_alloc(&parent->rphy->dev, phy_id); 802 /* FIXME: better error handling */ 803 BUG_ON(sas_port_add(phy->port) != 0); 804 805 806 switch (phy->attached_dev_type) { 807 case EDGE_DEV: 808 rphy = sas_expander_alloc(phy->port, 809 SAS_EDGE_EXPANDER_DEVICE); 810 break; 811 case FANOUT_DEV: 812 rphy = sas_expander_alloc(phy->port, 813 SAS_FANOUT_EXPANDER_DEVICE); 814 break; 815 default: 816 rphy = NULL; /* shut gcc up */ 817 BUG(); 818 } 819 port = parent->port; 820 child->rphy = rphy; 821 edev = rphy_to_expander_device(rphy); 822 child->dev_type = phy->attached_dev_type; 823 child->parent = parent; 824 child->port = port; 825 child->iproto = phy->attached_iproto; 826 child->tproto = phy->attached_tproto; 827 memcpy(child->sas_addr, phy->attached_sas_addr, SAS_ADDR_SIZE); 828 sas_hash_addr(child->hashed_sas_addr, child->sas_addr); 829 sas_ex_get_linkrate(parent, child, phy); 830 edev->level = parent_ex->level + 1; 831 parent->port->disc.max_level = max(parent->port->disc.max_level, 832 edev->level); 833 sas_init_dev(child); 834 sas_fill_in_rphy(child, rphy); 835 sas_rphy_add(rphy); 836 837 spin_lock_irq(&parent->port->dev_list_lock); 838 list_add_tail(&child->dev_list_node, &parent->port->dev_list); 839 spin_unlock_irq(&parent->port->dev_list_lock); 840 841 res = sas_discover_expander(child); 842 if (res) { 843 kfree(child); 844 return NULL; 845 } 846 list_add_tail(&child->siblings, &parent->ex_dev.children); 847 return child; 848 } 849 850 static int sas_ex_discover_dev(struct domain_device *dev, int phy_id) 851 { 852 struct expander_device *ex = &dev->ex_dev; 853 struct ex_phy *ex_phy = &ex->ex_phy[phy_id]; 854 struct domain_device *child = NULL; 855 int res = 0; 856 857 /* Phy state */ 858 if (ex_phy->linkrate == SAS_SATA_SPINUP_HOLD) { 859 if (!sas_smp_phy_control(dev, phy_id, PHY_FUNC_LINK_RESET, NULL)) 860 res = sas_ex_phy_discover(dev, phy_id); 861 if (res) 862 return res; 863 } 864 865 /* Parent and domain coherency */ 866 if (!dev->parent && (SAS_ADDR(ex_phy->attached_sas_addr) == 867 SAS_ADDR(dev->port->sas_addr))) { 868 sas_add_parent_port(dev, phy_id); 869 return 0; 870 } 871 if (dev->parent && (SAS_ADDR(ex_phy->attached_sas_addr) == 872 SAS_ADDR(dev->parent->sas_addr))) { 873 sas_add_parent_port(dev, phy_id); 874 if (ex_phy->routing_attr == TABLE_ROUTING) 875 sas_configure_phy(dev, phy_id, dev->port->sas_addr, 1); 876 return 0; 877 } 878 879 if (sas_dev_present_in_domain(dev->port, ex_phy->attached_sas_addr)) 880 sas_ex_disable_port(dev, ex_phy->attached_sas_addr); 881 882 if (ex_phy->attached_dev_type == NO_DEVICE) { 883 if (ex_phy->routing_attr == DIRECT_ROUTING) { 884 memset(ex_phy->attached_sas_addr, 0, SAS_ADDR_SIZE); 885 sas_configure_routing(dev, ex_phy->attached_sas_addr); 886 } 887 return 0; 888 } else if (ex_phy->linkrate == SAS_LINK_RATE_UNKNOWN) 889 return 0; 890 891 if (ex_phy->attached_dev_type != SAS_END_DEV && 892 ex_phy->attached_dev_type != FANOUT_DEV && 893 ex_phy->attached_dev_type != EDGE_DEV) { 894 SAS_DPRINTK("unknown device type(0x%x) attached to ex %016llx " 895 "phy 0x%x\n", ex_phy->attached_dev_type, 896 SAS_ADDR(dev->sas_addr), 897 phy_id); 898 return 0; 899 } 900 901 res = sas_configure_routing(dev, ex_phy->attached_sas_addr); 902 if (res) { 903 SAS_DPRINTK("configure routing for dev %016llx " 904 "reported 0x%x. Forgotten\n", 905 SAS_ADDR(ex_phy->attached_sas_addr), res); 906 sas_disable_routing(dev, ex_phy->attached_sas_addr); 907 return res; 908 } 909 910 res = sas_ex_join_wide_port(dev, phy_id); 911 if (!res) { 912 SAS_DPRINTK("Attaching ex phy%d to wide port %016llx\n", 913 phy_id, SAS_ADDR(ex_phy->attached_sas_addr)); 914 return res; 915 } 916 917 switch (ex_phy->attached_dev_type) { 918 case SAS_END_DEV: 919 child = sas_ex_discover_end_dev(dev, phy_id); 920 break; 921 case FANOUT_DEV: 922 if (SAS_ADDR(dev->port->disc.fanout_sas_addr)) { 923 SAS_DPRINTK("second fanout expander %016llx phy 0x%x " 924 "attached to ex %016llx phy 0x%x\n", 925 SAS_ADDR(ex_phy->attached_sas_addr), 926 ex_phy->attached_phy_id, 927 SAS_ADDR(dev->sas_addr), 928 phy_id); 929 sas_ex_disable_phy(dev, phy_id); 930 break; 931 } else 932 memcpy(dev->port->disc.fanout_sas_addr, 933 ex_phy->attached_sas_addr, SAS_ADDR_SIZE); 934 /* fallthrough */ 935 case EDGE_DEV: 936 child = sas_ex_discover_expander(dev, phy_id); 937 break; 938 default: 939 break; 940 } 941 942 if (child) { 943 int i; 944 945 for (i = 0; i < ex->num_phys; i++) { 946 if (ex->ex_phy[i].phy_state == PHY_VACANT || 947 ex->ex_phy[i].phy_state == PHY_NOT_PRESENT) 948 continue; 949 /* 950 * Due to races, the phy might not get added to the 951 * wide port, so we add the phy to the wide port here. 952 */ 953 if (SAS_ADDR(ex->ex_phy[i].attached_sas_addr) == 954 SAS_ADDR(child->sas_addr)) { 955 ex->ex_phy[i].phy_state= PHY_DEVICE_DISCOVERED; 956 res = sas_ex_join_wide_port(dev, i); 957 if (!res) 958 SAS_DPRINTK("Attaching ex phy%d to wide port %016llx\n", 959 i, SAS_ADDR(ex->ex_phy[i].attached_sas_addr)); 960 961 } 962 } 963 } 964 965 return res; 966 } 967 968 static int sas_find_sub_addr(struct domain_device *dev, u8 *sub_addr) 969 { 970 struct expander_device *ex = &dev->ex_dev; 971 int i; 972 973 for (i = 0; i < ex->num_phys; i++) { 974 struct ex_phy *phy = &ex->ex_phy[i]; 975 976 if (phy->phy_state == PHY_VACANT || 977 phy->phy_state == PHY_NOT_PRESENT) 978 continue; 979 980 if ((phy->attached_dev_type == EDGE_DEV || 981 phy->attached_dev_type == FANOUT_DEV) && 982 phy->routing_attr == SUBTRACTIVE_ROUTING) { 983 984 memcpy(sub_addr, phy->attached_sas_addr,SAS_ADDR_SIZE); 985 986 return 1; 987 } 988 } 989 return 0; 990 } 991 992 static int sas_check_level_subtractive_boundary(struct domain_device *dev) 993 { 994 struct expander_device *ex = &dev->ex_dev; 995 struct domain_device *child; 996 u8 sub_addr[8] = {0, }; 997 998 list_for_each_entry(child, &ex->children, siblings) { 999 if (child->dev_type != EDGE_DEV && 1000 child->dev_type != FANOUT_DEV) 1001 continue; 1002 if (sub_addr[0] == 0) { 1003 sas_find_sub_addr(child, sub_addr); 1004 continue; 1005 } else { 1006 u8 s2[8]; 1007 1008 if (sas_find_sub_addr(child, s2) && 1009 (SAS_ADDR(sub_addr) != SAS_ADDR(s2))) { 1010 1011 SAS_DPRINTK("ex %016llx->%016llx-?->%016llx " 1012 "diverges from subtractive " 1013 "boundary %016llx\n", 1014 SAS_ADDR(dev->sas_addr), 1015 SAS_ADDR(child->sas_addr), 1016 SAS_ADDR(s2), 1017 SAS_ADDR(sub_addr)); 1018 1019 sas_ex_disable_port(child, s2); 1020 } 1021 } 1022 } 1023 return 0; 1024 } 1025 /** 1026 * sas_ex_discover_devices -- discover devices attached to this expander 1027 * dev: pointer to the expander domain device 1028 * single: if you want to do a single phy, else set to -1; 1029 * 1030 * Configure this expander for use with its devices and register the 1031 * devices of this expander. 1032 */ 1033 static int sas_ex_discover_devices(struct domain_device *dev, int single) 1034 { 1035 struct expander_device *ex = &dev->ex_dev; 1036 int i = 0, end = ex->num_phys; 1037 int res = 0; 1038 1039 if (0 <= single && single < end) { 1040 i = single; 1041 end = i+1; 1042 } 1043 1044 for ( ; i < end; i++) { 1045 struct ex_phy *ex_phy = &ex->ex_phy[i]; 1046 1047 if (ex_phy->phy_state == PHY_VACANT || 1048 ex_phy->phy_state == PHY_NOT_PRESENT || 1049 ex_phy->phy_state == PHY_DEVICE_DISCOVERED) 1050 continue; 1051 1052 switch (ex_phy->linkrate) { 1053 case SAS_PHY_DISABLED: 1054 case SAS_PHY_RESET_PROBLEM: 1055 case SAS_SATA_PORT_SELECTOR: 1056 continue; 1057 default: 1058 res = sas_ex_discover_dev(dev, i); 1059 if (res) 1060 break; 1061 continue; 1062 } 1063 } 1064 1065 if (!res) 1066 sas_check_level_subtractive_boundary(dev); 1067 1068 return res; 1069 } 1070 1071 static int sas_check_ex_subtractive_boundary(struct domain_device *dev) 1072 { 1073 struct expander_device *ex = &dev->ex_dev; 1074 int i; 1075 u8 *sub_sas_addr = NULL; 1076 1077 if (dev->dev_type != EDGE_DEV) 1078 return 0; 1079 1080 for (i = 0; i < ex->num_phys; i++) { 1081 struct ex_phy *phy = &ex->ex_phy[i]; 1082 1083 if (phy->phy_state == PHY_VACANT || 1084 phy->phy_state == PHY_NOT_PRESENT) 1085 continue; 1086 1087 if ((phy->attached_dev_type == FANOUT_DEV || 1088 phy->attached_dev_type == EDGE_DEV) && 1089 phy->routing_attr == SUBTRACTIVE_ROUTING) { 1090 1091 if (!sub_sas_addr) 1092 sub_sas_addr = &phy->attached_sas_addr[0]; 1093 else if (SAS_ADDR(sub_sas_addr) != 1094 SAS_ADDR(phy->attached_sas_addr)) { 1095 1096 SAS_DPRINTK("ex %016llx phy 0x%x " 1097 "diverges(%016llx) on subtractive " 1098 "boundary(%016llx). Disabled\n", 1099 SAS_ADDR(dev->sas_addr), i, 1100 SAS_ADDR(phy->attached_sas_addr), 1101 SAS_ADDR(sub_sas_addr)); 1102 sas_ex_disable_phy(dev, i); 1103 } 1104 } 1105 } 1106 return 0; 1107 } 1108 1109 static void sas_print_parent_topology_bug(struct domain_device *child, 1110 struct ex_phy *parent_phy, 1111 struct ex_phy *child_phy) 1112 { 1113 static const char ra_char[] = { 1114 [DIRECT_ROUTING] = 'D', 1115 [SUBTRACTIVE_ROUTING] = 'S', 1116 [TABLE_ROUTING] = 'T', 1117 }; 1118 static const char *ex_type[] = { 1119 [EDGE_DEV] = "edge", 1120 [FANOUT_DEV] = "fanout", 1121 }; 1122 struct domain_device *parent = child->parent; 1123 1124 sas_printk("%s ex %016llx phy 0x%x <--> %s ex %016llx phy 0x%x " 1125 "has %c:%c routing link!\n", 1126 1127 ex_type[parent->dev_type], 1128 SAS_ADDR(parent->sas_addr), 1129 parent_phy->phy_id, 1130 1131 ex_type[child->dev_type], 1132 SAS_ADDR(child->sas_addr), 1133 child_phy->phy_id, 1134 1135 ra_char[parent_phy->routing_attr], 1136 ra_char[child_phy->routing_attr]); 1137 } 1138 1139 static int sas_check_eeds(struct domain_device *child, 1140 struct ex_phy *parent_phy, 1141 struct ex_phy *child_phy) 1142 { 1143 int res = 0; 1144 struct domain_device *parent = child->parent; 1145 1146 if (SAS_ADDR(parent->port->disc.fanout_sas_addr) != 0) { 1147 res = -ENODEV; 1148 SAS_DPRINTK("edge ex %016llx phy S:0x%x <--> edge ex %016llx " 1149 "phy S:0x%x, while there is a fanout ex %016llx\n", 1150 SAS_ADDR(parent->sas_addr), 1151 parent_phy->phy_id, 1152 SAS_ADDR(child->sas_addr), 1153 child_phy->phy_id, 1154 SAS_ADDR(parent->port->disc.fanout_sas_addr)); 1155 } else if (SAS_ADDR(parent->port->disc.eeds_a) == 0) { 1156 memcpy(parent->port->disc.eeds_a, parent->sas_addr, 1157 SAS_ADDR_SIZE); 1158 memcpy(parent->port->disc.eeds_b, child->sas_addr, 1159 SAS_ADDR_SIZE); 1160 } else if (((SAS_ADDR(parent->port->disc.eeds_a) == 1161 SAS_ADDR(parent->sas_addr)) || 1162 (SAS_ADDR(parent->port->disc.eeds_a) == 1163 SAS_ADDR(child->sas_addr))) 1164 && 1165 ((SAS_ADDR(parent->port->disc.eeds_b) == 1166 SAS_ADDR(parent->sas_addr)) || 1167 (SAS_ADDR(parent->port->disc.eeds_b) == 1168 SAS_ADDR(child->sas_addr)))) 1169 ; 1170 else { 1171 res = -ENODEV; 1172 SAS_DPRINTK("edge ex %016llx phy 0x%x <--> edge ex %016llx " 1173 "phy 0x%x link forms a third EEDS!\n", 1174 SAS_ADDR(parent->sas_addr), 1175 parent_phy->phy_id, 1176 SAS_ADDR(child->sas_addr), 1177 child_phy->phy_id); 1178 } 1179 1180 return res; 1181 } 1182 1183 /* Here we spill over 80 columns. It is intentional. 1184 */ 1185 static int sas_check_parent_topology(struct domain_device *child) 1186 { 1187 struct expander_device *child_ex = &child->ex_dev; 1188 struct expander_device *parent_ex; 1189 int i; 1190 int res = 0; 1191 1192 if (!child->parent) 1193 return 0; 1194 1195 if (child->parent->dev_type != EDGE_DEV && 1196 child->parent->dev_type != FANOUT_DEV) 1197 return 0; 1198 1199 parent_ex = &child->parent->ex_dev; 1200 1201 for (i = 0; i < parent_ex->num_phys; i++) { 1202 struct ex_phy *parent_phy = &parent_ex->ex_phy[i]; 1203 struct ex_phy *child_phy; 1204 1205 if (parent_phy->phy_state == PHY_VACANT || 1206 parent_phy->phy_state == PHY_NOT_PRESENT) 1207 continue; 1208 1209 if (SAS_ADDR(parent_phy->attached_sas_addr) != SAS_ADDR(child->sas_addr)) 1210 continue; 1211 1212 child_phy = &child_ex->ex_phy[parent_phy->attached_phy_id]; 1213 1214 switch (child->parent->dev_type) { 1215 case EDGE_DEV: 1216 if (child->dev_type == FANOUT_DEV) { 1217 if (parent_phy->routing_attr != SUBTRACTIVE_ROUTING || 1218 child_phy->routing_attr != TABLE_ROUTING) { 1219 sas_print_parent_topology_bug(child, parent_phy, child_phy); 1220 res = -ENODEV; 1221 } 1222 } else if (parent_phy->routing_attr == SUBTRACTIVE_ROUTING) { 1223 if (child_phy->routing_attr == SUBTRACTIVE_ROUTING) { 1224 res = sas_check_eeds(child, parent_phy, child_phy); 1225 } else if (child_phy->routing_attr != TABLE_ROUTING) { 1226 sas_print_parent_topology_bug(child, parent_phy, child_phy); 1227 res = -ENODEV; 1228 } 1229 } else if (parent_phy->routing_attr == TABLE_ROUTING && 1230 child_phy->routing_attr != SUBTRACTIVE_ROUTING) { 1231 sas_print_parent_topology_bug(child, parent_phy, child_phy); 1232 res = -ENODEV; 1233 } 1234 break; 1235 case FANOUT_DEV: 1236 if (parent_phy->routing_attr != TABLE_ROUTING || 1237 child_phy->routing_attr != SUBTRACTIVE_ROUTING) { 1238 sas_print_parent_topology_bug(child, parent_phy, child_phy); 1239 res = -ENODEV; 1240 } 1241 break; 1242 default: 1243 break; 1244 } 1245 } 1246 1247 return res; 1248 } 1249 1250 #define RRI_REQ_SIZE 16 1251 #define RRI_RESP_SIZE 44 1252 1253 static int sas_configure_present(struct domain_device *dev, int phy_id, 1254 u8 *sas_addr, int *index, int *present) 1255 { 1256 int i, res = 0; 1257 struct expander_device *ex = &dev->ex_dev; 1258 struct ex_phy *phy = &ex->ex_phy[phy_id]; 1259 u8 *rri_req; 1260 u8 *rri_resp; 1261 1262 *present = 0; 1263 *index = 0; 1264 1265 rri_req = alloc_smp_req(RRI_REQ_SIZE); 1266 if (!rri_req) 1267 return -ENOMEM; 1268 1269 rri_resp = alloc_smp_resp(RRI_RESP_SIZE); 1270 if (!rri_resp) { 1271 kfree(rri_req); 1272 return -ENOMEM; 1273 } 1274 1275 rri_req[1] = SMP_REPORT_ROUTE_INFO; 1276 rri_req[9] = phy_id; 1277 1278 for (i = 0; i < ex->max_route_indexes ; i++) { 1279 *(__be16 *)(rri_req+6) = cpu_to_be16(i); 1280 res = smp_execute_task(dev, rri_req, RRI_REQ_SIZE, rri_resp, 1281 RRI_RESP_SIZE); 1282 if (res) 1283 goto out; 1284 res = rri_resp[2]; 1285 if (res == SMP_RESP_NO_INDEX) { 1286 SAS_DPRINTK("overflow of indexes: dev %016llx " 1287 "phy 0x%x index 0x%x\n", 1288 SAS_ADDR(dev->sas_addr), phy_id, i); 1289 goto out; 1290 } else if (res != SMP_RESP_FUNC_ACC) { 1291 SAS_DPRINTK("%s: dev %016llx phy 0x%x index 0x%x " 1292 "result 0x%x\n", __func__, 1293 SAS_ADDR(dev->sas_addr), phy_id, i, res); 1294 goto out; 1295 } 1296 if (SAS_ADDR(sas_addr) != 0) { 1297 if (SAS_ADDR(rri_resp+16) == SAS_ADDR(sas_addr)) { 1298 *index = i; 1299 if ((rri_resp[12] & 0x80) == 0x80) 1300 *present = 0; 1301 else 1302 *present = 1; 1303 goto out; 1304 } else if (SAS_ADDR(rri_resp+16) == 0) { 1305 *index = i; 1306 *present = 0; 1307 goto out; 1308 } 1309 } else if (SAS_ADDR(rri_resp+16) == 0 && 1310 phy->last_da_index < i) { 1311 phy->last_da_index = i; 1312 *index = i; 1313 *present = 0; 1314 goto out; 1315 } 1316 } 1317 res = -1; 1318 out: 1319 kfree(rri_req); 1320 kfree(rri_resp); 1321 return res; 1322 } 1323 1324 #define CRI_REQ_SIZE 44 1325 #define CRI_RESP_SIZE 8 1326 1327 static int sas_configure_set(struct domain_device *dev, int phy_id, 1328 u8 *sas_addr, int index, int include) 1329 { 1330 int res; 1331 u8 *cri_req; 1332 u8 *cri_resp; 1333 1334 cri_req = alloc_smp_req(CRI_REQ_SIZE); 1335 if (!cri_req) 1336 return -ENOMEM; 1337 1338 cri_resp = alloc_smp_resp(CRI_RESP_SIZE); 1339 if (!cri_resp) { 1340 kfree(cri_req); 1341 return -ENOMEM; 1342 } 1343 1344 cri_req[1] = SMP_CONF_ROUTE_INFO; 1345 *(__be16 *)(cri_req+6) = cpu_to_be16(index); 1346 cri_req[9] = phy_id; 1347 if (SAS_ADDR(sas_addr) == 0 || !include) 1348 cri_req[12] |= 0x80; 1349 memcpy(cri_req+16, sas_addr, SAS_ADDR_SIZE); 1350 1351 res = smp_execute_task(dev, cri_req, CRI_REQ_SIZE, cri_resp, 1352 CRI_RESP_SIZE); 1353 if (res) 1354 goto out; 1355 res = cri_resp[2]; 1356 if (res == SMP_RESP_NO_INDEX) { 1357 SAS_DPRINTK("overflow of indexes: dev %016llx phy 0x%x " 1358 "index 0x%x\n", 1359 SAS_ADDR(dev->sas_addr), phy_id, index); 1360 } 1361 out: 1362 kfree(cri_req); 1363 kfree(cri_resp); 1364 return res; 1365 } 1366 1367 static int sas_configure_phy(struct domain_device *dev, int phy_id, 1368 u8 *sas_addr, int include) 1369 { 1370 int index; 1371 int present; 1372 int res; 1373 1374 res = sas_configure_present(dev, phy_id, sas_addr, &index, &present); 1375 if (res) 1376 return res; 1377 if (include ^ present) 1378 return sas_configure_set(dev, phy_id, sas_addr, index,include); 1379 1380 return res; 1381 } 1382 1383 /** 1384 * sas_configure_parent -- configure routing table of parent 1385 * parent: parent expander 1386 * child: child expander 1387 * sas_addr: SAS port identifier of device directly attached to child 1388 */ 1389 static int sas_configure_parent(struct domain_device *parent, 1390 struct domain_device *child, 1391 u8 *sas_addr, int include) 1392 { 1393 struct expander_device *ex_parent = &parent->ex_dev; 1394 int res = 0; 1395 int i; 1396 1397 if (parent->parent) { 1398 res = sas_configure_parent(parent->parent, parent, sas_addr, 1399 include); 1400 if (res) 1401 return res; 1402 } 1403 1404 if (ex_parent->conf_route_table == 0) { 1405 SAS_DPRINTK("ex %016llx has self-configuring routing table\n", 1406 SAS_ADDR(parent->sas_addr)); 1407 return 0; 1408 } 1409 1410 for (i = 0; i < ex_parent->num_phys; i++) { 1411 struct ex_phy *phy = &ex_parent->ex_phy[i]; 1412 1413 if ((phy->routing_attr == TABLE_ROUTING) && 1414 (SAS_ADDR(phy->attached_sas_addr) == 1415 SAS_ADDR(child->sas_addr))) { 1416 res = sas_configure_phy(parent, i, sas_addr, include); 1417 if (res) 1418 return res; 1419 } 1420 } 1421 1422 return res; 1423 } 1424 1425 /** 1426 * sas_configure_routing -- configure routing 1427 * dev: expander device 1428 * sas_addr: port identifier of device directly attached to the expander device 1429 */ 1430 static int sas_configure_routing(struct domain_device *dev, u8 *sas_addr) 1431 { 1432 if (dev->parent) 1433 return sas_configure_parent(dev->parent, dev, sas_addr, 1); 1434 return 0; 1435 } 1436 1437 static int sas_disable_routing(struct domain_device *dev, u8 *sas_addr) 1438 { 1439 if (dev->parent) 1440 return sas_configure_parent(dev->parent, dev, sas_addr, 0); 1441 return 0; 1442 } 1443 1444 /** 1445 * sas_discover_expander -- expander discovery 1446 * @ex: pointer to expander domain device 1447 * 1448 * See comment in sas_discover_sata(). 1449 */ 1450 static int sas_discover_expander(struct domain_device *dev) 1451 { 1452 int res; 1453 1454 res = sas_notify_lldd_dev_found(dev); 1455 if (res) 1456 return res; 1457 1458 res = sas_ex_general(dev); 1459 if (res) 1460 goto out_err; 1461 res = sas_ex_manuf_info(dev); 1462 if (res) 1463 goto out_err; 1464 1465 res = sas_expander_discover(dev); 1466 if (res) { 1467 SAS_DPRINTK("expander %016llx discovery failed(0x%x)\n", 1468 SAS_ADDR(dev->sas_addr), res); 1469 goto out_err; 1470 } 1471 1472 sas_check_ex_subtractive_boundary(dev); 1473 res = sas_check_parent_topology(dev); 1474 if (res) 1475 goto out_err; 1476 return 0; 1477 out_err: 1478 sas_notify_lldd_dev_gone(dev); 1479 return res; 1480 } 1481 1482 static int sas_ex_level_discovery(struct asd_sas_port *port, const int level) 1483 { 1484 int res = 0; 1485 struct domain_device *dev; 1486 1487 list_for_each_entry(dev, &port->dev_list, dev_list_node) { 1488 if (dev->dev_type == EDGE_DEV || 1489 dev->dev_type == FANOUT_DEV) { 1490 struct sas_expander_device *ex = 1491 rphy_to_expander_device(dev->rphy); 1492 1493 if (level == ex->level) 1494 res = sas_ex_discover_devices(dev, -1); 1495 else if (level > 0) 1496 res = sas_ex_discover_devices(port->port_dev, -1); 1497 1498 } 1499 } 1500 1501 return res; 1502 } 1503 1504 static int sas_ex_bfs_disc(struct asd_sas_port *port) 1505 { 1506 int res; 1507 int level; 1508 1509 do { 1510 level = port->disc.max_level; 1511 res = sas_ex_level_discovery(port, level); 1512 mb(); 1513 } while (level < port->disc.max_level); 1514 1515 return res; 1516 } 1517 1518 int sas_discover_root_expander(struct domain_device *dev) 1519 { 1520 int res; 1521 struct sas_expander_device *ex = rphy_to_expander_device(dev->rphy); 1522 1523 res = sas_rphy_add(dev->rphy); 1524 if (res) 1525 goto out_err; 1526 1527 ex->level = dev->port->disc.max_level; /* 0 */ 1528 res = sas_discover_expander(dev); 1529 if (res) 1530 goto out_err2; 1531 1532 sas_ex_bfs_disc(dev->port); 1533 1534 return res; 1535 1536 out_err2: 1537 sas_rphy_remove(dev->rphy); 1538 out_err: 1539 return res; 1540 } 1541 1542 /* ---------- Domain revalidation ---------- */ 1543 1544 static int sas_get_phy_discover(struct domain_device *dev, 1545 int phy_id, struct smp_resp *disc_resp) 1546 { 1547 int res; 1548 u8 *disc_req; 1549 1550 disc_req = alloc_smp_req(DISCOVER_REQ_SIZE); 1551 if (!disc_req) 1552 return -ENOMEM; 1553 1554 disc_req[1] = SMP_DISCOVER; 1555 disc_req[9] = phy_id; 1556 1557 res = smp_execute_task(dev, disc_req, DISCOVER_REQ_SIZE, 1558 disc_resp, DISCOVER_RESP_SIZE); 1559 if (res) 1560 goto out; 1561 else if (disc_resp->result != SMP_RESP_FUNC_ACC) { 1562 res = disc_resp->result; 1563 goto out; 1564 } 1565 out: 1566 kfree(disc_req); 1567 return res; 1568 } 1569 1570 static int sas_get_phy_change_count(struct domain_device *dev, 1571 int phy_id, int *pcc) 1572 { 1573 int res; 1574 struct smp_resp *disc_resp; 1575 1576 disc_resp = alloc_smp_resp(DISCOVER_RESP_SIZE); 1577 if (!disc_resp) 1578 return -ENOMEM; 1579 1580 res = sas_get_phy_discover(dev, phy_id, disc_resp); 1581 if (!res) 1582 *pcc = disc_resp->disc.change_count; 1583 1584 kfree(disc_resp); 1585 return res; 1586 } 1587 1588 static int sas_get_phy_attached_sas_addr(struct domain_device *dev, 1589 int phy_id, u8 *attached_sas_addr) 1590 { 1591 int res; 1592 struct smp_resp *disc_resp; 1593 struct discover_resp *dr; 1594 1595 disc_resp = alloc_smp_resp(DISCOVER_RESP_SIZE); 1596 if (!disc_resp) 1597 return -ENOMEM; 1598 dr = &disc_resp->disc; 1599 1600 res = sas_get_phy_discover(dev, phy_id, disc_resp); 1601 if (!res) { 1602 memcpy(attached_sas_addr,disc_resp->disc.attached_sas_addr,8); 1603 if (dr->attached_dev_type == 0) 1604 memset(attached_sas_addr, 0, 8); 1605 } 1606 kfree(disc_resp); 1607 return res; 1608 } 1609 1610 static int sas_find_bcast_phy(struct domain_device *dev, int *phy_id, 1611 int from_phy, bool update) 1612 { 1613 struct expander_device *ex = &dev->ex_dev; 1614 int res = 0; 1615 int i; 1616 1617 for (i = from_phy; i < ex->num_phys; i++) { 1618 int phy_change_count = 0; 1619 1620 res = sas_get_phy_change_count(dev, i, &phy_change_count); 1621 if (res) 1622 goto out; 1623 else if (phy_change_count != ex->ex_phy[i].phy_change_count) { 1624 if (update) 1625 ex->ex_phy[i].phy_change_count = 1626 phy_change_count; 1627 *phy_id = i; 1628 return 0; 1629 } 1630 } 1631 out: 1632 return res; 1633 } 1634 1635 static int sas_get_ex_change_count(struct domain_device *dev, int *ecc) 1636 { 1637 int res; 1638 u8 *rg_req; 1639 struct smp_resp *rg_resp; 1640 1641 rg_req = alloc_smp_req(RG_REQ_SIZE); 1642 if (!rg_req) 1643 return -ENOMEM; 1644 1645 rg_resp = alloc_smp_resp(RG_RESP_SIZE); 1646 if (!rg_resp) { 1647 kfree(rg_req); 1648 return -ENOMEM; 1649 } 1650 1651 rg_req[1] = SMP_REPORT_GENERAL; 1652 1653 res = smp_execute_task(dev, rg_req, RG_REQ_SIZE, rg_resp, 1654 RG_RESP_SIZE); 1655 if (res) 1656 goto out; 1657 if (rg_resp->result != SMP_RESP_FUNC_ACC) { 1658 res = rg_resp->result; 1659 goto out; 1660 } 1661 1662 *ecc = be16_to_cpu(rg_resp->rg.change_count); 1663 out: 1664 kfree(rg_resp); 1665 kfree(rg_req); 1666 return res; 1667 } 1668 /** 1669 * sas_find_bcast_dev - find the device issue BROADCAST(CHANGE). 1670 * @dev:domain device to be detect. 1671 * @src_dev: the device which originated BROADCAST(CHANGE). 1672 * 1673 * Add self-configuration expander suport. Suppose two expander cascading, 1674 * when the first level expander is self-configuring, hotplug the disks in 1675 * second level expander, BROADCAST(CHANGE) will not only be originated 1676 * in the second level expander, but also be originated in the first level 1677 * expander (see SAS protocol SAS 2r-14, 7.11 for detail), it is to say, 1678 * expander changed count in two level expanders will all increment at least 1679 * once, but the phy which chang count has changed is the source device which 1680 * we concerned. 1681 */ 1682 1683 static int sas_find_bcast_dev(struct domain_device *dev, 1684 struct domain_device **src_dev) 1685 { 1686 struct expander_device *ex = &dev->ex_dev; 1687 int ex_change_count = -1; 1688 int phy_id = -1; 1689 int res; 1690 struct domain_device *ch; 1691 1692 res = sas_get_ex_change_count(dev, &ex_change_count); 1693 if (res) 1694 goto out; 1695 if (ex_change_count != -1 && ex_change_count != ex->ex_change_count) { 1696 /* Just detect if this expander phys phy change count changed, 1697 * in order to determine if this expander originate BROADCAST, 1698 * and do not update phy change count field in our structure. 1699 */ 1700 res = sas_find_bcast_phy(dev, &phy_id, 0, false); 1701 if (phy_id != -1) { 1702 *src_dev = dev; 1703 ex->ex_change_count = ex_change_count; 1704 SAS_DPRINTK("Expander phy change count has changed\n"); 1705 return res; 1706 } else 1707 SAS_DPRINTK("Expander phys DID NOT change\n"); 1708 } 1709 list_for_each_entry(ch, &ex->children, siblings) { 1710 if (ch->dev_type == EDGE_DEV || ch->dev_type == FANOUT_DEV) { 1711 res = sas_find_bcast_dev(ch, src_dev); 1712 if (src_dev) 1713 return res; 1714 } 1715 } 1716 out: 1717 return res; 1718 } 1719 1720 static void sas_unregister_ex_tree(struct domain_device *dev) 1721 { 1722 struct expander_device *ex = &dev->ex_dev; 1723 struct domain_device *child, *n; 1724 1725 list_for_each_entry_safe(child, n, &ex->children, siblings) { 1726 if (child->dev_type == EDGE_DEV || 1727 child->dev_type == FANOUT_DEV) 1728 sas_unregister_ex_tree(child); 1729 else 1730 sas_unregister_dev(child); 1731 } 1732 sas_unregister_dev(dev); 1733 } 1734 1735 static void sas_unregister_devs_sas_addr(struct domain_device *parent, 1736 int phy_id, bool last) 1737 { 1738 struct expander_device *ex_dev = &parent->ex_dev; 1739 struct ex_phy *phy = &ex_dev->ex_phy[phy_id]; 1740 struct domain_device *child, *n; 1741 if (last) { 1742 list_for_each_entry_safe(child, n, 1743 &ex_dev->children, siblings) { 1744 if (SAS_ADDR(child->sas_addr) == 1745 SAS_ADDR(phy->attached_sas_addr)) { 1746 if (child->dev_type == EDGE_DEV || 1747 child->dev_type == FANOUT_DEV) 1748 sas_unregister_ex_tree(child); 1749 else 1750 sas_unregister_dev(child); 1751 break; 1752 } 1753 } 1754 sas_disable_routing(parent, phy->attached_sas_addr); 1755 } 1756 memset(phy->attached_sas_addr, 0, SAS_ADDR_SIZE); 1757 sas_port_delete_phy(phy->port, phy->phy); 1758 if (phy->port->num_phys == 0) 1759 sas_port_delete(phy->port); 1760 phy->port = NULL; 1761 } 1762 1763 static int sas_discover_bfs_by_root_level(struct domain_device *root, 1764 const int level) 1765 { 1766 struct expander_device *ex_root = &root->ex_dev; 1767 struct domain_device *child; 1768 int res = 0; 1769 1770 list_for_each_entry(child, &ex_root->children, siblings) { 1771 if (child->dev_type == EDGE_DEV || 1772 child->dev_type == FANOUT_DEV) { 1773 struct sas_expander_device *ex = 1774 rphy_to_expander_device(child->rphy); 1775 1776 if (level > ex->level) 1777 res = sas_discover_bfs_by_root_level(child, 1778 level); 1779 else if (level == ex->level) 1780 res = sas_ex_discover_devices(child, -1); 1781 } 1782 } 1783 return res; 1784 } 1785 1786 static int sas_discover_bfs_by_root(struct domain_device *dev) 1787 { 1788 int res; 1789 struct sas_expander_device *ex = rphy_to_expander_device(dev->rphy); 1790 int level = ex->level+1; 1791 1792 res = sas_ex_discover_devices(dev, -1); 1793 if (res) 1794 goto out; 1795 do { 1796 res = sas_discover_bfs_by_root_level(dev, level); 1797 mb(); 1798 level += 1; 1799 } while (level <= dev->port->disc.max_level); 1800 out: 1801 return res; 1802 } 1803 1804 static int sas_discover_new(struct domain_device *dev, int phy_id) 1805 { 1806 struct ex_phy *ex_phy = &dev->ex_dev.ex_phy[phy_id]; 1807 struct domain_device *child; 1808 bool found = false; 1809 int res, i; 1810 1811 SAS_DPRINTK("ex %016llx phy%d new device attached\n", 1812 SAS_ADDR(dev->sas_addr), phy_id); 1813 res = sas_ex_phy_discover(dev, phy_id); 1814 if (res) 1815 goto out; 1816 /* to support the wide port inserted */ 1817 for (i = 0; i < dev->ex_dev.num_phys; i++) { 1818 struct ex_phy *ex_phy_temp = &dev->ex_dev.ex_phy[i]; 1819 if (i == phy_id) 1820 continue; 1821 if (SAS_ADDR(ex_phy_temp->attached_sas_addr) == 1822 SAS_ADDR(ex_phy->attached_sas_addr)) { 1823 found = true; 1824 break; 1825 } 1826 } 1827 if (found) { 1828 sas_ex_join_wide_port(dev, phy_id); 1829 return 0; 1830 } 1831 res = sas_ex_discover_devices(dev, phy_id); 1832 if (!res) 1833 goto out; 1834 list_for_each_entry(child, &dev->ex_dev.children, siblings) { 1835 if (SAS_ADDR(child->sas_addr) == 1836 SAS_ADDR(ex_phy->attached_sas_addr)) { 1837 if (child->dev_type == EDGE_DEV || 1838 child->dev_type == FANOUT_DEV) 1839 res = sas_discover_bfs_by_root(child); 1840 break; 1841 } 1842 } 1843 out: 1844 return res; 1845 } 1846 1847 static int sas_rediscover_dev(struct domain_device *dev, int phy_id, bool last) 1848 { 1849 struct expander_device *ex = &dev->ex_dev; 1850 struct ex_phy *phy = &ex->ex_phy[phy_id]; 1851 u8 attached_sas_addr[8]; 1852 int res; 1853 1854 res = sas_get_phy_attached_sas_addr(dev, phy_id, attached_sas_addr); 1855 switch (res) { 1856 case SMP_RESP_NO_PHY: 1857 phy->phy_state = PHY_NOT_PRESENT; 1858 sas_unregister_devs_sas_addr(dev, phy_id, last); 1859 goto out; break; 1860 case SMP_RESP_PHY_VACANT: 1861 phy->phy_state = PHY_VACANT; 1862 sas_unregister_devs_sas_addr(dev, phy_id, last); 1863 goto out; break; 1864 case SMP_RESP_FUNC_ACC: 1865 break; 1866 } 1867 1868 if (SAS_ADDR(attached_sas_addr) == 0) { 1869 phy->phy_state = PHY_EMPTY; 1870 sas_unregister_devs_sas_addr(dev, phy_id, last); 1871 } else if (SAS_ADDR(attached_sas_addr) == 1872 SAS_ADDR(phy->attached_sas_addr)) { 1873 SAS_DPRINTK("ex %016llx phy 0x%x broadcast flutter\n", 1874 SAS_ADDR(dev->sas_addr), phy_id); 1875 sas_ex_phy_discover(dev, phy_id); 1876 } else 1877 res = sas_discover_new(dev, phy_id); 1878 out: 1879 return res; 1880 } 1881 1882 /** 1883 * sas_rediscover - revalidate the domain. 1884 * @dev:domain device to be detect. 1885 * @phy_id: the phy id will be detected. 1886 * 1887 * NOTE: this process _must_ quit (return) as soon as any connection 1888 * errors are encountered. Connection recovery is done elsewhere. 1889 * Discover process only interrogates devices in order to discover the 1890 * domain.For plugging out, we un-register the device only when it is 1891 * the last phy in the port, for other phys in this port, we just delete it 1892 * from the port.For inserting, we do discovery when it is the 1893 * first phy,for other phys in this port, we add it to the port to 1894 * forming the wide-port. 1895 */ 1896 static int sas_rediscover(struct domain_device *dev, const int phy_id) 1897 { 1898 struct expander_device *ex = &dev->ex_dev; 1899 struct ex_phy *changed_phy = &ex->ex_phy[phy_id]; 1900 int res = 0; 1901 int i; 1902 bool last = true; /* is this the last phy of the port */ 1903 1904 SAS_DPRINTK("ex %016llx phy%d originated BROADCAST(CHANGE)\n", 1905 SAS_ADDR(dev->sas_addr), phy_id); 1906 1907 if (SAS_ADDR(changed_phy->attached_sas_addr) != 0) { 1908 for (i = 0; i < ex->num_phys; i++) { 1909 struct ex_phy *phy = &ex->ex_phy[i]; 1910 1911 if (i == phy_id) 1912 continue; 1913 if (SAS_ADDR(phy->attached_sas_addr) == 1914 SAS_ADDR(changed_phy->attached_sas_addr)) { 1915 SAS_DPRINTK("phy%d part of wide port with " 1916 "phy%d\n", phy_id, i); 1917 last = false; 1918 break; 1919 } 1920 } 1921 res = sas_rediscover_dev(dev, phy_id, last); 1922 } else 1923 res = sas_discover_new(dev, phy_id); 1924 return res; 1925 } 1926 1927 /** 1928 * sas_revalidate_domain -- revalidate the domain 1929 * @port: port to the domain of interest 1930 * 1931 * NOTE: this process _must_ quit (return) as soon as any connection 1932 * errors are encountered. Connection recovery is done elsewhere. 1933 * Discover process only interrogates devices in order to discover the 1934 * domain. 1935 */ 1936 int sas_ex_revalidate_domain(struct domain_device *port_dev) 1937 { 1938 int res; 1939 struct domain_device *dev = NULL; 1940 1941 res = sas_find_bcast_dev(port_dev, &dev); 1942 if (res) 1943 goto out; 1944 if (dev) { 1945 struct expander_device *ex = &dev->ex_dev; 1946 int i = 0, phy_id; 1947 1948 do { 1949 phy_id = -1; 1950 res = sas_find_bcast_phy(dev, &phy_id, i, true); 1951 if (phy_id == -1) 1952 break; 1953 res = sas_rediscover(dev, phy_id); 1954 i = phy_id + 1; 1955 } while (i < ex->num_phys); 1956 } 1957 out: 1958 return res; 1959 } 1960 1961 int sas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy, 1962 struct request *req) 1963 { 1964 struct domain_device *dev; 1965 int ret, type; 1966 struct request *rsp = req->next_rq; 1967 1968 if (!rsp) { 1969 printk("%s: space for a smp response is missing\n", 1970 __func__); 1971 return -EINVAL; 1972 } 1973 1974 /* no rphy means no smp target support (ie aic94xx host) */ 1975 if (!rphy) 1976 return sas_smp_host_handler(shost, req, rsp); 1977 1978 type = rphy->identify.device_type; 1979 1980 if (type != SAS_EDGE_EXPANDER_DEVICE && 1981 type != SAS_FANOUT_EXPANDER_DEVICE) { 1982 printk("%s: can we send a smp request to a device?\n", 1983 __func__); 1984 return -EINVAL; 1985 } 1986 1987 dev = sas_find_dev_by_rphy(rphy); 1988 if (!dev) { 1989 printk("%s: fail to find a domain_device?\n", __func__); 1990 return -EINVAL; 1991 } 1992 1993 /* do we need to support multiple segments? */ 1994 if (req->bio->bi_vcnt > 1 || rsp->bio->bi_vcnt > 1) { 1995 printk("%s: multiple segments req %u %u, rsp %u %u\n", 1996 __func__, req->bio->bi_vcnt, blk_rq_bytes(req), 1997 rsp->bio->bi_vcnt, blk_rq_bytes(rsp)); 1998 return -EINVAL; 1999 } 2000 2001 ret = smp_execute_task(dev, bio_data(req->bio), blk_rq_bytes(req), 2002 bio_data(rsp->bio), blk_rq_bytes(rsp)); 2003 if (ret > 0) { 2004 /* positive number is the untransferred residual */ 2005 rsp->resid_len = ret; 2006 req->resid_len = 0; 2007 ret = 0; 2008 } else if (ret == 0) { 2009 rsp->resid_len = 0; 2010 req->resid_len = 0; 2011 } 2012 2013 return ret; 2014 } 2015