1 /* 2 * Copyright (C) 2017 NXP Semiconductors 3 * Copyright (C) 2017 Bin Meng <bmeng.cn@gmail.com> 4 * 5 * SPDX-License-Identifier: GPL-2.0+ 6 */ 7 8 #include <common.h> 9 #include <dm.h> 10 #include <errno.h> 11 #include <memalign.h> 12 #include <pci.h> 13 #include <dm/device-internal.h> 14 #include "nvme.h" 15 16 #define NVME_Q_DEPTH 2 17 #define NVME_AQ_DEPTH 2 18 #define NVME_SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) 19 #define NVME_CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) 20 #define ADMIN_TIMEOUT 60 21 #define IO_TIMEOUT 30 22 #define MAX_PRP_POOL 512 23 24 enum nvme_queue_id { 25 NVME_ADMIN_Q, 26 NVME_IO_Q, 27 NVME_Q_NUM, 28 }; 29 30 /* 31 * An NVM Express queue. Each device has at least two (one for admin 32 * commands and one for I/O commands). 33 */ 34 struct nvme_queue { 35 struct nvme_dev *dev; 36 struct nvme_command *sq_cmds; 37 struct nvme_completion *cqes; 38 wait_queue_head_t sq_full; 39 u32 __iomem *q_db; 40 u16 q_depth; 41 s16 cq_vector; 42 u16 sq_head; 43 u16 sq_tail; 44 u16 cq_head; 45 u16 qid; 46 u8 cq_phase; 47 u8 cqe_seen; 48 unsigned long cmdid_data[]; 49 }; 50 51 static int nvme_wait_ready(struct nvme_dev *dev, bool enabled) 52 { 53 u32 bit = enabled ? NVME_CSTS_RDY : 0; 54 int timeout; 55 ulong start; 56 57 /* Timeout field in the CAP register is in 500 millisecond units */ 58 timeout = NVME_CAP_TIMEOUT(dev->cap) * 500; 59 60 start = get_timer(0); 61 while (get_timer(start) < timeout) { 62 if ((readl(&dev->bar->csts) & NVME_CSTS_RDY) == bit) 63 return 0; 64 } 65 66 return -ETIME; 67 } 68 69 static int nvme_setup_prps(struct nvme_dev *dev, u64 *prp2, 70 int total_len, u64 dma_addr) 71 { 72 u32 page_size = dev->page_size; 73 int offset = dma_addr & (page_size - 1); 74 u64 *prp_pool; 75 int length = total_len; 76 int i, nprps; 77 length -= (page_size - offset); 78 79 if (length <= 0) { 80 *prp2 = 0; 81 return 0; 82 } 83 84 if (length) 85 dma_addr += (page_size - offset); 86 87 if (length <= page_size) { 88 *prp2 = dma_addr; 89 return 0; 90 } 91 92 nprps = DIV_ROUND_UP(length, page_size); 93 94 if (nprps > dev->prp_entry_num) { 95 free(dev->prp_pool); 96 dev->prp_pool = malloc(nprps << 3); 97 if (!dev->prp_pool) { 98 printf("Error: malloc prp_pool fail\n"); 99 return -ENOMEM; 100 } 101 dev->prp_entry_num = nprps; 102 } 103 104 prp_pool = dev->prp_pool; 105 i = 0; 106 while (nprps) { 107 if (i == ((page_size >> 3) - 1)) { 108 *(prp_pool + i) = cpu_to_le64((ulong)prp_pool + 109 page_size); 110 i = 0; 111 prp_pool += page_size; 112 } 113 *(prp_pool + i++) = cpu_to_le64(dma_addr); 114 dma_addr += page_size; 115 nprps--; 116 } 117 *prp2 = (ulong)dev->prp_pool; 118 119 return 0; 120 } 121 122 static __le16 nvme_get_cmd_id(void) 123 { 124 static unsigned short cmdid; 125 126 return cpu_to_le16((cmdid < USHRT_MAX) ? cmdid++ : 0); 127 } 128 129 static u16 nvme_read_completion_status(struct nvme_queue *nvmeq, u16 index) 130 { 131 u64 start = (ulong)&nvmeq->cqes[index]; 132 u64 stop = start + sizeof(struct nvme_completion); 133 134 invalidate_dcache_range(start, stop); 135 136 return le16_to_cpu(readw(&(nvmeq->cqes[index].status))); 137 } 138 139 /** 140 * nvme_submit_cmd() - copy a command into a queue and ring the doorbell 141 * 142 * @nvmeq: The queue to use 143 * @cmd: The command to send 144 */ 145 static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) 146 { 147 u16 tail = nvmeq->sq_tail; 148 149 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); 150 flush_dcache_range((ulong)&nvmeq->sq_cmds[tail], 151 (ulong)&nvmeq->sq_cmds[tail] + sizeof(*cmd)); 152 153 if (++tail == nvmeq->q_depth) 154 tail = 0; 155 writel(tail, nvmeq->q_db); 156 nvmeq->sq_tail = tail; 157 } 158 159 static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, 160 struct nvme_command *cmd, 161 u32 *result, unsigned timeout) 162 { 163 u16 head = nvmeq->cq_head; 164 u16 phase = nvmeq->cq_phase; 165 u16 status; 166 ulong start_time; 167 ulong timeout_us = timeout * 100000; 168 169 cmd->common.command_id = nvme_get_cmd_id(); 170 nvme_submit_cmd(nvmeq, cmd); 171 172 start_time = timer_get_us(); 173 174 for (;;) { 175 status = nvme_read_completion_status(nvmeq, head); 176 if ((status & 0x01) == phase) 177 break; 178 if (timeout_us > 0 && (timer_get_us() - start_time) 179 >= timeout_us) 180 return -ETIMEDOUT; 181 } 182 183 status >>= 1; 184 if (status) { 185 printf("ERROR: status = %x, phase = %d, head = %d\n", 186 status, phase, head); 187 status = 0; 188 if (++head == nvmeq->q_depth) { 189 head = 0; 190 phase = !phase; 191 } 192 writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 193 nvmeq->cq_head = head; 194 nvmeq->cq_phase = phase; 195 196 return -EIO; 197 } 198 199 if (result) 200 *result = le32_to_cpu(readl(&(nvmeq->cqes[head].result))); 201 202 if (++head == nvmeq->q_depth) { 203 head = 0; 204 phase = !phase; 205 } 206 writel(head, nvmeq->q_db + nvmeq->dev->db_stride); 207 nvmeq->cq_head = head; 208 nvmeq->cq_phase = phase; 209 210 return status; 211 } 212 213 static int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, 214 u32 *result) 215 { 216 return nvme_submit_sync_cmd(dev->queues[NVME_ADMIN_Q], cmd, 217 result, ADMIN_TIMEOUT); 218 } 219 220 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, 221 int qid, int depth) 222 { 223 struct nvme_queue *nvmeq = malloc(sizeof(*nvmeq)); 224 if (!nvmeq) 225 return NULL; 226 memset(nvmeq, 0, sizeof(*nvmeq)); 227 228 nvmeq->cqes = (void *)memalign(4096, NVME_CQ_SIZE(depth)); 229 if (!nvmeq->cqes) 230 goto free_nvmeq; 231 memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(depth)); 232 233 nvmeq->sq_cmds = (void *)memalign(4096, NVME_SQ_SIZE(depth)); 234 if (!nvmeq->sq_cmds) 235 goto free_queue; 236 memset((void *)nvmeq->sq_cmds, 0, NVME_SQ_SIZE(depth)); 237 238 nvmeq->dev = dev; 239 240 nvmeq->cq_head = 0; 241 nvmeq->cq_phase = 1; 242 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 243 nvmeq->q_depth = depth; 244 nvmeq->qid = qid; 245 dev->queue_count++; 246 dev->queues[qid] = nvmeq; 247 248 return nvmeq; 249 250 free_queue: 251 free((void *)nvmeq->cqes); 252 free_nvmeq: 253 free(nvmeq); 254 255 return NULL; 256 } 257 258 static int nvme_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) 259 { 260 struct nvme_command c; 261 262 memset(&c, 0, sizeof(c)); 263 c.delete_queue.opcode = opcode; 264 c.delete_queue.qid = cpu_to_le16(id); 265 266 return nvme_submit_admin_cmd(dev, &c, NULL); 267 } 268 269 static int nvme_delete_sq(struct nvme_dev *dev, u16 sqid) 270 { 271 return nvme_delete_queue(dev, nvme_admin_delete_sq, sqid); 272 } 273 274 static int nvme_delete_cq(struct nvme_dev *dev, u16 cqid) 275 { 276 return nvme_delete_queue(dev, nvme_admin_delete_cq, cqid); 277 } 278 279 static int nvme_enable_ctrl(struct nvme_dev *dev) 280 { 281 dev->ctrl_config &= ~NVME_CC_SHN_MASK; 282 dev->ctrl_config |= NVME_CC_ENABLE; 283 writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc); 284 285 return nvme_wait_ready(dev, true); 286 } 287 288 static int nvme_disable_ctrl(struct nvme_dev *dev) 289 { 290 dev->ctrl_config &= ~NVME_CC_SHN_MASK; 291 dev->ctrl_config &= ~NVME_CC_ENABLE; 292 writel(cpu_to_le32(dev->ctrl_config), &dev->bar->cc); 293 294 return nvme_wait_ready(dev, false); 295 } 296 297 static void nvme_free_queue(struct nvme_queue *nvmeq) 298 { 299 free((void *)nvmeq->cqes); 300 free(nvmeq->sq_cmds); 301 free(nvmeq); 302 } 303 304 static void nvme_free_queues(struct nvme_dev *dev, int lowest) 305 { 306 int i; 307 308 for (i = dev->queue_count - 1; i >= lowest; i--) { 309 struct nvme_queue *nvmeq = dev->queues[i]; 310 dev->queue_count--; 311 dev->queues[i] = NULL; 312 nvme_free_queue(nvmeq); 313 } 314 } 315 316 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 317 { 318 struct nvme_dev *dev = nvmeq->dev; 319 320 nvmeq->sq_tail = 0; 321 nvmeq->cq_head = 0; 322 nvmeq->cq_phase = 1; 323 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; 324 memset((void *)nvmeq->cqes, 0, NVME_CQ_SIZE(nvmeq->q_depth)); 325 flush_dcache_range((ulong)nvmeq->cqes, 326 (ulong)nvmeq->cqes + NVME_CQ_SIZE(nvmeq->q_depth)); 327 dev->online_queues++; 328 } 329 330 static int nvme_configure_admin_queue(struct nvme_dev *dev) 331 { 332 int result; 333 u32 aqa; 334 u64 cap = dev->cap; 335 struct nvme_queue *nvmeq; 336 /* most architectures use 4KB as the page size */ 337 unsigned page_shift = 12; 338 unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12; 339 unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12; 340 341 if (page_shift < dev_page_min) { 342 debug("Device minimum page size (%u) too large for host (%u)\n", 343 1 << dev_page_min, 1 << page_shift); 344 return -ENODEV; 345 } 346 347 if (page_shift > dev_page_max) { 348 debug("Device maximum page size (%u) smaller than host (%u)\n", 349 1 << dev_page_max, 1 << page_shift); 350 page_shift = dev_page_max; 351 } 352 353 result = nvme_disable_ctrl(dev); 354 if (result < 0) 355 return result; 356 357 nvmeq = dev->queues[NVME_ADMIN_Q]; 358 if (!nvmeq) { 359 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH); 360 if (!nvmeq) 361 return -ENOMEM; 362 } 363 364 aqa = nvmeq->q_depth - 1; 365 aqa |= aqa << 16; 366 aqa |= aqa << 16; 367 368 dev->page_size = 1 << page_shift; 369 370 dev->ctrl_config = NVME_CC_CSS_NVM; 371 dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT; 372 dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; 373 dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 374 375 writel(aqa, &dev->bar->aqa); 376 nvme_writeq((ulong)nvmeq->sq_cmds, &dev->bar->asq); 377 nvme_writeq((ulong)nvmeq->cqes, &dev->bar->acq); 378 379 result = nvme_enable_ctrl(dev); 380 if (result) 381 goto free_nvmeq; 382 383 nvmeq->cq_vector = 0; 384 385 nvme_init_queue(dev->queues[NVME_ADMIN_Q], 0); 386 387 return result; 388 389 free_nvmeq: 390 nvme_free_queues(dev, 0); 391 392 return result; 393 } 394 395 static int nvme_alloc_cq(struct nvme_dev *dev, u16 qid, 396 struct nvme_queue *nvmeq) 397 { 398 struct nvme_command c; 399 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED; 400 401 memset(&c, 0, sizeof(c)); 402 c.create_cq.opcode = nvme_admin_create_cq; 403 c.create_cq.prp1 = cpu_to_le64((ulong)nvmeq->cqes); 404 c.create_cq.cqid = cpu_to_le16(qid); 405 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 406 c.create_cq.cq_flags = cpu_to_le16(flags); 407 c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector); 408 409 return nvme_submit_admin_cmd(dev, &c, NULL); 410 } 411 412 static int nvme_alloc_sq(struct nvme_dev *dev, u16 qid, 413 struct nvme_queue *nvmeq) 414 { 415 struct nvme_command c; 416 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM; 417 418 memset(&c, 0, sizeof(c)); 419 c.create_sq.opcode = nvme_admin_create_sq; 420 c.create_sq.prp1 = cpu_to_le64((ulong)nvmeq->sq_cmds); 421 c.create_sq.sqid = cpu_to_le16(qid); 422 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1); 423 c.create_sq.sq_flags = cpu_to_le16(flags); 424 c.create_sq.cqid = cpu_to_le16(qid); 425 426 return nvme_submit_admin_cmd(dev, &c, NULL); 427 } 428 429 int nvme_identify(struct nvme_dev *dev, unsigned nsid, 430 unsigned cns, dma_addr_t dma_addr) 431 { 432 struct nvme_command c; 433 u32 page_size = dev->page_size; 434 int offset = dma_addr & (page_size - 1); 435 int length = sizeof(struct nvme_id_ctrl); 436 int ret; 437 438 memset(&c, 0, sizeof(c)); 439 c.identify.opcode = nvme_admin_identify; 440 c.identify.nsid = cpu_to_le32(nsid); 441 c.identify.prp1 = cpu_to_le64(dma_addr); 442 443 length -= (page_size - offset); 444 if (length <= 0) { 445 c.identify.prp2 = 0; 446 } else { 447 dma_addr += (page_size - offset); 448 c.identify.prp2 = cpu_to_le64(dma_addr); 449 } 450 451 c.identify.cns = cpu_to_le32(cns); 452 453 ret = nvme_submit_admin_cmd(dev, &c, NULL); 454 if (!ret) 455 invalidate_dcache_range(dma_addr, 456 dma_addr + sizeof(struct nvme_id_ctrl)); 457 458 return ret; 459 } 460 461 int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid, 462 dma_addr_t dma_addr, u32 *result) 463 { 464 struct nvme_command c; 465 466 memset(&c, 0, sizeof(c)); 467 c.features.opcode = nvme_admin_get_features; 468 c.features.nsid = cpu_to_le32(nsid); 469 c.features.prp1 = cpu_to_le64(dma_addr); 470 c.features.fid = cpu_to_le32(fid); 471 472 /* 473 * TODO: add cache invalidate operation when the size of 474 * the DMA buffer is known 475 */ 476 477 return nvme_submit_admin_cmd(dev, &c, result); 478 } 479 480 int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11, 481 dma_addr_t dma_addr, u32 *result) 482 { 483 struct nvme_command c; 484 485 memset(&c, 0, sizeof(c)); 486 c.features.opcode = nvme_admin_set_features; 487 c.features.prp1 = cpu_to_le64(dma_addr); 488 c.features.fid = cpu_to_le32(fid); 489 c.features.dword11 = cpu_to_le32(dword11); 490 491 /* 492 * TODO: add cache flush operation when the size of 493 * the DMA buffer is known 494 */ 495 496 return nvme_submit_admin_cmd(dev, &c, result); 497 } 498 499 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid) 500 { 501 struct nvme_dev *dev = nvmeq->dev; 502 int result; 503 504 nvmeq->cq_vector = qid - 1; 505 result = nvme_alloc_cq(dev, qid, nvmeq); 506 if (result < 0) 507 goto release_cq; 508 509 result = nvme_alloc_sq(dev, qid, nvmeq); 510 if (result < 0) 511 goto release_sq; 512 513 nvme_init_queue(nvmeq, qid); 514 515 return result; 516 517 release_sq: 518 nvme_delete_sq(dev, qid); 519 release_cq: 520 nvme_delete_cq(dev, qid); 521 522 return result; 523 } 524 525 static int nvme_set_queue_count(struct nvme_dev *dev, int count) 526 { 527 int status; 528 u32 result; 529 u32 q_count = (count - 1) | ((count - 1) << 16); 530 531 status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, 532 q_count, 0, &result); 533 534 if (status < 0) 535 return status; 536 if (status > 1) 537 return 0; 538 539 return min(result & 0xffff, result >> 16) + 1; 540 } 541 542 static void nvme_create_io_queues(struct nvme_dev *dev) 543 { 544 unsigned int i; 545 546 for (i = dev->queue_count; i <= dev->max_qid; i++) 547 if (!nvme_alloc_queue(dev, i, dev->q_depth)) 548 break; 549 550 for (i = dev->online_queues; i <= dev->queue_count - 1; i++) 551 if (nvme_create_queue(dev->queues[i], i)) 552 break; 553 } 554 555 static int nvme_setup_io_queues(struct nvme_dev *dev) 556 { 557 int nr_io_queues; 558 int result; 559 560 nr_io_queues = 1; 561 result = nvme_set_queue_count(dev, nr_io_queues); 562 if (result <= 0) 563 return result; 564 565 if (result < nr_io_queues) 566 nr_io_queues = result; 567 568 dev->max_qid = nr_io_queues; 569 570 /* Free previously allocated queues */ 571 nvme_free_queues(dev, nr_io_queues + 1); 572 nvme_create_io_queues(dev); 573 574 return 0; 575 } 576 577 static int nvme_get_info_from_identify(struct nvme_dev *dev) 578 { 579 ALLOC_CACHE_ALIGN_BUFFER(char, buf, sizeof(struct nvme_id_ctrl)); 580 struct nvme_id_ctrl *ctrl = (struct nvme_id_ctrl *)buf; 581 int ret; 582 int shift = NVME_CAP_MPSMIN(dev->cap) + 12; 583 584 ret = nvme_identify(dev, 0, 1, (dma_addr_t)ctrl); 585 if (ret) 586 return -EIO; 587 588 dev->nn = le32_to_cpu(ctrl->nn); 589 dev->vwc = ctrl->vwc; 590 memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); 591 memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); 592 memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); 593 if (ctrl->mdts) 594 dev->max_transfer_shift = (ctrl->mdts + shift); 595 else { 596 /* 597 * Maximum Data Transfer Size (MDTS) field indicates the maximum 598 * data transfer size between the host and the controller. The 599 * host should not submit a command that exceeds this transfer 600 * size. The value is in units of the minimum memory page size 601 * and is reported as a power of two (2^n). 602 * 603 * The spec also says: a value of 0h indicates no restrictions 604 * on transfer size. But in nvme_blk_read/write() below we have 605 * the following algorithm for maximum number of logic blocks 606 * per transfer: 607 * 608 * u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift); 609 * 610 * In order for lbas not to overflow, the maximum number is 15 611 * which means dev->max_transfer_shift = 15 + 9 (ns->lba_shift). 612 * Let's use 20 which provides 1MB size. 613 */ 614 dev->max_transfer_shift = 20; 615 } 616 617 return 0; 618 } 619 620 int nvme_scan_namespace(void) 621 { 622 struct uclass *uc; 623 struct udevice *dev; 624 int ret; 625 626 ret = uclass_get(UCLASS_NVME, &uc); 627 if (ret) 628 return ret; 629 630 uclass_foreach_dev(dev, uc) { 631 ret = device_probe(dev); 632 if (ret) 633 return ret; 634 } 635 636 return 0; 637 } 638 639 static int nvme_blk_probe(struct udevice *udev) 640 { 641 struct nvme_dev *ndev = dev_get_priv(udev->parent); 642 struct blk_desc *desc = dev_get_uclass_platdata(udev); 643 struct nvme_ns *ns = dev_get_priv(udev); 644 u8 flbas; 645 ALLOC_CACHE_ALIGN_BUFFER(char, buf, sizeof(struct nvme_id_ns)); 646 struct nvme_id_ns *id = (struct nvme_id_ns *)buf; 647 struct pci_child_platdata *pplat; 648 649 memset(ns, 0, sizeof(*ns)); 650 ns->dev = ndev; 651 /* extract the namespace id from the block device name */ 652 ns->ns_id = trailing_strtol(udev->name) + 1; 653 if (nvme_identify(ndev, ns->ns_id, 0, (dma_addr_t)id)) 654 return -EIO; 655 656 flbas = id->flbas & NVME_NS_FLBAS_LBA_MASK; 657 ns->flbas = flbas; 658 ns->lba_shift = id->lbaf[flbas].ds; 659 ns->mode_select_num_blocks = le64_to_cpu(id->nsze); 660 ns->mode_select_block_len = 1 << ns->lba_shift; 661 list_add(&ns->list, &ndev->namespaces); 662 663 desc->lba = ns->mode_select_num_blocks; 664 desc->log2blksz = ns->lba_shift; 665 desc->blksz = 1 << ns->lba_shift; 666 desc->bdev = udev; 667 pplat = dev_get_parent_platdata(udev->parent); 668 sprintf(desc->vendor, "0x%.4x", pplat->vendor); 669 memcpy(desc->product, ndev->serial, sizeof(ndev->serial)); 670 memcpy(desc->revision, ndev->firmware_rev, sizeof(ndev->firmware_rev)); 671 part_init(desc); 672 673 return 0; 674 } 675 676 static ulong nvme_blk_rw(struct udevice *udev, lbaint_t blknr, 677 lbaint_t blkcnt, void *buffer, bool read) 678 { 679 struct nvme_ns *ns = dev_get_priv(udev); 680 struct nvme_dev *dev = ns->dev; 681 struct nvme_command c; 682 struct blk_desc *desc = dev_get_uclass_platdata(udev); 683 int status; 684 u64 prp2; 685 u64 total_len = blkcnt << desc->log2blksz; 686 u64 temp_len = total_len; 687 688 u64 slba = blknr; 689 u16 lbas = 1 << (dev->max_transfer_shift - ns->lba_shift); 690 u64 total_lbas = blkcnt; 691 692 if (!read) 693 flush_dcache_range((unsigned long)buffer, 694 (unsigned long)buffer + total_len); 695 696 c.rw.opcode = read ? nvme_cmd_read : nvme_cmd_write; 697 c.rw.flags = 0; 698 c.rw.nsid = cpu_to_le32(ns->ns_id); 699 c.rw.control = 0; 700 c.rw.dsmgmt = 0; 701 c.rw.reftag = 0; 702 c.rw.apptag = 0; 703 c.rw.appmask = 0; 704 c.rw.metadata = 0; 705 706 while (total_lbas) { 707 if (total_lbas < lbas) { 708 lbas = (u16)total_lbas; 709 total_lbas = 0; 710 } else { 711 total_lbas -= lbas; 712 } 713 714 if (nvme_setup_prps(dev, &prp2, 715 lbas << ns->lba_shift, (ulong)buffer)) 716 return -EIO; 717 c.rw.slba = cpu_to_le64(slba); 718 slba += lbas; 719 c.rw.length = cpu_to_le16(lbas - 1); 720 c.rw.prp1 = cpu_to_le64((ulong)buffer); 721 c.rw.prp2 = cpu_to_le64(prp2); 722 status = nvme_submit_sync_cmd(dev->queues[NVME_IO_Q], 723 &c, NULL, IO_TIMEOUT); 724 if (status) 725 break; 726 temp_len -= lbas << ns->lba_shift; 727 buffer += lbas << ns->lba_shift; 728 } 729 730 if (read) 731 invalidate_dcache_range((unsigned long)buffer, 732 (unsigned long)buffer + total_len); 733 734 return (total_len - temp_len) >> desc->log2blksz; 735 } 736 737 static ulong nvme_blk_read(struct udevice *udev, lbaint_t blknr, 738 lbaint_t blkcnt, void *buffer) 739 { 740 return nvme_blk_rw(udev, blknr, blkcnt, buffer, true); 741 } 742 743 static ulong nvme_blk_write(struct udevice *udev, lbaint_t blknr, 744 lbaint_t blkcnt, const void *buffer) 745 { 746 return nvme_blk_rw(udev, blknr, blkcnt, (void *)buffer, false); 747 } 748 749 static const struct blk_ops nvme_blk_ops = { 750 .read = nvme_blk_read, 751 .write = nvme_blk_write, 752 }; 753 754 U_BOOT_DRIVER(nvme_blk) = { 755 .name = "nvme-blk", 756 .id = UCLASS_BLK, 757 .probe = nvme_blk_probe, 758 .ops = &nvme_blk_ops, 759 .priv_auto_alloc_size = sizeof(struct nvme_ns), 760 }; 761 762 static int nvme_bind(struct udevice *udev) 763 { 764 static int ndev_num; 765 char name[20]; 766 767 sprintf(name, "nvme#%d", ndev_num++); 768 769 return device_set_name(udev, name); 770 } 771 772 static int nvme_probe(struct udevice *udev) 773 { 774 int ret; 775 struct nvme_dev *ndev = dev_get_priv(udev); 776 777 ndev->instance = trailing_strtol(udev->name); 778 779 INIT_LIST_HEAD(&ndev->namespaces); 780 ndev->bar = dm_pci_map_bar(udev, PCI_BASE_ADDRESS_0, 781 PCI_REGION_MEM); 782 if (readl(&ndev->bar->csts) == -1) { 783 ret = -ENODEV; 784 printf("Error: %s: Out of memory!\n", udev->name); 785 goto free_nvme; 786 } 787 788 ndev->queues = malloc(NVME_Q_NUM * sizeof(struct nvme_queue *)); 789 if (!ndev->queues) { 790 ret = -ENOMEM; 791 printf("Error: %s: Out of memory!\n", udev->name); 792 goto free_nvme; 793 } 794 memset(ndev->queues, 0, 795 sizeof(NVME_Q_NUM * sizeof(struct nvme_queue *))); 796 797 ndev->prp_pool = malloc(MAX_PRP_POOL); 798 if (!ndev->prp_pool) { 799 ret = -ENOMEM; 800 printf("Error: %s: Out of memory!\n", udev->name); 801 goto free_nvme; 802 } 803 ndev->prp_entry_num = MAX_PRP_POOL >> 3; 804 805 ndev->cap = nvme_readq(&ndev->bar->cap); 806 ndev->q_depth = min_t(int, NVME_CAP_MQES(ndev->cap) + 1, NVME_Q_DEPTH); 807 ndev->db_stride = 1 << NVME_CAP_STRIDE(ndev->cap); 808 ndev->dbs = ((void __iomem *)ndev->bar) + 4096; 809 810 ret = nvme_configure_admin_queue(ndev); 811 if (ret) 812 goto free_queue; 813 814 ret = nvme_setup_io_queues(ndev); 815 if (ret) 816 goto free_queue; 817 818 nvme_get_info_from_identify(ndev); 819 820 return 0; 821 822 free_queue: 823 free((void *)ndev->queues); 824 free_nvme: 825 return ret; 826 } 827 828 U_BOOT_DRIVER(nvme) = { 829 .name = "nvme", 830 .id = UCLASS_NVME, 831 .bind = nvme_bind, 832 .probe = nvme_probe, 833 .priv_auto_alloc_size = sizeof(struct nvme_dev), 834 }; 835 836 struct pci_device_id nvme_supported[] = { 837 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, ~0) }, 838 {} 839 }; 840 841 U_BOOT_PCI_DEVICE(nvme, nvme_supported); 842