1 /* 2 * Block driver for s390 storage class memory. 3 * 4 * Copyright IBM Corp. 2012 5 * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com> 6 */ 7 8 #define KMSG_COMPONENT "scm_block" 9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 11 #include <linux/interrupt.h> 12 #include <linux/spinlock.h> 13 #include <linux/mempool.h> 14 #include <linux/module.h> 15 #include <linux/blkdev.h> 16 #include <linux/genhd.h> 17 #include <linux/slab.h> 18 #include <linux/list.h> 19 #include <asm/eadm.h> 20 #include "scm_blk.h" 21 22 debug_info_t *scm_debug; 23 static int scm_major; 24 static mempool_t *aidaw_pool; 25 static DEFINE_SPINLOCK(list_lock); 26 static LIST_HEAD(inactive_requests); 27 static unsigned int nr_requests = 64; 28 static unsigned int nr_requests_per_io = 8; 29 static atomic_t nr_devices = ATOMIC_INIT(0); 30 module_param(nr_requests, uint, S_IRUGO); 31 MODULE_PARM_DESC(nr_requests, "Number of parallel requests."); 32 33 module_param(nr_requests_per_io, uint, S_IRUGO); 34 MODULE_PARM_DESC(nr_requests_per_io, "Number of requests per IO."); 35 36 MODULE_DESCRIPTION("Block driver for s390 storage class memory."); 37 MODULE_LICENSE("GPL"); 38 MODULE_ALIAS("scm:scmdev*"); 39 40 static void __scm_free_rq(struct scm_request *scmrq) 41 { 42 struct aob_rq_header *aobrq = to_aobrq(scmrq); 43 44 free_page((unsigned long) scmrq->aob); 45 __scm_free_rq_cluster(scmrq); 46 kfree(scmrq->request); 47 kfree(aobrq); 48 } 49 50 static void scm_free_rqs(void) 51 { 52 struct list_head *iter, *safe; 53 struct scm_request *scmrq; 54 55 spin_lock_irq(&list_lock); 56 list_for_each_safe(iter, safe, &inactive_requests) { 57 scmrq = list_entry(iter, struct scm_request, list); 58 list_del(&scmrq->list); 59 __scm_free_rq(scmrq); 60 } 61 spin_unlock_irq(&list_lock); 62 63 mempool_destroy(aidaw_pool); 64 } 65 66 static int __scm_alloc_rq(void) 67 { 68 struct aob_rq_header *aobrq; 69 struct scm_request *scmrq; 70 71 aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL); 72 if (!aobrq) 73 return -ENOMEM; 74 75 scmrq = (void *) aobrq->data; 76 scmrq->aob = (void *) get_zeroed_page(GFP_DMA); 77 if (!scmrq->aob) 78 goto free; 79 80 scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]), 81 GFP_KERNEL); 82 if (!scmrq->request) 83 goto free; 84 85 if (__scm_alloc_rq_cluster(scmrq)) 86 goto free; 87 88 INIT_LIST_HEAD(&scmrq->list); 89 spin_lock_irq(&list_lock); 90 list_add(&scmrq->list, &inactive_requests); 91 spin_unlock_irq(&list_lock); 92 93 return 0; 94 free: 95 __scm_free_rq(scmrq); 96 return -ENOMEM; 97 } 98 99 static int scm_alloc_rqs(unsigned int nrqs) 100 { 101 int ret = 0; 102 103 aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0); 104 if (!aidaw_pool) 105 return -ENOMEM; 106 107 while (nrqs-- && !ret) 108 ret = __scm_alloc_rq(); 109 110 return ret; 111 } 112 113 static struct scm_request *scm_request_fetch(void) 114 { 115 struct scm_request *scmrq = NULL; 116 117 spin_lock(&list_lock); 118 if (list_empty(&inactive_requests)) 119 goto out; 120 scmrq = list_first_entry(&inactive_requests, struct scm_request, list); 121 list_del(&scmrq->list); 122 out: 123 spin_unlock(&list_lock); 124 return scmrq; 125 } 126 127 static void scm_request_done(struct scm_request *scmrq) 128 { 129 unsigned long flags; 130 struct msb *msb; 131 u64 aidaw; 132 int i; 133 134 for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) { 135 msb = &scmrq->aob->msb[i]; 136 aidaw = msb->data_addr; 137 138 if ((msb->flags & MSB_FLAG_IDA) && aidaw && 139 IS_ALIGNED(aidaw, PAGE_SIZE)) 140 mempool_free(virt_to_page(aidaw), aidaw_pool); 141 } 142 143 spin_lock_irqsave(&list_lock, flags); 144 list_add(&scmrq->list, &inactive_requests); 145 spin_unlock_irqrestore(&list_lock, flags); 146 } 147 148 static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req) 149 { 150 return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT; 151 } 152 153 static inline struct aidaw *scm_aidaw_alloc(void) 154 { 155 struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC); 156 157 return page ? page_address(page) : NULL; 158 } 159 160 static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw) 161 { 162 unsigned long _aidaw = (unsigned long) aidaw; 163 unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw; 164 165 return (bytes / sizeof(*aidaw)) * PAGE_SIZE; 166 } 167 168 struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes) 169 { 170 struct aidaw *aidaw; 171 172 if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes) 173 return scmrq->next_aidaw; 174 175 aidaw = scm_aidaw_alloc(); 176 if (aidaw) 177 memset(aidaw, 0, PAGE_SIZE); 178 return aidaw; 179 } 180 181 static int scm_request_prepare(struct scm_request *scmrq) 182 { 183 struct scm_blk_dev *bdev = scmrq->bdev; 184 struct scm_device *scmdev = bdev->gendisk->private_data; 185 int pos = scmrq->aob->request.msb_count; 186 struct msb *msb = &scmrq->aob->msb[pos]; 187 struct request *req = scmrq->request[pos]; 188 struct req_iterator iter; 189 struct aidaw *aidaw; 190 struct bio_vec bv; 191 192 aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req)); 193 if (!aidaw) 194 return -ENOMEM; 195 196 msb->bs = MSB_BS_4K; 197 scmrq->aob->request.msb_count++; 198 msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9); 199 msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE; 200 msb->flags |= MSB_FLAG_IDA; 201 msb->data_addr = (u64) aidaw; 202 203 rq_for_each_segment(bv, req, iter) { 204 WARN_ON(bv.bv_offset); 205 msb->blk_count += bv.bv_len >> 12; 206 aidaw->data_addr = (u64) page_address(bv.bv_page); 207 aidaw++; 208 } 209 210 scmrq->next_aidaw = aidaw; 211 return 0; 212 } 213 214 static inline void scm_request_set(struct scm_request *scmrq, 215 struct request *req) 216 { 217 scmrq->request[scmrq->aob->request.msb_count] = req; 218 } 219 220 static inline void scm_request_init(struct scm_blk_dev *bdev, 221 struct scm_request *scmrq) 222 { 223 struct aob_rq_header *aobrq = to_aobrq(scmrq); 224 struct aob *aob = scmrq->aob; 225 226 memset(scmrq->request, 0, 227 nr_requests_per_io * sizeof(scmrq->request[0])); 228 memset(aob, 0, sizeof(*aob)); 229 aobrq->scmdev = bdev->scmdev; 230 aob->request.cmd_code = ARQB_CMD_MOVE; 231 aob->request.data = (u64) aobrq; 232 scmrq->bdev = bdev; 233 scmrq->retries = 4; 234 scmrq->error = 0; 235 /* We don't use all msbs - place aidaws at the end of the aob page. */ 236 scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io]; 237 scm_request_cluster_init(scmrq); 238 } 239 240 static void scm_ensure_queue_restart(struct scm_blk_dev *bdev) 241 { 242 if (atomic_read(&bdev->queued_reqs)) { 243 /* Queue restart is triggered by the next interrupt. */ 244 return; 245 } 246 blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY); 247 } 248 249 void scm_request_requeue(struct scm_request *scmrq) 250 { 251 struct scm_blk_dev *bdev = scmrq->bdev; 252 int i; 253 254 scm_release_cluster(scmrq); 255 for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) 256 blk_requeue_request(bdev->rq, scmrq->request[i]); 257 258 atomic_dec(&bdev->queued_reqs); 259 scm_request_done(scmrq); 260 scm_ensure_queue_restart(bdev); 261 } 262 263 void scm_request_finish(struct scm_request *scmrq) 264 { 265 struct scm_blk_dev *bdev = scmrq->bdev; 266 int i; 267 268 scm_release_cluster(scmrq); 269 for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) 270 blk_end_request_all(scmrq->request[i], scmrq->error); 271 272 atomic_dec(&bdev->queued_reqs); 273 scm_request_done(scmrq); 274 } 275 276 static int scm_request_start(struct scm_request *scmrq) 277 { 278 struct scm_blk_dev *bdev = scmrq->bdev; 279 int ret; 280 281 atomic_inc(&bdev->queued_reqs); 282 if (!scmrq->aob->request.msb_count) { 283 scm_request_requeue(scmrq); 284 return -EINVAL; 285 } 286 287 ret = eadm_start_aob(scmrq->aob); 288 if (ret) { 289 SCM_LOG(5, "no subchannel"); 290 scm_request_requeue(scmrq); 291 } 292 return ret; 293 } 294 295 static void scm_blk_request(struct request_queue *rq) 296 { 297 struct scm_device *scmdev = rq->queuedata; 298 struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev); 299 struct scm_request *scmrq = NULL; 300 struct request *req; 301 302 while ((req = blk_peek_request(rq))) { 303 if (!scm_permit_request(bdev, req)) 304 goto out; 305 306 if (!scmrq) { 307 scmrq = scm_request_fetch(); 308 if (!scmrq) { 309 SCM_LOG(5, "no request"); 310 goto out; 311 } 312 scm_request_init(bdev, scmrq); 313 } 314 scm_request_set(scmrq, req); 315 316 if (!scm_reserve_cluster(scmrq)) { 317 SCM_LOG(5, "cluster busy"); 318 scm_request_set(scmrq, NULL); 319 if (scmrq->aob->request.msb_count) 320 goto out; 321 322 scm_request_done(scmrq); 323 return; 324 } 325 326 if (scm_need_cluster_request(scmrq)) { 327 if (scmrq->aob->request.msb_count) { 328 /* Start cluster requests separately. */ 329 scm_request_set(scmrq, NULL); 330 if (scm_request_start(scmrq)) 331 return; 332 } else { 333 atomic_inc(&bdev->queued_reqs); 334 blk_start_request(req); 335 scm_initiate_cluster_request(scmrq); 336 } 337 scmrq = NULL; 338 continue; 339 } 340 341 if (scm_request_prepare(scmrq)) { 342 SCM_LOG(5, "aidaw alloc failed"); 343 scm_request_set(scmrq, NULL); 344 goto out; 345 } 346 blk_start_request(req); 347 348 if (scmrq->aob->request.msb_count < nr_requests_per_io) 349 continue; 350 351 if (scm_request_start(scmrq)) 352 return; 353 354 scmrq = NULL; 355 } 356 out: 357 if (scmrq) 358 scm_request_start(scmrq); 359 else 360 scm_ensure_queue_restart(bdev); 361 } 362 363 static void __scmrq_log_error(struct scm_request *scmrq) 364 { 365 struct aob *aob = scmrq->aob; 366 367 if (scmrq->error == -ETIMEDOUT) 368 SCM_LOG(1, "Request timeout"); 369 else { 370 SCM_LOG(1, "Request error"); 371 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response)); 372 } 373 if (scmrq->retries) 374 SCM_LOG(1, "Retry request"); 375 else 376 pr_err("An I/O operation to SCM failed with rc=%d\n", 377 scmrq->error); 378 } 379 380 void scm_blk_irq(struct scm_device *scmdev, void *data, int error) 381 { 382 struct scm_request *scmrq = data; 383 struct scm_blk_dev *bdev = scmrq->bdev; 384 385 scmrq->error = error; 386 if (error) 387 __scmrq_log_error(scmrq); 388 389 spin_lock(&bdev->lock); 390 list_add_tail(&scmrq->list, &bdev->finished_requests); 391 spin_unlock(&bdev->lock); 392 tasklet_hi_schedule(&bdev->tasklet); 393 } 394 395 static void scm_blk_handle_error(struct scm_request *scmrq) 396 { 397 struct scm_blk_dev *bdev = scmrq->bdev; 398 unsigned long flags; 399 400 if (scmrq->error != -EIO) 401 goto restart; 402 403 /* For -EIO the response block is valid. */ 404 switch (scmrq->aob->response.eqc) { 405 case EQC_WR_PROHIBIT: 406 spin_lock_irqsave(&bdev->lock, flags); 407 if (bdev->state != SCM_WR_PROHIBIT) 408 pr_info("%lx: Write access to the SCM increment is suspended\n", 409 (unsigned long) bdev->scmdev->address); 410 bdev->state = SCM_WR_PROHIBIT; 411 spin_unlock_irqrestore(&bdev->lock, flags); 412 goto requeue; 413 default: 414 break; 415 } 416 417 restart: 418 if (!eadm_start_aob(scmrq->aob)) 419 return; 420 421 requeue: 422 spin_lock_irqsave(&bdev->rq_lock, flags); 423 scm_request_requeue(scmrq); 424 spin_unlock_irqrestore(&bdev->rq_lock, flags); 425 } 426 427 static void scm_blk_tasklet(struct scm_blk_dev *bdev) 428 { 429 struct scm_request *scmrq; 430 unsigned long flags; 431 432 spin_lock_irqsave(&bdev->lock, flags); 433 while (!list_empty(&bdev->finished_requests)) { 434 scmrq = list_first_entry(&bdev->finished_requests, 435 struct scm_request, list); 436 list_del(&scmrq->list); 437 spin_unlock_irqrestore(&bdev->lock, flags); 438 439 if (scmrq->error && scmrq->retries-- > 0) { 440 scm_blk_handle_error(scmrq); 441 442 /* Request restarted or requeued, handle next. */ 443 spin_lock_irqsave(&bdev->lock, flags); 444 continue; 445 } 446 447 if (scm_test_cluster_request(scmrq)) { 448 scm_cluster_request_irq(scmrq); 449 spin_lock_irqsave(&bdev->lock, flags); 450 continue; 451 } 452 453 scm_request_finish(scmrq); 454 spin_lock_irqsave(&bdev->lock, flags); 455 } 456 spin_unlock_irqrestore(&bdev->lock, flags); 457 /* Look out for more requests. */ 458 blk_run_queue(bdev->rq); 459 } 460 461 static const struct block_device_operations scm_blk_devops = { 462 .owner = THIS_MODULE, 463 }; 464 465 int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev) 466 { 467 struct request_queue *rq; 468 int len, ret = -ENOMEM; 469 unsigned int devindex, nr_max_blk; 470 471 devindex = atomic_inc_return(&nr_devices) - 1; 472 /* scma..scmz + scmaa..scmzz */ 473 if (devindex > 701) { 474 ret = -ENODEV; 475 goto out; 476 } 477 478 bdev->scmdev = scmdev; 479 bdev->state = SCM_OPER; 480 spin_lock_init(&bdev->rq_lock); 481 spin_lock_init(&bdev->lock); 482 INIT_LIST_HEAD(&bdev->finished_requests); 483 atomic_set(&bdev->queued_reqs, 0); 484 tasklet_init(&bdev->tasklet, 485 (void (*)(unsigned long)) scm_blk_tasklet, 486 (unsigned long) bdev); 487 488 rq = blk_init_queue(scm_blk_request, &bdev->rq_lock); 489 if (!rq) 490 goto out; 491 492 bdev->rq = rq; 493 nr_max_blk = min(scmdev->nr_max_block, 494 (unsigned int) (PAGE_SIZE / sizeof(struct aidaw))); 495 496 blk_queue_logical_block_size(rq, 1 << 12); 497 blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */ 498 blk_queue_max_segments(rq, nr_max_blk); 499 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq); 500 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq); 501 scm_blk_dev_cluster_setup(bdev); 502 503 bdev->gendisk = alloc_disk(SCM_NR_PARTS); 504 if (!bdev->gendisk) 505 goto out_queue; 506 507 rq->queuedata = scmdev; 508 bdev->gendisk->private_data = scmdev; 509 bdev->gendisk->fops = &scm_blk_devops; 510 bdev->gendisk->queue = rq; 511 bdev->gendisk->major = scm_major; 512 bdev->gendisk->first_minor = devindex * SCM_NR_PARTS; 513 514 len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm"); 515 if (devindex > 25) { 516 len += snprintf(bdev->gendisk->disk_name + len, 517 DISK_NAME_LEN - len, "%c", 518 'a' + (devindex / 26) - 1); 519 devindex = devindex % 26; 520 } 521 snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c", 522 'a' + devindex); 523 524 /* 512 byte sectors */ 525 set_capacity(bdev->gendisk, scmdev->size >> 9); 526 device_add_disk(&scmdev->dev, bdev->gendisk); 527 return 0; 528 529 out_queue: 530 blk_cleanup_queue(rq); 531 out: 532 atomic_dec(&nr_devices); 533 return ret; 534 } 535 536 void scm_blk_dev_cleanup(struct scm_blk_dev *bdev) 537 { 538 tasklet_kill(&bdev->tasklet); 539 del_gendisk(bdev->gendisk); 540 blk_cleanup_queue(bdev->gendisk->queue); 541 put_disk(bdev->gendisk); 542 } 543 544 void scm_blk_set_available(struct scm_blk_dev *bdev) 545 { 546 unsigned long flags; 547 548 spin_lock_irqsave(&bdev->lock, flags); 549 if (bdev->state == SCM_WR_PROHIBIT) 550 pr_info("%lx: Write access to the SCM increment is restored\n", 551 (unsigned long) bdev->scmdev->address); 552 bdev->state = SCM_OPER; 553 spin_unlock_irqrestore(&bdev->lock, flags); 554 } 555 556 static bool __init scm_blk_params_valid(void) 557 { 558 if (!nr_requests_per_io || nr_requests_per_io > 64) 559 return false; 560 561 return scm_cluster_size_valid(); 562 } 563 564 static int __init scm_blk_init(void) 565 { 566 int ret = -EINVAL; 567 568 if (!scm_blk_params_valid()) 569 goto out; 570 571 ret = register_blkdev(0, "scm"); 572 if (ret < 0) 573 goto out; 574 575 scm_major = ret; 576 ret = scm_alloc_rqs(nr_requests); 577 if (ret) 578 goto out_free; 579 580 scm_debug = debug_register("scm_log", 16, 1, 16); 581 if (!scm_debug) { 582 ret = -ENOMEM; 583 goto out_free; 584 } 585 586 debug_register_view(scm_debug, &debug_hex_ascii_view); 587 debug_set_level(scm_debug, 2); 588 589 ret = scm_drv_init(); 590 if (ret) 591 goto out_dbf; 592 593 return ret; 594 595 out_dbf: 596 debug_unregister(scm_debug); 597 out_free: 598 scm_free_rqs(); 599 unregister_blkdev(scm_major, "scm"); 600 out: 601 return ret; 602 } 603 module_init(scm_blk_init); 604 605 static void __exit scm_blk_cleanup(void) 606 { 607 scm_drv_cleanup(); 608 debug_unregister(scm_debug); 609 scm_free_rqs(); 610 unregister_blkdev(scm_major, "scm"); 611 } 612 module_exit(scm_blk_cleanup); 613