1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * VFIO based Physical Subchannel device driver 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/device.h> 16 #include <linux/slab.h> 17 #include <linux/uuid.h> 18 #include <linux/mdev.h> 19 20 #include <asm/isc.h> 21 22 #include "chp.h" 23 #include "ioasm.h" 24 #include "css.h" 25 #include "vfio_ccw_private.h" 26 27 struct workqueue_struct *vfio_ccw_work_q; 28 static struct kmem_cache *vfio_ccw_io_region; 29 static struct kmem_cache *vfio_ccw_cmd_region; 30 static struct kmem_cache *vfio_ccw_schib_region; 31 static struct kmem_cache *vfio_ccw_crw_region; 32 33 debug_info_t *vfio_ccw_debug_msg_id; 34 debug_info_t *vfio_ccw_debug_trace_id; 35 36 /* 37 * Helpers 38 */ 39 int vfio_ccw_sch_quiesce(struct subchannel *sch) 40 { 41 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 42 DECLARE_COMPLETION_ONSTACK(completion); 43 int iretry, ret = 0; 44 45 spin_lock_irq(sch->lock); 46 if (!sch->schib.pmcw.ena) 47 goto out_unlock; 48 ret = cio_disable_subchannel(sch); 49 if (ret != -EBUSY) 50 goto out_unlock; 51 52 iretry = 255; 53 do { 54 55 ret = cio_cancel_halt_clear(sch, &iretry); 56 57 if (ret == -EIO) { 58 pr_err("vfio_ccw: could not quiesce subchannel 0.%x.%04x!\n", 59 sch->schid.ssid, sch->schid.sch_no); 60 break; 61 } 62 63 /* 64 * Flush all I/O and wait for 65 * cancel/halt/clear completion. 66 */ 67 private->completion = &completion; 68 spin_unlock_irq(sch->lock); 69 70 if (ret == -EBUSY) 71 wait_for_completion_timeout(&completion, 3*HZ); 72 73 private->completion = NULL; 74 flush_workqueue(vfio_ccw_work_q); 75 spin_lock_irq(sch->lock); 76 ret = cio_disable_subchannel(sch); 77 } while (ret == -EBUSY); 78 out_unlock: 79 private->state = VFIO_CCW_STATE_NOT_OPER; 80 spin_unlock_irq(sch->lock); 81 return ret; 82 } 83 84 static void vfio_ccw_sch_io_todo(struct work_struct *work) 85 { 86 struct vfio_ccw_private *private; 87 struct irb *irb; 88 bool is_final; 89 bool cp_is_finished = false; 90 91 private = container_of(work, struct vfio_ccw_private, io_work); 92 irb = &private->irb; 93 94 is_final = !(scsw_actl(&irb->scsw) & 95 (SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT)); 96 if (scsw_is_solicited(&irb->scsw)) { 97 cp_update_scsw(&private->cp, &irb->scsw); 98 if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) { 99 cp_free(&private->cp); 100 cp_is_finished = true; 101 } 102 } 103 mutex_lock(&private->io_mutex); 104 memcpy(private->io_region->irb_area, irb, sizeof(*irb)); 105 mutex_unlock(&private->io_mutex); 106 107 /* 108 * Reset to IDLE only if processing of a channel program 109 * has finished. Do not overwrite a possible processing 110 * state if the final interrupt was for HSCH or CSCH. 111 */ 112 if (private->mdev && cp_is_finished) 113 private->state = VFIO_CCW_STATE_IDLE; 114 115 if (private->io_trigger) 116 eventfd_signal(private->io_trigger, 1); 117 } 118 119 static void vfio_ccw_crw_todo(struct work_struct *work) 120 { 121 struct vfio_ccw_private *private; 122 123 private = container_of(work, struct vfio_ccw_private, crw_work); 124 125 if (!list_empty(&private->crw) && private->crw_trigger) 126 eventfd_signal(private->crw_trigger, 1); 127 } 128 129 /* 130 * Css driver callbacks 131 */ 132 static void vfio_ccw_sch_irq(struct subchannel *sch) 133 { 134 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 135 136 inc_irq_stat(IRQIO_CIO); 137 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_INTERRUPT); 138 } 139 140 static struct vfio_ccw_private *vfio_ccw_alloc_private(struct subchannel *sch) 141 { 142 struct vfio_ccw_private *private; 143 144 private = kzalloc(sizeof(*private), GFP_KERNEL); 145 if (!private) 146 return ERR_PTR(-ENOMEM); 147 148 private->sch = sch; 149 mutex_init(&private->io_mutex); 150 private->state = VFIO_CCW_STATE_NOT_OPER; 151 INIT_LIST_HEAD(&private->crw); 152 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); 153 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); 154 atomic_set(&private->avail, 1); 155 156 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1), 157 GFP_KERNEL); 158 if (!private->cp.guest_cp) 159 goto out_free_private; 160 161 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region, 162 GFP_KERNEL | GFP_DMA); 163 if (!private->io_region) 164 goto out_free_cp; 165 166 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region, 167 GFP_KERNEL | GFP_DMA); 168 if (!private->cmd_region) 169 goto out_free_io; 170 171 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region, 172 GFP_KERNEL | GFP_DMA); 173 174 if (!private->schib_region) 175 goto out_free_cmd; 176 177 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region, 178 GFP_KERNEL | GFP_DMA); 179 180 if (!private->crw_region) 181 goto out_free_schib; 182 return private; 183 184 out_free_schib: 185 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 186 out_free_cmd: 187 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 188 out_free_io: 189 kmem_cache_free(vfio_ccw_io_region, private->io_region); 190 out_free_cp: 191 kfree(private->cp.guest_cp); 192 out_free_private: 193 mutex_destroy(&private->io_mutex); 194 kfree(private); 195 return ERR_PTR(-ENOMEM); 196 } 197 198 static void vfio_ccw_free_private(struct vfio_ccw_private *private) 199 { 200 struct vfio_ccw_crw *crw, *temp; 201 202 list_for_each_entry_safe(crw, temp, &private->crw, next) { 203 list_del(&crw->next); 204 kfree(crw); 205 } 206 207 kmem_cache_free(vfio_ccw_crw_region, private->crw_region); 208 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 209 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 210 kmem_cache_free(vfio_ccw_io_region, private->io_region); 211 kfree(private->cp.guest_cp); 212 mutex_destroy(&private->io_mutex); 213 kfree(private); 214 } 215 216 static int vfio_ccw_sch_probe(struct subchannel *sch) 217 { 218 struct pmcw *pmcw = &sch->schib.pmcw; 219 struct vfio_ccw_private *private; 220 int ret = -ENOMEM; 221 222 if (pmcw->qf) { 223 dev_warn(&sch->dev, "vfio: ccw: does not support QDIO: %s\n", 224 dev_name(&sch->dev)); 225 return -ENODEV; 226 } 227 228 private = vfio_ccw_alloc_private(sch); 229 if (IS_ERR(private)) 230 return PTR_ERR(private); 231 232 dev_set_drvdata(&sch->dev, private); 233 234 spin_lock_irq(sch->lock); 235 sch->isc = VFIO_CCW_ISC; 236 ret = cio_enable_subchannel(sch, (u32)(unsigned long)sch); 237 spin_unlock_irq(sch->lock); 238 if (ret) 239 goto out_free; 240 241 private->state = VFIO_CCW_STATE_STANDBY; 242 243 ret = vfio_ccw_mdev_reg(sch); 244 if (ret) 245 goto out_disable; 246 247 if (dev_get_uevent_suppress(&sch->dev)) { 248 dev_set_uevent_suppress(&sch->dev, 0); 249 kobject_uevent(&sch->dev.kobj, KOBJ_ADD); 250 } 251 252 VFIO_CCW_MSG_EVENT(4, "bound to subchannel %x.%x.%04x\n", 253 sch->schid.cssid, sch->schid.ssid, 254 sch->schid.sch_no); 255 return 0; 256 257 out_disable: 258 cio_disable_subchannel(sch); 259 out_free: 260 dev_set_drvdata(&sch->dev, NULL); 261 vfio_ccw_free_private(private); 262 return ret; 263 } 264 265 static void vfio_ccw_sch_remove(struct subchannel *sch) 266 { 267 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 268 269 vfio_ccw_sch_quiesce(sch); 270 vfio_ccw_mdev_unreg(sch); 271 272 dev_set_drvdata(&sch->dev, NULL); 273 274 vfio_ccw_free_private(private); 275 276 VFIO_CCW_MSG_EVENT(4, "unbound from subchannel %x.%x.%04x\n", 277 sch->schid.cssid, sch->schid.ssid, 278 sch->schid.sch_no); 279 } 280 281 static void vfio_ccw_sch_shutdown(struct subchannel *sch) 282 { 283 vfio_ccw_sch_quiesce(sch); 284 } 285 286 /** 287 * vfio_ccw_sch_event - process subchannel event 288 * @sch: subchannel 289 * @process: non-zero if function is called in process context 290 * 291 * An unspecified event occurred for this subchannel. Adjust data according 292 * to the current operational state of the subchannel. Return zero when the 293 * event has been handled sufficiently or -EAGAIN when this function should 294 * be called again in process context. 295 */ 296 static int vfio_ccw_sch_event(struct subchannel *sch, int process) 297 { 298 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 299 unsigned long flags; 300 int rc = -EAGAIN; 301 302 spin_lock_irqsave(sch->lock, flags); 303 if (!device_is_registered(&sch->dev)) 304 goto out_unlock; 305 306 if (work_pending(&sch->todo_work)) 307 goto out_unlock; 308 309 if (cio_update_schib(sch)) { 310 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER); 311 rc = 0; 312 goto out_unlock; 313 } 314 315 private = dev_get_drvdata(&sch->dev); 316 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 317 private->state = private->mdev ? VFIO_CCW_STATE_IDLE : 318 VFIO_CCW_STATE_STANDBY; 319 } 320 rc = 0; 321 322 out_unlock: 323 spin_unlock_irqrestore(sch->lock, flags); 324 325 return rc; 326 } 327 328 static void vfio_ccw_queue_crw(struct vfio_ccw_private *private, 329 unsigned int rsc, 330 unsigned int erc, 331 unsigned int rsid) 332 { 333 struct vfio_ccw_crw *crw; 334 335 /* 336 * If unable to allocate a CRW, just drop the event and 337 * carry on. The guest will either see a later one or 338 * learn when it issues its own store subchannel. 339 */ 340 crw = kzalloc(sizeof(*crw), GFP_ATOMIC); 341 if (!crw) 342 return; 343 344 /* 345 * Build the CRW based on the inputs given to us. 346 */ 347 crw->crw.rsc = rsc; 348 crw->crw.erc = erc; 349 crw->crw.rsid = rsid; 350 351 list_add_tail(&crw->next, &private->crw); 352 queue_work(vfio_ccw_work_q, &private->crw_work); 353 } 354 355 static int vfio_ccw_chp_event(struct subchannel *sch, 356 struct chp_link *link, int event) 357 { 358 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 359 int mask = chp_ssd_get_mask(&sch->ssd_info, link); 360 int retry = 255; 361 362 if (!private || !mask) 363 return 0; 364 365 trace_vfio_ccw_chp_event(private->sch->schid, mask, event); 366 VFIO_CCW_MSG_EVENT(2, "%pUl (%x.%x.%04x): mask=0x%x event=%d\n", 367 mdev_uuid(private->mdev), sch->schid.cssid, 368 sch->schid.ssid, sch->schid.sch_no, 369 mask, event); 370 371 if (cio_update_schib(sch)) 372 return -ENODEV; 373 374 switch (event) { 375 case CHP_VARY_OFF: 376 /* Path logically turned off */ 377 sch->opm &= ~mask; 378 sch->lpm &= ~mask; 379 if (sch->schib.pmcw.lpum & mask) 380 cio_cancel_halt_clear(sch, &retry); 381 break; 382 case CHP_OFFLINE: 383 /* Path is gone */ 384 if (sch->schib.pmcw.lpum & mask) 385 cio_cancel_halt_clear(sch, &retry); 386 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_PERRN, 387 link->chpid.id); 388 break; 389 case CHP_VARY_ON: 390 /* Path logically turned on */ 391 sch->opm |= mask; 392 sch->lpm |= mask; 393 break; 394 case CHP_ONLINE: 395 /* Path became available */ 396 sch->lpm |= mask & sch->opm; 397 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_INIT, 398 link->chpid.id); 399 break; 400 } 401 402 return 0; 403 } 404 405 static struct css_device_id vfio_ccw_sch_ids[] = { 406 { .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, }, 407 { /* end of list */ }, 408 }; 409 MODULE_DEVICE_TABLE(css, vfio_ccw_sch_ids); 410 411 static struct css_driver vfio_ccw_sch_driver = { 412 .drv = { 413 .name = "vfio_ccw", 414 .owner = THIS_MODULE, 415 }, 416 .subchannel_type = vfio_ccw_sch_ids, 417 .irq = vfio_ccw_sch_irq, 418 .probe = vfio_ccw_sch_probe, 419 .remove = vfio_ccw_sch_remove, 420 .shutdown = vfio_ccw_sch_shutdown, 421 .sch_event = vfio_ccw_sch_event, 422 .chp_event = vfio_ccw_chp_event, 423 }; 424 425 static int __init vfio_ccw_debug_init(void) 426 { 427 vfio_ccw_debug_msg_id = debug_register("vfio_ccw_msg", 16, 1, 428 11 * sizeof(long)); 429 if (!vfio_ccw_debug_msg_id) 430 goto out_unregister; 431 debug_register_view(vfio_ccw_debug_msg_id, &debug_sprintf_view); 432 debug_set_level(vfio_ccw_debug_msg_id, 2); 433 vfio_ccw_debug_trace_id = debug_register("vfio_ccw_trace", 16, 1, 16); 434 if (!vfio_ccw_debug_trace_id) 435 goto out_unregister; 436 debug_register_view(vfio_ccw_debug_trace_id, &debug_hex_ascii_view); 437 debug_set_level(vfio_ccw_debug_trace_id, 2); 438 return 0; 439 440 out_unregister: 441 debug_unregister(vfio_ccw_debug_msg_id); 442 debug_unregister(vfio_ccw_debug_trace_id); 443 return -1; 444 } 445 446 static void vfio_ccw_debug_exit(void) 447 { 448 debug_unregister(vfio_ccw_debug_msg_id); 449 debug_unregister(vfio_ccw_debug_trace_id); 450 } 451 452 static void vfio_ccw_destroy_regions(void) 453 { 454 kmem_cache_destroy(vfio_ccw_crw_region); 455 kmem_cache_destroy(vfio_ccw_schib_region); 456 kmem_cache_destroy(vfio_ccw_cmd_region); 457 kmem_cache_destroy(vfio_ccw_io_region); 458 } 459 460 static int __init vfio_ccw_sch_init(void) 461 { 462 int ret; 463 464 ret = vfio_ccw_debug_init(); 465 if (ret) 466 return ret; 467 468 vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw"); 469 if (!vfio_ccw_work_q) { 470 ret = -ENOMEM; 471 goto out_regions; 472 } 473 474 vfio_ccw_io_region = kmem_cache_create_usercopy("vfio_ccw_io_region", 475 sizeof(struct ccw_io_region), 0, 476 SLAB_ACCOUNT, 0, 477 sizeof(struct ccw_io_region), NULL); 478 if (!vfio_ccw_io_region) { 479 ret = -ENOMEM; 480 goto out_regions; 481 } 482 483 vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region", 484 sizeof(struct ccw_cmd_region), 0, 485 SLAB_ACCOUNT, 0, 486 sizeof(struct ccw_cmd_region), NULL); 487 if (!vfio_ccw_cmd_region) { 488 ret = -ENOMEM; 489 goto out_regions; 490 } 491 492 vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region", 493 sizeof(struct ccw_schib_region), 0, 494 SLAB_ACCOUNT, 0, 495 sizeof(struct ccw_schib_region), NULL); 496 497 if (!vfio_ccw_schib_region) { 498 ret = -ENOMEM; 499 goto out_regions; 500 } 501 502 vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region", 503 sizeof(struct ccw_crw_region), 0, 504 SLAB_ACCOUNT, 0, 505 sizeof(struct ccw_crw_region), NULL); 506 507 if (!vfio_ccw_crw_region) { 508 ret = -ENOMEM; 509 goto out_regions; 510 } 511 512 ret = mdev_register_driver(&vfio_ccw_mdev_driver); 513 if (ret) 514 goto out_regions; 515 516 isc_register(VFIO_CCW_ISC); 517 ret = css_driver_register(&vfio_ccw_sch_driver); 518 if (ret) { 519 isc_unregister(VFIO_CCW_ISC); 520 goto out_driver; 521 } 522 523 return ret; 524 525 out_driver: 526 mdev_unregister_driver(&vfio_ccw_mdev_driver); 527 out_regions: 528 vfio_ccw_destroy_regions(); 529 destroy_workqueue(vfio_ccw_work_q); 530 vfio_ccw_debug_exit(); 531 return ret; 532 } 533 534 static void __exit vfio_ccw_sch_exit(void) 535 { 536 css_driver_unregister(&vfio_ccw_sch_driver); 537 mdev_unregister_driver(&vfio_ccw_mdev_driver); 538 isc_unregister(VFIO_CCW_ISC); 539 vfio_ccw_destroy_regions(); 540 destroy_workqueue(vfio_ccw_work_q); 541 vfio_ccw_debug_exit(); 542 } 543 module_init(vfio_ccw_sch_init); 544 module_exit(vfio_ccw_sch_exit); 545 546 MODULE_LICENSE("GPL v2"); 547