1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * VFIO based Physical Subchannel device driver 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/slab.h> 16 #include <linux/mdev.h> 17 18 #include <asm/isc.h> 19 20 #include "chp.h" 21 #include "ioasm.h" 22 #include "css.h" 23 #include "vfio_ccw_private.h" 24 25 struct workqueue_struct *vfio_ccw_work_q; 26 static struct kmem_cache *vfio_ccw_io_region; 27 static struct kmem_cache *vfio_ccw_cmd_region; 28 static struct kmem_cache *vfio_ccw_schib_region; 29 static struct kmem_cache *vfio_ccw_crw_region; 30 31 debug_info_t *vfio_ccw_debug_msg_id; 32 debug_info_t *vfio_ccw_debug_trace_id; 33 34 /* 35 * Helpers 36 */ 37 int vfio_ccw_sch_quiesce(struct subchannel *sch) 38 { 39 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 40 DECLARE_COMPLETION_ONSTACK(completion); 41 int iretry, ret = 0; 42 43 iretry = 255; 44 do { 45 46 ret = cio_cancel_halt_clear(sch, &iretry); 47 48 if (ret == -EIO) { 49 pr_err("vfio_ccw: could not quiesce subchannel 0.%x.%04x!\n", 50 sch->schid.ssid, sch->schid.sch_no); 51 break; 52 } 53 54 /* 55 * Flush all I/O and wait for 56 * cancel/halt/clear completion. 57 */ 58 private->completion = &completion; 59 spin_unlock_irq(sch->lock); 60 61 if (ret == -EBUSY) 62 wait_for_completion_timeout(&completion, 3*HZ); 63 64 private->completion = NULL; 65 flush_workqueue(vfio_ccw_work_q); 66 spin_lock_irq(sch->lock); 67 ret = cio_disable_subchannel(sch); 68 } while (ret == -EBUSY); 69 70 return ret; 71 } 72 73 static void vfio_ccw_sch_io_todo(struct work_struct *work) 74 { 75 struct vfio_ccw_private *private; 76 struct irb *irb; 77 bool is_final; 78 bool cp_is_finished = false; 79 80 private = container_of(work, struct vfio_ccw_private, io_work); 81 irb = &private->irb; 82 83 is_final = !(scsw_actl(&irb->scsw) & 84 (SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT)); 85 if (scsw_is_solicited(&irb->scsw)) { 86 cp_update_scsw(&private->cp, &irb->scsw); 87 if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) { 88 cp_free(&private->cp); 89 cp_is_finished = true; 90 } 91 } 92 mutex_lock(&private->io_mutex); 93 memcpy(private->io_region->irb_area, irb, sizeof(*irb)); 94 mutex_unlock(&private->io_mutex); 95 96 /* 97 * Reset to IDLE only if processing of a channel program 98 * has finished. Do not overwrite a possible processing 99 * state if the interrupt was unsolicited, or if the final 100 * interrupt was for HSCH or CSCH. 101 */ 102 if (cp_is_finished) 103 private->state = VFIO_CCW_STATE_IDLE; 104 105 if (private->io_trigger) 106 eventfd_signal(private->io_trigger, 1); 107 } 108 109 static void vfio_ccw_crw_todo(struct work_struct *work) 110 { 111 struct vfio_ccw_private *private; 112 113 private = container_of(work, struct vfio_ccw_private, crw_work); 114 115 if (!list_empty(&private->crw) && private->crw_trigger) 116 eventfd_signal(private->crw_trigger, 1); 117 } 118 119 /* 120 * Css driver callbacks 121 */ 122 static void vfio_ccw_sch_irq(struct subchannel *sch) 123 { 124 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 125 126 inc_irq_stat(IRQIO_CIO); 127 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_INTERRUPT); 128 } 129 130 static struct vfio_ccw_private *vfio_ccw_alloc_private(struct subchannel *sch) 131 { 132 struct vfio_ccw_private *private; 133 134 private = kzalloc(sizeof(*private), GFP_KERNEL); 135 if (!private) 136 return ERR_PTR(-ENOMEM); 137 138 private->sch = sch; 139 mutex_init(&private->io_mutex); 140 private->state = VFIO_CCW_STATE_STANDBY; 141 INIT_LIST_HEAD(&private->crw); 142 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); 143 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); 144 145 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1), 146 GFP_KERNEL); 147 if (!private->cp.guest_cp) 148 goto out_free_private; 149 150 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region, 151 GFP_KERNEL | GFP_DMA); 152 if (!private->io_region) 153 goto out_free_cp; 154 155 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region, 156 GFP_KERNEL | GFP_DMA); 157 if (!private->cmd_region) 158 goto out_free_io; 159 160 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region, 161 GFP_KERNEL | GFP_DMA); 162 163 if (!private->schib_region) 164 goto out_free_cmd; 165 166 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region, 167 GFP_KERNEL | GFP_DMA); 168 169 if (!private->crw_region) 170 goto out_free_schib; 171 return private; 172 173 out_free_schib: 174 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 175 out_free_cmd: 176 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 177 out_free_io: 178 kmem_cache_free(vfio_ccw_io_region, private->io_region); 179 out_free_cp: 180 kfree(private->cp.guest_cp); 181 out_free_private: 182 mutex_destroy(&private->io_mutex); 183 kfree(private); 184 return ERR_PTR(-ENOMEM); 185 } 186 187 static void vfio_ccw_free_private(struct vfio_ccw_private *private) 188 { 189 struct vfio_ccw_crw *crw, *temp; 190 191 list_for_each_entry_safe(crw, temp, &private->crw, next) { 192 list_del(&crw->next); 193 kfree(crw); 194 } 195 196 kmem_cache_free(vfio_ccw_crw_region, private->crw_region); 197 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 198 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 199 kmem_cache_free(vfio_ccw_io_region, private->io_region); 200 kfree(private->cp.guest_cp); 201 mutex_destroy(&private->io_mutex); 202 kfree(private); 203 } 204 static int vfio_ccw_sch_probe(struct subchannel *sch) 205 { 206 struct pmcw *pmcw = &sch->schib.pmcw; 207 struct vfio_ccw_private *private; 208 int ret = -ENOMEM; 209 210 if (pmcw->qf) { 211 dev_warn(&sch->dev, "vfio: ccw: does not support QDIO: %s\n", 212 dev_name(&sch->dev)); 213 return -ENODEV; 214 } 215 216 private = vfio_ccw_alloc_private(sch); 217 if (IS_ERR(private)) 218 return PTR_ERR(private); 219 220 dev_set_drvdata(&sch->dev, private); 221 222 private->mdev_type.sysfs_name = "io"; 223 private->mdev_type.pretty_name = "I/O subchannel (Non-QDIO)"; 224 private->mdev_types[0] = &private->mdev_type; 225 ret = mdev_register_parent(&private->parent, &sch->dev, 226 &vfio_ccw_mdev_driver, 227 private->mdev_types, 1); 228 if (ret) 229 goto out_free; 230 231 VFIO_CCW_MSG_EVENT(4, "bound to subchannel %x.%x.%04x\n", 232 sch->schid.cssid, sch->schid.ssid, 233 sch->schid.sch_no); 234 return 0; 235 236 out_free: 237 dev_set_drvdata(&sch->dev, NULL); 238 vfio_ccw_free_private(private); 239 return ret; 240 } 241 242 static void vfio_ccw_sch_remove(struct subchannel *sch) 243 { 244 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 245 246 mdev_unregister_parent(&private->parent); 247 248 dev_set_drvdata(&sch->dev, NULL); 249 250 vfio_ccw_free_private(private); 251 252 VFIO_CCW_MSG_EVENT(4, "unbound from subchannel %x.%x.%04x\n", 253 sch->schid.cssid, sch->schid.ssid, 254 sch->schid.sch_no); 255 } 256 257 static void vfio_ccw_sch_shutdown(struct subchannel *sch) 258 { 259 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 260 261 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 262 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER); 263 } 264 265 /** 266 * vfio_ccw_sch_event - process subchannel event 267 * @sch: subchannel 268 * @process: non-zero if function is called in process context 269 * 270 * An unspecified event occurred for this subchannel. Adjust data according 271 * to the current operational state of the subchannel. Return zero when the 272 * event has been handled sufficiently or -EAGAIN when this function should 273 * be called again in process context. 274 */ 275 static int vfio_ccw_sch_event(struct subchannel *sch, int process) 276 { 277 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 278 unsigned long flags; 279 int rc = -EAGAIN; 280 281 spin_lock_irqsave(sch->lock, flags); 282 if (!device_is_registered(&sch->dev)) 283 goto out_unlock; 284 285 if (work_pending(&sch->todo_work)) 286 goto out_unlock; 287 288 rc = 0; 289 290 if (cio_update_schib(sch)) 291 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER); 292 293 out_unlock: 294 spin_unlock_irqrestore(sch->lock, flags); 295 296 return rc; 297 } 298 299 static void vfio_ccw_queue_crw(struct vfio_ccw_private *private, 300 unsigned int rsc, 301 unsigned int erc, 302 unsigned int rsid) 303 { 304 struct vfio_ccw_crw *crw; 305 306 /* 307 * If unable to allocate a CRW, just drop the event and 308 * carry on. The guest will either see a later one or 309 * learn when it issues its own store subchannel. 310 */ 311 crw = kzalloc(sizeof(*crw), GFP_ATOMIC); 312 if (!crw) 313 return; 314 315 /* 316 * Build the CRW based on the inputs given to us. 317 */ 318 crw->crw.rsc = rsc; 319 crw->crw.erc = erc; 320 crw->crw.rsid = rsid; 321 322 list_add_tail(&crw->next, &private->crw); 323 queue_work(vfio_ccw_work_q, &private->crw_work); 324 } 325 326 static int vfio_ccw_chp_event(struct subchannel *sch, 327 struct chp_link *link, int event) 328 { 329 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); 330 int mask = chp_ssd_get_mask(&sch->ssd_info, link); 331 int retry = 255; 332 333 if (!private || !mask) 334 return 0; 335 336 trace_vfio_ccw_chp_event(private->sch->schid, mask, event); 337 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: mask=0x%x event=%d\n", 338 sch->schid.cssid, 339 sch->schid.ssid, sch->schid.sch_no, 340 mask, event); 341 342 if (cio_update_schib(sch)) 343 return -ENODEV; 344 345 switch (event) { 346 case CHP_VARY_OFF: 347 /* Path logically turned off */ 348 sch->opm &= ~mask; 349 sch->lpm &= ~mask; 350 if (sch->schib.pmcw.lpum & mask) 351 cio_cancel_halt_clear(sch, &retry); 352 break; 353 case CHP_OFFLINE: 354 /* Path is gone */ 355 if (sch->schib.pmcw.lpum & mask) 356 cio_cancel_halt_clear(sch, &retry); 357 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_PERRN, 358 link->chpid.id); 359 break; 360 case CHP_VARY_ON: 361 /* Path logically turned on */ 362 sch->opm |= mask; 363 sch->lpm |= mask; 364 break; 365 case CHP_ONLINE: 366 /* Path became available */ 367 sch->lpm |= mask & sch->opm; 368 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_INIT, 369 link->chpid.id); 370 break; 371 } 372 373 return 0; 374 } 375 376 static struct css_device_id vfio_ccw_sch_ids[] = { 377 { .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, }, 378 { /* end of list */ }, 379 }; 380 MODULE_DEVICE_TABLE(css, vfio_ccw_sch_ids); 381 382 static struct css_driver vfio_ccw_sch_driver = { 383 .drv = { 384 .name = "vfio_ccw", 385 .owner = THIS_MODULE, 386 }, 387 .subchannel_type = vfio_ccw_sch_ids, 388 .irq = vfio_ccw_sch_irq, 389 .probe = vfio_ccw_sch_probe, 390 .remove = vfio_ccw_sch_remove, 391 .shutdown = vfio_ccw_sch_shutdown, 392 .sch_event = vfio_ccw_sch_event, 393 .chp_event = vfio_ccw_chp_event, 394 }; 395 396 static int __init vfio_ccw_debug_init(void) 397 { 398 vfio_ccw_debug_msg_id = debug_register("vfio_ccw_msg", 16, 1, 399 11 * sizeof(long)); 400 if (!vfio_ccw_debug_msg_id) 401 goto out_unregister; 402 debug_register_view(vfio_ccw_debug_msg_id, &debug_sprintf_view); 403 debug_set_level(vfio_ccw_debug_msg_id, 2); 404 vfio_ccw_debug_trace_id = debug_register("vfio_ccw_trace", 16, 1, 16); 405 if (!vfio_ccw_debug_trace_id) 406 goto out_unregister; 407 debug_register_view(vfio_ccw_debug_trace_id, &debug_hex_ascii_view); 408 debug_set_level(vfio_ccw_debug_trace_id, 2); 409 return 0; 410 411 out_unregister: 412 debug_unregister(vfio_ccw_debug_msg_id); 413 debug_unregister(vfio_ccw_debug_trace_id); 414 return -1; 415 } 416 417 static void vfio_ccw_debug_exit(void) 418 { 419 debug_unregister(vfio_ccw_debug_msg_id); 420 debug_unregister(vfio_ccw_debug_trace_id); 421 } 422 423 static void vfio_ccw_destroy_regions(void) 424 { 425 kmem_cache_destroy(vfio_ccw_crw_region); 426 kmem_cache_destroy(vfio_ccw_schib_region); 427 kmem_cache_destroy(vfio_ccw_cmd_region); 428 kmem_cache_destroy(vfio_ccw_io_region); 429 } 430 431 static int __init vfio_ccw_sch_init(void) 432 { 433 int ret; 434 435 ret = vfio_ccw_debug_init(); 436 if (ret) 437 return ret; 438 439 vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw"); 440 if (!vfio_ccw_work_q) { 441 ret = -ENOMEM; 442 goto out_regions; 443 } 444 445 vfio_ccw_io_region = kmem_cache_create_usercopy("vfio_ccw_io_region", 446 sizeof(struct ccw_io_region), 0, 447 SLAB_ACCOUNT, 0, 448 sizeof(struct ccw_io_region), NULL); 449 if (!vfio_ccw_io_region) { 450 ret = -ENOMEM; 451 goto out_regions; 452 } 453 454 vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region", 455 sizeof(struct ccw_cmd_region), 0, 456 SLAB_ACCOUNT, 0, 457 sizeof(struct ccw_cmd_region), NULL); 458 if (!vfio_ccw_cmd_region) { 459 ret = -ENOMEM; 460 goto out_regions; 461 } 462 463 vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region", 464 sizeof(struct ccw_schib_region), 0, 465 SLAB_ACCOUNT, 0, 466 sizeof(struct ccw_schib_region), NULL); 467 468 if (!vfio_ccw_schib_region) { 469 ret = -ENOMEM; 470 goto out_regions; 471 } 472 473 vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region", 474 sizeof(struct ccw_crw_region), 0, 475 SLAB_ACCOUNT, 0, 476 sizeof(struct ccw_crw_region), NULL); 477 478 if (!vfio_ccw_crw_region) { 479 ret = -ENOMEM; 480 goto out_regions; 481 } 482 483 ret = mdev_register_driver(&vfio_ccw_mdev_driver); 484 if (ret) 485 goto out_regions; 486 487 isc_register(VFIO_CCW_ISC); 488 ret = css_driver_register(&vfio_ccw_sch_driver); 489 if (ret) { 490 isc_unregister(VFIO_CCW_ISC); 491 goto out_driver; 492 } 493 494 return ret; 495 496 out_driver: 497 mdev_unregister_driver(&vfio_ccw_mdev_driver); 498 out_regions: 499 vfio_ccw_destroy_regions(); 500 destroy_workqueue(vfio_ccw_work_q); 501 vfio_ccw_debug_exit(); 502 return ret; 503 } 504 505 static void __exit vfio_ccw_sch_exit(void) 506 { 507 css_driver_unregister(&vfio_ccw_sch_driver); 508 mdev_unregister_driver(&vfio_ccw_mdev_driver); 509 isc_unregister(VFIO_CCW_ISC); 510 vfio_ccw_destroy_regions(); 511 destroy_workqueue(vfio_ccw_work_q); 512 vfio_ccw_debug_exit(); 513 } 514 module_init(vfio_ccw_sch_init); 515 module_exit(vfio_ccw_sch_exit); 516 517 MODULE_LICENSE("GPL v2"); 518