1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2017 IBM Corp. 3 #include <linux/module.h> 4 #include <linux/pci.h> 5 #include <linux/idr.h> 6 #include <asm/pnv-ocxl.h> 7 #include "ocxl_internal.h" 8 9 /* 10 * Any opencapi device which wants to use this 'generic' driver should 11 * use the 0x062B device ID. Vendors should define the subsystem 12 * vendor/device ID to help differentiate devices. 13 */ 14 static const struct pci_device_id ocxl_pci_tbl[] = { 15 { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x062B), }, 16 { } 17 }; 18 MODULE_DEVICE_TABLE(pci, ocxl_pci_tbl); 19 20 21 static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn) 22 { 23 return (get_device(&fn->dev) == NULL) ? NULL : fn; 24 } 25 26 static void ocxl_fn_put(struct ocxl_fn *fn) 27 { 28 put_device(&fn->dev); 29 } 30 31 struct ocxl_afu *ocxl_afu_get(struct ocxl_afu *afu) 32 { 33 return (get_device(&afu->dev) == NULL) ? NULL : afu; 34 } 35 36 void ocxl_afu_put(struct ocxl_afu *afu) 37 { 38 put_device(&afu->dev); 39 } 40 41 static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn) 42 { 43 struct ocxl_afu *afu; 44 45 afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL); 46 if (!afu) 47 return NULL; 48 49 mutex_init(&afu->contexts_lock); 50 mutex_init(&afu->afu_control_lock); 51 idr_init(&afu->contexts_idr); 52 afu->fn = fn; 53 ocxl_fn_get(fn); 54 return afu; 55 } 56 57 static void free_afu(struct ocxl_afu *afu) 58 { 59 idr_destroy(&afu->contexts_idr); 60 ocxl_fn_put(afu->fn); 61 kfree(afu); 62 } 63 64 static void free_afu_dev(struct device *dev) 65 { 66 struct ocxl_afu *afu = to_ocxl_afu(dev); 67 68 ocxl_unregister_afu(afu); 69 free_afu(afu); 70 } 71 72 static int set_afu_device(struct ocxl_afu *afu, const char *location) 73 { 74 struct ocxl_fn *fn = afu->fn; 75 int rc; 76 77 afu->dev.parent = &fn->dev; 78 afu->dev.release = free_afu_dev; 79 rc = dev_set_name(&afu->dev, "%s.%s.%hhu", afu->config.name, location, 80 afu->config.idx); 81 return rc; 82 } 83 84 static int assign_afu_actag(struct ocxl_afu *afu, struct pci_dev *dev) 85 { 86 struct ocxl_fn *fn = afu->fn; 87 int actag_count, actag_offset; 88 89 /* 90 * if there were not enough actags for the function, each afu 91 * reduces its count as well 92 */ 93 actag_count = afu->config.actag_supported * 94 fn->actag_enabled / fn->actag_supported; 95 actag_offset = ocxl_actag_afu_alloc(fn, actag_count); 96 if (actag_offset < 0) { 97 dev_err(&afu->dev, "Can't allocate %d actags for AFU: %d\n", 98 actag_count, actag_offset); 99 return actag_offset; 100 } 101 afu->actag_base = fn->actag_base + actag_offset; 102 afu->actag_enabled = actag_count; 103 104 ocxl_config_set_afu_actag(dev, afu->config.dvsec_afu_control_pos, 105 afu->actag_base, afu->actag_enabled); 106 dev_dbg(&afu->dev, "actag base=%d enabled=%d\n", 107 afu->actag_base, afu->actag_enabled); 108 return 0; 109 } 110 111 static void reclaim_afu_actag(struct ocxl_afu *afu) 112 { 113 struct ocxl_fn *fn = afu->fn; 114 int start_offset, size; 115 116 start_offset = afu->actag_base - fn->actag_base; 117 size = afu->actag_enabled; 118 ocxl_actag_afu_free(afu->fn, start_offset, size); 119 } 120 121 static int assign_afu_pasid(struct ocxl_afu *afu, struct pci_dev *dev) 122 { 123 struct ocxl_fn *fn = afu->fn; 124 int pasid_count, pasid_offset; 125 126 /* 127 * We only support the case where the function configuration 128 * requested enough PASIDs to cover all AFUs. 129 */ 130 pasid_count = 1 << afu->config.pasid_supported_log; 131 pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count); 132 if (pasid_offset < 0) { 133 dev_err(&afu->dev, "Can't allocate %d PASIDs for AFU: %d\n", 134 pasid_count, pasid_offset); 135 return pasid_offset; 136 } 137 afu->pasid_base = fn->pasid_base + pasid_offset; 138 afu->pasid_count = 0; 139 afu->pasid_max = pasid_count; 140 141 ocxl_config_set_afu_pasid(dev, afu->config.dvsec_afu_control_pos, 142 afu->pasid_base, 143 afu->config.pasid_supported_log); 144 dev_dbg(&afu->dev, "PASID base=%d, enabled=%d\n", 145 afu->pasid_base, pasid_count); 146 return 0; 147 } 148 149 static void reclaim_afu_pasid(struct ocxl_afu *afu) 150 { 151 struct ocxl_fn *fn = afu->fn; 152 int start_offset, size; 153 154 start_offset = afu->pasid_base - fn->pasid_base; 155 size = 1 << afu->config.pasid_supported_log; 156 ocxl_pasid_afu_free(afu->fn, start_offset, size); 157 } 158 159 static int reserve_fn_bar(struct ocxl_fn *fn, int bar) 160 { 161 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 162 int rc, idx; 163 164 if (bar != 0 && bar != 2 && bar != 4) 165 return -EINVAL; 166 167 idx = bar >> 1; 168 if (fn->bar_used[idx]++ == 0) { 169 rc = pci_request_region(dev, bar, "ocxl"); 170 if (rc) 171 return rc; 172 } 173 return 0; 174 } 175 176 static void release_fn_bar(struct ocxl_fn *fn, int bar) 177 { 178 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 179 int idx; 180 181 if (bar != 0 && bar != 2 && bar != 4) 182 return; 183 184 idx = bar >> 1; 185 if (--fn->bar_used[idx] == 0) 186 pci_release_region(dev, bar); 187 WARN_ON(fn->bar_used[idx] < 0); 188 } 189 190 static int map_mmio_areas(struct ocxl_afu *afu, struct pci_dev *dev) 191 { 192 int rc; 193 194 rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar); 195 if (rc) 196 return rc; 197 198 rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar); 199 if (rc) { 200 release_fn_bar(afu->fn, afu->config.global_mmio_bar); 201 return rc; 202 } 203 204 afu->global_mmio_start = 205 pci_resource_start(dev, afu->config.global_mmio_bar) + 206 afu->config.global_mmio_offset; 207 afu->pp_mmio_start = 208 pci_resource_start(dev, afu->config.pp_mmio_bar) + 209 afu->config.pp_mmio_offset; 210 211 afu->global_mmio_ptr = ioremap(afu->global_mmio_start, 212 afu->config.global_mmio_size); 213 if (!afu->global_mmio_ptr) { 214 release_fn_bar(afu->fn, afu->config.pp_mmio_bar); 215 release_fn_bar(afu->fn, afu->config.global_mmio_bar); 216 dev_err(&dev->dev, "Error mapping global mmio area\n"); 217 return -ENOMEM; 218 } 219 220 /* 221 * Leave an empty page between the per-process mmio area and 222 * the AFU interrupt mappings 223 */ 224 afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE; 225 return 0; 226 } 227 228 static void unmap_mmio_areas(struct ocxl_afu *afu) 229 { 230 if (afu->global_mmio_ptr) { 231 iounmap(afu->global_mmio_ptr); 232 afu->global_mmio_ptr = NULL; 233 } 234 afu->global_mmio_start = 0; 235 afu->pp_mmio_start = 0; 236 release_fn_bar(afu->fn, afu->config.pp_mmio_bar); 237 release_fn_bar(afu->fn, afu->config.global_mmio_bar); 238 } 239 240 static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev) 241 { 242 int rc; 243 244 rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx); 245 if (rc) 246 return rc; 247 248 rc = set_afu_device(afu, dev_name(&dev->dev)); 249 if (rc) 250 return rc; 251 252 rc = assign_afu_actag(afu, dev); 253 if (rc) 254 return rc; 255 256 rc = assign_afu_pasid(afu, dev); 257 if (rc) { 258 reclaim_afu_actag(afu); 259 return rc; 260 } 261 262 rc = map_mmio_areas(afu, dev); 263 if (rc) { 264 reclaim_afu_pasid(afu); 265 reclaim_afu_actag(afu); 266 return rc; 267 } 268 return 0; 269 } 270 271 static void deconfigure_afu(struct ocxl_afu *afu) 272 { 273 unmap_mmio_areas(afu); 274 reclaim_afu_pasid(afu); 275 reclaim_afu_actag(afu); 276 } 277 278 static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu) 279 { 280 int rc; 281 282 ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1); 283 /* 284 * Char device creation is the last step, as processes can 285 * call our driver immediately, so all our inits must be finished. 286 */ 287 rc = ocxl_create_cdev(afu); 288 if (rc) 289 return rc; 290 return 0; 291 } 292 293 static void deactivate_afu(struct ocxl_afu *afu) 294 { 295 struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent); 296 297 ocxl_destroy_cdev(afu); 298 ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0); 299 } 300 301 static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx) 302 { 303 int rc; 304 struct ocxl_afu *afu; 305 306 afu = alloc_afu(fn); 307 if (!afu) 308 return -ENOMEM; 309 310 rc = configure_afu(afu, afu_idx, dev); 311 if (rc) { 312 free_afu(afu); 313 return rc; 314 } 315 316 rc = ocxl_register_afu(afu); 317 if (rc) 318 goto err; 319 320 rc = ocxl_sysfs_add_afu(afu); 321 if (rc) 322 goto err; 323 324 rc = activate_afu(dev, afu); 325 if (rc) 326 goto err_sys; 327 328 list_add_tail(&afu->list, &fn->afu_list); 329 return 0; 330 331 err_sys: 332 ocxl_sysfs_remove_afu(afu); 333 err: 334 deconfigure_afu(afu); 335 device_unregister(&afu->dev); 336 return rc; 337 } 338 339 static void remove_afu(struct ocxl_afu *afu) 340 { 341 list_del(&afu->list); 342 ocxl_context_detach_all(afu); 343 deactivate_afu(afu); 344 ocxl_sysfs_remove_afu(afu); 345 deconfigure_afu(afu); 346 device_unregister(&afu->dev); 347 } 348 349 static struct ocxl_fn *alloc_function(struct pci_dev *dev) 350 { 351 struct ocxl_fn *fn; 352 353 fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL); 354 if (!fn) 355 return NULL; 356 357 INIT_LIST_HEAD(&fn->afu_list); 358 INIT_LIST_HEAD(&fn->pasid_list); 359 INIT_LIST_HEAD(&fn->actag_list); 360 return fn; 361 } 362 363 static void free_function(struct ocxl_fn *fn) 364 { 365 WARN_ON(!list_empty(&fn->afu_list)); 366 WARN_ON(!list_empty(&fn->pasid_list)); 367 kfree(fn); 368 } 369 370 static void free_function_dev(struct device *dev) 371 { 372 struct ocxl_fn *fn = to_ocxl_function(dev); 373 374 free_function(fn); 375 } 376 377 static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev) 378 { 379 int rc; 380 381 fn->dev.parent = &dev->dev; 382 fn->dev.release = free_function_dev; 383 rc = dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev)); 384 if (rc) 385 return rc; 386 pci_set_drvdata(dev, fn); 387 return 0; 388 } 389 390 static int assign_function_actag(struct ocxl_fn *fn) 391 { 392 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 393 u16 base, enabled, supported; 394 int rc; 395 396 rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported); 397 if (rc) 398 return rc; 399 400 fn->actag_base = base; 401 fn->actag_enabled = enabled; 402 fn->actag_supported = supported; 403 404 ocxl_config_set_actag(dev, fn->config.dvsec_function_pos, 405 fn->actag_base, fn->actag_enabled); 406 dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n", 407 fn->actag_base, fn->actag_enabled); 408 return 0; 409 } 410 411 static int set_function_pasid(struct ocxl_fn *fn) 412 { 413 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 414 int rc, desired_count, max_count; 415 416 /* A function may not require any PASID */ 417 if (fn->config.max_pasid_log < 0) 418 return 0; 419 420 rc = ocxl_config_get_pasid_info(dev, &max_count); 421 if (rc) 422 return rc; 423 424 desired_count = 1 << fn->config.max_pasid_log; 425 426 if (desired_count > max_count) { 427 dev_err(&fn->dev, 428 "Function requires more PASIDs than is available (%d vs. %d)\n", 429 desired_count, max_count); 430 return -ENOSPC; 431 } 432 433 fn->pasid_base = 0; 434 return 0; 435 } 436 437 static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev) 438 { 439 int rc; 440 441 rc = pci_enable_device(dev); 442 if (rc) { 443 dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc); 444 return rc; 445 } 446 447 /* 448 * Once it has been confirmed to work on our hardware, we 449 * should reset the function, to force the adapter to restart 450 * from scratch. 451 * A function reset would also reset all its AFUs. 452 * 453 * Some hints for implementation: 454 * 455 * - there's not status bit to know when the reset is done. We 456 * should try reading the config space to know when it's 457 * done. 458 * - probably something like: 459 * Reset 460 * wait 100ms 461 * issue config read 462 * allow device up to 1 sec to return success on config 463 * read before declaring it broken 464 * 465 * Some shared logic on the card (CFG, TLX) won't be reset, so 466 * there's no guarantee that it will be enough. 467 */ 468 rc = ocxl_config_read_function(dev, &fn->config); 469 if (rc) 470 return rc; 471 472 rc = set_function_device(fn, dev); 473 if (rc) 474 return rc; 475 476 rc = assign_function_actag(fn); 477 if (rc) 478 return rc; 479 480 rc = set_function_pasid(fn); 481 if (rc) 482 return rc; 483 484 rc = ocxl_link_setup(dev, 0, &fn->link); 485 if (rc) 486 return rc; 487 488 rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos); 489 if (rc) { 490 ocxl_link_release(dev, fn->link); 491 return rc; 492 } 493 return 0; 494 } 495 496 static void deconfigure_function(struct ocxl_fn *fn) 497 { 498 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 499 500 ocxl_link_release(dev, fn->link); 501 pci_disable_device(dev); 502 } 503 504 static struct ocxl_fn *init_function(struct pci_dev *dev) 505 { 506 struct ocxl_fn *fn; 507 int rc; 508 509 fn = alloc_function(dev); 510 if (!fn) 511 return ERR_PTR(-ENOMEM); 512 513 rc = configure_function(fn, dev); 514 if (rc) { 515 free_function(fn); 516 return ERR_PTR(rc); 517 } 518 519 rc = device_register(&fn->dev); 520 if (rc) { 521 deconfigure_function(fn); 522 put_device(&fn->dev); 523 return ERR_PTR(rc); 524 } 525 return fn; 526 } 527 528 static void remove_function(struct ocxl_fn *fn) 529 { 530 deconfigure_function(fn); 531 device_unregister(&fn->dev); 532 } 533 534 static int ocxl_probe(struct pci_dev *dev, const struct pci_device_id *id) 535 { 536 int rc, afu_count = 0; 537 u8 afu; 538 struct ocxl_fn *fn; 539 540 if (!radix_enabled()) { 541 dev_err(&dev->dev, "Unsupported memory model (hash)\n"); 542 return -ENODEV; 543 } 544 545 fn = init_function(dev); 546 if (IS_ERR(fn)) { 547 dev_err(&dev->dev, "function init failed: %li\n", 548 PTR_ERR(fn)); 549 return PTR_ERR(fn); 550 } 551 552 for (afu = 0; afu <= fn->config.max_afu_index; afu++) { 553 rc = ocxl_config_check_afu_index(dev, &fn->config, afu); 554 if (rc > 0) { 555 rc = init_afu(dev, fn, afu); 556 if (rc) { 557 dev_err(&dev->dev, 558 "Can't initialize AFU index %d\n", afu); 559 continue; 560 } 561 afu_count++; 562 } 563 } 564 dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count); 565 return 0; 566 } 567 568 static void ocxl_remove(struct pci_dev *dev) 569 { 570 struct ocxl_afu *afu, *tmp; 571 struct ocxl_fn *fn = pci_get_drvdata(dev); 572 573 list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) { 574 remove_afu(afu); 575 } 576 remove_function(fn); 577 } 578 579 struct pci_driver ocxl_pci_driver = { 580 .name = "ocxl", 581 .id_table = ocxl_pci_tbl, 582 .probe = ocxl_probe, 583 .remove = ocxl_remove, 584 .shutdown = ocxl_remove, 585 }; 586