1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2019 IBM Corp. 3 #include <linux/idr.h> 4 #include "ocxl_internal.h" 5 6 static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn) 7 { 8 return (get_device(&fn->dev) == NULL) ? NULL : fn; 9 } 10 11 static void ocxl_fn_put(struct ocxl_fn *fn) 12 { 13 put_device(&fn->dev); 14 } 15 16 static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn) 17 { 18 struct ocxl_afu *afu; 19 20 afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL); 21 if (!afu) 22 return NULL; 23 24 kref_init(&afu->kref); 25 mutex_init(&afu->contexts_lock); 26 mutex_init(&afu->afu_control_lock); 27 idr_init(&afu->contexts_idr); 28 afu->fn = fn; 29 ocxl_fn_get(fn); 30 return afu; 31 } 32 33 static void free_afu(struct kref *kref) 34 { 35 struct ocxl_afu *afu = container_of(kref, struct ocxl_afu, kref); 36 37 idr_destroy(&afu->contexts_idr); 38 ocxl_fn_put(afu->fn); 39 kfree(afu); 40 } 41 42 void ocxl_afu_get(struct ocxl_afu *afu) 43 { 44 kref_get(&afu->kref); 45 } 46 EXPORT_SYMBOL_GPL(ocxl_afu_get); 47 48 void ocxl_afu_put(struct ocxl_afu *afu) 49 { 50 kref_put(&afu->kref, free_afu); 51 } 52 EXPORT_SYMBOL_GPL(ocxl_afu_put); 53 54 static int assign_afu_actag(struct ocxl_afu *afu) 55 { 56 struct ocxl_fn *fn = afu->fn; 57 int actag_count, actag_offset; 58 struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); 59 60 /* 61 * if there were not enough actags for the function, each afu 62 * reduces its count as well 63 */ 64 actag_count = afu->config.actag_supported * 65 fn->actag_enabled / fn->actag_supported; 66 actag_offset = ocxl_actag_afu_alloc(fn, actag_count); 67 if (actag_offset < 0) { 68 dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n", 69 actag_count, actag_offset); 70 return actag_offset; 71 } 72 afu->actag_base = fn->actag_base + actag_offset; 73 afu->actag_enabled = actag_count; 74 75 ocxl_config_set_afu_actag(pci_dev, afu->config.dvsec_afu_control_pos, 76 afu->actag_base, afu->actag_enabled); 77 dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n", 78 afu->actag_base, afu->actag_enabled); 79 return 0; 80 } 81 82 static void reclaim_afu_actag(struct ocxl_afu *afu) 83 { 84 struct ocxl_fn *fn = afu->fn; 85 int start_offset, size; 86 87 start_offset = afu->actag_base - fn->actag_base; 88 size = afu->actag_enabled; 89 ocxl_actag_afu_free(afu->fn, start_offset, size); 90 } 91 92 static int assign_afu_pasid(struct ocxl_afu *afu) 93 { 94 struct ocxl_fn *fn = afu->fn; 95 int pasid_count, pasid_offset; 96 struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); 97 98 /* 99 * We only support the case where the function configuration 100 * requested enough PASIDs to cover all AFUs. 101 */ 102 pasid_count = 1 << afu->config.pasid_supported_log; 103 pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count); 104 if (pasid_offset < 0) { 105 dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n", 106 pasid_count, pasid_offset); 107 return pasid_offset; 108 } 109 afu->pasid_base = fn->pasid_base + pasid_offset; 110 afu->pasid_count = 0; 111 afu->pasid_max = pasid_count; 112 113 ocxl_config_set_afu_pasid(pci_dev, afu->config.dvsec_afu_control_pos, 114 afu->pasid_base, 115 afu->config.pasid_supported_log); 116 dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n", 117 afu->pasid_base, pasid_count); 118 return 0; 119 } 120 121 static void reclaim_afu_pasid(struct ocxl_afu *afu) 122 { 123 struct ocxl_fn *fn = afu->fn; 124 int start_offset, size; 125 126 start_offset = afu->pasid_base - fn->pasid_base; 127 size = 1 << afu->config.pasid_supported_log; 128 ocxl_pasid_afu_free(afu->fn, start_offset, size); 129 } 130 131 static int reserve_fn_bar(struct ocxl_fn *fn, int bar) 132 { 133 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 134 int rc, idx; 135 136 if (bar != 0 && bar != 2 && bar != 4) 137 return -EINVAL; 138 139 idx = bar >> 1; 140 if (fn->bar_used[idx]++ == 0) { 141 rc = pci_request_region(dev, bar, "ocxl"); 142 if (rc) 143 return rc; 144 } 145 return 0; 146 } 147 148 static void release_fn_bar(struct ocxl_fn *fn, int bar) 149 { 150 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 151 int idx; 152 153 if (bar != 0 && bar != 2 && bar != 4) 154 return; 155 156 idx = bar >> 1; 157 if (--fn->bar_used[idx] == 0) 158 pci_release_region(dev, bar); 159 WARN_ON(fn->bar_used[idx] < 0); 160 } 161 162 static int map_mmio_areas(struct ocxl_afu *afu) 163 { 164 int rc; 165 struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent); 166 167 rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar); 168 if (rc) 169 return rc; 170 171 rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar); 172 if (rc) { 173 release_fn_bar(afu->fn, afu->config.global_mmio_bar); 174 return rc; 175 } 176 177 afu->global_mmio_start = 178 pci_resource_start(pci_dev, afu->config.global_mmio_bar) + 179 afu->config.global_mmio_offset; 180 afu->pp_mmio_start = 181 pci_resource_start(pci_dev, afu->config.pp_mmio_bar) + 182 afu->config.pp_mmio_offset; 183 184 afu->global_mmio_ptr = ioremap(afu->global_mmio_start, 185 afu->config.global_mmio_size); 186 if (!afu->global_mmio_ptr) { 187 release_fn_bar(afu->fn, afu->config.pp_mmio_bar); 188 release_fn_bar(afu->fn, afu->config.global_mmio_bar); 189 dev_err(&pci_dev->dev, "Error mapping global mmio area\n"); 190 return -ENOMEM; 191 } 192 193 /* 194 * Leave an empty page between the per-process mmio area and 195 * the AFU interrupt mappings 196 */ 197 afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE; 198 return 0; 199 } 200 201 static void unmap_mmio_areas(struct ocxl_afu *afu) 202 { 203 if (afu->global_mmio_ptr) { 204 iounmap(afu->global_mmio_ptr); 205 afu->global_mmio_ptr = NULL; 206 } 207 afu->global_mmio_start = 0; 208 afu->pp_mmio_start = 0; 209 release_fn_bar(afu->fn, afu->config.pp_mmio_bar); 210 release_fn_bar(afu->fn, afu->config.global_mmio_bar); 211 } 212 213 static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev) 214 { 215 int rc; 216 217 rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx); 218 if (rc) 219 return rc; 220 221 rc = assign_afu_actag(afu); 222 if (rc) 223 return rc; 224 225 rc = assign_afu_pasid(afu); 226 if (rc) 227 goto err_free_actag; 228 229 rc = map_mmio_areas(afu); 230 if (rc) 231 goto err_free_pasid; 232 233 return 0; 234 235 err_free_pasid: 236 reclaim_afu_pasid(afu); 237 err_free_actag: 238 reclaim_afu_actag(afu); 239 return rc; 240 } 241 242 static void deconfigure_afu(struct ocxl_afu *afu) 243 { 244 unmap_mmio_areas(afu); 245 reclaim_afu_pasid(afu); 246 reclaim_afu_actag(afu); 247 } 248 249 static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu) 250 { 251 ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1); 252 253 return 0; 254 } 255 256 static void deactivate_afu(struct ocxl_afu *afu) 257 { 258 struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent); 259 260 ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0); 261 } 262 263 static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx) 264 { 265 int rc; 266 struct ocxl_afu *afu; 267 268 afu = alloc_afu(fn); 269 if (!afu) 270 return -ENOMEM; 271 272 rc = configure_afu(afu, afu_idx, dev); 273 if (rc) { 274 ocxl_afu_put(afu); 275 return rc; 276 } 277 278 rc = activate_afu(dev, afu); 279 if (rc) { 280 deconfigure_afu(afu); 281 ocxl_afu_put(afu); 282 return rc; 283 } 284 285 list_add_tail(&afu->list, &fn->afu_list); 286 287 return 0; 288 } 289 290 static void remove_afu(struct ocxl_afu *afu) 291 { 292 list_del(&afu->list); 293 ocxl_context_detach_all(afu); 294 deactivate_afu(afu); 295 deconfigure_afu(afu); 296 ocxl_afu_put(afu); // matches the implicit get in alloc_afu 297 } 298 299 static struct ocxl_fn *alloc_function(void) 300 { 301 struct ocxl_fn *fn; 302 303 fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL); 304 if (!fn) 305 return NULL; 306 307 INIT_LIST_HEAD(&fn->afu_list); 308 INIT_LIST_HEAD(&fn->pasid_list); 309 INIT_LIST_HEAD(&fn->actag_list); 310 311 return fn; 312 } 313 314 static void free_function(struct ocxl_fn *fn) 315 { 316 WARN_ON(!list_empty(&fn->afu_list)); 317 WARN_ON(!list_empty(&fn->pasid_list)); 318 kfree(fn); 319 } 320 321 static void free_function_dev(struct device *dev) 322 { 323 struct ocxl_fn *fn = container_of(dev, struct ocxl_fn, dev); 324 325 free_function(fn); 326 } 327 328 static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev) 329 { 330 fn->dev.parent = &dev->dev; 331 fn->dev.release = free_function_dev; 332 return dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev)); 333 } 334 335 static int assign_function_actag(struct ocxl_fn *fn) 336 { 337 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 338 u16 base, enabled, supported; 339 int rc; 340 341 rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported); 342 if (rc) 343 return rc; 344 345 fn->actag_base = base; 346 fn->actag_enabled = enabled; 347 fn->actag_supported = supported; 348 349 ocxl_config_set_actag(dev, fn->config.dvsec_function_pos, 350 fn->actag_base, fn->actag_enabled); 351 dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n", 352 fn->actag_base, fn->actag_enabled); 353 return 0; 354 } 355 356 static int set_function_pasid(struct ocxl_fn *fn) 357 { 358 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 359 int rc, desired_count, max_count; 360 361 /* A function may not require any PASID */ 362 if (fn->config.max_pasid_log < 0) 363 return 0; 364 365 rc = ocxl_config_get_pasid_info(dev, &max_count); 366 if (rc) 367 return rc; 368 369 desired_count = 1 << fn->config.max_pasid_log; 370 371 if (desired_count > max_count) { 372 dev_err(&fn->dev, 373 "Function requires more PASIDs than is available (%d vs. %d)\n", 374 desired_count, max_count); 375 return -ENOSPC; 376 } 377 378 fn->pasid_base = 0; 379 return 0; 380 } 381 382 static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev) 383 { 384 int rc; 385 386 rc = pci_enable_device(dev); 387 if (rc) { 388 dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc); 389 return rc; 390 } 391 392 /* 393 * Once it has been confirmed to work on our hardware, we 394 * should reset the function, to force the adapter to restart 395 * from scratch. 396 * A function reset would also reset all its AFUs. 397 * 398 * Some hints for implementation: 399 * 400 * - there's not status bit to know when the reset is done. We 401 * should try reading the config space to know when it's 402 * done. 403 * - probably something like: 404 * Reset 405 * wait 100ms 406 * issue config read 407 * allow device up to 1 sec to return success on config 408 * read before declaring it broken 409 * 410 * Some shared logic on the card (CFG, TLX) won't be reset, so 411 * there's no guarantee that it will be enough. 412 */ 413 rc = ocxl_config_read_function(dev, &fn->config); 414 if (rc) 415 return rc; 416 417 rc = set_function_device(fn, dev); 418 if (rc) 419 return rc; 420 421 rc = assign_function_actag(fn); 422 if (rc) 423 return rc; 424 425 rc = set_function_pasid(fn); 426 if (rc) 427 return rc; 428 429 rc = ocxl_link_setup(dev, 0, &fn->link); 430 if (rc) 431 return rc; 432 433 rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos); 434 if (rc) { 435 ocxl_link_release(dev, fn->link); 436 return rc; 437 } 438 return 0; 439 } 440 441 static void deconfigure_function(struct ocxl_fn *fn) 442 { 443 struct pci_dev *dev = to_pci_dev(fn->dev.parent); 444 445 ocxl_link_release(dev, fn->link); 446 pci_disable_device(dev); 447 } 448 449 static struct ocxl_fn *init_function(struct pci_dev *dev) 450 { 451 struct ocxl_fn *fn; 452 int rc; 453 454 fn = alloc_function(); 455 if (!fn) 456 return ERR_PTR(-ENOMEM); 457 458 rc = configure_function(fn, dev); 459 if (rc) { 460 free_function(fn); 461 return ERR_PTR(rc); 462 } 463 464 rc = device_register(&fn->dev); 465 if (rc) { 466 deconfigure_function(fn); 467 put_device(&fn->dev); 468 return ERR_PTR(rc); 469 } 470 return fn; 471 } 472 473 // Device detection & initialisation 474 475 struct ocxl_fn *ocxl_function_open(struct pci_dev *dev) 476 { 477 int rc, afu_count = 0; 478 u8 afu; 479 struct ocxl_fn *fn; 480 481 if (!radix_enabled()) { 482 dev_err(&dev->dev, "Unsupported memory model (hash)\n"); 483 return ERR_PTR(-ENODEV); 484 } 485 486 fn = init_function(dev); 487 if (IS_ERR(fn)) { 488 dev_err(&dev->dev, "function init failed: %li\n", 489 PTR_ERR(fn)); 490 return fn; 491 } 492 493 for (afu = 0; afu <= fn->config.max_afu_index; afu++) { 494 rc = ocxl_config_check_afu_index(dev, &fn->config, afu); 495 if (rc > 0) { 496 rc = init_afu(dev, fn, afu); 497 if (rc) { 498 dev_err(&dev->dev, 499 "Can't initialize AFU index %d\n", afu); 500 continue; 501 } 502 afu_count++; 503 } 504 } 505 dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count); 506 return fn; 507 } 508 EXPORT_SYMBOL_GPL(ocxl_function_open); 509 510 struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn) 511 { 512 return &fn->afu_list; 513 } 514 EXPORT_SYMBOL_GPL(ocxl_function_afu_list); 515 516 struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx) 517 { 518 struct ocxl_afu *afu; 519 520 list_for_each_entry(afu, &fn->afu_list, list) { 521 if (afu->config.idx == afu_idx) 522 return afu; 523 } 524 525 return NULL; 526 } 527 EXPORT_SYMBOL_GPL(ocxl_function_fetch_afu); 528 529 const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn) 530 { 531 return &fn->config; 532 } 533 EXPORT_SYMBOL_GPL(ocxl_function_config); 534 535 void ocxl_function_close(struct ocxl_fn *fn) 536 { 537 struct ocxl_afu *afu, *tmp; 538 539 list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) { 540 remove_afu(afu); 541 } 542 543 deconfigure_function(fn); 544 device_unregister(&fn->dev); 545 } 546 EXPORT_SYMBOL_GPL(ocxl_function_close); 547 548 // AFU Metadata 549 550 struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu) 551 { 552 return &afu->config; 553 } 554 EXPORT_SYMBOL_GPL(ocxl_afu_config); 555 556 void ocxl_afu_set_private(struct ocxl_afu *afu, void *private) 557 { 558 afu->private = private; 559 } 560 EXPORT_SYMBOL_GPL(ocxl_afu_set_private); 561 562 void *ocxl_afu_get_private(struct ocxl_afu *afu) 563 { 564 if (afu) 565 return afu->private; 566 567 return NULL; 568 } 569 EXPORT_SYMBOL_GPL(ocxl_afu_get_private); 570