1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2014 IBM Corp. 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/sched.h> 8 #include <linux/sched/clock.h> 9 #include <linux/slab.h> 10 #include <linux/mutex.h> 11 #include <linux/mm.h> 12 #include <linux/uaccess.h> 13 #include <linux/delay.h> 14 #include <linux/irqdomain.h> 15 #include <asm/synch.h> 16 #include <asm/switch_to.h> 17 #include <misc/cxl-base.h> 18 19 #include "cxl.h" 20 #include "trace.h" 21 22 static int afu_control(struct cxl_afu *afu, u64 command, u64 clear, 23 u64 result, u64 mask, bool enabled) 24 { 25 u64 AFU_Cntl; 26 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 27 int rc = 0; 28 29 spin_lock(&afu->afu_cntl_lock); 30 pr_devel("AFU command starting: %llx\n", command); 31 32 trace_cxl_afu_ctrl(afu, command); 33 34 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 35 cxl_p2n_write(afu, CXL_AFU_Cntl_An, (AFU_Cntl & ~clear) | command); 36 37 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 38 while ((AFU_Cntl & mask) != result) { 39 if (time_after_eq(jiffies, timeout)) { 40 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n"); 41 rc = -EBUSY; 42 goto out; 43 } 44 45 if (!cxl_ops->link_ok(afu->adapter, afu)) { 46 afu->enabled = enabled; 47 rc = -EIO; 48 goto out; 49 } 50 51 pr_devel_ratelimited("AFU control... (0x%016llx)\n", 52 AFU_Cntl | command); 53 cpu_relax(); 54 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 55 } 56 57 if (AFU_Cntl & CXL_AFU_Cntl_An_RA) { 58 /* 59 * Workaround for a bug in the XSL used in the Mellanox CX4 60 * that fails to clear the RA bit after an AFU reset, 61 * preventing subsequent AFU resets from working. 62 */ 63 cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl & ~CXL_AFU_Cntl_An_RA); 64 } 65 66 pr_devel("AFU command complete: %llx\n", command); 67 afu->enabled = enabled; 68 out: 69 trace_cxl_afu_ctrl_done(afu, command, rc); 70 spin_unlock(&afu->afu_cntl_lock); 71 72 return rc; 73 } 74 75 static int afu_enable(struct cxl_afu *afu) 76 { 77 pr_devel("AFU enable request\n"); 78 79 return afu_control(afu, CXL_AFU_Cntl_An_E, 0, 80 CXL_AFU_Cntl_An_ES_Enabled, 81 CXL_AFU_Cntl_An_ES_MASK, true); 82 } 83 84 int cxl_afu_disable(struct cxl_afu *afu) 85 { 86 pr_devel("AFU disable request\n"); 87 88 return afu_control(afu, 0, CXL_AFU_Cntl_An_E, 89 CXL_AFU_Cntl_An_ES_Disabled, 90 CXL_AFU_Cntl_An_ES_MASK, false); 91 } 92 93 /* This will disable as well as reset */ 94 static int native_afu_reset(struct cxl_afu *afu) 95 { 96 int rc; 97 u64 serr; 98 99 pr_devel("AFU reset request\n"); 100 101 rc = afu_control(afu, CXL_AFU_Cntl_An_RA, 0, 102 CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled, 103 CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK, 104 false); 105 106 /* 107 * Re-enable any masked interrupts when the AFU is not 108 * activated to avoid side effects after attaching a process 109 * in dedicated mode. 110 */ 111 if (afu->current_mode == 0) { 112 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An); 113 serr &= ~CXL_PSL_SERR_An_IRQ_MASKS; 114 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr); 115 } 116 117 return rc; 118 } 119 120 static int native_afu_check_and_enable(struct cxl_afu *afu) 121 { 122 if (!cxl_ops->link_ok(afu->adapter, afu)) { 123 WARN(1, "Refusing to enable afu while link down!\n"); 124 return -EIO; 125 } 126 if (afu->enabled) 127 return 0; 128 return afu_enable(afu); 129 } 130 131 int cxl_psl_purge(struct cxl_afu *afu) 132 { 133 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 134 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 135 u64 dsisr, dar; 136 u64 start, end; 137 u64 trans_fault = 0x0ULL; 138 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 139 int rc = 0; 140 141 trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc); 142 143 pr_devel("PSL purge request\n"); 144 145 if (cxl_is_power8()) 146 trans_fault = CXL_PSL_DSISR_TRANS; 147 if (cxl_is_power9()) 148 trans_fault = CXL_PSL9_DSISR_An_TF; 149 150 if (!cxl_ops->link_ok(afu->adapter, afu)) { 151 dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n"); 152 rc = -EIO; 153 goto out; 154 } 155 156 if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) { 157 WARN(1, "psl_purge request while AFU not disabled!\n"); 158 cxl_afu_disable(afu); 159 } 160 161 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 162 PSL_CNTL | CXL_PSL_SCNTL_An_Pc); 163 start = local_clock(); 164 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 165 while ((PSL_CNTL & CXL_PSL_SCNTL_An_Ps_MASK) 166 == CXL_PSL_SCNTL_An_Ps_Pending) { 167 if (time_after_eq(jiffies, timeout)) { 168 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n"); 169 rc = -EBUSY; 170 goto out; 171 } 172 if (!cxl_ops->link_ok(afu->adapter, afu)) { 173 rc = -EIO; 174 goto out; 175 } 176 177 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 178 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx PSL_DSISR: 0x%016llx\n", 179 PSL_CNTL, dsisr); 180 181 if (dsisr & trans_fault) { 182 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 183 dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n", 184 dsisr, dar); 185 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); 186 } else if (dsisr) { 187 dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n", 188 dsisr); 189 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); 190 } else { 191 cpu_relax(); 192 } 193 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 194 } 195 end = local_clock(); 196 pr_devel("PSL purged in %lld ns\n", end - start); 197 198 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 199 PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc); 200 out: 201 trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc); 202 return rc; 203 } 204 205 static int spa_max_procs(int spa_size) 206 { 207 /* 208 * From the CAIA: 209 * end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255 210 * Most of that junk is really just an overly-complicated way of saying 211 * the last 256 bytes are __aligned(128), so it's really: 212 * end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255 213 * and 214 * end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1 215 * so 216 * sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256 217 * Ignore the alignment (which is safe in this case as long as we are 218 * careful with our rounding) and solve for n: 219 */ 220 return ((spa_size / 8) - 96) / 17; 221 } 222 223 static int cxl_alloc_spa(struct cxl_afu *afu, int mode) 224 { 225 unsigned spa_size; 226 227 /* Work out how many pages to allocate */ 228 afu->native->spa_order = -1; 229 do { 230 afu->native->spa_order++; 231 spa_size = (1 << afu->native->spa_order) * PAGE_SIZE; 232 233 if (spa_size > 0x100000) { 234 dev_warn(&afu->dev, "num_of_processes too large for the SPA, limiting to %i (0x%x)\n", 235 afu->native->spa_max_procs, afu->native->spa_size); 236 if (mode != CXL_MODE_DEDICATED) 237 afu->num_procs = afu->native->spa_max_procs; 238 break; 239 } 240 241 afu->native->spa_size = spa_size; 242 afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size); 243 } while (afu->native->spa_max_procs < afu->num_procs); 244 245 if (!(afu->native->spa = (struct cxl_process_element *) 246 __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) { 247 pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n"); 248 return -ENOMEM; 249 } 250 pr_devel("spa pages: %i afu->spa_max_procs: %i afu->num_procs: %i\n", 251 1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs); 252 253 return 0; 254 } 255 256 static void attach_spa(struct cxl_afu *afu) 257 { 258 u64 spap; 259 260 afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa + 261 ((afu->native->spa_max_procs + 3) * 128)); 262 263 spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr; 264 spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size; 265 spap |= CXL_PSL_SPAP_V; 266 pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", 267 afu->native->spa, afu->native->spa_max_procs, 268 afu->native->sw_command_status, spap); 269 cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap); 270 } 271 272 static inline void detach_spa(struct cxl_afu *afu) 273 { 274 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); 275 } 276 277 void cxl_release_spa(struct cxl_afu *afu) 278 { 279 if (afu->native->spa) { 280 free_pages((unsigned long) afu->native->spa, 281 afu->native->spa_order); 282 afu->native->spa = NULL; 283 } 284 } 285 286 /* 287 * Invalidation of all ERAT entries is no longer required by CAIA2. Use 288 * only for debug. 289 */ 290 int cxl_invalidate_all_psl9(struct cxl *adapter) 291 { 292 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 293 u64 ierat; 294 295 pr_devel("CXL adapter - invalidation of all ERAT entries\n"); 296 297 /* Invalidates all ERAT entries for Radix or HPT */ 298 ierat = CXL_XSL9_IERAT_IALL; 299 if (radix_enabled()) 300 ierat |= CXL_XSL9_IERAT_INVR; 301 cxl_p1_write(adapter, CXL_XSL9_IERAT, ierat); 302 303 while (cxl_p1_read(adapter, CXL_XSL9_IERAT) & CXL_XSL9_IERAT_IINPROG) { 304 if (time_after_eq(jiffies, timeout)) { 305 dev_warn(&adapter->dev, 306 "WARNING: CXL adapter invalidation of all ERAT entries timed out!\n"); 307 return -EBUSY; 308 } 309 if (!cxl_ops->link_ok(adapter, NULL)) 310 return -EIO; 311 cpu_relax(); 312 } 313 return 0; 314 } 315 316 int cxl_invalidate_all_psl8(struct cxl *adapter) 317 { 318 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 319 320 pr_devel("CXL adapter wide TLBIA & SLBIA\n"); 321 322 cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A); 323 324 cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL); 325 while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) { 326 if (time_after_eq(jiffies, timeout)) { 327 dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n"); 328 return -EBUSY; 329 } 330 if (!cxl_ops->link_ok(adapter, NULL)) 331 return -EIO; 332 cpu_relax(); 333 } 334 335 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL); 336 while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) { 337 if (time_after_eq(jiffies, timeout)) { 338 dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n"); 339 return -EBUSY; 340 } 341 if (!cxl_ops->link_ok(adapter, NULL)) 342 return -EIO; 343 cpu_relax(); 344 } 345 return 0; 346 } 347 348 int cxl_data_cache_flush(struct cxl *adapter) 349 { 350 u64 reg; 351 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 352 353 /* 354 * Do a datacache flush only if datacache is available. 355 * In case of PSL9D datacache absent hence flush operation. 356 * would timeout. 357 */ 358 if (adapter->native->no_data_cache) { 359 pr_devel("No PSL data cache. Ignoring cache flush req.\n"); 360 return 0; 361 } 362 363 pr_devel("Flushing data cache\n"); 364 reg = cxl_p1_read(adapter, CXL_PSL_Control); 365 reg |= CXL_PSL_Control_Fr; 366 cxl_p1_write(adapter, CXL_PSL_Control, reg); 367 368 reg = cxl_p1_read(adapter, CXL_PSL_Control); 369 while ((reg & CXL_PSL_Control_Fs_MASK) != CXL_PSL_Control_Fs_Complete) { 370 if (time_after_eq(jiffies, timeout)) { 371 dev_warn(&adapter->dev, "WARNING: cache flush timed out!\n"); 372 return -EBUSY; 373 } 374 375 if (!cxl_ops->link_ok(adapter, NULL)) { 376 dev_warn(&adapter->dev, "WARNING: link down when flushing cache\n"); 377 return -EIO; 378 } 379 cpu_relax(); 380 reg = cxl_p1_read(adapter, CXL_PSL_Control); 381 } 382 383 reg &= ~CXL_PSL_Control_Fr; 384 cxl_p1_write(adapter, CXL_PSL_Control, reg); 385 return 0; 386 } 387 388 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1) 389 { 390 int rc; 391 392 /* 1. Disable SSTP by writing 0 to SSTP1[V] */ 393 cxl_p2n_write(afu, CXL_SSTP1_An, 0); 394 395 /* 2. Invalidate all SLB entries */ 396 if ((rc = cxl_afu_slbia(afu))) 397 return rc; 398 399 /* 3. Set SSTP0_An */ 400 cxl_p2n_write(afu, CXL_SSTP0_An, sstp0); 401 402 /* 4. Set SSTP1_An */ 403 cxl_p2n_write(afu, CXL_SSTP1_An, sstp1); 404 405 return 0; 406 } 407 408 /* Using per slice version may improve performance here. (ie. SLBIA_An) */ 409 static void slb_invalid(struct cxl_context *ctx) 410 { 411 struct cxl *adapter = ctx->afu->adapter; 412 u64 slbia; 413 414 WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex)); 415 416 cxl_p1_write(adapter, CXL_PSL_LBISEL, 417 ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) | 418 be32_to_cpu(ctx->elem->lpid)); 419 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID); 420 421 while (1) { 422 if (!cxl_ops->link_ok(adapter, NULL)) 423 break; 424 slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA); 425 if (!(slbia & CXL_TLB_SLB_P)) 426 break; 427 cpu_relax(); 428 } 429 } 430 431 static int do_process_element_cmd(struct cxl_context *ctx, 432 u64 cmd, u64 pe_state) 433 { 434 u64 state; 435 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 436 int rc = 0; 437 438 trace_cxl_llcmd(ctx, cmd); 439 440 WARN_ON(!ctx->afu->enabled); 441 442 ctx->elem->software_state = cpu_to_be32(pe_state); 443 smp_wmb(); 444 *(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe); 445 smp_mb(); 446 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe); 447 while (1) { 448 if (time_after_eq(jiffies, timeout)) { 449 dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n"); 450 rc = -EBUSY; 451 goto out; 452 } 453 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) { 454 dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n"); 455 rc = -EIO; 456 goto out; 457 } 458 state = be64_to_cpup(ctx->afu->native->sw_command_status); 459 if (state == ~0ULL) { 460 pr_err("cxl: Error adding process element to AFU\n"); 461 rc = -1; 462 goto out; 463 } 464 if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK | CXL_SPA_SW_LINK_MASK)) == 465 (cmd | (cmd >> 16) | ctx->pe)) 466 break; 467 /* 468 * The command won't finish in the PSL if there are 469 * outstanding DSIs. Hence we need to yield here in 470 * case there are outstanding DSIs that we need to 471 * service. Tuning possiblity: we could wait for a 472 * while before sched 473 */ 474 schedule(); 475 476 } 477 out: 478 trace_cxl_llcmd_done(ctx, cmd, rc); 479 return rc; 480 } 481 482 static int add_process_element(struct cxl_context *ctx) 483 { 484 int rc = 0; 485 486 mutex_lock(&ctx->afu->native->spa_mutex); 487 pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe); 488 if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V))) 489 ctx->pe_inserted = true; 490 pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe); 491 mutex_unlock(&ctx->afu->native->spa_mutex); 492 return rc; 493 } 494 495 static int terminate_process_element(struct cxl_context *ctx) 496 { 497 int rc = 0; 498 499 /* fast path terminate if it's already invalid */ 500 if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V))) 501 return rc; 502 503 mutex_lock(&ctx->afu->native->spa_mutex); 504 pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe); 505 /* We could be asked to terminate when the hw is down. That 506 * should always succeed: it's not running if the hw has gone 507 * away and is being reset. 508 */ 509 if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) 510 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE, 511 CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T); 512 ctx->elem->software_state = 0; /* Remove Valid bit */ 513 pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe); 514 mutex_unlock(&ctx->afu->native->spa_mutex); 515 return rc; 516 } 517 518 static int remove_process_element(struct cxl_context *ctx) 519 { 520 int rc = 0; 521 522 mutex_lock(&ctx->afu->native->spa_mutex); 523 pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe); 524 525 /* We could be asked to remove when the hw is down. Again, if 526 * the hw is down, the PE is gone, so we succeed. 527 */ 528 if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) 529 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0); 530 531 if (!rc) 532 ctx->pe_inserted = false; 533 if (cxl_is_power8()) 534 slb_invalid(ctx); 535 pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe); 536 mutex_unlock(&ctx->afu->native->spa_mutex); 537 538 return rc; 539 } 540 541 void cxl_assign_psn_space(struct cxl_context *ctx) 542 { 543 if (!ctx->afu->pp_size || ctx->master) { 544 ctx->psn_phys = ctx->afu->psn_phys; 545 ctx->psn_size = ctx->afu->adapter->ps_size; 546 } else { 547 ctx->psn_phys = ctx->afu->psn_phys + 548 (ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe); 549 ctx->psn_size = ctx->afu->pp_size; 550 } 551 } 552 553 static int activate_afu_directed(struct cxl_afu *afu) 554 { 555 int rc; 556 557 dev_info(&afu->dev, "Activating AFU directed mode\n"); 558 559 afu->num_procs = afu->max_procs_virtualised; 560 if (afu->native->spa == NULL) { 561 if (cxl_alloc_spa(afu, CXL_MODE_DIRECTED)) 562 return -ENOMEM; 563 } 564 attach_spa(afu); 565 566 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU); 567 if (cxl_is_power8()) 568 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 569 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); 570 571 afu->current_mode = CXL_MODE_DIRECTED; 572 573 if ((rc = cxl_chardev_m_afu_add(afu))) 574 return rc; 575 576 if ((rc = cxl_sysfs_afu_m_add(afu))) 577 goto err; 578 579 if ((rc = cxl_chardev_s_afu_add(afu))) 580 goto err1; 581 582 return 0; 583 err1: 584 cxl_sysfs_afu_m_remove(afu); 585 err: 586 cxl_chardev_afu_remove(afu); 587 return rc; 588 } 589 590 #ifdef CONFIG_CPU_LITTLE_ENDIAN 591 #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE) 592 #else 593 #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE)) 594 #endif 595 596 u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9) 597 { 598 u64 sr = 0; 599 600 set_endian(sr); 601 if (master) 602 sr |= CXL_PSL_SR_An_MP; 603 if (mfspr(SPRN_LPCR) & LPCR_TC) 604 sr |= CXL_PSL_SR_An_TC; 605 606 if (kernel) { 607 if (!real_mode) 608 sr |= CXL_PSL_SR_An_R; 609 sr |= (mfmsr() & MSR_SF) | CXL_PSL_SR_An_HV; 610 } else { 611 sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R; 612 if (radix_enabled()) 613 sr |= CXL_PSL_SR_An_HV; 614 else 615 sr &= ~(CXL_PSL_SR_An_HV); 616 if (!test_tsk_thread_flag(current, TIF_32BIT)) 617 sr |= CXL_PSL_SR_An_SF; 618 } 619 if (p9) { 620 if (radix_enabled()) 621 sr |= CXL_PSL_SR_An_XLAT_ror; 622 else 623 sr |= CXL_PSL_SR_An_XLAT_hpt; 624 } 625 return sr; 626 } 627 628 static u64 calculate_sr(struct cxl_context *ctx) 629 { 630 return cxl_calculate_sr(ctx->master, ctx->kernel, false, 631 cxl_is_power9()); 632 } 633 634 static void update_ivtes_directed(struct cxl_context *ctx) 635 { 636 bool need_update = (ctx->status == STARTED); 637 int r; 638 639 if (need_update) { 640 WARN_ON(terminate_process_element(ctx)); 641 WARN_ON(remove_process_element(ctx)); 642 } 643 644 for (r = 0; r < CXL_IRQ_RANGES; r++) { 645 ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]); 646 ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]); 647 } 648 649 /* 650 * Theoretically we could use the update llcmd, instead of a 651 * terminate/remove/add (or if an atomic update was required we could 652 * do a suspend/update/resume), however it seems there might be issues 653 * with the update llcmd on some cards (including those using an XSL on 654 * an ASIC) so for now it's safest to go with the commands that are 655 * known to work. In the future if we come across a situation where the 656 * card may be performing transactions using the same PE while we are 657 * doing this update we might need to revisit this. 658 */ 659 if (need_update) 660 WARN_ON(add_process_element(ctx)); 661 } 662 663 static int process_element_entry_psl9(struct cxl_context *ctx, u64 wed, u64 amr) 664 { 665 u32 pid; 666 int rc; 667 668 cxl_assign_psn_space(ctx); 669 670 ctx->elem->ctxtime = 0; /* disable */ 671 ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID)); 672 ctx->elem->haurp = 0; /* disable */ 673 674 if (ctx->kernel) 675 pid = 0; 676 else { 677 if (ctx->mm == NULL) { 678 pr_devel("%s: unable to get mm for pe=%d pid=%i\n", 679 __func__, ctx->pe, pid_nr(ctx->pid)); 680 return -EINVAL; 681 } 682 pid = ctx->mm->context.id; 683 } 684 685 /* Assign a unique TIDR (thread id) for the current thread */ 686 if (!(ctx->tidr) && (ctx->assign_tidr)) { 687 rc = set_thread_tidr(current); 688 if (rc) 689 return -ENODEV; 690 ctx->tidr = current->thread.tidr; 691 pr_devel("%s: current tidr: %d\n", __func__, ctx->tidr); 692 } 693 694 ctx->elem->common.tid = cpu_to_be32(ctx->tidr); 695 ctx->elem->common.pid = cpu_to_be32(pid); 696 697 ctx->elem->sr = cpu_to_be64(calculate_sr(ctx)); 698 699 ctx->elem->common.csrp = 0; /* disable */ 700 701 cxl_prefault(ctx, wed); 702 703 /* 704 * Ensure we have the multiplexed PSL interrupt set up to take faults 705 * for kernel contexts that may not have allocated any AFU IRQs at all: 706 */ 707 if (ctx->irqs.range[0] == 0) { 708 ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq; 709 ctx->irqs.range[0] = 1; 710 } 711 712 ctx->elem->common.amr = cpu_to_be64(amr); 713 ctx->elem->common.wed = cpu_to_be64(wed); 714 715 return 0; 716 } 717 718 int cxl_attach_afu_directed_psl9(struct cxl_context *ctx, u64 wed, u64 amr) 719 { 720 int result; 721 722 /* fill the process element entry */ 723 result = process_element_entry_psl9(ctx, wed, amr); 724 if (result) 725 return result; 726 727 update_ivtes_directed(ctx); 728 729 /* first guy needs to enable */ 730 result = cxl_ops->afu_check_and_enable(ctx->afu); 731 if (result) 732 return result; 733 734 return add_process_element(ctx); 735 } 736 737 int cxl_attach_afu_directed_psl8(struct cxl_context *ctx, u64 wed, u64 amr) 738 { 739 u32 pid; 740 int result; 741 742 cxl_assign_psn_space(ctx); 743 744 ctx->elem->ctxtime = 0; /* disable */ 745 ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID)); 746 ctx->elem->haurp = 0; /* disable */ 747 ctx->elem->u.sdr = cpu_to_be64(mfspr(SPRN_SDR1)); 748 749 pid = current->pid; 750 if (ctx->kernel) 751 pid = 0; 752 ctx->elem->common.tid = 0; 753 ctx->elem->common.pid = cpu_to_be32(pid); 754 755 ctx->elem->sr = cpu_to_be64(calculate_sr(ctx)); 756 757 ctx->elem->common.csrp = 0; /* disable */ 758 ctx->elem->common.u.psl8.aurp0 = 0; /* disable */ 759 ctx->elem->common.u.psl8.aurp1 = 0; /* disable */ 760 761 cxl_prefault(ctx, wed); 762 763 ctx->elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0); 764 ctx->elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1); 765 766 /* 767 * Ensure we have the multiplexed PSL interrupt set up to take faults 768 * for kernel contexts that may not have allocated any AFU IRQs at all: 769 */ 770 if (ctx->irqs.range[0] == 0) { 771 ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq; 772 ctx->irqs.range[0] = 1; 773 } 774 775 update_ivtes_directed(ctx); 776 777 ctx->elem->common.amr = cpu_to_be64(amr); 778 ctx->elem->common.wed = cpu_to_be64(wed); 779 780 /* first guy needs to enable */ 781 if ((result = cxl_ops->afu_check_and_enable(ctx->afu))) 782 return result; 783 784 return add_process_element(ctx); 785 } 786 787 static int deactivate_afu_directed(struct cxl_afu *afu) 788 { 789 dev_info(&afu->dev, "Deactivating AFU directed mode\n"); 790 791 afu->current_mode = 0; 792 afu->num_procs = 0; 793 794 cxl_sysfs_afu_m_remove(afu); 795 cxl_chardev_afu_remove(afu); 796 797 /* 798 * The CAIA section 2.2.1 indicates that the procedure for starting and 799 * stopping an AFU in AFU directed mode is AFU specific, which is not 800 * ideal since this code is generic and with one exception has no 801 * knowledge of the AFU. This is in contrast to the procedure for 802 * disabling a dedicated process AFU, which is documented to just 803 * require a reset. The architecture does indicate that both an AFU 804 * reset and an AFU disable should result in the AFU being disabled and 805 * we do both followed by a PSL purge for safety. 806 * 807 * Notably we used to have some issues with the disable sequence on PSL 808 * cards, which is why we ended up using this heavy weight procedure in 809 * the first place, however a bug was discovered that had rendered the 810 * disable operation ineffective, so it is conceivable that was the 811 * sole explanation for those difficulties. Careful regression testing 812 * is recommended if anyone attempts to remove or reorder these 813 * operations. 814 * 815 * The XSL on the Mellanox CX4 behaves a little differently from the 816 * PSL based cards and will time out an AFU reset if the AFU is still 817 * enabled. That card is special in that we do have a means to identify 818 * it from this code, so in that case we skip the reset and just use a 819 * disable/purge to avoid the timeout and corresponding noise in the 820 * kernel log. 821 */ 822 if (afu->adapter->native->sl_ops->needs_reset_before_disable) 823 cxl_ops->afu_reset(afu); 824 cxl_afu_disable(afu); 825 cxl_psl_purge(afu); 826 827 return 0; 828 } 829 830 int cxl_activate_dedicated_process_psl9(struct cxl_afu *afu) 831 { 832 dev_info(&afu->dev, "Activating dedicated process mode\n"); 833 834 /* 835 * If XSL is set to dedicated mode (Set in PSL_SCNTL reg), the 836 * XSL and AFU are programmed to work with a single context. 837 * The context information should be configured in the SPA area 838 * index 0 (so PSL_SPAP must be configured before enabling the 839 * AFU). 840 */ 841 afu->num_procs = 1; 842 if (afu->native->spa == NULL) { 843 if (cxl_alloc_spa(afu, CXL_MODE_DEDICATED)) 844 return -ENOMEM; 845 } 846 attach_spa(afu); 847 848 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); 849 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); 850 851 afu->current_mode = CXL_MODE_DEDICATED; 852 853 return cxl_chardev_d_afu_add(afu); 854 } 855 856 int cxl_activate_dedicated_process_psl8(struct cxl_afu *afu) 857 { 858 dev_info(&afu->dev, "Activating dedicated process mode\n"); 859 860 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); 861 862 cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */ 863 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */ 864 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 865 cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID)); 866 cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */ 867 cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1)); 868 869 cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */ 870 cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */ 871 cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */ 872 873 afu->current_mode = CXL_MODE_DEDICATED; 874 afu->num_procs = 1; 875 876 return cxl_chardev_d_afu_add(afu); 877 } 878 879 void cxl_update_dedicated_ivtes_psl9(struct cxl_context *ctx) 880 { 881 int r; 882 883 for (r = 0; r < CXL_IRQ_RANGES; r++) { 884 ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]); 885 ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]); 886 } 887 } 888 889 void cxl_update_dedicated_ivtes_psl8(struct cxl_context *ctx) 890 { 891 struct cxl_afu *afu = ctx->afu; 892 893 cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 894 (((u64)ctx->irqs.offset[0] & 0xffff) << 48) | 895 (((u64)ctx->irqs.offset[1] & 0xffff) << 32) | 896 (((u64)ctx->irqs.offset[2] & 0xffff) << 16) | 897 ((u64)ctx->irqs.offset[3] & 0xffff)); 898 cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64) 899 (((u64)ctx->irqs.range[0] & 0xffff) << 48) | 900 (((u64)ctx->irqs.range[1] & 0xffff) << 32) | 901 (((u64)ctx->irqs.range[2] & 0xffff) << 16) | 902 ((u64)ctx->irqs.range[3] & 0xffff)); 903 } 904 905 int cxl_attach_dedicated_process_psl9(struct cxl_context *ctx, u64 wed, u64 amr) 906 { 907 struct cxl_afu *afu = ctx->afu; 908 int result; 909 910 /* fill the process element entry */ 911 result = process_element_entry_psl9(ctx, wed, amr); 912 if (result) 913 return result; 914 915 if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes) 916 afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx); 917 918 ctx->elem->software_state = cpu_to_be32(CXL_PE_SOFTWARE_STATE_V); 919 /* 920 * Ideally we should do a wmb() here to make sure the changes to the 921 * PE are visible to the card before we call afu_enable. 922 * On ppc64 though all mmios are preceded by a 'sync' instruction hence 923 * we dont dont need one here. 924 */ 925 926 result = cxl_ops->afu_reset(afu); 927 if (result) 928 return result; 929 930 return afu_enable(afu); 931 } 932 933 int cxl_attach_dedicated_process_psl8(struct cxl_context *ctx, u64 wed, u64 amr) 934 { 935 struct cxl_afu *afu = ctx->afu; 936 u64 pid; 937 int rc; 938 939 pid = (u64)current->pid << 32; 940 if (ctx->kernel) 941 pid = 0; 942 cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid); 943 944 cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx)); 945 946 if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1))) 947 return rc; 948 949 cxl_prefault(ctx, wed); 950 951 if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes) 952 afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx); 953 954 cxl_p2n_write(afu, CXL_PSL_AMR_An, amr); 955 956 /* master only context for dedicated */ 957 cxl_assign_psn_space(ctx); 958 959 if ((rc = cxl_ops->afu_reset(afu))) 960 return rc; 961 962 cxl_p2n_write(afu, CXL_PSL_WED_An, wed); 963 964 return afu_enable(afu); 965 } 966 967 static int deactivate_dedicated_process(struct cxl_afu *afu) 968 { 969 dev_info(&afu->dev, "Deactivating dedicated process mode\n"); 970 971 afu->current_mode = 0; 972 afu->num_procs = 0; 973 974 cxl_chardev_afu_remove(afu); 975 976 return 0; 977 } 978 979 static int native_afu_deactivate_mode(struct cxl_afu *afu, int mode) 980 { 981 if (mode == CXL_MODE_DIRECTED) 982 return deactivate_afu_directed(afu); 983 if (mode == CXL_MODE_DEDICATED) 984 return deactivate_dedicated_process(afu); 985 return 0; 986 } 987 988 static int native_afu_activate_mode(struct cxl_afu *afu, int mode) 989 { 990 if (!mode) 991 return 0; 992 if (!(mode & afu->modes_supported)) 993 return -EINVAL; 994 995 if (!cxl_ops->link_ok(afu->adapter, afu)) { 996 WARN(1, "Device link is down, refusing to activate!\n"); 997 return -EIO; 998 } 999 1000 if (mode == CXL_MODE_DIRECTED) 1001 return activate_afu_directed(afu); 1002 if ((mode == CXL_MODE_DEDICATED) && 1003 (afu->adapter->native->sl_ops->activate_dedicated_process)) 1004 return afu->adapter->native->sl_ops->activate_dedicated_process(afu); 1005 1006 return -EINVAL; 1007 } 1008 1009 static int native_attach_process(struct cxl_context *ctx, bool kernel, 1010 u64 wed, u64 amr) 1011 { 1012 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) { 1013 WARN(1, "Device link is down, refusing to attach process!\n"); 1014 return -EIO; 1015 } 1016 1017 ctx->kernel = kernel; 1018 if ((ctx->afu->current_mode == CXL_MODE_DIRECTED) && 1019 (ctx->afu->adapter->native->sl_ops->attach_afu_directed)) 1020 return ctx->afu->adapter->native->sl_ops->attach_afu_directed(ctx, wed, amr); 1021 1022 if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) && 1023 (ctx->afu->adapter->native->sl_ops->attach_dedicated_process)) 1024 return ctx->afu->adapter->native->sl_ops->attach_dedicated_process(ctx, wed, amr); 1025 1026 return -EINVAL; 1027 } 1028 1029 static inline int detach_process_native_dedicated(struct cxl_context *ctx) 1030 { 1031 /* 1032 * The CAIA section 2.1.1 indicates that we need to do an AFU reset to 1033 * stop the AFU in dedicated mode (we therefore do not make that 1034 * optional like we do in the afu directed path). It does not indicate 1035 * that we need to do an explicit disable (which should occur 1036 * implicitly as part of the reset) or purge, but we do these as well 1037 * to be on the safe side. 1038 * 1039 * Notably we used to have some issues with the disable sequence 1040 * (before the sequence was spelled out in the architecture) which is 1041 * why we were so heavy weight in the first place, however a bug was 1042 * discovered that had rendered the disable operation ineffective, so 1043 * it is conceivable that was the sole explanation for those 1044 * difficulties. Point is, we should be careful and do some regression 1045 * testing if we ever attempt to remove any part of this procedure. 1046 */ 1047 cxl_ops->afu_reset(ctx->afu); 1048 cxl_afu_disable(ctx->afu); 1049 cxl_psl_purge(ctx->afu); 1050 return 0; 1051 } 1052 1053 static void native_update_ivtes(struct cxl_context *ctx) 1054 { 1055 if (ctx->afu->current_mode == CXL_MODE_DIRECTED) 1056 return update_ivtes_directed(ctx); 1057 if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) && 1058 (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)) 1059 return ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx); 1060 WARN(1, "native_update_ivtes: Bad mode\n"); 1061 } 1062 1063 static inline int detach_process_native_afu_directed(struct cxl_context *ctx) 1064 { 1065 if (!ctx->pe_inserted) 1066 return 0; 1067 if (terminate_process_element(ctx)) 1068 return -1; 1069 if (remove_process_element(ctx)) 1070 return -1; 1071 1072 return 0; 1073 } 1074 1075 static int native_detach_process(struct cxl_context *ctx) 1076 { 1077 trace_cxl_detach(ctx); 1078 1079 if (ctx->afu->current_mode == CXL_MODE_DEDICATED) 1080 return detach_process_native_dedicated(ctx); 1081 1082 return detach_process_native_afu_directed(ctx); 1083 } 1084 1085 static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info) 1086 { 1087 /* If the adapter has gone away, we can't get any meaningful 1088 * information. 1089 */ 1090 if (!cxl_ops->link_ok(afu->adapter, afu)) 1091 return -EIO; 1092 1093 info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 1094 info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 1095 if (cxl_is_power8()) 1096 info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An); 1097 info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An); 1098 info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); 1099 info->proc_handle = 0; 1100 1101 return 0; 1102 } 1103 1104 void cxl_native_irq_dump_regs_psl9(struct cxl_context *ctx) 1105 { 1106 u64 fir1, serr; 1107 1108 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL9_FIR1); 1109 1110 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1); 1111 if (ctx->afu->adapter->native->sl_ops->register_serr_irq) { 1112 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An); 1113 cxl_afu_decode_psl_serr(ctx->afu, serr); 1114 } 1115 } 1116 1117 void cxl_native_irq_dump_regs_psl8(struct cxl_context *ctx) 1118 { 1119 u64 fir1, fir2, fir_slice, serr, afu_debug; 1120 1121 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1); 1122 fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2); 1123 fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An); 1124 afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An); 1125 1126 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1); 1127 dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2); 1128 if (ctx->afu->adapter->native->sl_ops->register_serr_irq) { 1129 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An); 1130 cxl_afu_decode_psl_serr(ctx->afu, serr); 1131 } 1132 dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice); 1133 dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug); 1134 } 1135 1136 static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx, 1137 u64 dsisr, u64 errstat) 1138 { 1139 1140 dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat); 1141 1142 if (ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers) 1143 ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers(ctx); 1144 1145 if (ctx->afu->adapter->native->sl_ops->debugfs_stop_trace) { 1146 dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n"); 1147 ctx->afu->adapter->native->sl_ops->debugfs_stop_trace(ctx->afu->adapter); 1148 } 1149 1150 return cxl_ops->ack_irq(ctx, 0, errstat); 1151 } 1152 1153 static bool cxl_is_translation_fault(struct cxl_afu *afu, u64 dsisr) 1154 { 1155 if ((cxl_is_power8()) && (dsisr & CXL_PSL_DSISR_TRANS)) 1156 return true; 1157 1158 if ((cxl_is_power9()) && (dsisr & CXL_PSL9_DSISR_An_TF)) 1159 return true; 1160 1161 return false; 1162 } 1163 1164 irqreturn_t cxl_fail_irq_psl(struct cxl_afu *afu, struct cxl_irq_info *irq_info) 1165 { 1166 if (cxl_is_translation_fault(afu, irq_info->dsisr)) 1167 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); 1168 else 1169 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); 1170 1171 return IRQ_HANDLED; 1172 } 1173 1174 static irqreturn_t native_irq_multiplexed(int irq, void *data) 1175 { 1176 struct cxl_afu *afu = data; 1177 struct cxl_context *ctx; 1178 struct cxl_irq_info irq_info; 1179 u64 phreg = cxl_p2n_read(afu, CXL_PSL_PEHandle_An); 1180 int ph, ret = IRQ_HANDLED, res; 1181 1182 /* check if eeh kicked in while the interrupt was in flight */ 1183 if (unlikely(phreg == ~0ULL)) { 1184 dev_warn(&afu->dev, 1185 "Ignoring slice interrupt(%d) due to fenced card", 1186 irq); 1187 return IRQ_HANDLED; 1188 } 1189 /* Mask the pe-handle from register value */ 1190 ph = phreg & 0xffff; 1191 if ((res = native_get_irq_info(afu, &irq_info))) { 1192 WARN(1, "Unable to get CXL IRQ Info: %i\n", res); 1193 if (afu->adapter->native->sl_ops->fail_irq) 1194 return afu->adapter->native->sl_ops->fail_irq(afu, &irq_info); 1195 return ret; 1196 } 1197 1198 rcu_read_lock(); 1199 ctx = idr_find(&afu->contexts_idr, ph); 1200 if (ctx) { 1201 if (afu->adapter->native->sl_ops->handle_interrupt) 1202 ret = afu->adapter->native->sl_ops->handle_interrupt(irq, ctx, &irq_info); 1203 rcu_read_unlock(); 1204 return ret; 1205 } 1206 rcu_read_unlock(); 1207 1208 WARN(1, "Unable to demultiplex CXL PSL IRQ for PE %i DSISR %016llx DAR" 1209 " %016llx\n(Possible AFU HW issue - was a term/remove acked" 1210 " with outstanding transactions?)\n", ph, irq_info.dsisr, 1211 irq_info.dar); 1212 if (afu->adapter->native->sl_ops->fail_irq) 1213 ret = afu->adapter->native->sl_ops->fail_irq(afu, &irq_info); 1214 return ret; 1215 } 1216 1217 static void native_irq_wait(struct cxl_context *ctx) 1218 { 1219 u64 dsisr; 1220 int timeout = 1000; 1221 int ph; 1222 1223 /* 1224 * Wait until no further interrupts are presented by the PSL 1225 * for this context. 1226 */ 1227 while (timeout--) { 1228 ph = cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) & 0xffff; 1229 if (ph != ctx->pe) 1230 return; 1231 dsisr = cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An); 1232 if (cxl_is_power8() && 1233 ((dsisr & CXL_PSL_DSISR_PENDING) == 0)) 1234 return; 1235 if (cxl_is_power9() && 1236 ((dsisr & CXL_PSL9_DSISR_PENDING) == 0)) 1237 return; 1238 /* 1239 * We are waiting for the workqueue to process our 1240 * irq, so need to let that run here. 1241 */ 1242 msleep(1); 1243 } 1244 1245 dev_warn(&ctx->afu->dev, "WARNING: waiting on DSI for PE %i" 1246 " DSISR %016llx!\n", ph, dsisr); 1247 return; 1248 } 1249 1250 static irqreturn_t native_slice_irq_err(int irq, void *data) 1251 { 1252 struct cxl_afu *afu = data; 1253 u64 errstat, serr, afu_error, dsisr; 1254 u64 fir_slice, afu_debug, irq_mask; 1255 1256 /* 1257 * slice err interrupt is only used with full PSL (no XSL) 1258 */ 1259 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An); 1260 errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); 1261 afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An); 1262 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 1263 cxl_afu_decode_psl_serr(afu, serr); 1264 1265 if (cxl_is_power8()) { 1266 fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An); 1267 afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An); 1268 dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice); 1269 dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug); 1270 } 1271 dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat); 1272 dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error); 1273 dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr); 1274 1275 /* mask off the IRQ so it won't retrigger until the AFU is reset */ 1276 irq_mask = (serr & CXL_PSL_SERR_An_IRQS) >> 32; 1277 serr |= irq_mask; 1278 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr); 1279 dev_info(&afu->dev, "Further such interrupts will be masked until the AFU is reset\n"); 1280 1281 return IRQ_HANDLED; 1282 } 1283 1284 void cxl_native_err_irq_dump_regs_psl9(struct cxl *adapter) 1285 { 1286 u64 fir1; 1287 1288 fir1 = cxl_p1_read(adapter, CXL_PSL9_FIR1); 1289 dev_crit(&adapter->dev, "PSL_FIR: 0x%016llx\n", fir1); 1290 } 1291 1292 void cxl_native_err_irq_dump_regs_psl8(struct cxl *adapter) 1293 { 1294 u64 fir1, fir2; 1295 1296 fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1); 1297 fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2); 1298 dev_crit(&adapter->dev, 1299 "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", 1300 fir1, fir2); 1301 } 1302 1303 static irqreturn_t native_irq_err(int irq, void *data) 1304 { 1305 struct cxl *adapter = data; 1306 u64 err_ivte; 1307 1308 WARN(1, "CXL ERROR interrupt %i\n", irq); 1309 1310 err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE); 1311 dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%016llx\n", err_ivte); 1312 1313 if (adapter->native->sl_ops->debugfs_stop_trace) { 1314 dev_crit(&adapter->dev, "STOPPING CXL TRACE\n"); 1315 adapter->native->sl_ops->debugfs_stop_trace(adapter); 1316 } 1317 1318 if (adapter->native->sl_ops->err_irq_dump_registers) 1319 adapter->native->sl_ops->err_irq_dump_registers(adapter); 1320 1321 return IRQ_HANDLED; 1322 } 1323 1324 int cxl_native_register_psl_err_irq(struct cxl *adapter) 1325 { 1326 int rc; 1327 1328 adapter->irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err", 1329 dev_name(&adapter->dev)); 1330 if (!adapter->irq_name) 1331 return -ENOMEM; 1332 1333 if ((rc = cxl_register_one_irq(adapter, native_irq_err, adapter, 1334 &adapter->native->err_hwirq, 1335 &adapter->native->err_virq, 1336 adapter->irq_name))) { 1337 kfree(adapter->irq_name); 1338 adapter->irq_name = NULL; 1339 return rc; 1340 } 1341 1342 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->native->err_hwirq & 0xffff); 1343 1344 return 0; 1345 } 1346 1347 void cxl_native_release_psl_err_irq(struct cxl *adapter) 1348 { 1349 if (adapter->native->err_virq == 0 || 1350 adapter->native->err_virq != 1351 irq_find_mapping(NULL, adapter->native->err_hwirq)) 1352 return; 1353 1354 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000); 1355 cxl_unmap_irq(adapter->native->err_virq, adapter); 1356 cxl_ops->release_one_irq(adapter, adapter->native->err_hwirq); 1357 kfree(adapter->irq_name); 1358 adapter->native->err_virq = 0; 1359 } 1360 1361 int cxl_native_register_serr_irq(struct cxl_afu *afu) 1362 { 1363 u64 serr; 1364 int rc; 1365 1366 afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err", 1367 dev_name(&afu->dev)); 1368 if (!afu->err_irq_name) 1369 return -ENOMEM; 1370 1371 if ((rc = cxl_register_one_irq(afu->adapter, native_slice_irq_err, afu, 1372 &afu->serr_hwirq, 1373 &afu->serr_virq, afu->err_irq_name))) { 1374 kfree(afu->err_irq_name); 1375 afu->err_irq_name = NULL; 1376 return rc; 1377 } 1378 1379 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An); 1380 if (cxl_is_power8()) 1381 serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff); 1382 if (cxl_is_power9()) { 1383 /* 1384 * By default, all errors are masked. So don't set all masks. 1385 * Slice errors will be transfered. 1386 */ 1387 serr = (serr & ~0xff0000007fffffffULL) | (afu->serr_hwirq & 0xffff); 1388 } 1389 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr); 1390 1391 return 0; 1392 } 1393 1394 void cxl_native_release_serr_irq(struct cxl_afu *afu) 1395 { 1396 if (afu->serr_virq == 0 || 1397 afu->serr_virq != irq_find_mapping(NULL, afu->serr_hwirq)) 1398 return; 1399 1400 cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000); 1401 cxl_unmap_irq(afu->serr_virq, afu); 1402 cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq); 1403 kfree(afu->err_irq_name); 1404 afu->serr_virq = 0; 1405 } 1406 1407 int cxl_native_register_psl_irq(struct cxl_afu *afu) 1408 { 1409 int rc; 1410 1411 afu->psl_irq_name = kasprintf(GFP_KERNEL, "cxl-%s", 1412 dev_name(&afu->dev)); 1413 if (!afu->psl_irq_name) 1414 return -ENOMEM; 1415 1416 if ((rc = cxl_register_one_irq(afu->adapter, native_irq_multiplexed, 1417 afu, &afu->native->psl_hwirq, &afu->native->psl_virq, 1418 afu->psl_irq_name))) { 1419 kfree(afu->psl_irq_name); 1420 afu->psl_irq_name = NULL; 1421 } 1422 return rc; 1423 } 1424 1425 void cxl_native_release_psl_irq(struct cxl_afu *afu) 1426 { 1427 if (afu->native->psl_virq == 0 || 1428 afu->native->psl_virq != 1429 irq_find_mapping(NULL, afu->native->psl_hwirq)) 1430 return; 1431 1432 cxl_unmap_irq(afu->native->psl_virq, afu); 1433 cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq); 1434 kfree(afu->psl_irq_name); 1435 afu->native->psl_virq = 0; 1436 } 1437 1438 static void recover_psl_err(struct cxl_afu *afu, u64 errstat) 1439 { 1440 u64 dsisr; 1441 1442 pr_devel("RECOVERING FROM PSL ERROR... (0x%016llx)\n", errstat); 1443 1444 /* Clear PSL_DSISR[PE] */ 1445 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 1446 cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE); 1447 1448 /* Write 1s to clear error status bits */ 1449 cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat); 1450 } 1451 1452 static int native_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask) 1453 { 1454 trace_cxl_psl_irq_ack(ctx, tfc); 1455 if (tfc) 1456 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc); 1457 if (psl_reset_mask) 1458 recover_psl_err(ctx->afu, psl_reset_mask); 1459 1460 return 0; 1461 } 1462 1463 int cxl_check_error(struct cxl_afu *afu) 1464 { 1465 return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL); 1466 } 1467 1468 static bool native_support_attributes(const char *attr_name, 1469 enum cxl_attrs type) 1470 { 1471 return true; 1472 } 1473 1474 static int native_afu_cr_read64(struct cxl_afu *afu, int cr, u64 off, u64 *out) 1475 { 1476 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu))) 1477 return -EIO; 1478 if (unlikely(off >= afu->crs_len)) 1479 return -ERANGE; 1480 *out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset + 1481 (cr * afu->crs_len) + off); 1482 return 0; 1483 } 1484 1485 static int native_afu_cr_read32(struct cxl_afu *afu, int cr, u64 off, u32 *out) 1486 { 1487 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu))) 1488 return -EIO; 1489 if (unlikely(off >= afu->crs_len)) 1490 return -ERANGE; 1491 *out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset + 1492 (cr * afu->crs_len) + off); 1493 return 0; 1494 } 1495 1496 static int native_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off, u16 *out) 1497 { 1498 u64 aligned_off = off & ~0x3L; 1499 u32 val; 1500 int rc; 1501 1502 rc = native_afu_cr_read32(afu, cr, aligned_off, &val); 1503 if (!rc) 1504 *out = (val >> ((off & 0x3) * 8)) & 0xffff; 1505 return rc; 1506 } 1507 1508 static int native_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off, u8 *out) 1509 { 1510 u64 aligned_off = off & ~0x3L; 1511 u32 val; 1512 int rc; 1513 1514 rc = native_afu_cr_read32(afu, cr, aligned_off, &val); 1515 if (!rc) 1516 *out = (val >> ((off & 0x3) * 8)) & 0xff; 1517 return rc; 1518 } 1519 1520 static int native_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in) 1521 { 1522 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu))) 1523 return -EIO; 1524 if (unlikely(off >= afu->crs_len)) 1525 return -ERANGE; 1526 out_le32(afu->native->afu_desc_mmio + afu->crs_offset + 1527 (cr * afu->crs_len) + off, in); 1528 return 0; 1529 } 1530 1531 static int native_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in) 1532 { 1533 u64 aligned_off = off & ~0x3L; 1534 u32 val32, mask, shift; 1535 int rc; 1536 1537 rc = native_afu_cr_read32(afu, cr, aligned_off, &val32); 1538 if (rc) 1539 return rc; 1540 shift = (off & 0x3) * 8; 1541 WARN_ON(shift == 24); 1542 mask = 0xffff << shift; 1543 val32 = (val32 & ~mask) | (in << shift); 1544 1545 rc = native_afu_cr_write32(afu, cr, aligned_off, val32); 1546 return rc; 1547 } 1548 1549 static int native_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in) 1550 { 1551 u64 aligned_off = off & ~0x3L; 1552 u32 val32, mask, shift; 1553 int rc; 1554 1555 rc = native_afu_cr_read32(afu, cr, aligned_off, &val32); 1556 if (rc) 1557 return rc; 1558 shift = (off & 0x3) * 8; 1559 mask = 0xff << shift; 1560 val32 = (val32 & ~mask) | (in << shift); 1561 1562 rc = native_afu_cr_write32(afu, cr, aligned_off, val32); 1563 return rc; 1564 } 1565 1566 const struct cxl_backend_ops cxl_native_ops = { 1567 .module = THIS_MODULE, 1568 .adapter_reset = cxl_pci_reset, 1569 .alloc_one_irq = cxl_pci_alloc_one_irq, 1570 .release_one_irq = cxl_pci_release_one_irq, 1571 .alloc_irq_ranges = cxl_pci_alloc_irq_ranges, 1572 .release_irq_ranges = cxl_pci_release_irq_ranges, 1573 .setup_irq = cxl_pci_setup_irq, 1574 .handle_psl_slice_error = native_handle_psl_slice_error, 1575 .psl_interrupt = NULL, 1576 .ack_irq = native_ack_irq, 1577 .irq_wait = native_irq_wait, 1578 .attach_process = native_attach_process, 1579 .detach_process = native_detach_process, 1580 .update_ivtes = native_update_ivtes, 1581 .support_attributes = native_support_attributes, 1582 .link_ok = cxl_adapter_link_ok, 1583 .release_afu = cxl_pci_release_afu, 1584 .afu_read_err_buffer = cxl_pci_afu_read_err_buffer, 1585 .afu_check_and_enable = native_afu_check_and_enable, 1586 .afu_activate_mode = native_afu_activate_mode, 1587 .afu_deactivate_mode = native_afu_deactivate_mode, 1588 .afu_reset = native_afu_reset, 1589 .afu_cr_read8 = native_afu_cr_read8, 1590 .afu_cr_read16 = native_afu_cr_read16, 1591 .afu_cr_read32 = native_afu_cr_read32, 1592 .afu_cr_read64 = native_afu_cr_read64, 1593 .afu_cr_write8 = native_afu_cr_write8, 1594 .afu_cr_write16 = native_afu_cr_write16, 1595 .afu_cr_write32 = native_afu_cr_write32, 1596 .read_adapter_vpd = cxl_pci_read_adapter_vpd, 1597 }; 1598