1 /* 2 * Copyright 2014 IBM Corp. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #include <linux/spinlock.h> 11 #include <linux/sched.h> 12 #include <linux/sched/clock.h> 13 #include <linux/slab.h> 14 #include <linux/mutex.h> 15 #include <linux/mm.h> 16 #include <linux/uaccess.h> 17 #include <linux/delay.h> 18 #include <asm/synch.h> 19 #include <asm/switch_to.h> 20 #include <misc/cxl-base.h> 21 22 #include "cxl.h" 23 #include "trace.h" 24 25 static int afu_control(struct cxl_afu *afu, u64 command, u64 clear, 26 u64 result, u64 mask, bool enabled) 27 { 28 u64 AFU_Cntl; 29 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 30 int rc = 0; 31 32 spin_lock(&afu->afu_cntl_lock); 33 pr_devel("AFU command starting: %llx\n", command); 34 35 trace_cxl_afu_ctrl(afu, command); 36 37 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 38 cxl_p2n_write(afu, CXL_AFU_Cntl_An, (AFU_Cntl & ~clear) | command); 39 40 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 41 while ((AFU_Cntl & mask) != result) { 42 if (time_after_eq(jiffies, timeout)) { 43 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n"); 44 rc = -EBUSY; 45 goto out; 46 } 47 48 if (!cxl_ops->link_ok(afu->adapter, afu)) { 49 afu->enabled = enabled; 50 rc = -EIO; 51 goto out; 52 } 53 54 pr_devel_ratelimited("AFU control... (0x%016llx)\n", 55 AFU_Cntl | command); 56 cpu_relax(); 57 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 58 } 59 60 if (AFU_Cntl & CXL_AFU_Cntl_An_RA) { 61 /* 62 * Workaround for a bug in the XSL used in the Mellanox CX4 63 * that fails to clear the RA bit after an AFU reset, 64 * preventing subsequent AFU resets from working. 65 */ 66 cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl & ~CXL_AFU_Cntl_An_RA); 67 } 68 69 pr_devel("AFU command complete: %llx\n", command); 70 afu->enabled = enabled; 71 out: 72 trace_cxl_afu_ctrl_done(afu, command, rc); 73 spin_unlock(&afu->afu_cntl_lock); 74 75 return rc; 76 } 77 78 static int afu_enable(struct cxl_afu *afu) 79 { 80 pr_devel("AFU enable request\n"); 81 82 return afu_control(afu, CXL_AFU_Cntl_An_E, 0, 83 CXL_AFU_Cntl_An_ES_Enabled, 84 CXL_AFU_Cntl_An_ES_MASK, true); 85 } 86 87 int cxl_afu_disable(struct cxl_afu *afu) 88 { 89 pr_devel("AFU disable request\n"); 90 91 return afu_control(afu, 0, CXL_AFU_Cntl_An_E, 92 CXL_AFU_Cntl_An_ES_Disabled, 93 CXL_AFU_Cntl_An_ES_MASK, false); 94 } 95 96 /* This will disable as well as reset */ 97 static int native_afu_reset(struct cxl_afu *afu) 98 { 99 int rc; 100 u64 serr; 101 102 pr_devel("AFU reset request\n"); 103 104 rc = afu_control(afu, CXL_AFU_Cntl_An_RA, 0, 105 CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled, 106 CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK, 107 false); 108 109 /* 110 * Re-enable any masked interrupts when the AFU is not 111 * activated to avoid side effects after attaching a process 112 * in dedicated mode. 113 */ 114 if (afu->current_mode == 0) { 115 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An); 116 serr &= ~CXL_PSL_SERR_An_IRQ_MASKS; 117 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr); 118 } 119 120 return rc; 121 } 122 123 static int native_afu_check_and_enable(struct cxl_afu *afu) 124 { 125 if (!cxl_ops->link_ok(afu->adapter, afu)) { 126 WARN(1, "Refusing to enable afu while link down!\n"); 127 return -EIO; 128 } 129 if (afu->enabled) 130 return 0; 131 return afu_enable(afu); 132 } 133 134 int cxl_psl_purge(struct cxl_afu *afu) 135 { 136 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 137 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 138 u64 dsisr, dar; 139 u64 start, end; 140 u64 trans_fault = 0x0ULL; 141 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 142 int rc = 0; 143 144 trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc); 145 146 pr_devel("PSL purge request\n"); 147 148 if (cxl_is_power8()) 149 trans_fault = CXL_PSL_DSISR_TRANS; 150 if (cxl_is_power9()) 151 trans_fault = CXL_PSL9_DSISR_An_TF; 152 153 if (!cxl_ops->link_ok(afu->adapter, afu)) { 154 dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n"); 155 rc = -EIO; 156 goto out; 157 } 158 159 if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) { 160 WARN(1, "psl_purge request while AFU not disabled!\n"); 161 cxl_afu_disable(afu); 162 } 163 164 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 165 PSL_CNTL | CXL_PSL_SCNTL_An_Pc); 166 start = local_clock(); 167 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 168 while ((PSL_CNTL & CXL_PSL_SCNTL_An_Ps_MASK) 169 == CXL_PSL_SCNTL_An_Ps_Pending) { 170 if (time_after_eq(jiffies, timeout)) { 171 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n"); 172 rc = -EBUSY; 173 goto out; 174 } 175 if (!cxl_ops->link_ok(afu->adapter, afu)) { 176 rc = -EIO; 177 goto out; 178 } 179 180 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 181 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx PSL_DSISR: 0x%016llx\n", 182 PSL_CNTL, dsisr); 183 184 if (dsisr & trans_fault) { 185 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 186 dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n", 187 dsisr, dar); 188 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); 189 } else if (dsisr) { 190 dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n", 191 dsisr); 192 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); 193 } else { 194 cpu_relax(); 195 } 196 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 197 } 198 end = local_clock(); 199 pr_devel("PSL purged in %lld ns\n", end - start); 200 201 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 202 PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc); 203 out: 204 trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc); 205 return rc; 206 } 207 208 static int spa_max_procs(int spa_size) 209 { 210 /* 211 * From the CAIA: 212 * end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255 213 * Most of that junk is really just an overly-complicated way of saying 214 * the last 256 bytes are __aligned(128), so it's really: 215 * end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255 216 * and 217 * end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1 218 * so 219 * sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256 220 * Ignore the alignment (which is safe in this case as long as we are 221 * careful with our rounding) and solve for n: 222 */ 223 return ((spa_size / 8) - 96) / 17; 224 } 225 226 static int cxl_alloc_spa(struct cxl_afu *afu, int mode) 227 { 228 unsigned spa_size; 229 230 /* Work out how many pages to allocate */ 231 afu->native->spa_order = -1; 232 do { 233 afu->native->spa_order++; 234 spa_size = (1 << afu->native->spa_order) * PAGE_SIZE; 235 236 if (spa_size > 0x100000) { 237 dev_warn(&afu->dev, "num_of_processes too large for the SPA, limiting to %i (0x%x)\n", 238 afu->native->spa_max_procs, afu->native->spa_size); 239 if (mode != CXL_MODE_DEDICATED) 240 afu->num_procs = afu->native->spa_max_procs; 241 break; 242 } 243 244 afu->native->spa_size = spa_size; 245 afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size); 246 } while (afu->native->spa_max_procs < afu->num_procs); 247 248 if (!(afu->native->spa = (struct cxl_process_element *) 249 __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) { 250 pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n"); 251 return -ENOMEM; 252 } 253 pr_devel("spa pages: %i afu->spa_max_procs: %i afu->num_procs: %i\n", 254 1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs); 255 256 return 0; 257 } 258 259 static void attach_spa(struct cxl_afu *afu) 260 { 261 u64 spap; 262 263 afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa + 264 ((afu->native->spa_max_procs + 3) * 128)); 265 266 spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr; 267 spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size; 268 spap |= CXL_PSL_SPAP_V; 269 pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", 270 afu->native->spa, afu->native->spa_max_procs, 271 afu->native->sw_command_status, spap); 272 cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap); 273 } 274 275 static inline void detach_spa(struct cxl_afu *afu) 276 { 277 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); 278 } 279 280 void cxl_release_spa(struct cxl_afu *afu) 281 { 282 if (afu->native->spa) { 283 free_pages((unsigned long) afu->native->spa, 284 afu->native->spa_order); 285 afu->native->spa = NULL; 286 } 287 } 288 289 /* 290 * Invalidation of all ERAT entries is no longer required by CAIA2. Use 291 * only for debug. 292 */ 293 int cxl_invalidate_all_psl9(struct cxl *adapter) 294 { 295 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 296 u64 ierat; 297 298 pr_devel("CXL adapter - invalidation of all ERAT entries\n"); 299 300 /* Invalidates all ERAT entries for Radix or HPT */ 301 ierat = CXL_XSL9_IERAT_IALL; 302 if (radix_enabled()) 303 ierat |= CXL_XSL9_IERAT_INVR; 304 cxl_p1_write(adapter, CXL_XSL9_IERAT, ierat); 305 306 while (cxl_p1_read(adapter, CXL_XSL9_IERAT) & CXL_XSL9_IERAT_IINPROG) { 307 if (time_after_eq(jiffies, timeout)) { 308 dev_warn(&adapter->dev, 309 "WARNING: CXL adapter invalidation of all ERAT entries timed out!\n"); 310 return -EBUSY; 311 } 312 if (!cxl_ops->link_ok(adapter, NULL)) 313 return -EIO; 314 cpu_relax(); 315 } 316 return 0; 317 } 318 319 int cxl_invalidate_all_psl8(struct cxl *adapter) 320 { 321 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 322 323 pr_devel("CXL adapter wide TLBIA & SLBIA\n"); 324 325 cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A); 326 327 cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL); 328 while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) { 329 if (time_after_eq(jiffies, timeout)) { 330 dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n"); 331 return -EBUSY; 332 } 333 if (!cxl_ops->link_ok(adapter, NULL)) 334 return -EIO; 335 cpu_relax(); 336 } 337 338 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL); 339 while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) { 340 if (time_after_eq(jiffies, timeout)) { 341 dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n"); 342 return -EBUSY; 343 } 344 if (!cxl_ops->link_ok(adapter, NULL)) 345 return -EIO; 346 cpu_relax(); 347 } 348 return 0; 349 } 350 351 int cxl_data_cache_flush(struct cxl *adapter) 352 { 353 u64 reg; 354 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 355 356 pr_devel("Flushing data cache\n"); 357 358 reg = cxl_p1_read(adapter, CXL_PSL_Control); 359 reg |= CXL_PSL_Control_Fr; 360 cxl_p1_write(adapter, CXL_PSL_Control, reg); 361 362 reg = cxl_p1_read(adapter, CXL_PSL_Control); 363 while ((reg & CXL_PSL_Control_Fs_MASK) != CXL_PSL_Control_Fs_Complete) { 364 if (time_after_eq(jiffies, timeout)) { 365 dev_warn(&adapter->dev, "WARNING: cache flush timed out!\n"); 366 return -EBUSY; 367 } 368 369 if (!cxl_ops->link_ok(adapter, NULL)) { 370 dev_warn(&adapter->dev, "WARNING: link down when flushing cache\n"); 371 return -EIO; 372 } 373 cpu_relax(); 374 reg = cxl_p1_read(adapter, CXL_PSL_Control); 375 } 376 377 reg &= ~CXL_PSL_Control_Fr; 378 cxl_p1_write(adapter, CXL_PSL_Control, reg); 379 return 0; 380 } 381 382 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1) 383 { 384 int rc; 385 386 /* 1. Disable SSTP by writing 0 to SSTP1[V] */ 387 cxl_p2n_write(afu, CXL_SSTP1_An, 0); 388 389 /* 2. Invalidate all SLB entries */ 390 if ((rc = cxl_afu_slbia(afu))) 391 return rc; 392 393 /* 3. Set SSTP0_An */ 394 cxl_p2n_write(afu, CXL_SSTP0_An, sstp0); 395 396 /* 4. Set SSTP1_An */ 397 cxl_p2n_write(afu, CXL_SSTP1_An, sstp1); 398 399 return 0; 400 } 401 402 /* Using per slice version may improve performance here. (ie. SLBIA_An) */ 403 static void slb_invalid(struct cxl_context *ctx) 404 { 405 struct cxl *adapter = ctx->afu->adapter; 406 u64 slbia; 407 408 WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex)); 409 410 cxl_p1_write(adapter, CXL_PSL_LBISEL, 411 ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) | 412 be32_to_cpu(ctx->elem->lpid)); 413 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID); 414 415 while (1) { 416 if (!cxl_ops->link_ok(adapter, NULL)) 417 break; 418 slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA); 419 if (!(slbia & CXL_TLB_SLB_P)) 420 break; 421 cpu_relax(); 422 } 423 } 424 425 static int do_process_element_cmd(struct cxl_context *ctx, 426 u64 cmd, u64 pe_state) 427 { 428 u64 state; 429 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 430 int rc = 0; 431 432 trace_cxl_llcmd(ctx, cmd); 433 434 WARN_ON(!ctx->afu->enabled); 435 436 ctx->elem->software_state = cpu_to_be32(pe_state); 437 smp_wmb(); 438 *(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe); 439 smp_mb(); 440 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe); 441 while (1) { 442 if (time_after_eq(jiffies, timeout)) { 443 dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n"); 444 rc = -EBUSY; 445 goto out; 446 } 447 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) { 448 dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n"); 449 rc = -EIO; 450 goto out; 451 } 452 state = be64_to_cpup(ctx->afu->native->sw_command_status); 453 if (state == ~0ULL) { 454 pr_err("cxl: Error adding process element to AFU\n"); 455 rc = -1; 456 goto out; 457 } 458 if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK | CXL_SPA_SW_LINK_MASK)) == 459 (cmd | (cmd >> 16) | ctx->pe)) 460 break; 461 /* 462 * The command won't finish in the PSL if there are 463 * outstanding DSIs. Hence we need to yield here in 464 * case there are outstanding DSIs that we need to 465 * service. Tuning possiblity: we could wait for a 466 * while before sched 467 */ 468 schedule(); 469 470 } 471 out: 472 trace_cxl_llcmd_done(ctx, cmd, rc); 473 return rc; 474 } 475 476 static int add_process_element(struct cxl_context *ctx) 477 { 478 int rc = 0; 479 480 mutex_lock(&ctx->afu->native->spa_mutex); 481 pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe); 482 if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V))) 483 ctx->pe_inserted = true; 484 pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe); 485 mutex_unlock(&ctx->afu->native->spa_mutex); 486 return rc; 487 } 488 489 static int terminate_process_element(struct cxl_context *ctx) 490 { 491 int rc = 0; 492 493 /* fast path terminate if it's already invalid */ 494 if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V))) 495 return rc; 496 497 mutex_lock(&ctx->afu->native->spa_mutex); 498 pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe); 499 /* We could be asked to terminate when the hw is down. That 500 * should always succeed: it's not running if the hw has gone 501 * away and is being reset. 502 */ 503 if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) 504 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE, 505 CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T); 506 ctx->elem->software_state = 0; /* Remove Valid bit */ 507 pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe); 508 mutex_unlock(&ctx->afu->native->spa_mutex); 509 return rc; 510 } 511 512 static int remove_process_element(struct cxl_context *ctx) 513 { 514 int rc = 0; 515 516 mutex_lock(&ctx->afu->native->spa_mutex); 517 pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe); 518 519 /* We could be asked to remove when the hw is down. Again, if 520 * the hw is down, the PE is gone, so we succeed. 521 */ 522 if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) 523 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0); 524 525 if (!rc) 526 ctx->pe_inserted = false; 527 if (cxl_is_power8()) 528 slb_invalid(ctx); 529 pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe); 530 mutex_unlock(&ctx->afu->native->spa_mutex); 531 532 return rc; 533 } 534 535 void cxl_assign_psn_space(struct cxl_context *ctx) 536 { 537 if (!ctx->afu->pp_size || ctx->master) { 538 ctx->psn_phys = ctx->afu->psn_phys; 539 ctx->psn_size = ctx->afu->adapter->ps_size; 540 } else { 541 ctx->psn_phys = ctx->afu->psn_phys + 542 (ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe); 543 ctx->psn_size = ctx->afu->pp_size; 544 } 545 } 546 547 static int activate_afu_directed(struct cxl_afu *afu) 548 { 549 int rc; 550 551 dev_info(&afu->dev, "Activating AFU directed mode\n"); 552 553 afu->num_procs = afu->max_procs_virtualised; 554 if (afu->native->spa == NULL) { 555 if (cxl_alloc_spa(afu, CXL_MODE_DIRECTED)) 556 return -ENOMEM; 557 } 558 attach_spa(afu); 559 560 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU); 561 if (cxl_is_power8()) 562 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 563 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); 564 565 afu->current_mode = CXL_MODE_DIRECTED; 566 567 if ((rc = cxl_chardev_m_afu_add(afu))) 568 return rc; 569 570 if ((rc = cxl_sysfs_afu_m_add(afu))) 571 goto err; 572 573 if ((rc = cxl_chardev_s_afu_add(afu))) 574 goto err1; 575 576 return 0; 577 err1: 578 cxl_sysfs_afu_m_remove(afu); 579 err: 580 cxl_chardev_afu_remove(afu); 581 return rc; 582 } 583 584 #ifdef CONFIG_CPU_LITTLE_ENDIAN 585 #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE) 586 #else 587 #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE)) 588 #endif 589 590 u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9) 591 { 592 u64 sr = 0; 593 594 set_endian(sr); 595 if (master) 596 sr |= CXL_PSL_SR_An_MP; 597 if (mfspr(SPRN_LPCR) & LPCR_TC) 598 sr |= CXL_PSL_SR_An_TC; 599 if (kernel) { 600 if (!real_mode) 601 sr |= CXL_PSL_SR_An_R; 602 sr |= (mfmsr() & MSR_SF) | CXL_PSL_SR_An_HV; 603 } else { 604 sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R; 605 if (radix_enabled()) 606 sr |= CXL_PSL_SR_An_HV; 607 else 608 sr &= ~(CXL_PSL_SR_An_HV); 609 if (!test_tsk_thread_flag(current, TIF_32BIT)) 610 sr |= CXL_PSL_SR_An_SF; 611 } 612 if (p9) { 613 if (radix_enabled()) 614 sr |= CXL_PSL_SR_An_XLAT_ror; 615 else 616 sr |= CXL_PSL_SR_An_XLAT_hpt; 617 } 618 return sr; 619 } 620 621 static u64 calculate_sr(struct cxl_context *ctx) 622 { 623 return cxl_calculate_sr(ctx->master, ctx->kernel, ctx->real_mode, 624 cxl_is_power9()); 625 } 626 627 static void update_ivtes_directed(struct cxl_context *ctx) 628 { 629 bool need_update = (ctx->status == STARTED); 630 int r; 631 632 if (need_update) { 633 WARN_ON(terminate_process_element(ctx)); 634 WARN_ON(remove_process_element(ctx)); 635 } 636 637 for (r = 0; r < CXL_IRQ_RANGES; r++) { 638 ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]); 639 ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]); 640 } 641 642 /* 643 * Theoretically we could use the update llcmd, instead of a 644 * terminate/remove/add (or if an atomic update was required we could 645 * do a suspend/update/resume), however it seems there might be issues 646 * with the update llcmd on some cards (including those using an XSL on 647 * an ASIC) so for now it's safest to go with the commands that are 648 * known to work. In the future if we come across a situation where the 649 * card may be performing transactions using the same PE while we are 650 * doing this update we might need to revisit this. 651 */ 652 if (need_update) 653 WARN_ON(add_process_element(ctx)); 654 } 655 656 static int process_element_entry_psl9(struct cxl_context *ctx, u64 wed, u64 amr) 657 { 658 u32 pid; 659 int rc; 660 661 cxl_assign_psn_space(ctx); 662 663 ctx->elem->ctxtime = 0; /* disable */ 664 ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID)); 665 ctx->elem->haurp = 0; /* disable */ 666 667 if (ctx->kernel) 668 pid = 0; 669 else { 670 if (ctx->mm == NULL) { 671 pr_devel("%s: unable to get mm for pe=%d pid=%i\n", 672 __func__, ctx->pe, pid_nr(ctx->pid)); 673 return -EINVAL; 674 } 675 pid = ctx->mm->context.id; 676 } 677 678 /* Assign a unique TIDR (thread id) for the current thread */ 679 if (!(ctx->tidr) && (ctx->assign_tidr)) { 680 rc = set_thread_tidr(current); 681 if (rc) 682 return -ENODEV; 683 ctx->tidr = current->thread.tidr; 684 pr_devel("%s: current tidr: %d\n", __func__, ctx->tidr); 685 } 686 687 ctx->elem->common.tid = cpu_to_be32(ctx->tidr); 688 ctx->elem->common.pid = cpu_to_be32(pid); 689 690 ctx->elem->sr = cpu_to_be64(calculate_sr(ctx)); 691 692 ctx->elem->common.csrp = 0; /* disable */ 693 694 cxl_prefault(ctx, wed); 695 696 /* 697 * Ensure we have the multiplexed PSL interrupt set up to take faults 698 * for kernel contexts that may not have allocated any AFU IRQs at all: 699 */ 700 if (ctx->irqs.range[0] == 0) { 701 ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq; 702 ctx->irqs.range[0] = 1; 703 } 704 705 ctx->elem->common.amr = cpu_to_be64(amr); 706 ctx->elem->common.wed = cpu_to_be64(wed); 707 708 return 0; 709 } 710 711 int cxl_attach_afu_directed_psl9(struct cxl_context *ctx, u64 wed, u64 amr) 712 { 713 int result; 714 715 /* fill the process element entry */ 716 result = process_element_entry_psl9(ctx, wed, amr); 717 if (result) 718 return result; 719 720 update_ivtes_directed(ctx); 721 722 /* first guy needs to enable */ 723 result = cxl_ops->afu_check_and_enable(ctx->afu); 724 if (result) 725 return result; 726 727 return add_process_element(ctx); 728 } 729 730 int cxl_attach_afu_directed_psl8(struct cxl_context *ctx, u64 wed, u64 amr) 731 { 732 u32 pid; 733 int result; 734 735 cxl_assign_psn_space(ctx); 736 737 ctx->elem->ctxtime = 0; /* disable */ 738 ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID)); 739 ctx->elem->haurp = 0; /* disable */ 740 ctx->elem->u.sdr = cpu_to_be64(mfspr(SPRN_SDR1)); 741 742 pid = current->pid; 743 if (ctx->kernel) 744 pid = 0; 745 ctx->elem->common.tid = 0; 746 ctx->elem->common.pid = cpu_to_be32(pid); 747 748 ctx->elem->sr = cpu_to_be64(calculate_sr(ctx)); 749 750 ctx->elem->common.csrp = 0; /* disable */ 751 ctx->elem->common.u.psl8.aurp0 = 0; /* disable */ 752 ctx->elem->common.u.psl8.aurp1 = 0; /* disable */ 753 754 cxl_prefault(ctx, wed); 755 756 ctx->elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0); 757 ctx->elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1); 758 759 /* 760 * Ensure we have the multiplexed PSL interrupt set up to take faults 761 * for kernel contexts that may not have allocated any AFU IRQs at all: 762 */ 763 if (ctx->irqs.range[0] == 0) { 764 ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq; 765 ctx->irqs.range[0] = 1; 766 } 767 768 update_ivtes_directed(ctx); 769 770 ctx->elem->common.amr = cpu_to_be64(amr); 771 ctx->elem->common.wed = cpu_to_be64(wed); 772 773 /* first guy needs to enable */ 774 if ((result = cxl_ops->afu_check_and_enable(ctx->afu))) 775 return result; 776 777 return add_process_element(ctx); 778 } 779 780 static int deactivate_afu_directed(struct cxl_afu *afu) 781 { 782 dev_info(&afu->dev, "Deactivating AFU directed mode\n"); 783 784 afu->current_mode = 0; 785 afu->num_procs = 0; 786 787 cxl_sysfs_afu_m_remove(afu); 788 cxl_chardev_afu_remove(afu); 789 790 /* 791 * The CAIA section 2.2.1 indicates that the procedure for starting and 792 * stopping an AFU in AFU directed mode is AFU specific, which is not 793 * ideal since this code is generic and with one exception has no 794 * knowledge of the AFU. This is in contrast to the procedure for 795 * disabling a dedicated process AFU, which is documented to just 796 * require a reset. The architecture does indicate that both an AFU 797 * reset and an AFU disable should result in the AFU being disabled and 798 * we do both followed by a PSL purge for safety. 799 * 800 * Notably we used to have some issues with the disable sequence on PSL 801 * cards, which is why we ended up using this heavy weight procedure in 802 * the first place, however a bug was discovered that had rendered the 803 * disable operation ineffective, so it is conceivable that was the 804 * sole explanation for those difficulties. Careful regression testing 805 * is recommended if anyone attempts to remove or reorder these 806 * operations. 807 * 808 * The XSL on the Mellanox CX4 behaves a little differently from the 809 * PSL based cards and will time out an AFU reset if the AFU is still 810 * enabled. That card is special in that we do have a means to identify 811 * it from this code, so in that case we skip the reset and just use a 812 * disable/purge to avoid the timeout and corresponding noise in the 813 * kernel log. 814 */ 815 if (afu->adapter->native->sl_ops->needs_reset_before_disable) 816 cxl_ops->afu_reset(afu); 817 cxl_afu_disable(afu); 818 cxl_psl_purge(afu); 819 820 return 0; 821 } 822 823 int cxl_activate_dedicated_process_psl9(struct cxl_afu *afu) 824 { 825 dev_info(&afu->dev, "Activating dedicated process mode\n"); 826 827 /* 828 * If XSL is set to dedicated mode (Set in PSL_SCNTL reg), the 829 * XSL and AFU are programmed to work with a single context. 830 * The context information should be configured in the SPA area 831 * index 0 (so PSL_SPAP must be configured before enabling the 832 * AFU). 833 */ 834 afu->num_procs = 1; 835 if (afu->native->spa == NULL) { 836 if (cxl_alloc_spa(afu, CXL_MODE_DEDICATED)) 837 return -ENOMEM; 838 } 839 attach_spa(afu); 840 841 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); 842 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); 843 844 afu->current_mode = CXL_MODE_DEDICATED; 845 846 return cxl_chardev_d_afu_add(afu); 847 } 848 849 int cxl_activate_dedicated_process_psl8(struct cxl_afu *afu) 850 { 851 dev_info(&afu->dev, "Activating dedicated process mode\n"); 852 853 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); 854 855 cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */ 856 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */ 857 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 858 cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID)); 859 cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */ 860 cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1)); 861 862 cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */ 863 cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */ 864 cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */ 865 866 afu->current_mode = CXL_MODE_DEDICATED; 867 afu->num_procs = 1; 868 869 return cxl_chardev_d_afu_add(afu); 870 } 871 872 void cxl_update_dedicated_ivtes_psl9(struct cxl_context *ctx) 873 { 874 int r; 875 876 for (r = 0; r < CXL_IRQ_RANGES; r++) { 877 ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]); 878 ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]); 879 } 880 } 881 882 void cxl_update_dedicated_ivtes_psl8(struct cxl_context *ctx) 883 { 884 struct cxl_afu *afu = ctx->afu; 885 886 cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 887 (((u64)ctx->irqs.offset[0] & 0xffff) << 48) | 888 (((u64)ctx->irqs.offset[1] & 0xffff) << 32) | 889 (((u64)ctx->irqs.offset[2] & 0xffff) << 16) | 890 ((u64)ctx->irqs.offset[3] & 0xffff)); 891 cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64) 892 (((u64)ctx->irqs.range[0] & 0xffff) << 48) | 893 (((u64)ctx->irqs.range[1] & 0xffff) << 32) | 894 (((u64)ctx->irqs.range[2] & 0xffff) << 16) | 895 ((u64)ctx->irqs.range[3] & 0xffff)); 896 } 897 898 int cxl_attach_dedicated_process_psl9(struct cxl_context *ctx, u64 wed, u64 amr) 899 { 900 struct cxl_afu *afu = ctx->afu; 901 int result; 902 903 /* fill the process element entry */ 904 result = process_element_entry_psl9(ctx, wed, amr); 905 if (result) 906 return result; 907 908 if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes) 909 afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx); 910 911 ctx->elem->software_state = cpu_to_be32(CXL_PE_SOFTWARE_STATE_V); 912 /* 913 * Ideally we should do a wmb() here to make sure the changes to the 914 * PE are visible to the card before we call afu_enable. 915 * On ppc64 though all mmios are preceded by a 'sync' instruction hence 916 * we dont dont need one here. 917 */ 918 919 result = cxl_ops->afu_reset(afu); 920 if (result) 921 return result; 922 923 return afu_enable(afu); 924 } 925 926 int cxl_attach_dedicated_process_psl8(struct cxl_context *ctx, u64 wed, u64 amr) 927 { 928 struct cxl_afu *afu = ctx->afu; 929 u64 pid; 930 int rc; 931 932 pid = (u64)current->pid << 32; 933 if (ctx->kernel) 934 pid = 0; 935 cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid); 936 937 cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx)); 938 939 if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1))) 940 return rc; 941 942 cxl_prefault(ctx, wed); 943 944 if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes) 945 afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx); 946 947 cxl_p2n_write(afu, CXL_PSL_AMR_An, amr); 948 949 /* master only context for dedicated */ 950 cxl_assign_psn_space(ctx); 951 952 if ((rc = cxl_ops->afu_reset(afu))) 953 return rc; 954 955 cxl_p2n_write(afu, CXL_PSL_WED_An, wed); 956 957 return afu_enable(afu); 958 } 959 960 static int deactivate_dedicated_process(struct cxl_afu *afu) 961 { 962 dev_info(&afu->dev, "Deactivating dedicated process mode\n"); 963 964 afu->current_mode = 0; 965 afu->num_procs = 0; 966 967 cxl_chardev_afu_remove(afu); 968 969 return 0; 970 } 971 972 static int native_afu_deactivate_mode(struct cxl_afu *afu, int mode) 973 { 974 if (mode == CXL_MODE_DIRECTED) 975 return deactivate_afu_directed(afu); 976 if (mode == CXL_MODE_DEDICATED) 977 return deactivate_dedicated_process(afu); 978 return 0; 979 } 980 981 static int native_afu_activate_mode(struct cxl_afu *afu, int mode) 982 { 983 if (!mode) 984 return 0; 985 if (!(mode & afu->modes_supported)) 986 return -EINVAL; 987 988 if (!cxl_ops->link_ok(afu->adapter, afu)) { 989 WARN(1, "Device link is down, refusing to activate!\n"); 990 return -EIO; 991 } 992 993 if (mode == CXL_MODE_DIRECTED) 994 return activate_afu_directed(afu); 995 if ((mode == CXL_MODE_DEDICATED) && 996 (afu->adapter->native->sl_ops->activate_dedicated_process)) 997 return afu->adapter->native->sl_ops->activate_dedicated_process(afu); 998 999 return -EINVAL; 1000 } 1001 1002 static int native_attach_process(struct cxl_context *ctx, bool kernel, 1003 u64 wed, u64 amr) 1004 { 1005 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) { 1006 WARN(1, "Device link is down, refusing to attach process!\n"); 1007 return -EIO; 1008 } 1009 1010 ctx->kernel = kernel; 1011 if ((ctx->afu->current_mode == CXL_MODE_DIRECTED) && 1012 (ctx->afu->adapter->native->sl_ops->attach_afu_directed)) 1013 return ctx->afu->adapter->native->sl_ops->attach_afu_directed(ctx, wed, amr); 1014 1015 if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) && 1016 (ctx->afu->adapter->native->sl_ops->attach_dedicated_process)) 1017 return ctx->afu->adapter->native->sl_ops->attach_dedicated_process(ctx, wed, amr); 1018 1019 return -EINVAL; 1020 } 1021 1022 static inline int detach_process_native_dedicated(struct cxl_context *ctx) 1023 { 1024 /* 1025 * The CAIA section 2.1.1 indicates that we need to do an AFU reset to 1026 * stop the AFU in dedicated mode (we therefore do not make that 1027 * optional like we do in the afu directed path). It does not indicate 1028 * that we need to do an explicit disable (which should occur 1029 * implicitly as part of the reset) or purge, but we do these as well 1030 * to be on the safe side. 1031 * 1032 * Notably we used to have some issues with the disable sequence 1033 * (before the sequence was spelled out in the architecture) which is 1034 * why we were so heavy weight in the first place, however a bug was 1035 * discovered that had rendered the disable operation ineffective, so 1036 * it is conceivable that was the sole explanation for those 1037 * difficulties. Point is, we should be careful and do some regression 1038 * testing if we ever attempt to remove any part of this procedure. 1039 */ 1040 cxl_ops->afu_reset(ctx->afu); 1041 cxl_afu_disable(ctx->afu); 1042 cxl_psl_purge(ctx->afu); 1043 return 0; 1044 } 1045 1046 static void native_update_ivtes(struct cxl_context *ctx) 1047 { 1048 if (ctx->afu->current_mode == CXL_MODE_DIRECTED) 1049 return update_ivtes_directed(ctx); 1050 if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) && 1051 (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)) 1052 return ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx); 1053 WARN(1, "native_update_ivtes: Bad mode\n"); 1054 } 1055 1056 static inline int detach_process_native_afu_directed(struct cxl_context *ctx) 1057 { 1058 if (!ctx->pe_inserted) 1059 return 0; 1060 if (terminate_process_element(ctx)) 1061 return -1; 1062 if (remove_process_element(ctx)) 1063 return -1; 1064 1065 return 0; 1066 } 1067 1068 static int native_detach_process(struct cxl_context *ctx) 1069 { 1070 trace_cxl_detach(ctx); 1071 1072 if (ctx->afu->current_mode == CXL_MODE_DEDICATED) 1073 return detach_process_native_dedicated(ctx); 1074 1075 return detach_process_native_afu_directed(ctx); 1076 } 1077 1078 static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info) 1079 { 1080 /* If the adapter has gone away, we can't get any meaningful 1081 * information. 1082 */ 1083 if (!cxl_ops->link_ok(afu->adapter, afu)) 1084 return -EIO; 1085 1086 info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 1087 info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 1088 if (cxl_is_power8()) 1089 info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An); 1090 info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An); 1091 info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); 1092 info->proc_handle = 0; 1093 1094 return 0; 1095 } 1096 1097 void cxl_native_irq_dump_regs_psl9(struct cxl_context *ctx) 1098 { 1099 u64 fir1, serr; 1100 1101 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL9_FIR1); 1102 1103 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1); 1104 if (ctx->afu->adapter->native->sl_ops->register_serr_irq) { 1105 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An); 1106 cxl_afu_decode_psl_serr(ctx->afu, serr); 1107 } 1108 } 1109 1110 void cxl_native_irq_dump_regs_psl8(struct cxl_context *ctx) 1111 { 1112 u64 fir1, fir2, fir_slice, serr, afu_debug; 1113 1114 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1); 1115 fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2); 1116 fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An); 1117 afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An); 1118 1119 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1); 1120 dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2); 1121 if (ctx->afu->adapter->native->sl_ops->register_serr_irq) { 1122 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An); 1123 cxl_afu_decode_psl_serr(ctx->afu, serr); 1124 } 1125 dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice); 1126 dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug); 1127 } 1128 1129 static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx, 1130 u64 dsisr, u64 errstat) 1131 { 1132 1133 dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat); 1134 1135 if (ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers) 1136 ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers(ctx); 1137 1138 if (ctx->afu->adapter->native->sl_ops->debugfs_stop_trace) { 1139 dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n"); 1140 ctx->afu->adapter->native->sl_ops->debugfs_stop_trace(ctx->afu->adapter); 1141 } 1142 1143 return cxl_ops->ack_irq(ctx, 0, errstat); 1144 } 1145 1146 static bool cxl_is_translation_fault(struct cxl_afu *afu, u64 dsisr) 1147 { 1148 if ((cxl_is_power8()) && (dsisr & CXL_PSL_DSISR_TRANS)) 1149 return true; 1150 1151 if ((cxl_is_power9()) && (dsisr & CXL_PSL9_DSISR_An_TF)) 1152 return true; 1153 1154 return false; 1155 } 1156 1157 irqreturn_t cxl_fail_irq_psl(struct cxl_afu *afu, struct cxl_irq_info *irq_info) 1158 { 1159 if (cxl_is_translation_fault(afu, irq_info->dsisr)) 1160 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); 1161 else 1162 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); 1163 1164 return IRQ_HANDLED; 1165 } 1166 1167 static irqreturn_t native_irq_multiplexed(int irq, void *data) 1168 { 1169 struct cxl_afu *afu = data; 1170 struct cxl_context *ctx; 1171 struct cxl_irq_info irq_info; 1172 u64 phreg = cxl_p2n_read(afu, CXL_PSL_PEHandle_An); 1173 int ph, ret = IRQ_HANDLED, res; 1174 1175 /* check if eeh kicked in while the interrupt was in flight */ 1176 if (unlikely(phreg == ~0ULL)) { 1177 dev_warn(&afu->dev, 1178 "Ignoring slice interrupt(%d) due to fenced card", 1179 irq); 1180 return IRQ_HANDLED; 1181 } 1182 /* Mask the pe-handle from register value */ 1183 ph = phreg & 0xffff; 1184 if ((res = native_get_irq_info(afu, &irq_info))) { 1185 WARN(1, "Unable to get CXL IRQ Info: %i\n", res); 1186 if (afu->adapter->native->sl_ops->fail_irq) 1187 return afu->adapter->native->sl_ops->fail_irq(afu, &irq_info); 1188 return ret; 1189 } 1190 1191 rcu_read_lock(); 1192 ctx = idr_find(&afu->contexts_idr, ph); 1193 if (ctx) { 1194 if (afu->adapter->native->sl_ops->handle_interrupt) 1195 ret = afu->adapter->native->sl_ops->handle_interrupt(irq, ctx, &irq_info); 1196 rcu_read_unlock(); 1197 return ret; 1198 } 1199 rcu_read_unlock(); 1200 1201 WARN(1, "Unable to demultiplex CXL PSL IRQ for PE %i DSISR %016llx DAR" 1202 " %016llx\n(Possible AFU HW issue - was a term/remove acked" 1203 " with outstanding transactions?)\n", ph, irq_info.dsisr, 1204 irq_info.dar); 1205 if (afu->adapter->native->sl_ops->fail_irq) 1206 ret = afu->adapter->native->sl_ops->fail_irq(afu, &irq_info); 1207 return ret; 1208 } 1209 1210 static void native_irq_wait(struct cxl_context *ctx) 1211 { 1212 u64 dsisr; 1213 int timeout = 1000; 1214 int ph; 1215 1216 /* 1217 * Wait until no further interrupts are presented by the PSL 1218 * for this context. 1219 */ 1220 while (timeout--) { 1221 ph = cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) & 0xffff; 1222 if (ph != ctx->pe) 1223 return; 1224 dsisr = cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An); 1225 if (cxl_is_power8() && 1226 ((dsisr & CXL_PSL_DSISR_PENDING) == 0)) 1227 return; 1228 if (cxl_is_power9() && 1229 ((dsisr & CXL_PSL9_DSISR_PENDING) == 0)) 1230 return; 1231 /* 1232 * We are waiting for the workqueue to process our 1233 * irq, so need to let that run here. 1234 */ 1235 msleep(1); 1236 } 1237 1238 dev_warn(&ctx->afu->dev, "WARNING: waiting on DSI for PE %i" 1239 " DSISR %016llx!\n", ph, dsisr); 1240 return; 1241 } 1242 1243 static irqreturn_t native_slice_irq_err(int irq, void *data) 1244 { 1245 struct cxl_afu *afu = data; 1246 u64 errstat, serr, afu_error, dsisr; 1247 u64 fir_slice, afu_debug, irq_mask; 1248 1249 /* 1250 * slice err interrupt is only used with full PSL (no XSL) 1251 */ 1252 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An); 1253 errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); 1254 afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An); 1255 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 1256 cxl_afu_decode_psl_serr(afu, serr); 1257 1258 if (cxl_is_power8()) { 1259 fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An); 1260 afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An); 1261 dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice); 1262 dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug); 1263 } 1264 dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat); 1265 dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error); 1266 dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr); 1267 1268 /* mask off the IRQ so it won't retrigger until the AFU is reset */ 1269 irq_mask = (serr & CXL_PSL_SERR_An_IRQS) >> 32; 1270 serr |= irq_mask; 1271 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr); 1272 dev_info(&afu->dev, "Further such interrupts will be masked until the AFU is reset\n"); 1273 1274 return IRQ_HANDLED; 1275 } 1276 1277 void cxl_native_err_irq_dump_regs_psl9(struct cxl *adapter) 1278 { 1279 u64 fir1; 1280 1281 fir1 = cxl_p1_read(adapter, CXL_PSL9_FIR1); 1282 dev_crit(&adapter->dev, "PSL_FIR: 0x%016llx\n", fir1); 1283 } 1284 1285 void cxl_native_err_irq_dump_regs_psl8(struct cxl *adapter) 1286 { 1287 u64 fir1, fir2; 1288 1289 fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1); 1290 fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2); 1291 dev_crit(&adapter->dev, 1292 "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", 1293 fir1, fir2); 1294 } 1295 1296 static irqreturn_t native_irq_err(int irq, void *data) 1297 { 1298 struct cxl *adapter = data; 1299 u64 err_ivte; 1300 1301 WARN(1, "CXL ERROR interrupt %i\n", irq); 1302 1303 err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE); 1304 dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%016llx\n", err_ivte); 1305 1306 if (adapter->native->sl_ops->debugfs_stop_trace) { 1307 dev_crit(&adapter->dev, "STOPPING CXL TRACE\n"); 1308 adapter->native->sl_ops->debugfs_stop_trace(adapter); 1309 } 1310 1311 if (adapter->native->sl_ops->err_irq_dump_registers) 1312 adapter->native->sl_ops->err_irq_dump_registers(adapter); 1313 1314 return IRQ_HANDLED; 1315 } 1316 1317 int cxl_native_register_psl_err_irq(struct cxl *adapter) 1318 { 1319 int rc; 1320 1321 adapter->irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err", 1322 dev_name(&adapter->dev)); 1323 if (!adapter->irq_name) 1324 return -ENOMEM; 1325 1326 if ((rc = cxl_register_one_irq(adapter, native_irq_err, adapter, 1327 &adapter->native->err_hwirq, 1328 &adapter->native->err_virq, 1329 adapter->irq_name))) { 1330 kfree(adapter->irq_name); 1331 adapter->irq_name = NULL; 1332 return rc; 1333 } 1334 1335 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->native->err_hwirq & 0xffff); 1336 1337 return 0; 1338 } 1339 1340 void cxl_native_release_psl_err_irq(struct cxl *adapter) 1341 { 1342 if (adapter->native->err_virq == 0 || 1343 adapter->native->err_virq != 1344 irq_find_mapping(NULL, adapter->native->err_hwirq)) 1345 return; 1346 1347 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000); 1348 cxl_unmap_irq(adapter->native->err_virq, adapter); 1349 cxl_ops->release_one_irq(adapter, adapter->native->err_hwirq); 1350 kfree(adapter->irq_name); 1351 adapter->native->err_virq = 0; 1352 } 1353 1354 int cxl_native_register_serr_irq(struct cxl_afu *afu) 1355 { 1356 u64 serr; 1357 int rc; 1358 1359 afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err", 1360 dev_name(&afu->dev)); 1361 if (!afu->err_irq_name) 1362 return -ENOMEM; 1363 1364 if ((rc = cxl_register_one_irq(afu->adapter, native_slice_irq_err, afu, 1365 &afu->serr_hwirq, 1366 &afu->serr_virq, afu->err_irq_name))) { 1367 kfree(afu->err_irq_name); 1368 afu->err_irq_name = NULL; 1369 return rc; 1370 } 1371 1372 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An); 1373 if (cxl_is_power8()) 1374 serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff); 1375 if (cxl_is_power9()) { 1376 /* 1377 * By default, all errors are masked. So don't set all masks. 1378 * Slice errors will be transfered. 1379 */ 1380 serr = (serr & ~0xff0000007fffffffULL) | (afu->serr_hwirq & 0xffff); 1381 } 1382 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr); 1383 1384 return 0; 1385 } 1386 1387 void cxl_native_release_serr_irq(struct cxl_afu *afu) 1388 { 1389 if (afu->serr_virq == 0 || 1390 afu->serr_virq != irq_find_mapping(NULL, afu->serr_hwirq)) 1391 return; 1392 1393 cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000); 1394 cxl_unmap_irq(afu->serr_virq, afu); 1395 cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq); 1396 kfree(afu->err_irq_name); 1397 afu->serr_virq = 0; 1398 } 1399 1400 int cxl_native_register_psl_irq(struct cxl_afu *afu) 1401 { 1402 int rc; 1403 1404 afu->psl_irq_name = kasprintf(GFP_KERNEL, "cxl-%s", 1405 dev_name(&afu->dev)); 1406 if (!afu->psl_irq_name) 1407 return -ENOMEM; 1408 1409 if ((rc = cxl_register_one_irq(afu->adapter, native_irq_multiplexed, 1410 afu, &afu->native->psl_hwirq, &afu->native->psl_virq, 1411 afu->psl_irq_name))) { 1412 kfree(afu->psl_irq_name); 1413 afu->psl_irq_name = NULL; 1414 } 1415 return rc; 1416 } 1417 1418 void cxl_native_release_psl_irq(struct cxl_afu *afu) 1419 { 1420 if (afu->native->psl_virq == 0 || 1421 afu->native->psl_virq != 1422 irq_find_mapping(NULL, afu->native->psl_hwirq)) 1423 return; 1424 1425 cxl_unmap_irq(afu->native->psl_virq, afu); 1426 cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq); 1427 kfree(afu->psl_irq_name); 1428 afu->native->psl_virq = 0; 1429 } 1430 1431 static void recover_psl_err(struct cxl_afu *afu, u64 errstat) 1432 { 1433 u64 dsisr; 1434 1435 pr_devel("RECOVERING FROM PSL ERROR... (0x%016llx)\n", errstat); 1436 1437 /* Clear PSL_DSISR[PE] */ 1438 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 1439 cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE); 1440 1441 /* Write 1s to clear error status bits */ 1442 cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat); 1443 } 1444 1445 static int native_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask) 1446 { 1447 trace_cxl_psl_irq_ack(ctx, tfc); 1448 if (tfc) 1449 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc); 1450 if (psl_reset_mask) 1451 recover_psl_err(ctx->afu, psl_reset_mask); 1452 1453 return 0; 1454 } 1455 1456 int cxl_check_error(struct cxl_afu *afu) 1457 { 1458 return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL); 1459 } 1460 1461 static bool native_support_attributes(const char *attr_name, 1462 enum cxl_attrs type) 1463 { 1464 return true; 1465 } 1466 1467 static int native_afu_cr_read64(struct cxl_afu *afu, int cr, u64 off, u64 *out) 1468 { 1469 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu))) 1470 return -EIO; 1471 if (unlikely(off >= afu->crs_len)) 1472 return -ERANGE; 1473 *out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset + 1474 (cr * afu->crs_len) + off); 1475 return 0; 1476 } 1477 1478 static int native_afu_cr_read32(struct cxl_afu *afu, int cr, u64 off, u32 *out) 1479 { 1480 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu))) 1481 return -EIO; 1482 if (unlikely(off >= afu->crs_len)) 1483 return -ERANGE; 1484 *out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset + 1485 (cr * afu->crs_len) + off); 1486 return 0; 1487 } 1488 1489 static int native_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off, u16 *out) 1490 { 1491 u64 aligned_off = off & ~0x3L; 1492 u32 val; 1493 int rc; 1494 1495 rc = native_afu_cr_read32(afu, cr, aligned_off, &val); 1496 if (!rc) 1497 *out = (val >> ((off & 0x3) * 8)) & 0xffff; 1498 return rc; 1499 } 1500 1501 static int native_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off, u8 *out) 1502 { 1503 u64 aligned_off = off & ~0x3L; 1504 u32 val; 1505 int rc; 1506 1507 rc = native_afu_cr_read32(afu, cr, aligned_off, &val); 1508 if (!rc) 1509 *out = (val >> ((off & 0x3) * 8)) & 0xff; 1510 return rc; 1511 } 1512 1513 static int native_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in) 1514 { 1515 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu))) 1516 return -EIO; 1517 if (unlikely(off >= afu->crs_len)) 1518 return -ERANGE; 1519 out_le32(afu->native->afu_desc_mmio + afu->crs_offset + 1520 (cr * afu->crs_len) + off, in); 1521 return 0; 1522 } 1523 1524 static int native_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in) 1525 { 1526 u64 aligned_off = off & ~0x3L; 1527 u32 val32, mask, shift; 1528 int rc; 1529 1530 rc = native_afu_cr_read32(afu, cr, aligned_off, &val32); 1531 if (rc) 1532 return rc; 1533 shift = (off & 0x3) * 8; 1534 WARN_ON(shift == 24); 1535 mask = 0xffff << shift; 1536 val32 = (val32 & ~mask) | (in << shift); 1537 1538 rc = native_afu_cr_write32(afu, cr, aligned_off, val32); 1539 return rc; 1540 } 1541 1542 static int native_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in) 1543 { 1544 u64 aligned_off = off & ~0x3L; 1545 u32 val32, mask, shift; 1546 int rc; 1547 1548 rc = native_afu_cr_read32(afu, cr, aligned_off, &val32); 1549 if (rc) 1550 return rc; 1551 shift = (off & 0x3) * 8; 1552 mask = 0xff << shift; 1553 val32 = (val32 & ~mask) | (in << shift); 1554 1555 rc = native_afu_cr_write32(afu, cr, aligned_off, val32); 1556 return rc; 1557 } 1558 1559 const struct cxl_backend_ops cxl_native_ops = { 1560 .module = THIS_MODULE, 1561 .adapter_reset = cxl_pci_reset, 1562 .alloc_one_irq = cxl_pci_alloc_one_irq, 1563 .release_one_irq = cxl_pci_release_one_irq, 1564 .alloc_irq_ranges = cxl_pci_alloc_irq_ranges, 1565 .release_irq_ranges = cxl_pci_release_irq_ranges, 1566 .setup_irq = cxl_pci_setup_irq, 1567 .handle_psl_slice_error = native_handle_psl_slice_error, 1568 .psl_interrupt = NULL, 1569 .ack_irq = native_ack_irq, 1570 .irq_wait = native_irq_wait, 1571 .attach_process = native_attach_process, 1572 .detach_process = native_detach_process, 1573 .update_ivtes = native_update_ivtes, 1574 .support_attributes = native_support_attributes, 1575 .link_ok = cxl_adapter_link_ok, 1576 .release_afu = cxl_pci_release_afu, 1577 .afu_read_err_buffer = cxl_pci_afu_read_err_buffer, 1578 .afu_check_and_enable = native_afu_check_and_enable, 1579 .afu_activate_mode = native_afu_activate_mode, 1580 .afu_deactivate_mode = native_afu_deactivate_mode, 1581 .afu_reset = native_afu_reset, 1582 .afu_cr_read8 = native_afu_cr_read8, 1583 .afu_cr_read16 = native_afu_cr_read16, 1584 .afu_cr_read32 = native_afu_cr_read32, 1585 .afu_cr_read64 = native_afu_cr_read64, 1586 .afu_cr_write8 = native_afu_cr_write8, 1587 .afu_cr_write16 = native_afu_cr_write16, 1588 .afu_cr_write32 = native_afu_cr_write32, 1589 .read_adapter_vpd = cxl_pci_read_adapter_vpd, 1590 }; 1591