1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/sched/clock.h> 7 8 #include "i915_drv.h" 9 #include "i915_irq.h" 10 #include "intel_breadcrumbs.h" 11 #include "intel_gt.h" 12 #include "intel_gt_irq.h" 13 #include "intel_lrc_reg.h" 14 #include "intel_uncore.h" 15 #include "intel_rps.h" 16 17 static void guc_irq_handler(struct intel_guc *guc, u16 iir) 18 { 19 if (iir & GUC_INTR_GUC2HOST) 20 intel_guc_to_host_event_handler(guc); 21 } 22 23 static u32 24 gen11_gt_engine_identity(struct intel_gt *gt, 25 const unsigned int bank, const unsigned int bit) 26 { 27 void __iomem * const regs = gt->uncore->regs; 28 u32 timeout_ts; 29 u32 ident; 30 31 lockdep_assert_held(>->irq_lock); 32 33 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit)); 34 35 /* 36 * NB: Specs do not specify how long to spin wait, 37 * so we do ~100us as an educated guess. 38 */ 39 timeout_ts = (local_clock() >> 10) + 100; 40 do { 41 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank)); 42 } while (!(ident & GEN11_INTR_DATA_VALID) && 43 !time_after32(local_clock() >> 10, timeout_ts)); 44 45 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) { 46 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", 47 bank, bit, ident); 48 return 0; 49 } 50 51 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank), 52 GEN11_INTR_DATA_VALID); 53 54 return ident; 55 } 56 57 static void 58 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, 59 const u16 iir) 60 { 61 if (instance == OTHER_GUC_INSTANCE) 62 return guc_irq_handler(>->uc.guc, iir); 63 64 if (instance == OTHER_GTPM_INSTANCE) 65 return gen11_rps_irq_handler(>->rps, iir); 66 67 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", 68 instance, iir); 69 } 70 71 static void 72 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class, 73 const u8 instance, const u16 iir) 74 { 75 struct intel_engine_cs *engine; 76 77 if (instance <= MAX_ENGINE_INSTANCE) 78 engine = gt->engine_class[class][instance]; 79 else 80 engine = NULL; 81 82 if (likely(engine)) 83 return intel_engine_cs_irq(engine, iir); 84 85 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n", 86 class, instance); 87 } 88 89 static void 90 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity) 91 { 92 const u8 class = GEN11_INTR_ENGINE_CLASS(identity); 93 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity); 94 const u16 intr = GEN11_INTR_ENGINE_INTR(identity); 95 96 if (unlikely(!intr)) 97 return; 98 99 if (class <= COPY_ENGINE_CLASS) 100 return gen11_engine_irq_handler(gt, class, instance, intr); 101 102 if (class == OTHER_CLASS) 103 return gen11_other_irq_handler(gt, instance, intr); 104 105 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n", 106 class, instance, intr); 107 } 108 109 static void 110 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank) 111 { 112 void __iomem * const regs = gt->uncore->regs; 113 unsigned long intr_dw; 114 unsigned int bit; 115 116 lockdep_assert_held(>->irq_lock); 117 118 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); 119 120 for_each_set_bit(bit, &intr_dw, 32) { 121 const u32 ident = gen11_gt_engine_identity(gt, bank, bit); 122 123 gen11_gt_identity_handler(gt, ident); 124 } 125 126 /* Clear must be after shared has been served for engine */ 127 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw); 128 } 129 130 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl) 131 { 132 unsigned int bank; 133 134 spin_lock(>->irq_lock); 135 136 for (bank = 0; bank < 2; bank++) { 137 if (master_ctl & GEN11_GT_DW_IRQ(bank)) 138 gen11_gt_bank_handler(gt, bank); 139 } 140 141 spin_unlock(>->irq_lock); 142 } 143 144 bool gen11_gt_reset_one_iir(struct intel_gt *gt, 145 const unsigned int bank, const unsigned int bit) 146 { 147 void __iomem * const regs = gt->uncore->regs; 148 u32 dw; 149 150 lockdep_assert_held(>->irq_lock); 151 152 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); 153 if (dw & BIT(bit)) { 154 /* 155 * According to the BSpec, DW_IIR bits cannot be cleared without 156 * first servicing the Selector & Shared IIR registers. 157 */ 158 gen11_gt_engine_identity(gt, bank, bit); 159 160 /* 161 * We locked GT INT DW by reading it. If we want to (try 162 * to) recover from this successfully, we need to clear 163 * our bit, otherwise we are locking the register for 164 * everybody. 165 */ 166 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit)); 167 168 return true; 169 } 170 171 return false; 172 } 173 174 void gen11_gt_irq_reset(struct intel_gt *gt) 175 { 176 struct intel_uncore *uncore = gt->uncore; 177 178 /* Disable RCS, BCS, VCS and VECS class engines. */ 179 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0); 180 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0); 181 182 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ 183 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0); 184 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0); 185 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0); 186 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0); 187 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5)) 188 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~0); 189 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7)) 190 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~0); 191 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0); 192 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) 193 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0); 194 195 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); 196 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); 197 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0); 198 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0); 199 } 200 201 void gen11_gt_irq_postinstall(struct intel_gt *gt) 202 { 203 struct intel_uncore *uncore = gt->uncore; 204 u32 irqs = GT_RENDER_USER_INTERRUPT; 205 u32 dmask; 206 u32 smask; 207 208 if (!intel_uc_wants_guc_submission(>->uc)) 209 irqs |= GT_CS_MASTER_ERROR_INTERRUPT | 210 GT_CONTEXT_SWITCH_INTERRUPT | 211 GT_WAIT_SEMAPHORE_INTERRUPT; 212 213 dmask = irqs << 16 | irqs; 214 smask = irqs << 16; 215 216 BUILD_BUG_ON(irqs & 0xffff0000); 217 218 /* Enable RCS, BCS, VCS and VECS class interrupts. */ 219 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask); 220 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask); 221 222 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ 223 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask); 224 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask); 225 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask); 226 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask); 227 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5)) 228 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask); 229 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7)) 230 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask); 231 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask); 232 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) 233 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask); 234 /* 235 * RPS interrupts will get enabled/disabled on demand when RPS itself 236 * is enabled/disabled. 237 */ 238 gt->pm_ier = 0x0; 239 gt->pm_imr = ~gt->pm_ier; 240 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); 241 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); 242 243 /* Same thing for GuC interrupts */ 244 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0); 245 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0); 246 } 247 248 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) 249 { 250 if (gt_iir & GT_RENDER_USER_INTERRUPT) 251 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], 252 gt_iir); 253 254 if (gt_iir & ILK_BSD_USER_INTERRUPT) 255 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], 256 gt_iir); 257 } 258 259 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) 260 { 261 if (!HAS_L3_DPF(gt->i915)) 262 return; 263 264 spin_lock(>->irq_lock); 265 gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915)); 266 spin_unlock(>->irq_lock); 267 268 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1) 269 gt->i915->l3_parity.which_slice |= 1 << 1; 270 271 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT) 272 gt->i915->l3_parity.which_slice |= 1 << 0; 273 274 schedule_work(>->i915->l3_parity.error_work); 275 } 276 277 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) 278 { 279 if (gt_iir & GT_RENDER_USER_INTERRUPT) 280 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], 281 gt_iir); 282 283 if (gt_iir & GT_BSD_USER_INTERRUPT) 284 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], 285 gt_iir >> 12); 286 287 if (gt_iir & GT_BLT_USER_INTERRUPT) 288 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0], 289 gt_iir >> 22); 290 291 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | 292 GT_BSD_CS_ERROR_INTERRUPT | 293 GT_CS_MASTER_ERROR_INTERRUPT)) 294 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir); 295 296 if (gt_iir & GT_PARITY_ERROR(gt->i915)) 297 gen7_parity_error_irq_handler(gt, gt_iir); 298 } 299 300 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl) 301 { 302 void __iomem * const regs = gt->uncore->regs; 303 u32 iir; 304 305 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { 306 iir = raw_reg_read(regs, GEN8_GT_IIR(0)); 307 if (likely(iir)) { 308 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0], 309 iir >> GEN8_RCS_IRQ_SHIFT); 310 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0], 311 iir >> GEN8_BCS_IRQ_SHIFT); 312 raw_reg_write(regs, GEN8_GT_IIR(0), iir); 313 } 314 } 315 316 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) { 317 iir = raw_reg_read(regs, GEN8_GT_IIR(1)); 318 if (likely(iir)) { 319 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0], 320 iir >> GEN8_VCS0_IRQ_SHIFT); 321 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1], 322 iir >> GEN8_VCS1_IRQ_SHIFT); 323 raw_reg_write(regs, GEN8_GT_IIR(1), iir); 324 } 325 } 326 327 if (master_ctl & GEN8_GT_VECS_IRQ) { 328 iir = raw_reg_read(regs, GEN8_GT_IIR(3)); 329 if (likely(iir)) { 330 intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0], 331 iir >> GEN8_VECS_IRQ_SHIFT); 332 raw_reg_write(regs, GEN8_GT_IIR(3), iir); 333 } 334 } 335 336 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { 337 iir = raw_reg_read(regs, GEN8_GT_IIR(2)); 338 if (likely(iir)) { 339 gen6_rps_irq_handler(>->rps, iir); 340 guc_irq_handler(>->uc.guc, iir >> 16); 341 raw_reg_write(regs, GEN8_GT_IIR(2), iir); 342 } 343 } 344 } 345 346 void gen8_gt_irq_reset(struct intel_gt *gt) 347 { 348 struct intel_uncore *uncore = gt->uncore; 349 350 GEN8_IRQ_RESET_NDX(uncore, GT, 0); 351 GEN8_IRQ_RESET_NDX(uncore, GT, 1); 352 GEN8_IRQ_RESET_NDX(uncore, GT, 2); 353 GEN8_IRQ_RESET_NDX(uncore, GT, 3); 354 } 355 356 void gen8_gt_irq_postinstall(struct intel_gt *gt) 357 { 358 /* These are interrupts we'll toggle with the ring mask register */ 359 const u32 irqs = 360 GT_CS_MASTER_ERROR_INTERRUPT | 361 GT_RENDER_USER_INTERRUPT | 362 GT_CONTEXT_SWITCH_INTERRUPT | 363 GT_WAIT_SEMAPHORE_INTERRUPT; 364 const u32 gt_interrupts[] = { 365 irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT, 366 irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT, 367 0, 368 irqs << GEN8_VECS_IRQ_SHIFT, 369 }; 370 struct intel_uncore *uncore = gt->uncore; 371 372 gt->pm_ier = 0x0; 373 gt->pm_imr = ~gt->pm_ier; 374 GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]); 375 GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]); 376 /* 377 * RPS interrupts will get enabled/disabled on demand when RPS itself 378 * is enabled/disabled. Same wil be the case for GuC interrupts. 379 */ 380 GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier); 381 GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]); 382 } 383 384 static void gen5_gt_update_irq(struct intel_gt *gt, 385 u32 interrupt_mask, 386 u32 enabled_irq_mask) 387 { 388 lockdep_assert_held(>->irq_lock); 389 390 GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask); 391 392 gt->gt_imr &= ~interrupt_mask; 393 gt->gt_imr |= (~enabled_irq_mask & interrupt_mask); 394 intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr); 395 } 396 397 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask) 398 { 399 gen5_gt_update_irq(gt, mask, mask); 400 intel_uncore_posting_read_fw(gt->uncore, GTIMR); 401 } 402 403 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask) 404 { 405 gen5_gt_update_irq(gt, mask, 0); 406 } 407 408 void gen5_gt_irq_reset(struct intel_gt *gt) 409 { 410 struct intel_uncore *uncore = gt->uncore; 411 412 GEN3_IRQ_RESET(uncore, GT); 413 if (GRAPHICS_VER(gt->i915) >= 6) 414 GEN3_IRQ_RESET(uncore, GEN6_PM); 415 } 416 417 void gen5_gt_irq_postinstall(struct intel_gt *gt) 418 { 419 struct intel_uncore *uncore = gt->uncore; 420 u32 pm_irqs = 0; 421 u32 gt_irqs = 0; 422 423 gt->gt_imr = ~0; 424 if (HAS_L3_DPF(gt->i915)) { 425 /* L3 parity interrupt is always unmasked. */ 426 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915); 427 gt_irqs |= GT_PARITY_ERROR(gt->i915); 428 } 429 430 gt_irqs |= GT_RENDER_USER_INTERRUPT; 431 if (GRAPHICS_VER(gt->i915) == 5) 432 gt_irqs |= ILK_BSD_USER_INTERRUPT; 433 else 434 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; 435 436 GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs); 437 438 if (GRAPHICS_VER(gt->i915) >= 6) { 439 /* 440 * RPS interrupts will get enabled/disabled on demand when RPS 441 * itself is enabled/disabled. 442 */ 443 if (HAS_ENGINE(gt, VECS0)) { 444 pm_irqs |= PM_VEBOX_USER_INTERRUPT; 445 gt->pm_ier |= PM_VEBOX_USER_INTERRUPT; 446 } 447 448 gt->pm_imr = 0xffffffff; 449 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs); 450 } 451 } 452