1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <linux/sched/clock.h> 8 9 #include "i915_drv.h" 10 #include "i915_irq.h" 11 #include "intel_gt.h" 12 #include "intel_gt_irq.h" 13 #include "intel_uncore.h" 14 #include "intel_rps.h" 15 16 static void guc_irq_handler(struct intel_guc *guc, u16 iir) 17 { 18 if (iir & GUC_INTR_GUC2HOST) 19 intel_guc_to_host_event_handler(guc); 20 } 21 22 static void 23 cs_irq_handler(struct intel_engine_cs *engine, u32 iir) 24 { 25 bool tasklet = false; 26 27 if (iir & GT_CONTEXT_SWITCH_INTERRUPT) 28 tasklet = true; 29 30 if (iir & GT_RENDER_USER_INTERRUPT) { 31 intel_engine_signal_breadcrumbs(engine); 32 tasklet |= intel_engine_needs_breadcrumb_tasklet(engine); 33 } 34 35 if (tasklet) 36 tasklet_hi_schedule(&engine->execlists.tasklet); 37 } 38 39 static u32 40 gen11_gt_engine_identity(struct intel_gt *gt, 41 const unsigned int bank, const unsigned int bit) 42 { 43 void __iomem * const regs = gt->uncore->regs; 44 u32 timeout_ts; 45 u32 ident; 46 47 lockdep_assert_held(>->irq_lock); 48 49 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit)); 50 51 /* 52 * NB: Specs do not specify how long to spin wait, 53 * so we do ~100us as an educated guess. 54 */ 55 timeout_ts = (local_clock() >> 10) + 100; 56 do { 57 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank)); 58 } while (!(ident & GEN11_INTR_DATA_VALID) && 59 !time_after32(local_clock() >> 10, timeout_ts)); 60 61 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) { 62 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", 63 bank, bit, ident); 64 return 0; 65 } 66 67 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank), 68 GEN11_INTR_DATA_VALID); 69 70 return ident; 71 } 72 73 static void 74 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, 75 const u16 iir) 76 { 77 if (instance == OTHER_GUC_INSTANCE) 78 return guc_irq_handler(>->uc.guc, iir); 79 80 if (instance == OTHER_GTPM_INSTANCE) 81 return gen11_rps_irq_handler(>->rps, iir); 82 83 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", 84 instance, iir); 85 } 86 87 static void 88 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class, 89 const u8 instance, const u16 iir) 90 { 91 struct intel_engine_cs *engine; 92 93 if (instance <= MAX_ENGINE_INSTANCE) 94 engine = gt->engine_class[class][instance]; 95 else 96 engine = NULL; 97 98 if (likely(engine)) 99 return cs_irq_handler(engine, iir); 100 101 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n", 102 class, instance); 103 } 104 105 static void 106 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity) 107 { 108 const u8 class = GEN11_INTR_ENGINE_CLASS(identity); 109 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity); 110 const u16 intr = GEN11_INTR_ENGINE_INTR(identity); 111 112 if (unlikely(!intr)) 113 return; 114 115 if (class <= COPY_ENGINE_CLASS) 116 return gen11_engine_irq_handler(gt, class, instance, intr); 117 118 if (class == OTHER_CLASS) 119 return gen11_other_irq_handler(gt, instance, intr); 120 121 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n", 122 class, instance, intr); 123 } 124 125 static void 126 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank) 127 { 128 void __iomem * const regs = gt->uncore->regs; 129 unsigned long intr_dw; 130 unsigned int bit; 131 132 lockdep_assert_held(>->irq_lock); 133 134 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); 135 136 for_each_set_bit(bit, &intr_dw, 32) { 137 const u32 ident = gen11_gt_engine_identity(gt, bank, bit); 138 139 gen11_gt_identity_handler(gt, ident); 140 } 141 142 /* Clear must be after shared has been served for engine */ 143 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw); 144 } 145 146 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl) 147 { 148 unsigned int bank; 149 150 spin_lock(>->irq_lock); 151 152 for (bank = 0; bank < 2; bank++) { 153 if (master_ctl & GEN11_GT_DW_IRQ(bank)) 154 gen11_gt_bank_handler(gt, bank); 155 } 156 157 spin_unlock(>->irq_lock); 158 } 159 160 bool gen11_gt_reset_one_iir(struct intel_gt *gt, 161 const unsigned int bank, const unsigned int bit) 162 { 163 void __iomem * const regs = gt->uncore->regs; 164 u32 dw; 165 166 lockdep_assert_held(>->irq_lock); 167 168 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); 169 if (dw & BIT(bit)) { 170 /* 171 * According to the BSpec, DW_IIR bits cannot be cleared without 172 * first servicing the Selector & Shared IIR registers. 173 */ 174 gen11_gt_engine_identity(gt, bank, bit); 175 176 /* 177 * We locked GT INT DW by reading it. If we want to (try 178 * to) recover from this successfully, we need to clear 179 * our bit, otherwise we are locking the register for 180 * everybody. 181 */ 182 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit)); 183 184 return true; 185 } 186 187 return false; 188 } 189 190 void gen11_gt_irq_reset(struct intel_gt *gt) 191 { 192 struct intel_uncore *uncore = gt->uncore; 193 194 /* Disable RCS, BCS, VCS and VECS class engines. */ 195 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0); 196 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0); 197 198 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ 199 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0); 200 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0); 201 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0); 202 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0); 203 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0); 204 205 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); 206 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); 207 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0); 208 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0); 209 } 210 211 void gen11_gt_irq_postinstall(struct intel_gt *gt) 212 { 213 const u32 irqs = GT_RENDER_USER_INTERRUPT | GT_CONTEXT_SWITCH_INTERRUPT; 214 struct intel_uncore *uncore = gt->uncore; 215 const u32 dmask = irqs << 16 | irqs; 216 const u32 smask = irqs << 16; 217 218 BUILD_BUG_ON(irqs & 0xffff0000); 219 220 /* Enable RCS, BCS, VCS and VECS class interrupts. */ 221 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask); 222 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask); 223 224 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ 225 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask); 226 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask); 227 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask); 228 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask); 229 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask); 230 231 /* 232 * RPS interrupts will get enabled/disabled on demand when RPS itself 233 * is enabled/disabled. 234 */ 235 gt->pm_ier = 0x0; 236 gt->pm_imr = ~gt->pm_ier; 237 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); 238 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); 239 240 /* Same thing for GuC interrupts */ 241 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0); 242 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0); 243 } 244 245 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) 246 { 247 if (gt_iir & GT_RENDER_USER_INTERRUPT) 248 intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]); 249 if (gt_iir & ILK_BSD_USER_INTERRUPT) 250 intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]); 251 } 252 253 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) 254 { 255 if (!HAS_L3_DPF(gt->i915)) 256 return; 257 258 spin_lock(>->irq_lock); 259 gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915)); 260 spin_unlock(>->irq_lock); 261 262 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1) 263 gt->i915->l3_parity.which_slice |= 1 << 1; 264 265 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT) 266 gt->i915->l3_parity.which_slice |= 1 << 0; 267 268 schedule_work(>->i915->l3_parity.error_work); 269 } 270 271 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) 272 { 273 if (gt_iir & GT_RENDER_USER_INTERRUPT) 274 intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]); 275 if (gt_iir & GT_BSD_USER_INTERRUPT) 276 intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]); 277 if (gt_iir & GT_BLT_USER_INTERRUPT) 278 intel_engine_signal_breadcrumbs(gt->engine_class[COPY_ENGINE_CLASS][0]); 279 280 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | 281 GT_BSD_CS_ERROR_INTERRUPT | 282 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) 283 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir); 284 285 if (gt_iir & GT_PARITY_ERROR(gt->i915)) 286 gen7_parity_error_irq_handler(gt, gt_iir); 287 } 288 289 void gen8_gt_irq_ack(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4]) 290 { 291 void __iomem * const regs = gt->uncore->regs; 292 293 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { 294 gt_iir[0] = raw_reg_read(regs, GEN8_GT_IIR(0)); 295 if (likely(gt_iir[0])) 296 raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]); 297 } 298 299 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) { 300 gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1)); 301 if (likely(gt_iir[1])) 302 raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]); 303 } 304 305 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { 306 gt_iir[2] = raw_reg_read(regs, GEN8_GT_IIR(2)); 307 if (likely(gt_iir[2])) 308 raw_reg_write(regs, GEN8_GT_IIR(2), gt_iir[2]); 309 } 310 311 if (master_ctl & GEN8_GT_VECS_IRQ) { 312 gt_iir[3] = raw_reg_read(regs, GEN8_GT_IIR(3)); 313 if (likely(gt_iir[3])) 314 raw_reg_write(regs, GEN8_GT_IIR(3), gt_iir[3]); 315 } 316 } 317 318 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl, u32 gt_iir[4]) 319 { 320 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { 321 cs_irq_handler(gt->engine_class[RENDER_CLASS][0], 322 gt_iir[0] >> GEN8_RCS_IRQ_SHIFT); 323 cs_irq_handler(gt->engine_class[COPY_ENGINE_CLASS][0], 324 gt_iir[0] >> GEN8_BCS_IRQ_SHIFT); 325 } 326 327 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) { 328 cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][0], 329 gt_iir[1] >> GEN8_VCS0_IRQ_SHIFT); 330 cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][1], 331 gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT); 332 } 333 334 if (master_ctl & GEN8_GT_VECS_IRQ) { 335 cs_irq_handler(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0], 336 gt_iir[3] >> GEN8_VECS_IRQ_SHIFT); 337 } 338 339 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { 340 gen6_rps_irq_handler(>->rps, gt_iir[2]); 341 guc_irq_handler(>->uc.guc, gt_iir[2] >> 16); 342 } 343 } 344 345 void gen8_gt_irq_reset(struct intel_gt *gt) 346 { 347 struct intel_uncore *uncore = gt->uncore; 348 349 GEN8_IRQ_RESET_NDX(uncore, GT, 0); 350 GEN8_IRQ_RESET_NDX(uncore, GT, 1); 351 GEN8_IRQ_RESET_NDX(uncore, GT, 2); 352 GEN8_IRQ_RESET_NDX(uncore, GT, 3); 353 } 354 355 void gen8_gt_irq_postinstall(struct intel_gt *gt) 356 { 357 struct intel_uncore *uncore = gt->uncore; 358 359 /* These are interrupts we'll toggle with the ring mask register */ 360 u32 gt_interrupts[] = { 361 (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT | 362 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT | 363 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT | 364 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT), 365 366 (GT_RENDER_USER_INTERRUPT << GEN8_VCS0_IRQ_SHIFT | 367 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS0_IRQ_SHIFT | 368 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | 369 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT), 370 371 0, 372 373 (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT | 374 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT) 375 }; 376 377 gt->pm_ier = 0x0; 378 gt->pm_imr = ~gt->pm_ier; 379 GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]); 380 GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]); 381 /* 382 * RPS interrupts will get enabled/disabled on demand when RPS itself 383 * is enabled/disabled. Same wil be the case for GuC interrupts. 384 */ 385 GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier); 386 GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]); 387 } 388 389 static void gen5_gt_update_irq(struct intel_gt *gt, 390 u32 interrupt_mask, 391 u32 enabled_irq_mask) 392 { 393 lockdep_assert_held(>->irq_lock); 394 395 GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask); 396 397 gt->gt_imr &= ~interrupt_mask; 398 gt->gt_imr |= (~enabled_irq_mask & interrupt_mask); 399 intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr); 400 } 401 402 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask) 403 { 404 gen5_gt_update_irq(gt, mask, mask); 405 intel_uncore_posting_read_fw(gt->uncore, GTIMR); 406 } 407 408 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask) 409 { 410 gen5_gt_update_irq(gt, mask, 0); 411 } 412 413 void gen5_gt_irq_reset(struct intel_gt *gt) 414 { 415 struct intel_uncore *uncore = gt->uncore; 416 417 GEN3_IRQ_RESET(uncore, GT); 418 if (INTEL_GEN(gt->i915) >= 6) 419 GEN3_IRQ_RESET(uncore, GEN6_PM); 420 } 421 422 void gen5_gt_irq_postinstall(struct intel_gt *gt) 423 { 424 struct intel_uncore *uncore = gt->uncore; 425 u32 pm_irqs = 0; 426 u32 gt_irqs = 0; 427 428 gt->gt_imr = ~0; 429 if (HAS_L3_DPF(gt->i915)) { 430 /* L3 parity interrupt is always unmasked. */ 431 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915); 432 gt_irqs |= GT_PARITY_ERROR(gt->i915); 433 } 434 435 gt_irqs |= GT_RENDER_USER_INTERRUPT; 436 if (IS_GEN(gt->i915, 5)) 437 gt_irqs |= ILK_BSD_USER_INTERRUPT; 438 else 439 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; 440 441 GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs); 442 443 if (INTEL_GEN(gt->i915) >= 6) { 444 /* 445 * RPS interrupts will get enabled/disabled on demand when RPS 446 * itself is enabled/disabled. 447 */ 448 if (HAS_ENGINE(gt->i915, VECS0)) { 449 pm_irqs |= PM_VEBOX_USER_INTERRUPT; 450 gt->pm_ier |= PM_VEBOX_USER_INTERRUPT; 451 } 452 453 gt->pm_imr = 0xffffffff; 454 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs); 455 } 456 } 457