1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "chan.h" 23 #include "runl.h" 24 25 #include "gk104.h" 26 #include "cgrp.h" 27 #include "changk104.h" 28 29 #include <core/memory.h> 30 #include <subdev/mc.h> 31 32 #include <nvif/class.h> 33 34 static const struct nvkm_chan_func 35 tu102_chan = { 36 }; 37 38 static bool 39 tu102_runl_pending(struct nvkm_runl *runl) 40 { 41 struct nvkm_device *device = runl->fifo->engine.subdev.device; 42 43 return nvkm_rd32(device, 0x002b0c + (runl->id * 0x10)) & 0x00008000; 44 } 45 46 static void 47 tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, 48 struct nvkm_memory *mem, int nr) 49 { 50 struct nvkm_device *device = fifo->base.engine.subdev.device; 51 u64 addr = nvkm_memory_addr(mem); 52 /*XXX: target? */ 53 54 nvkm_wr32(device, 0x002b00 + (runl * 0x10), lower_32_bits(addr)); 55 nvkm_wr32(device, 0x002b04 + (runl * 0x10), upper_32_bits(addr)); 56 nvkm_wr32(device, 0x002b08 + (runl * 0x10), nr); 57 } 58 59 static const struct gk104_fifo_runlist_func 60 tu102_fifo_runlist = { 61 .size = 16, 62 .cgrp = gv100_fifo_runlist_cgrp, 63 .chan = gv100_fifo_runlist_chan, 64 .commit = tu102_fifo_runlist_commit, 65 }; 66 67 static const struct nvkm_runl_func 68 tu102_runl = { 69 .wait = nv50_runl_wait, 70 .pending = tu102_runl_pending, 71 .block = gk104_runl_block, 72 .allow = gk104_runl_allow, 73 }; 74 75 static const struct nvkm_enum 76 tu102_fifo_mmu_fault_engine[] = { 77 { 0x01, "DISPLAY" }, 78 { 0x03, "PTP" }, 79 { 0x06, "PWR_PMU" }, 80 { 0x08, "IFB", NULL, NVKM_ENGINE_IFB }, 81 { 0x09, "PERF" }, 82 { 0x1f, "PHYSICAL" }, 83 { 0x20, "HOST0" }, 84 { 0x21, "HOST1" }, 85 { 0x22, "HOST2" }, 86 { 0x23, "HOST3" }, 87 { 0x24, "HOST4" }, 88 { 0x25, "HOST5" }, 89 { 0x26, "HOST6" }, 90 { 0x27, "HOST7" }, 91 { 0x28, "HOST8" }, 92 { 0x29, "HOST9" }, 93 { 0x2a, "HOST10" }, 94 { 0x2b, "HOST11" }, 95 { 0x2c, "HOST12" }, 96 { 0x2d, "HOST13" }, 97 { 0x2e, "HOST14" }, 98 { 0x80, "BAR1", NULL, NVKM_SUBDEV_BAR }, 99 { 0xc0, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, 100 {} 101 }; 102 103 static void 104 tu102_fifo_recover_work(struct work_struct *w) 105 { 106 struct gk104_fifo *fifo = container_of(w, typeof(*fifo), recover.work); 107 struct nvkm_device *device = fifo->base.engine.subdev.device; 108 struct nvkm_engine *engine; 109 unsigned long flags; 110 u32 engm, runm, todo; 111 int engn, runl; 112 113 spin_lock_irqsave(&fifo->base.lock, flags); 114 runm = fifo->recover.runm; 115 engm = fifo->recover.engm; 116 fifo->recover.engm = 0; 117 fifo->recover.runm = 0; 118 spin_unlock_irqrestore(&fifo->base.lock, flags); 119 120 nvkm_mask(device, 0x002630, runm, runm); 121 122 for (todo = engm; engn = __ffs(todo), todo; todo &= ~BIT(engn)) { 123 if ((engine = fifo->engine[engn].engine)) { 124 nvkm_subdev_fini(&engine->subdev, false); 125 WARN_ON(nvkm_subdev_init(&engine->subdev)); 126 } 127 } 128 129 for (todo = runm; runl = __ffs(todo), todo; todo &= ~BIT(runl)) 130 gk104_fifo_runlist_update(fifo, runl); 131 132 nvkm_mask(device, 0x002630, runm, 0x00000000); 133 } 134 135 static void tu102_fifo_recover_engn(struct gk104_fifo *fifo, int engn); 136 137 static void 138 tu102_fifo_recover_runl(struct gk104_fifo *fifo, int runl) 139 { 140 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 141 struct nvkm_device *device = subdev->device; 142 const u32 runm = BIT(runl); 143 144 assert_spin_locked(&fifo->base.lock); 145 if (fifo->recover.runm & runm) 146 return; 147 fifo->recover.runm |= runm; 148 149 /* Block runlist to prevent channel assignment(s) from changing. */ 150 nvkm_mask(device, 0x002630, runm, runm); 151 152 /* Schedule recovery. */ 153 nvkm_warn(subdev, "runlist %d: scheduled for recovery\n", runl); 154 schedule_work(&fifo->recover.work); 155 } 156 157 static struct gk104_fifo_chan * 158 tu102_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid) 159 { 160 struct gk104_fifo_chan *chan; 161 struct nvkm_fifo_cgrp *cgrp; 162 163 list_for_each_entry(chan, &fifo->runlist[runl].chan, head) { 164 if (chan->base.chid == chid) { 165 list_del_init(&chan->head); 166 return chan; 167 } 168 } 169 170 list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) { 171 if (cgrp->id == chid) { 172 chan = list_first_entry(&cgrp->chan, typeof(*chan), head); 173 list_del_init(&chan->head); 174 if (!--cgrp->chan_nr) 175 list_del_init(&cgrp->head); 176 return chan; 177 } 178 } 179 180 return NULL; 181 } 182 183 static void 184 tu102_fifo_recover_chan(struct nvkm_fifo *base, int chid) 185 { 186 struct gk104_fifo *fifo = gk104_fifo(base); 187 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 188 struct nvkm_device *device = subdev->device; 189 const u32 stat = nvkm_rd32(device, 0x800004 + (chid * 0x08)); 190 const u32 runl = (stat & 0x000f0000) >> 16; 191 const bool used = (stat & 0x00000001); 192 unsigned long engn, engm = fifo->runlist[runl].engm; 193 struct gk104_fifo_chan *chan; 194 195 assert_spin_locked(&fifo->base.lock); 196 if (!used) 197 return; 198 199 /* Lookup SW state for channel, and mark it as dead. */ 200 chan = tu102_fifo_recover_chid(fifo, runl, chid); 201 if (chan) { 202 chan->killed = true; 203 nvkm_fifo_kevent(&fifo->base, chid); 204 } 205 206 /* Disable channel. */ 207 nvkm_wr32(device, 0x800004 + (chid * 0x08), stat | 0x00000800); 208 nvkm_warn(subdev, "channel %d: killed\n", chid); 209 210 /* Block channel assignments from changing during recovery. */ 211 tu102_fifo_recover_runl(fifo, runl); 212 213 /* Schedule recovery for any engines the channel is on. */ 214 for_each_set_bit(engn, &engm, fifo->engine_nr) { 215 struct gk104_fifo_engine_status status; 216 217 gk104_fifo_engine_status(fifo, engn, &status); 218 if (!status.chan || status.chan->id != chid) 219 continue; 220 tu102_fifo_recover_engn(fifo, engn); 221 } 222 } 223 224 static void 225 tu102_fifo_recover_engn(struct gk104_fifo *fifo, int engn) 226 { 227 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 228 struct nvkm_device *device = subdev->device; 229 const u32 runl = fifo->engine[engn].runl; 230 const u32 engm = BIT(engn); 231 struct gk104_fifo_engine_status status; 232 233 assert_spin_locked(&fifo->base.lock); 234 if (fifo->recover.engm & engm) 235 return; 236 fifo->recover.engm |= engm; 237 238 /* Block channel assignments from changing during recovery. */ 239 tu102_fifo_recover_runl(fifo, runl); 240 241 /* Determine which channel (if any) is currently on the engine. */ 242 gk104_fifo_engine_status(fifo, engn, &status); 243 if (status.chan) { 244 /* The channel is not longer viable, kill it. */ 245 tu102_fifo_recover_chan(&fifo->base, status.chan->id); 246 } 247 248 /* Preempt the runlist */ 249 nvkm_wr32(device, 0x2638, BIT(runl)); 250 251 /* Schedule recovery. */ 252 nvkm_warn(subdev, "engine %d: scheduled for recovery\n", engn); 253 schedule_work(&fifo->recover.work); 254 } 255 256 const struct nvkm_fifo_func_mmu_fault 257 tu102_fifo_mmu_fault = { 258 .recover = gf100_fifo_mmu_fault_recover, 259 .access = gv100_fifo_mmu_fault_access, 260 .engine = tu102_fifo_mmu_fault_engine, 261 .reason = gv100_fifo_mmu_fault_reason, 262 .hubclient = gv100_fifo_mmu_fault_hubclient, 263 .gpcclient = gv100_fifo_mmu_fault_gpcclient, 264 }; 265 266 static void 267 tu102_fifo_intr_ctxsw_timeout(struct gk104_fifo *fifo) 268 { 269 struct nvkm_device *device = fifo->base.engine.subdev.device; 270 unsigned long flags, engm; 271 u32 engn; 272 273 spin_lock_irqsave(&fifo->base.lock, flags); 274 275 engm = nvkm_rd32(device, 0x2a30); 276 nvkm_wr32(device, 0x2a30, engm); 277 278 for_each_set_bit(engn, &engm, 32) 279 tu102_fifo_recover_engn(fifo, engn); 280 281 spin_unlock_irqrestore(&fifo->base.lock, flags); 282 } 283 284 static void 285 tu102_fifo_intr_sched(struct nvkm_fifo *fifo) 286 { 287 struct nvkm_subdev *subdev = &fifo->engine.subdev; 288 u32 intr = nvkm_rd32(subdev->device, 0x00254c); 289 u32 code = intr & 0x000000ff; 290 291 nvkm_error(subdev, "SCHED_ERROR %02x\n", code); 292 } 293 294 static irqreturn_t 295 tu102_fifo_intr(struct nvkm_inth *inth) 296 { 297 struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); 298 struct nvkm_subdev *subdev = &fifo->engine.subdev; 299 struct nvkm_device *device = subdev->device; 300 u32 mask = nvkm_rd32(device, 0x002140); 301 u32 stat = nvkm_rd32(device, 0x002100) & mask; 302 303 if (stat & 0x00000001) { 304 gk104_fifo_intr_bind(fifo); 305 nvkm_wr32(device, 0x002100, 0x00000001); 306 stat &= ~0x00000001; 307 } 308 309 if (stat & 0x00000002) { 310 tu102_fifo_intr_ctxsw_timeout(gk104_fifo(fifo)); 311 stat &= ~0x00000002; 312 } 313 314 if (stat & 0x00000100) { 315 tu102_fifo_intr_sched(fifo); 316 nvkm_wr32(device, 0x002100, 0x00000100); 317 stat &= ~0x00000100; 318 } 319 320 if (stat & 0x00010000) { 321 gk104_fifo_intr_chsw(fifo); 322 nvkm_wr32(device, 0x002100, 0x00010000); 323 stat &= ~0x00010000; 324 } 325 326 if (stat & 0x20000000) { 327 if (gf100_fifo_intr_pbdma(fifo)) 328 stat &= ~0x20000000; 329 } 330 331 if (stat & 0x40000000) { 332 gk104_fifo_intr_runlist(fifo); 333 stat &= ~0x40000000; 334 } 335 336 if (stat & 0x80000000) { 337 nvkm_wr32(device, 0x002100, 0x80000000); 338 nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); 339 stat &= ~0x80000000; 340 } 341 342 if (stat) { 343 nvkm_error(subdev, "INTR %08x\n", stat); 344 spin_lock(&fifo->lock); 345 nvkm_mask(device, 0x002140, stat, 0x00000000); 346 spin_unlock(&fifo->lock); 347 nvkm_wr32(device, 0x002100, stat); 348 } 349 350 return IRQ_HANDLED; 351 } 352 353 static void 354 tu102_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) 355 { 356 /* Not directly related to PBDMAs, but, enables doorbell to function. */ 357 nvkm_mask(fifo->engine.subdev.device, 0xb65000, 0x80000000, 0x80000000); 358 } 359 360 static const struct nvkm_fifo_func 361 tu102_fifo = { 362 .dtor = gk104_fifo_dtor, 363 .oneinit = gk104_fifo_oneinit, 364 .chid_nr = gm200_fifo_chid_nr, 365 .chid_ctor = gk110_fifo_chid_ctor, 366 .runq_nr = gm200_fifo_runq_nr, 367 .runl_ctor = gk104_fifo_runl_ctor, 368 .init = gk104_fifo_init, 369 .init_pbdmas = tu102_fifo_init_pbdmas, 370 .fini = gk104_fifo_fini, 371 .intr = tu102_fifo_intr, 372 .mmu_fault = &tu102_fifo_mmu_fault, 373 .engine_id = gk104_fifo_engine_id, 374 .recover_chan = tu102_fifo_recover_chan, 375 .runlist = &tu102_fifo_runlist, 376 .nonstall = &gf100_fifo_nonstall, 377 .runl = &tu102_runl, 378 .runq = &gv100_runq, 379 .engn = &gv100_engn, 380 .engn_ce = &gv100_engn_ce, 381 .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, 382 .chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new }, 383 }; 384 385 int 386 tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 387 struct nvkm_fifo **pfifo) 388 { 389 struct gk104_fifo *fifo; 390 391 if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 392 return -ENOMEM; 393 fifo->func = &tu102_fifo; 394 INIT_WORK(&fifo->recover.work, tu102_fifo_recover_work); 395 *pfifo = &fifo->base; 396 397 return nvkm_fifo_ctor(&tu102_fifo, device, type, inst, &fifo->base); 398 } 399