1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "gk104.h" 23 #include "cgrp.h" 24 #include "changk104.h" 25 26 #include <core/client.h> 27 #include <core/gpuobj.h> 28 #include <subdev/bar.h> 29 #include <subdev/fault.h> 30 #include <subdev/top.h> 31 #include <subdev/timer.h> 32 #include <engine/sw.h> 33 34 #include <nvif/class.h> 35 36 static void 37 tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, 38 struct nvkm_memory *mem, int nr) 39 { 40 struct nvkm_device *device = fifo->base.engine.subdev.device; 41 u64 addr = nvkm_memory_addr(mem); 42 /*XXX: target? */ 43 44 nvkm_wr32(device, 0x002b00 + (runl * 0x10), lower_32_bits(addr)); 45 nvkm_wr32(device, 0x002b04 + (runl * 0x10), upper_32_bits(addr)); 46 nvkm_wr32(device, 0x002b08 + (runl * 0x10), nr); 47 48 /*XXX: how to wait? can you even wait? */ 49 } 50 51 static const struct gk104_fifo_runlist_func 52 tu102_fifo_runlist = { 53 .size = 16, 54 .cgrp = gv100_fifo_runlist_cgrp, 55 .chan = gv100_fifo_runlist_chan, 56 .commit = tu102_fifo_runlist_commit, 57 }; 58 59 static const struct nvkm_enum 60 tu102_fifo_fault_engine[] = { 61 { 0x01, "DISPLAY" }, 62 { 0x03, "PTP" }, 63 { 0x06, "PWR_PMU" }, 64 { 0x08, "IFB", NULL, NVKM_ENGINE_IFB }, 65 { 0x09, "PERF" }, 66 { 0x1f, "PHYSICAL" }, 67 { 0x20, "HOST0" }, 68 { 0x21, "HOST1" }, 69 { 0x22, "HOST2" }, 70 { 0x23, "HOST3" }, 71 { 0x24, "HOST4" }, 72 { 0x25, "HOST5" }, 73 { 0x26, "HOST6" }, 74 { 0x27, "HOST7" }, 75 { 0x28, "HOST8" }, 76 { 0x29, "HOST9" }, 77 { 0x2a, "HOST10" }, 78 { 0x2b, "HOST11" }, 79 { 0x2c, "HOST12" }, 80 { 0x2d, "HOST13" }, 81 { 0x2e, "HOST14" }, 82 { 0x80, "BAR1", NULL, NVKM_SUBDEV_BAR }, 83 { 0xc0, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, 84 {} 85 }; 86 87 static void 88 tu102_fifo_pbdma_init(struct gk104_fifo *fifo) 89 { 90 struct nvkm_device *device = fifo->base.engine.subdev.device; 91 const u32 mask = (1 << fifo->pbdma_nr) - 1; 92 /*XXX: this is a bit of a guess at this point in time. */ 93 nvkm_mask(device, 0xb65000, 0x80000fff, 0x80000000 | mask); 94 } 95 96 static const struct gk104_fifo_pbdma_func 97 tu102_fifo_pbdma = { 98 .nr = gm200_fifo_pbdma_nr, 99 .init = tu102_fifo_pbdma_init, 100 .init_timeout = gk208_fifo_pbdma_init_timeout, 101 }; 102 103 static const struct gk104_fifo_func 104 tu102_fifo = { 105 .pbdma = &tu102_fifo_pbdma, 106 .fault.access = gv100_fifo_fault_access, 107 .fault.engine = tu102_fifo_fault_engine, 108 .fault.reason = gv100_fifo_fault_reason, 109 .fault.hubclient = gv100_fifo_fault_hubclient, 110 .fault.gpcclient = gv100_fifo_fault_gpcclient, 111 .runlist = &tu102_fifo_runlist, 112 .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu102_fifo_gpfifo_new }, 113 .cgrp_force = true, 114 }; 115 116 static void 117 tu102_fifo_recover_work(struct work_struct *w) 118 { 119 struct gk104_fifo *fifo = container_of(w, typeof(*fifo), recover.work); 120 struct nvkm_device *device = fifo->base.engine.subdev.device; 121 struct nvkm_engine *engine; 122 unsigned long flags; 123 u32 engm, runm, todo; 124 int engn, runl; 125 126 spin_lock_irqsave(&fifo->base.lock, flags); 127 runm = fifo->recover.runm; 128 engm = fifo->recover.engm; 129 fifo->recover.engm = 0; 130 fifo->recover.runm = 0; 131 spin_unlock_irqrestore(&fifo->base.lock, flags); 132 133 nvkm_mask(device, 0x002630, runm, runm); 134 135 for (todo = engm; engn = __ffs(todo), todo; todo &= ~BIT(engn)) { 136 if ((engine = fifo->engine[engn].engine)) { 137 nvkm_subdev_fini(&engine->subdev, false); 138 WARN_ON(nvkm_subdev_init(&engine->subdev)); 139 } 140 } 141 142 for (todo = runm; runl = __ffs(todo), todo; todo &= ~BIT(runl)) 143 gk104_fifo_runlist_update(fifo, runl); 144 145 nvkm_mask(device, 0x002630, runm, 0x00000000); 146 } 147 148 static void tu102_fifo_recover_engn(struct gk104_fifo *fifo, int engn); 149 150 static void 151 tu102_fifo_recover_runl(struct gk104_fifo *fifo, int runl) 152 { 153 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 154 struct nvkm_device *device = subdev->device; 155 const u32 runm = BIT(runl); 156 157 assert_spin_locked(&fifo->base.lock); 158 if (fifo->recover.runm & runm) 159 return; 160 fifo->recover.runm |= runm; 161 162 /* Block runlist to prevent channel assignment(s) from changing. */ 163 nvkm_mask(device, 0x002630, runm, runm); 164 165 /* Schedule recovery. */ 166 nvkm_warn(subdev, "runlist %d: scheduled for recovery\n", runl); 167 schedule_work(&fifo->recover.work); 168 } 169 170 static struct gk104_fifo_chan * 171 tu102_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid) 172 { 173 struct gk104_fifo_chan *chan; 174 struct nvkm_fifo_cgrp *cgrp; 175 176 list_for_each_entry(chan, &fifo->runlist[runl].chan, head) { 177 if (chan->base.chid == chid) { 178 list_del_init(&chan->head); 179 return chan; 180 } 181 } 182 183 list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) { 184 if (cgrp->id == chid) { 185 chan = list_first_entry(&cgrp->chan, typeof(*chan), head); 186 list_del_init(&chan->head); 187 if (!--cgrp->chan_nr) 188 list_del_init(&cgrp->head); 189 return chan; 190 } 191 } 192 193 return NULL; 194 } 195 196 static void 197 tu102_fifo_recover_chan(struct nvkm_fifo *base, int chid) 198 { 199 struct gk104_fifo *fifo = gk104_fifo(base); 200 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 201 struct nvkm_device *device = subdev->device; 202 const u32 stat = nvkm_rd32(device, 0x800004 + (chid * 0x08)); 203 const u32 runl = (stat & 0x000f0000) >> 16; 204 const bool used = (stat & 0x00000001); 205 unsigned long engn, engm = fifo->runlist[runl].engm; 206 struct gk104_fifo_chan *chan; 207 208 assert_spin_locked(&fifo->base.lock); 209 if (!used) 210 return; 211 212 /* Lookup SW state for channel, and mark it as dead. */ 213 chan = tu102_fifo_recover_chid(fifo, runl, chid); 214 if (chan) { 215 chan->killed = true; 216 nvkm_fifo_kevent(&fifo->base, chid); 217 } 218 219 /* Disable channel. */ 220 nvkm_wr32(device, 0x800004 + (chid * 0x08), stat | 0x00000800); 221 nvkm_warn(subdev, "channel %d: killed\n", chid); 222 223 /* Block channel assignments from changing during recovery. */ 224 tu102_fifo_recover_runl(fifo, runl); 225 226 /* Schedule recovery for any engines the channel is on. */ 227 for_each_set_bit(engn, &engm, fifo->engine_nr) { 228 struct gk104_fifo_engine_status status; 229 230 gk104_fifo_engine_status(fifo, engn, &status); 231 if (!status.chan || status.chan->id != chid) 232 continue; 233 tu102_fifo_recover_engn(fifo, engn); 234 } 235 } 236 237 static void 238 tu102_fifo_recover_engn(struct gk104_fifo *fifo, int engn) 239 { 240 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 241 struct nvkm_device *device = subdev->device; 242 const u32 runl = fifo->engine[engn].runl; 243 const u32 engm = BIT(engn); 244 struct gk104_fifo_engine_status status; 245 246 assert_spin_locked(&fifo->base.lock); 247 if (fifo->recover.engm & engm) 248 return; 249 fifo->recover.engm |= engm; 250 251 /* Block channel assignments from changing during recovery. */ 252 tu102_fifo_recover_runl(fifo, runl); 253 254 /* Determine which channel (if any) is currently on the engine. */ 255 gk104_fifo_engine_status(fifo, engn, &status); 256 if (status.chan) { 257 /* The channel is not longer viable, kill it. */ 258 tu102_fifo_recover_chan(&fifo->base, status.chan->id); 259 } 260 261 /* Preempt the runlist */ 262 nvkm_wr32(device, 0x2638, BIT(runl)); 263 264 /* Schedule recovery. */ 265 nvkm_warn(subdev, "engine %d: scheduled for recovery\n", engn); 266 schedule_work(&fifo->recover.work); 267 } 268 269 static void 270 tu102_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) 271 { 272 struct gk104_fifo *fifo = gk104_fifo(base); 273 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 274 struct nvkm_device *device = subdev->device; 275 const struct nvkm_enum *er, *ee, *ec, *ea; 276 struct nvkm_engine *engine = NULL; 277 struct nvkm_fifo_chan *chan; 278 unsigned long flags; 279 const char *en = ""; 280 char ct[8] = "HUB/"; 281 int engn; 282 283 er = nvkm_enum_find(fifo->func->fault.reason, info->reason); 284 ee = nvkm_enum_find(fifo->func->fault.engine, info->engine); 285 if (info->hub) { 286 ec = nvkm_enum_find(fifo->func->fault.hubclient, info->client); 287 } else { 288 ec = nvkm_enum_find(fifo->func->fault.gpcclient, info->client); 289 snprintf(ct, sizeof(ct), "GPC%d/", info->gpc); 290 } 291 ea = nvkm_enum_find(fifo->func->fault.access, info->access); 292 293 if (ee && ee->data2) { 294 switch (ee->data2) { 295 case NVKM_SUBDEV_BAR: 296 nvkm_bar_bar1_reset(device); 297 break; 298 case NVKM_SUBDEV_INSTMEM: 299 nvkm_bar_bar2_reset(device); 300 break; 301 case NVKM_ENGINE_IFB: 302 nvkm_mask(device, 0x001718, 0x00000000, 0x00000000); 303 break; 304 default: 305 engine = nvkm_device_engine(device, ee->data2, 0); 306 break; 307 } 308 } 309 310 if (ee == NULL) { 311 struct nvkm_subdev *subdev = nvkm_top_fault(device, info->engine); 312 if (subdev) { 313 if (subdev->func == &nvkm_engine) 314 engine = container_of(subdev, typeof(*engine), subdev); 315 en = engine->subdev.name; 316 } 317 } else { 318 en = ee->name; 319 } 320 321 spin_lock_irqsave(&fifo->base.lock, flags); 322 chan = nvkm_fifo_chan_inst_locked(&fifo->base, info->inst); 323 324 nvkm_error(subdev, 325 "fault %02x [%s] at %016llx engine %02x [%s] client %02x " 326 "[%s%s] reason %02x [%s] on channel %d [%010llx %s]\n", 327 info->access, ea ? ea->name : "", info->addr, 328 info->engine, ee ? ee->name : en, 329 info->client, ct, ec ? ec->name : "", 330 info->reason, er ? er->name : "", chan ? chan->chid : -1, 331 info->inst, chan ? chan->object.client->name : "unknown"); 332 333 /* Kill the channel that caused the fault. */ 334 if (chan) 335 tu102_fifo_recover_chan(&fifo->base, chan->chid); 336 337 /* Channel recovery will probably have already done this for the 338 * correct engine(s), but just in case we can't find the channel 339 * information... 340 */ 341 for (engn = 0; engn < fifo->engine_nr && engine; engn++) { 342 if (fifo->engine[engn].engine == engine) { 343 tu102_fifo_recover_engn(fifo, engn); 344 break; 345 } 346 } 347 348 spin_unlock_irqrestore(&fifo->base.lock, flags); 349 } 350 351 static void 352 tu102_fifo_intr_ctxsw_timeout(struct gk104_fifo *fifo) 353 { 354 struct nvkm_device *device = fifo->base.engine.subdev.device; 355 unsigned long flags, engm; 356 u32 engn; 357 358 spin_lock_irqsave(&fifo->base.lock, flags); 359 360 engm = nvkm_rd32(device, 0x2a30); 361 nvkm_wr32(device, 0x2a30, engm); 362 363 for_each_set_bit(engn, &engm, 32) 364 tu102_fifo_recover_engn(fifo, engn); 365 366 spin_unlock_irqrestore(&fifo->base.lock, flags); 367 } 368 369 static void 370 tu102_fifo_intr_sched(struct gk104_fifo *fifo) 371 { 372 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 373 struct nvkm_device *device = subdev->device; 374 u32 intr = nvkm_rd32(device, 0x00254c); 375 u32 code = intr & 0x000000ff; 376 377 nvkm_error(subdev, "SCHED_ERROR %02x\n", code); 378 } 379 380 static void 381 tu102_fifo_intr(struct nvkm_fifo *base) 382 { 383 struct gk104_fifo *fifo = gk104_fifo(base); 384 struct nvkm_subdev *subdev = &fifo->base.engine.subdev; 385 struct nvkm_device *device = subdev->device; 386 u32 mask = nvkm_rd32(device, 0x002140); 387 u32 stat = nvkm_rd32(device, 0x002100) & mask; 388 389 if (stat & 0x00000001) { 390 gk104_fifo_intr_bind(fifo); 391 nvkm_wr32(device, 0x002100, 0x00000001); 392 stat &= ~0x00000001; 393 } 394 395 if (stat & 0x00000002) { 396 tu102_fifo_intr_ctxsw_timeout(fifo); 397 stat &= ~0x00000002; 398 } 399 400 if (stat & 0x00000100) { 401 tu102_fifo_intr_sched(fifo); 402 nvkm_wr32(device, 0x002100, 0x00000100); 403 stat &= ~0x00000100; 404 } 405 406 if (stat & 0x00010000) { 407 gk104_fifo_intr_chsw(fifo); 408 nvkm_wr32(device, 0x002100, 0x00010000); 409 stat &= ~0x00010000; 410 } 411 412 if (stat & 0x20000000) { 413 u32 mask = nvkm_rd32(device, 0x0025a0); 414 415 while (mask) { 416 u32 unit = __ffs(mask); 417 418 gk104_fifo_intr_pbdma_0(fifo, unit); 419 gk104_fifo_intr_pbdma_1(fifo, unit); 420 nvkm_wr32(device, 0x0025a0, (1 << unit)); 421 mask &= ~(1 << unit); 422 } 423 stat &= ~0x20000000; 424 } 425 426 if (stat & 0x40000000) { 427 gk104_fifo_intr_runlist(fifo); 428 stat &= ~0x40000000; 429 } 430 431 if (stat & 0x80000000) { 432 nvkm_wr32(device, 0x002100, 0x80000000); 433 gk104_fifo_intr_engine(fifo); 434 stat &= ~0x80000000; 435 } 436 437 if (stat) { 438 nvkm_error(subdev, "INTR %08x\n", stat); 439 nvkm_mask(device, 0x002140, stat, 0x00000000); 440 nvkm_wr32(device, 0x002100, stat); 441 } 442 } 443 444 static const struct nvkm_fifo_func 445 tu102_fifo_ = { 446 .dtor = gk104_fifo_dtor, 447 .oneinit = gk104_fifo_oneinit, 448 .info = gk104_fifo_info, 449 .init = gk104_fifo_init, 450 .fini = gk104_fifo_fini, 451 .intr = tu102_fifo_intr, 452 .fault = tu102_fifo_fault, 453 .engine_id = gk104_fifo_engine_id, 454 .id_engine = gk104_fifo_id_engine, 455 .uevent_init = gk104_fifo_uevent_init, 456 .uevent_fini = gk104_fifo_uevent_fini, 457 .recover_chan = tu102_fifo_recover_chan, 458 .class_get = gk104_fifo_class_get, 459 .class_new = gk104_fifo_class_new, 460 }; 461 462 int 463 tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, 464 struct nvkm_fifo **pfifo) 465 { 466 struct gk104_fifo *fifo; 467 468 if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 469 return -ENOMEM; 470 fifo->func = &tu102_fifo; 471 INIT_WORK(&fifo->recover.work, tu102_fifo_recover_work); 472 *pfifo = &fifo->base; 473 474 return nvkm_fifo_ctor(&tu102_fifo_, device, type, inst, 4096, &fifo->base); 475 } 476