1 /* 2 * Copyright 2012 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include "priv.h" 25 #include "chan.h" 26 #include "chid.h" 27 #include "runl.h" 28 #include "runq.h" 29 30 #include <core/gpuobj.h> 31 #include <subdev/bar.h> 32 #include <subdev/mc.h> 33 #include <subdev/mmu.h> 34 35 #include <nvif/cl0080.h> 36 #include <nvif/unpack.h> 37 38 bool nvkm_fifo_ctxsw_in_progress(struct nvkm_engine * engine)39 nvkm_fifo_ctxsw_in_progress(struct nvkm_engine *engine) 40 { 41 struct nvkm_runl *runl; 42 struct nvkm_engn *engn; 43 44 nvkm_runl_foreach(runl, engine->subdev.device->fifo) { 45 nvkm_runl_foreach_engn(engn, runl) { 46 if (engn->engine == engine) 47 return engn->func->chsw ? engn->func->chsw(engn) : false; 48 } 49 } 50 51 return false; 52 } 53 54 void nvkm_fifo_pause(struct nvkm_fifo * fifo,unsigned long * flags)55 nvkm_fifo_pause(struct nvkm_fifo *fifo, unsigned long *flags) 56 { 57 return fifo->func->pause(fifo, flags); 58 } 59 60 void nvkm_fifo_start(struct nvkm_fifo * fifo,unsigned long * flags)61 nvkm_fifo_start(struct nvkm_fifo *fifo, unsigned long *flags) 62 { 63 return fifo->func->start(fifo, flags); 64 } 65 66 void nvkm_fifo_fault(struct nvkm_fifo * fifo,struct nvkm_fault_data * info)67 nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info) 68 { 69 return fifo->func->mmu_fault->recover(fifo, info); 70 } 71 72 static int nvkm_fifo_class_new(struct nvkm_device * device,const struct nvkm_oclass * oclass,void * argv,u32 argc,struct nvkm_object ** pobject)73 nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, 74 void *argv, u32 argc, struct nvkm_object **pobject) 75 { 76 struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); 77 78 if (oclass->engn == &fifo->func->cgrp.user) 79 return nvkm_ucgrp_new(fifo, oclass, argv, argc, pobject); 80 81 if (oclass->engn == &fifo->func->chan.user) 82 return nvkm_uchan_new(fifo, NULL, oclass, argv, argc, pobject); 83 84 WARN_ON(1); 85 return -ENOSYS; 86 } 87 88 static const struct nvkm_device_oclass 89 nvkm_fifo_class = { 90 .ctor = nvkm_fifo_class_new, 91 }; 92 93 static int nvkm_fifo_class_get(struct nvkm_oclass * oclass,int index,const struct nvkm_device_oclass ** class)94 nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class) 95 { 96 struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); 97 const struct nvkm_fifo_func_cgrp *cgrp = &fifo->func->cgrp; 98 const struct nvkm_fifo_func_chan *chan = &fifo->func->chan; 99 int c = 0; 100 101 /* *_CHANNEL_GROUP_* */ 102 if (cgrp->user.oclass) { 103 if (c++ == index) { 104 oclass->base = cgrp->user; 105 oclass->engn = &fifo->func->cgrp.user; 106 *class = &nvkm_fifo_class; 107 return 0; 108 } 109 } 110 111 /* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */ 112 if (chan->user.oclass) { 113 if (c++ == index) { 114 oclass->base = chan->user; 115 oclass->engn = &fifo->func->chan.user; 116 *class = &nvkm_fifo_class; 117 return 0; 118 } 119 } 120 121 return c; 122 } 123 124 static int nvkm_fifo_fini(struct nvkm_engine * engine,bool suspend)125 nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend) 126 { 127 struct nvkm_fifo *fifo = nvkm_fifo(engine); 128 struct nvkm_runl *runl; 129 130 nvkm_inth_block(&fifo->engine.subdev.inth); 131 132 nvkm_runl_foreach(runl, fifo) 133 nvkm_runl_fini(runl); 134 135 return 0; 136 } 137 138 static int nvkm_fifo_init(struct nvkm_engine * engine)139 nvkm_fifo_init(struct nvkm_engine *engine) 140 { 141 struct nvkm_fifo *fifo = nvkm_fifo(engine); 142 struct nvkm_runq *runq; 143 struct nvkm_runl *runl; 144 u32 mask = 0; 145 146 if (fifo->func->init_pbdmas) { 147 nvkm_runq_foreach(runq, fifo) 148 mask |= BIT(runq->id); 149 150 fifo->func->init_pbdmas(fifo, mask); 151 152 nvkm_runq_foreach(runq, fifo) 153 runq->func->init(runq); 154 } 155 156 nvkm_runl_foreach(runl, fifo) { 157 if (runl->func->init) 158 runl->func->init(runl); 159 } 160 161 if (fifo->func->init) 162 fifo->func->init(fifo); 163 164 nvkm_inth_allow(&fifo->engine.subdev.inth); 165 return 0; 166 } 167 168 static int nvkm_fifo_info(struct nvkm_engine * engine,u64 mthd,u64 * data)169 nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) 170 { 171 struct nvkm_fifo *fifo = nvkm_fifo(engine); 172 struct nvkm_runl *runl; 173 struct nvkm_engn *engn; 174 int ret; 175 176 ret = nvkm_subdev_oneinit(&fifo->engine.subdev); 177 if (ret) 178 return ret; 179 180 switch (mthd) { 181 case NV_DEVICE_HOST_CHANNELS: *data = fifo->chid ? fifo->chid->nr : 0; return 0; 182 case NV_DEVICE_HOST_RUNLISTS: 183 *data = 0; 184 nvkm_runl_foreach(runl, fifo) 185 *data |= BIT(runl->id); 186 return 0; 187 case NV_DEVICE_HOST_RUNLIST_ENGINES: 188 runl = nvkm_runl_get(fifo, *data, 0); 189 if (runl) { 190 *data = 0; 191 nvkm_runl_foreach_engn(engn, runl) { 192 #define CASE(n) case NVKM_ENGINE_##n: *data |= NV_DEVICE_HOST_RUNLIST_ENGINES_##n; break 193 switch (engn->engine->subdev.type) { 194 case NVKM_ENGINE_DMAOBJ: 195 break; 196 CASE(SW ); 197 CASE(GR ); 198 CASE(MPEG ); 199 CASE(ME ); 200 CASE(CIPHER); 201 CASE(BSP ); 202 CASE(VP ); 203 CASE(CE ); 204 CASE(SEC ); 205 CASE(MSVLD ); 206 CASE(MSPDEC); 207 CASE(MSPPP ); 208 CASE(MSENC ); 209 CASE(VIC ); 210 CASE(SEC2 ); 211 CASE(NVDEC ); 212 CASE(NVENC ); 213 default: 214 WARN_ON(1); 215 break; 216 } 217 #undef CASE 218 } 219 return 0; 220 } 221 return -EINVAL; 222 case NV_DEVICE_HOST_RUNLIST_CHANNELS: 223 if (!fifo->chid) { 224 runl = nvkm_runl_get(fifo, *data, 0); 225 if (runl) { 226 *data = runl->chid->nr; 227 return 0; 228 } 229 } 230 return -EINVAL; 231 default: 232 break; 233 } 234 235 return -ENOSYS; 236 } 237 238 static int nvkm_fifo_oneinit(struct nvkm_engine * engine)239 nvkm_fifo_oneinit(struct nvkm_engine *engine) 240 { 241 struct nvkm_subdev *subdev = &engine->subdev; 242 struct nvkm_device *device = subdev->device; 243 struct nvkm_fifo *fifo = nvkm_fifo(engine); 244 struct nvkm_runl *runl; 245 struct nvkm_engn *engn; 246 int ret, nr, i; 247 248 /* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */ 249 if (fifo->func->chid_nr) { 250 ret = fifo->func->chid_ctor(fifo, fifo->func->chid_nr(fifo)); 251 if (ret) 252 return ret; 253 } 254 255 /* Create runqueues for each PBDMA. */ 256 if (fifo->func->runq_nr) { 257 for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) { 258 if (!nvkm_runq_new(fifo, i)) 259 return -ENOMEM; 260 } 261 } 262 263 /* Create runlists. */ 264 ret = fifo->func->runl_ctor(fifo); 265 if (ret) 266 return ret; 267 268 nvkm_runl_foreach(runl, fifo) { 269 RUNL_DEBUG(runl, "chan:%06x", runl->chan); 270 nvkm_runl_foreach_engn(engn, runl) { 271 ENGN_DEBUG(engn, ""); 272 } 273 } 274 275 /* Register interrupt handler. */ 276 if (fifo->func->intr) { 277 ret = nvkm_inth_add(&device->mc->intr, NVKM_INTR_SUBDEV, NVKM_INTR_PRIO_NORMAL, 278 subdev, fifo->func->intr, &subdev->inth); 279 if (ret) { 280 nvkm_error(subdev, "intr %d\n", ret); 281 return ret; 282 } 283 } 284 285 /* Initialise non-stall intr handling. */ 286 if (fifo->func->nonstall) { 287 if (fifo->func->nonstall_ctor) { 288 ret = fifo->func->nonstall_ctor(fifo); 289 if (ret < 0) { 290 nvkm_error(subdev, "nonstall %d\n", ret); 291 return ret; 292 } 293 } else { 294 ret = 1; 295 } 296 297 ret = nvkm_event_init(fifo->func->nonstall, &fifo->engine.subdev, 1, ret, 298 &fifo->nonstall.event); 299 if (ret) 300 return ret; 301 } 302 303 /* Allocate USERD + BAR1 polling area. */ 304 if (fifo->func->chan.func->userd->bar == 1) { 305 struct nvkm_vmm *bar1 = nvkm_bar_bar1_vmm(device); 306 307 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, fifo->chid->nr * 308 fifo->func->chan.func->userd->size, 0, true, 309 &fifo->userd.mem); 310 if (ret) 311 return ret; 312 313 ret = nvkm_vmm_get(bar1, 12, nvkm_memory_size(fifo->userd.mem), &fifo->userd.bar1); 314 if (ret) 315 return ret; 316 317 ret = nvkm_memory_map(fifo->userd.mem, 0, bar1, fifo->userd.bar1, NULL, 0); 318 if (ret) 319 return ret; 320 } 321 322 return 0; 323 } 324 325 static void nvkm_fifo_preinit(struct nvkm_engine * engine)326 nvkm_fifo_preinit(struct nvkm_engine *engine) 327 { 328 nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO, 0); 329 } 330 331 static void * nvkm_fifo_dtor(struct nvkm_engine * engine)332 nvkm_fifo_dtor(struct nvkm_engine *engine) 333 { 334 struct nvkm_fifo *fifo = nvkm_fifo(engine); 335 struct nvkm_runl *runl, *runt; 336 struct nvkm_runq *runq, *rtmp; 337 338 if (fifo->userd.bar1) 339 nvkm_vmm_put(nvkm_bar_bar1_vmm(engine->subdev.device), &fifo->userd.bar1); 340 nvkm_memory_unref(&fifo->userd.mem); 341 342 list_for_each_entry_safe(runl, runt, &fifo->runls, head) 343 nvkm_runl_del(runl); 344 list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head) 345 nvkm_runq_del(runq); 346 347 nvkm_chid_unref(&fifo->cgid); 348 nvkm_chid_unref(&fifo->chid); 349 350 nvkm_event_fini(&fifo->nonstall.event); 351 mutex_destroy(&fifo->mutex); 352 return fifo; 353 } 354 355 static const struct nvkm_engine_func 356 nvkm_fifo = { 357 .dtor = nvkm_fifo_dtor, 358 .preinit = nvkm_fifo_preinit, 359 .oneinit = nvkm_fifo_oneinit, 360 .info = nvkm_fifo_info, 361 .init = nvkm_fifo_init, 362 .fini = nvkm_fifo_fini, 363 .base.sclass = nvkm_fifo_class_get, 364 }; 365 366 int nvkm_fifo_new_(const struct nvkm_fifo_func * func,struct nvkm_device * device,enum nvkm_subdev_type type,int inst,struct nvkm_fifo ** pfifo)367 nvkm_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, 368 enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) 369 { 370 struct nvkm_fifo *fifo; 371 372 if (!(fifo = *pfifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) 373 return -ENOMEM; 374 375 fifo->func = func; 376 INIT_LIST_HEAD(&fifo->runqs); 377 INIT_LIST_HEAD(&fifo->runls); 378 /*TODO: Needs to be >CTXSW_TIMEOUT, so RC can recover before this is hit. 379 * CTXSW_TIMEOUT HW default seems to differ between GPUs, so just a 380 * large number for now until we support changing it. 381 */ 382 fifo->timeout.chan_msec = 10000; 383 spin_lock_init(&fifo->lock); 384 mutex_init(&fifo->mutex); 385 386 return nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine); 387 } 388