1 /* 2 * Copyright 2017 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "uvmm.h" 23 #include "umem.h" 24 #include "ummu.h" 25 26 #include <core/client.h> 27 #include <core/memory.h> 28 29 #include <nvif/if000c.h> 30 #include <nvif/unpack.h> 31 32 static const struct nvkm_object_func nvkm_uvmm; 33 struct nvkm_vmm * 34 nvkm_uvmm_search(struct nvkm_client *client, u64 handle) 35 { 36 struct nvkm_object *object; 37 38 object = nvkm_object_search(client, handle, &nvkm_uvmm); 39 if (IS_ERR(object)) 40 return (void *)object; 41 42 return nvkm_uvmm(object)->vmm; 43 } 44 45 static int 46 nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) 47 { 48 struct nvkm_client *client = uvmm->object.client; 49 union { 50 struct nvif_vmm_pfnclr_v0 v0; 51 } *args = argv; 52 struct nvkm_vmm *vmm = uvmm->vmm; 53 int ret = -ENOSYS; 54 u64 addr, size; 55 56 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 57 addr = args->v0.addr; 58 size = args->v0.size; 59 } else 60 return ret; 61 62 if (!client->super) 63 return -ENOENT; 64 65 if (size) { 66 mutex_lock(&vmm->mutex); 67 ret = nvkm_vmm_pfn_unmap(vmm, addr, size); 68 mutex_unlock(&vmm->mutex); 69 } 70 71 return ret; 72 } 73 74 static int 75 nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) 76 { 77 struct nvkm_client *client = uvmm->object.client; 78 union { 79 struct nvif_vmm_pfnmap_v0 v0; 80 } *args = argv; 81 struct nvkm_vmm *vmm = uvmm->vmm; 82 int ret = -ENOSYS; 83 u64 addr, size, *phys; 84 u8 page; 85 86 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { 87 page = args->v0.page; 88 addr = args->v0.addr; 89 size = args->v0.size; 90 phys = args->v0.phys; 91 if (argc != (size >> page) * sizeof(args->v0.phys[0])) 92 return -EINVAL; 93 } else 94 return ret; 95 96 if (!client->super) 97 return -ENOENT; 98 99 if (size) { 100 mutex_lock(&vmm->mutex); 101 ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys); 102 mutex_unlock(&vmm->mutex); 103 } 104 105 return ret; 106 } 107 108 static int 109 nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) 110 { 111 struct nvkm_client *client = uvmm->object.client; 112 union { 113 struct nvif_vmm_unmap_v0 v0; 114 } *args = argv; 115 struct nvkm_vmm *vmm = uvmm->vmm; 116 struct nvkm_vma *vma; 117 int ret = -ENOSYS; 118 u64 addr; 119 120 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 121 addr = args->v0.addr; 122 } else 123 return ret; 124 125 mutex_lock(&vmm->mutex); 126 vma = nvkm_vmm_node_search(vmm, addr); 127 if (ret = -ENOENT, !vma || vma->addr != addr) { 128 VMM_DEBUG(vmm, "lookup %016llx: %016llx", 129 addr, vma ? vma->addr : ~0ULL); 130 goto done; 131 } 132 133 if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { 134 VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, 135 vma->user, !client->super, vma->busy); 136 goto done; 137 } 138 139 if (ret = -EINVAL, !vma->memory) { 140 VMM_DEBUG(vmm, "unmapped"); 141 goto done; 142 } 143 144 nvkm_vmm_unmap_locked(vmm, vma, false); 145 ret = 0; 146 done: 147 mutex_unlock(&vmm->mutex); 148 return ret; 149 } 150 151 static int 152 nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) 153 { 154 struct nvkm_client *client = uvmm->object.client; 155 union { 156 struct nvif_vmm_map_v0 v0; 157 } *args = argv; 158 u64 addr, size, handle, offset; 159 struct nvkm_vmm *vmm = uvmm->vmm; 160 struct nvkm_vma *vma; 161 struct nvkm_memory *memory; 162 int ret = -ENOSYS; 163 164 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { 165 addr = args->v0.addr; 166 size = args->v0.size; 167 handle = args->v0.memory; 168 offset = args->v0.offset; 169 } else 170 return ret; 171 172 memory = nvkm_umem_search(client, handle); 173 if (IS_ERR(memory)) { 174 VMM_DEBUG(vmm, "memory %016llx %ld\n", handle, PTR_ERR(memory)); 175 return PTR_ERR(memory); 176 } 177 178 mutex_lock(&vmm->mutex); 179 if (ret = -ENOENT, !(vma = nvkm_vmm_node_search(vmm, addr))) { 180 VMM_DEBUG(vmm, "lookup %016llx", addr); 181 goto fail; 182 } 183 184 if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { 185 VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, 186 vma->user, !client->super, vma->busy); 187 goto fail; 188 } 189 190 if (ret = -EINVAL, vma->mapped && !vma->memory) { 191 VMM_DEBUG(vmm, "pfnmap %016llx", addr); 192 goto fail; 193 } 194 195 if (ret = -EINVAL, vma->addr != addr || vma->size != size) { 196 if (addr + size > vma->addr + vma->size || vma->memory || 197 (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) { 198 VMM_DEBUG(vmm, "split %d %d %d " 199 "%016llx %016llx %016llx %016llx", 200 !!vma->memory, vma->refd, vma->mapref, 201 addr, size, vma->addr, (u64)vma->size); 202 goto fail; 203 } 204 205 vma = nvkm_vmm_node_split(vmm, vma, addr, size); 206 if (!vma) { 207 ret = -ENOMEM; 208 goto fail; 209 } 210 } 211 vma->busy = true; 212 mutex_unlock(&vmm->mutex); 213 214 ret = nvkm_memory_map(memory, offset, vmm, vma, argv, argc); 215 if (ret == 0) { 216 /* Successful map will clear vma->busy. */ 217 nvkm_memory_unref(&memory); 218 return 0; 219 } 220 221 mutex_lock(&vmm->mutex); 222 vma->busy = false; 223 nvkm_vmm_unmap_region(vmm, vma); 224 fail: 225 mutex_unlock(&vmm->mutex); 226 nvkm_memory_unref(&memory); 227 return ret; 228 } 229 230 static int 231 nvkm_uvmm_mthd_put(struct nvkm_uvmm *uvmm, void *argv, u32 argc) 232 { 233 struct nvkm_client *client = uvmm->object.client; 234 union { 235 struct nvif_vmm_put_v0 v0; 236 } *args = argv; 237 struct nvkm_vmm *vmm = uvmm->vmm; 238 struct nvkm_vma *vma; 239 int ret = -ENOSYS; 240 u64 addr; 241 242 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 243 addr = args->v0.addr; 244 } else 245 return ret; 246 247 mutex_lock(&vmm->mutex); 248 vma = nvkm_vmm_node_search(vmm, args->v0.addr); 249 if (ret = -ENOENT, !vma || vma->addr != addr || vma->part) { 250 VMM_DEBUG(vmm, "lookup %016llx: %016llx %d", addr, 251 vma ? vma->addr : ~0ULL, vma ? vma->part : 0); 252 goto done; 253 } 254 255 if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { 256 VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, 257 vma->user, !client->super, vma->busy); 258 goto done; 259 } 260 261 nvkm_vmm_put_locked(vmm, vma); 262 ret = 0; 263 done: 264 mutex_unlock(&vmm->mutex); 265 return ret; 266 } 267 268 static int 269 nvkm_uvmm_mthd_get(struct nvkm_uvmm *uvmm, void *argv, u32 argc) 270 { 271 struct nvkm_client *client = uvmm->object.client; 272 union { 273 struct nvif_vmm_get_v0 v0; 274 } *args = argv; 275 struct nvkm_vmm *vmm = uvmm->vmm; 276 struct nvkm_vma *vma; 277 int ret = -ENOSYS; 278 bool getref, mapref, sparse; 279 u8 page, align; 280 u64 size; 281 282 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 283 getref = args->v0.type == NVIF_VMM_GET_V0_PTES; 284 mapref = args->v0.type == NVIF_VMM_GET_V0_ADDR; 285 sparse = args->v0.sparse; 286 page = args->v0.page; 287 align = args->v0.align; 288 size = args->v0.size; 289 } else 290 return ret; 291 292 mutex_lock(&vmm->mutex); 293 ret = nvkm_vmm_get_locked(vmm, getref, mapref, sparse, 294 page, align, size, &vma); 295 mutex_unlock(&vmm->mutex); 296 if (ret) 297 return ret; 298 299 args->v0.addr = vma->addr; 300 vma->user = !client->super; 301 return ret; 302 } 303 304 static int 305 nvkm_uvmm_mthd_page(struct nvkm_uvmm *uvmm, void *argv, u32 argc) 306 { 307 union { 308 struct nvif_vmm_page_v0 v0; 309 } *args = argv; 310 const struct nvkm_vmm_page *page; 311 int ret = -ENOSYS; 312 u8 type, index, nr; 313 314 page = uvmm->vmm->func->page; 315 for (nr = 0; page[nr].shift; nr++); 316 317 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { 318 if ((index = args->v0.index) >= nr) 319 return -EINVAL; 320 type = page[index].type; 321 args->v0.shift = page[index].shift; 322 args->v0.sparse = !!(type & NVKM_VMM_PAGE_SPARSE); 323 args->v0.vram = !!(type & NVKM_VMM_PAGE_VRAM); 324 args->v0.host = !!(type & NVKM_VMM_PAGE_HOST); 325 args->v0.comp = !!(type & NVKM_VMM_PAGE_COMP); 326 } else 327 return -ENOSYS; 328 329 return 0; 330 } 331 332 static int 333 nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) 334 { 335 struct nvkm_uvmm *uvmm = nvkm_uvmm(object); 336 switch (mthd) { 337 case NVIF_VMM_V0_PAGE : return nvkm_uvmm_mthd_page (uvmm, argv, argc); 338 case NVIF_VMM_V0_GET : return nvkm_uvmm_mthd_get (uvmm, argv, argc); 339 case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc); 340 case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc); 341 case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc); 342 case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc); 343 case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc); 344 case NVIF_VMM_V0_MTHD(0x00) ... NVIF_VMM_V0_MTHD(0x7f): 345 if (uvmm->vmm->func->mthd) { 346 return uvmm->vmm->func->mthd(uvmm->vmm, 347 uvmm->object.client, 348 mthd, argv, argc); 349 } 350 break; 351 default: 352 break; 353 } 354 return -EINVAL; 355 } 356 357 static void * 358 nvkm_uvmm_dtor(struct nvkm_object *object) 359 { 360 struct nvkm_uvmm *uvmm = nvkm_uvmm(object); 361 nvkm_vmm_unref(&uvmm->vmm); 362 return uvmm; 363 } 364 365 static const struct nvkm_object_func 366 nvkm_uvmm = { 367 .dtor = nvkm_uvmm_dtor, 368 .mthd = nvkm_uvmm_mthd, 369 }; 370 371 int 372 nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, 373 struct nvkm_object **pobject) 374 { 375 struct nvkm_mmu *mmu = nvkm_ummu(oclass->parent)->mmu; 376 const bool more = oclass->base.maxver >= 0; 377 union { 378 struct nvif_vmm_v0 v0; 379 } *args = argv; 380 const struct nvkm_vmm_page *page; 381 struct nvkm_uvmm *uvmm; 382 int ret = -ENOSYS; 383 u64 addr, size; 384 bool managed; 385 386 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) { 387 managed = args->v0.managed != 0; 388 addr = args->v0.addr; 389 size = args->v0.size; 390 } else 391 return ret; 392 393 if (!(uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL))) 394 return -ENOMEM; 395 nvkm_object_ctor(&nvkm_uvmm, oclass, &uvmm->object); 396 *pobject = &uvmm->object; 397 398 if (!mmu->vmm) { 399 ret = mmu->func->vmm.ctor(mmu, managed, addr, size, argv, argc, 400 NULL, "user", &uvmm->vmm); 401 if (ret) 402 return ret; 403 404 uvmm->vmm->debug = max(uvmm->vmm->debug, oclass->client->debug); 405 } else { 406 if (size) 407 return -EINVAL; 408 409 uvmm->vmm = nvkm_vmm_ref(mmu->vmm); 410 } 411 412 page = uvmm->vmm->func->page; 413 args->v0.page_nr = 0; 414 while (page && (page++)->shift) 415 args->v0.page_nr++; 416 args->v0.addr = uvmm->vmm->start; 417 args->v0.size = uvmm->vmm->limit; 418 return 0; 419 } 420