1 /*
2  * Copyright 2010 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24 #include "priv.h"
25 
26 #include <core/gpuobj.h>
27 #include <subdev/fb.h>
28 
29 void
30 nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node)
31 {
32 	struct nvkm_vm *vm = vma->vm;
33 	struct nvkm_mmu *mmu = vm->mmu;
34 	struct nvkm_mm_node *r = node->mem;
35 	int big = vma->node->type != mmu->func->spg_shift;
36 	u32 offset = vma->node->offset + (delta >> 12);
37 	u32 bits = vma->node->type - 12;
38 	u32 pde  = (offset >> mmu->func->pgt_bits) - vm->fpde;
39 	u32 pte  = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
40 	u32 max  = 1 << (mmu->func->pgt_bits - bits);
41 	u32 end, len;
42 
43 	delta = 0;
44 	while (r) {
45 		u64 phys = (u64)r->offset << 12;
46 		u32 num  = r->length >> bits;
47 
48 		while (num) {
49 			struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
50 
51 			end = (pte + num);
52 			if (unlikely(end >= max))
53 				end = max;
54 			len = end - pte;
55 
56 			mmu->func->map(vma, pgt, node, pte, len, phys, delta);
57 
58 			num -= len;
59 			pte += len;
60 			if (unlikely(end >= max)) {
61 				phys += len << (bits + 12);
62 				pde++;
63 				pte = 0;
64 			}
65 
66 			delta += (u64)len << vma->node->type;
67 		}
68 		r = r->next;
69 	};
70 
71 	mmu->func->flush(vm);
72 }
73 
74 static void
75 nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length,
76 		     struct nvkm_mem *mem)
77 {
78 	struct nvkm_vm *vm = vma->vm;
79 	struct nvkm_mmu *mmu = vm->mmu;
80 	int big = vma->node->type != mmu->func->spg_shift;
81 	u32 offset = vma->node->offset + (delta >> 12);
82 	u32 bits = vma->node->type - 12;
83 	u32 num  = length >> vma->node->type;
84 	u32 pde  = (offset >> mmu->func->pgt_bits) - vm->fpde;
85 	u32 pte  = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
86 	u32 max  = 1 << (mmu->func->pgt_bits - bits);
87 	unsigned m, sglen;
88 	u32 end, len;
89 	int i;
90 	struct scatterlist *sg;
91 
92 	for_each_sg(mem->sg->sgl, sg, mem->sg->nents, i) {
93 		struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
94 		sglen = sg_dma_len(sg) >> PAGE_SHIFT;
95 
96 		end = pte + sglen;
97 		if (unlikely(end >= max))
98 			end = max;
99 		len = end - pte;
100 
101 		for (m = 0; m < len; m++) {
102 			dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
103 
104 			mmu->func->map_sg(vma, pgt, mem, pte, 1, &addr);
105 			num--;
106 			pte++;
107 
108 			if (num == 0)
109 				goto finish;
110 		}
111 		if (unlikely(end >= max)) {
112 			pde++;
113 			pte = 0;
114 		}
115 		if (m < sglen) {
116 			for (; m < sglen; m++) {
117 				dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
118 
119 				mmu->func->map_sg(vma, pgt, mem, pte, 1, &addr);
120 				num--;
121 				pte++;
122 				if (num == 0)
123 					goto finish;
124 			}
125 		}
126 
127 	}
128 finish:
129 	mmu->func->flush(vm);
130 }
131 
132 static void
133 nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length,
134 	       struct nvkm_mem *mem)
135 {
136 	struct nvkm_vm *vm = vma->vm;
137 	struct nvkm_mmu *mmu = vm->mmu;
138 	dma_addr_t *list = mem->pages;
139 	int big = vma->node->type != mmu->func->spg_shift;
140 	u32 offset = vma->node->offset + (delta >> 12);
141 	u32 bits = vma->node->type - 12;
142 	u32 num  = length >> vma->node->type;
143 	u32 pde  = (offset >> mmu->func->pgt_bits) - vm->fpde;
144 	u32 pte  = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
145 	u32 max  = 1 << (mmu->func->pgt_bits - bits);
146 	u32 end, len;
147 
148 	while (num) {
149 		struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
150 
151 		end = (pte + num);
152 		if (unlikely(end >= max))
153 			end = max;
154 		len = end - pte;
155 
156 		mmu->func->map_sg(vma, pgt, mem, pte, len, list);
157 
158 		num  -= len;
159 		pte  += len;
160 		list += len;
161 		if (unlikely(end >= max)) {
162 			pde++;
163 			pte = 0;
164 		}
165 	}
166 
167 	mmu->func->flush(vm);
168 }
169 
170 void
171 nvkm_vm_map(struct nvkm_vma *vma, struct nvkm_mem *node)
172 {
173 	if (node->sg)
174 		nvkm_vm_map_sg_table(vma, 0, node->size << 12, node);
175 	else
176 	if (node->pages)
177 		nvkm_vm_map_sg(vma, 0, node->size << 12, node);
178 	else
179 		nvkm_vm_map_at(vma, 0, node);
180 }
181 
182 void
183 nvkm_vm_unmap_at(struct nvkm_vma *vma, u64 delta, u64 length)
184 {
185 	struct nvkm_vm *vm = vma->vm;
186 	struct nvkm_mmu *mmu = vm->mmu;
187 	int big = vma->node->type != mmu->func->spg_shift;
188 	u32 offset = vma->node->offset + (delta >> 12);
189 	u32 bits = vma->node->type - 12;
190 	u32 num  = length >> vma->node->type;
191 	u32 pde  = (offset >> mmu->func->pgt_bits) - vm->fpde;
192 	u32 pte  = (offset & ((1 << mmu->func->pgt_bits) - 1)) >> bits;
193 	u32 max  = 1 << (mmu->func->pgt_bits - bits);
194 	u32 end, len;
195 
196 	while (num) {
197 		struct nvkm_memory *pgt = vm->pgt[pde].mem[big];
198 
199 		end = (pte + num);
200 		if (unlikely(end >= max))
201 			end = max;
202 		len = end - pte;
203 
204 		mmu->func->unmap(vma, pgt, pte, len);
205 
206 		num -= len;
207 		pte += len;
208 		if (unlikely(end >= max)) {
209 			pde++;
210 			pte = 0;
211 		}
212 	}
213 
214 	mmu->func->flush(vm);
215 }
216 
217 void
218 nvkm_vm_unmap(struct nvkm_vma *vma)
219 {
220 	nvkm_vm_unmap_at(vma, 0, (u64)vma->node->length << 12);
221 }
222 
223 static void
224 nvkm_vm_unmap_pgt(struct nvkm_vm *vm, int big, u32 fpde, u32 lpde)
225 {
226 	struct nvkm_mmu *mmu = vm->mmu;
227 	struct nvkm_vm_pgd *vpgd;
228 	struct nvkm_vm_pgt *vpgt;
229 	struct nvkm_memory *pgt;
230 	u32 pde;
231 
232 	for (pde = fpde; pde <= lpde; pde++) {
233 		vpgt = &vm->pgt[pde - vm->fpde];
234 		if (--vpgt->refcount[big])
235 			continue;
236 
237 		pgt = vpgt->mem[big];
238 		vpgt->mem[big] = NULL;
239 
240 		list_for_each_entry(vpgd, &vm->pgd_list, head) {
241 			mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem);
242 		}
243 
244 		nvkm_memory_del(&pgt);
245 	}
246 }
247 
248 static int
249 nvkm_vm_map_pgt(struct nvkm_vm *vm, u32 pde, u32 type)
250 {
251 	struct nvkm_mmu *mmu = vm->mmu;
252 	struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
253 	struct nvkm_vm_pgd *vpgd;
254 	int big = (type != mmu->func->spg_shift);
255 	u32 pgt_size;
256 	int ret;
257 
258 	pgt_size  = (1 << (mmu->func->pgt_bits + 12)) >> type;
259 	pgt_size *= 8;
260 
261 	ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
262 			      pgt_size, 0x1000, true, &vpgt->mem[big]);
263 	if (unlikely(ret))
264 		return ret;
265 
266 	list_for_each_entry(vpgd, &vm->pgd_list, head) {
267 		mmu->func->map_pgt(vpgd->obj, pde, vpgt->mem);
268 	}
269 
270 	vpgt->refcount[big]++;
271 	return 0;
272 }
273 
274 int
275 nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access,
276 	    struct nvkm_vma *vma)
277 {
278 	struct nvkm_mmu *mmu = vm->mmu;
279 	u32 align = (1 << page_shift) >> 12;
280 	u32 msize = size >> 12;
281 	u32 fpde, lpde, pde;
282 	int ret;
283 
284 	mutex_lock(&vm->mutex);
285 	ret = nvkm_mm_head(&vm->mm, 0, page_shift, msize, msize, align,
286 			   &vma->node);
287 	if (unlikely(ret != 0)) {
288 		mutex_unlock(&vm->mutex);
289 		return ret;
290 	}
291 
292 	fpde = (vma->node->offset >> mmu->func->pgt_bits);
293 	lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits;
294 
295 	for (pde = fpde; pde <= lpde; pde++) {
296 		struct nvkm_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
297 		int big = (vma->node->type != mmu->func->spg_shift);
298 
299 		if (likely(vpgt->refcount[big])) {
300 			vpgt->refcount[big]++;
301 			continue;
302 		}
303 
304 		ret = nvkm_vm_map_pgt(vm, pde, vma->node->type);
305 		if (ret) {
306 			if (pde != fpde)
307 				nvkm_vm_unmap_pgt(vm, big, fpde, pde - 1);
308 			nvkm_mm_free(&vm->mm, &vma->node);
309 			mutex_unlock(&vm->mutex);
310 			return ret;
311 		}
312 	}
313 	mutex_unlock(&vm->mutex);
314 
315 	vma->vm = NULL;
316 	nvkm_vm_ref(vm, &vma->vm, NULL);
317 	vma->offset = (u64)vma->node->offset << 12;
318 	vma->access = access;
319 	return 0;
320 }
321 
322 void
323 nvkm_vm_put(struct nvkm_vma *vma)
324 {
325 	struct nvkm_mmu *mmu;
326 	struct nvkm_vm *vm;
327 	u32 fpde, lpde;
328 
329 	if (unlikely(vma->node == NULL))
330 		return;
331 	vm = vma->vm;
332 	mmu = vm->mmu;
333 
334 	fpde = (vma->node->offset >> mmu->func->pgt_bits);
335 	lpde = (vma->node->offset + vma->node->length - 1) >> mmu->func->pgt_bits;
336 
337 	mutex_lock(&vm->mutex);
338 	nvkm_vm_unmap_pgt(vm, vma->node->type != mmu->func->spg_shift, fpde, lpde);
339 	nvkm_mm_free(&vm->mm, &vma->node);
340 	mutex_unlock(&vm->mutex);
341 
342 	nvkm_vm_ref(NULL, &vma->vm, NULL);
343 }
344 
345 int
346 nvkm_vm_boot(struct nvkm_vm *vm, u64 size)
347 {
348 	struct nvkm_mmu *mmu = vm->mmu;
349 	struct nvkm_memory *pgt;
350 	int ret;
351 
352 	ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
353 			      (size >> mmu->func->spg_shift) * 8, 0x1000, true, &pgt);
354 	if (ret == 0) {
355 		vm->pgt[0].refcount[0] = 1;
356 		vm->pgt[0].mem[0] = pgt;
357 		nvkm_memory_boot(pgt, vm);
358 	}
359 
360 	return ret;
361 }
362 
363 int
364 nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset,
365 	       u32 block, struct lock_class_key *key, struct nvkm_vm **pvm)
366 {
367 	static struct lock_class_key _key;
368 	struct nvkm_vm *vm;
369 	u64 mm_length = (offset + length) - mm_offset;
370 	int ret;
371 
372 	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
373 	if (!vm)
374 		return -ENOMEM;
375 
376 	__mutex_init(&vm->mutex, "&vm->mutex", key ? key : &_key);
377 	INIT_LIST_HEAD(&vm->pgd_list);
378 	vm->mmu = mmu;
379 	kref_init(&vm->refcount);
380 	vm->fpde = offset >> (mmu->func->pgt_bits + 12);
381 	vm->lpde = (offset + length - 1) >> (mmu->func->pgt_bits + 12);
382 
383 	vm->pgt  = vzalloc((vm->lpde - vm->fpde + 1) * sizeof(*vm->pgt));
384 	if (!vm->pgt) {
385 		kfree(vm);
386 		return -ENOMEM;
387 	}
388 
389 	ret = nvkm_mm_init(&vm->mm, mm_offset >> 12, mm_length >> 12,
390 			   block >> 12);
391 	if (ret) {
392 		vfree(vm->pgt);
393 		kfree(vm);
394 		return ret;
395 	}
396 
397 	*pvm = vm;
398 
399 	return 0;
400 }
401 
402 int
403 nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset,
404 	    struct lock_class_key *key, struct nvkm_vm **pvm)
405 {
406 	struct nvkm_mmu *mmu = device->mmu;
407 	if (!mmu->func->create)
408 		return -EINVAL;
409 	return mmu->func->create(mmu, offset, length, mm_offset, key, pvm);
410 }
411 
412 static int
413 nvkm_vm_link(struct nvkm_vm *vm, struct nvkm_gpuobj *pgd)
414 {
415 	struct nvkm_mmu *mmu = vm->mmu;
416 	struct nvkm_vm_pgd *vpgd;
417 	int i;
418 
419 	if (!pgd)
420 		return 0;
421 
422 	vpgd = kzalloc(sizeof(*vpgd), GFP_KERNEL);
423 	if (!vpgd)
424 		return -ENOMEM;
425 
426 	vpgd->obj = pgd;
427 
428 	mutex_lock(&vm->mutex);
429 	for (i = vm->fpde; i <= vm->lpde; i++)
430 		mmu->func->map_pgt(pgd, i, vm->pgt[i - vm->fpde].mem);
431 	list_add(&vpgd->head, &vm->pgd_list);
432 	mutex_unlock(&vm->mutex);
433 	return 0;
434 }
435 
436 static void
437 nvkm_vm_unlink(struct nvkm_vm *vm, struct nvkm_gpuobj *mpgd)
438 {
439 	struct nvkm_vm_pgd *vpgd, *tmp;
440 
441 	if (!mpgd)
442 		return;
443 
444 	mutex_lock(&vm->mutex);
445 	list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
446 		if (vpgd->obj == mpgd) {
447 			list_del(&vpgd->head);
448 			kfree(vpgd);
449 			break;
450 		}
451 	}
452 	mutex_unlock(&vm->mutex);
453 }
454 
455 static void
456 nvkm_vm_del(struct kref *kref)
457 {
458 	struct nvkm_vm *vm = container_of(kref, typeof(*vm), refcount);
459 	struct nvkm_vm_pgd *vpgd, *tmp;
460 
461 	list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
462 		nvkm_vm_unlink(vm, vpgd->obj);
463 	}
464 
465 	nvkm_mm_fini(&vm->mm);
466 	vfree(vm->pgt);
467 	kfree(vm);
468 }
469 
470 int
471 nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_gpuobj *pgd)
472 {
473 	if (ref) {
474 		int ret = nvkm_vm_link(ref, pgd);
475 		if (ret)
476 			return ret;
477 
478 		kref_get(&ref->refcount);
479 	}
480 
481 	if (*ptr) {
482 		nvkm_vm_unlink(*ptr, pgd);
483 		kref_put(&(*ptr)->refcount, nvkm_vm_del);
484 	}
485 
486 	*ptr = ref;
487 	return 0;
488 }
489 
490 static int
491 nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
492 {
493 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
494 	if (mmu->func->oneinit)
495 		return mmu->func->oneinit(mmu);
496 	return 0;
497 }
498 
499 static int
500 nvkm_mmu_init(struct nvkm_subdev *subdev)
501 {
502 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
503 	if (mmu->func->init)
504 		mmu->func->init(mmu);
505 	return 0;
506 }
507 
508 static void *
509 nvkm_mmu_dtor(struct nvkm_subdev *subdev)
510 {
511 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
512 	if (mmu->func->dtor)
513 		return mmu->func->dtor(mmu);
514 	return mmu;
515 }
516 
517 static const struct nvkm_subdev_func
518 nvkm_mmu = {
519 	.dtor = nvkm_mmu_dtor,
520 	.oneinit = nvkm_mmu_oneinit,
521 	.init = nvkm_mmu_init,
522 };
523 
524 void
525 nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device,
526 	      int index, struct nvkm_mmu *mmu)
527 {
528 	nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev);
529 	mmu->func = func;
530 	mmu->limit = func->limit;
531 	mmu->dma_bits = func->dma_bits;
532 	mmu->lpg_shift = func->lpg_shift;
533 }
534 
535 int
536 nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device,
537 	      int index, struct nvkm_mmu **pmmu)
538 {
539 	if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL)))
540 		return -ENOMEM;
541 	nvkm_mmu_ctor(func, device, index, *pmmu);
542 	return 0;
543 }
544