1 /*
2  * Copyright 2010 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24 #include "ummu.h"
25 #include "vmm.h"
26 
27 #include <subdev/bar.h>
28 #include <subdev/fb.h>
29 
30 #include <nvif/if500d.h>
31 #include <nvif/if900d.h>
32 
33 struct nvkm_mmu_ptp {
34 	struct nvkm_mmu_pt *pt;
35 	struct list_head head;
36 	u8  shift;
37 	u16 mask;
38 	u16 free;
39 };
40 
41 static void
42 nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt)
43 {
44 	const int slot = pt->base >> pt->ptp->shift;
45 	struct nvkm_mmu_ptp *ptp = pt->ptp;
46 
47 	/* If there were no free slots in the parent allocation before,
48 	 * there will be now, so return PTP to the cache.
49 	 */
50 	if (!ptp->free)
51 		list_add(&ptp->head, &mmu->ptp.list);
52 	ptp->free |= BIT(slot);
53 
54 	/* If there's no more sub-allocations, destroy PTP. */
55 	if (ptp->free == ptp->mask) {
56 		nvkm_mmu_ptc_put(mmu, force, &ptp->pt);
57 		list_del(&ptp->head);
58 		kfree(ptp);
59 	}
60 
61 	kfree(pt);
62 }
63 
64 struct nvkm_mmu_pt *
65 nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero)
66 {
67 	struct nvkm_mmu_pt *pt;
68 	struct nvkm_mmu_ptp *ptp;
69 	int slot;
70 
71 	if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL)))
72 		return NULL;
73 
74 	ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head);
75 	if (!ptp) {
76 		/* Need to allocate a new parent to sub-allocate from. */
77 		if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) {
78 			kfree(pt);
79 			return NULL;
80 		}
81 
82 		ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false);
83 		if (!ptp->pt) {
84 			kfree(ptp);
85 			kfree(pt);
86 			return NULL;
87 		}
88 
89 		ptp->shift = order_base_2(size);
90 		slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift;
91 		ptp->mask = (1 << slot) - 1;
92 		ptp->free = ptp->mask;
93 		list_add(&ptp->head, &mmu->ptp.list);
94 	}
95 	pt->ptp = ptp;
96 	pt->sub = true;
97 
98 	/* Sub-allocate from parent object, removing PTP from cache
99 	 * if there's no more free slots left.
100 	 */
101 	slot = __ffs(ptp->free);
102 	ptp->free &= ~BIT(slot);
103 	if (!ptp->free)
104 		list_del(&ptp->head);
105 
106 	pt->memory = pt->ptp->pt->memory;
107 	pt->base = slot << ptp->shift;
108 	pt->addr = pt->ptp->pt->addr + pt->base;
109 	return pt;
110 }
111 
112 struct nvkm_mmu_ptc {
113 	struct list_head head;
114 	struct list_head item;
115 	u32 size;
116 	u32 refs;
117 };
118 
119 static inline struct nvkm_mmu_ptc *
120 nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size)
121 {
122 	struct nvkm_mmu_ptc *ptc;
123 
124 	list_for_each_entry(ptc, &mmu->ptc.list, head) {
125 		if (ptc->size == size)
126 			return ptc;
127 	}
128 
129 	ptc = kmalloc(sizeof(*ptc), GFP_KERNEL);
130 	if (ptc) {
131 		INIT_LIST_HEAD(&ptc->item);
132 		ptc->size = size;
133 		ptc->refs = 0;
134 		list_add(&ptc->head, &mmu->ptc.list);
135 	}
136 
137 	return ptc;
138 }
139 
140 void
141 nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt)
142 {
143 	struct nvkm_mmu_pt *pt = *ppt;
144 	if (pt) {
145 		/* Handle sub-allocated page tables. */
146 		if (pt->sub) {
147 			mutex_lock(&mmu->ptp.mutex);
148 			nvkm_mmu_ptp_put(mmu, force, pt);
149 			mutex_unlock(&mmu->ptp.mutex);
150 			return;
151 		}
152 
153 		/* Either cache or free the object. */
154 		mutex_lock(&mmu->ptc.mutex);
155 		if (pt->ptc->refs < 8 /* Heuristic. */ && !force) {
156 			list_add_tail(&pt->head, &pt->ptc->item);
157 			pt->ptc->refs++;
158 		} else {
159 			nvkm_memory_unref(&pt->memory);
160 			kfree(pt);
161 		}
162 		mutex_unlock(&mmu->ptc.mutex);
163 	}
164 }
165 
166 struct nvkm_mmu_pt *
167 nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero)
168 {
169 	struct nvkm_mmu_ptc *ptc;
170 	struct nvkm_mmu_pt *pt;
171 	int ret;
172 
173 	/* Sub-allocated page table (ie. GP100 LPT). */
174 	if (align < 0x1000) {
175 		mutex_lock(&mmu->ptp.mutex);
176 		pt = nvkm_mmu_ptp_get(mmu, align, zero);
177 		mutex_unlock(&mmu->ptp.mutex);
178 		return pt;
179 	}
180 
181 	/* Lookup cache for this page table size. */
182 	mutex_lock(&mmu->ptc.mutex);
183 	ptc = nvkm_mmu_ptc_find(mmu, size);
184 	if (!ptc) {
185 		mutex_unlock(&mmu->ptc.mutex);
186 		return NULL;
187 	}
188 
189 	/* If there's a free PT in the cache, reuse it. */
190 	pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head);
191 	if (pt) {
192 		if (zero)
193 			nvkm_fo64(pt->memory, 0, 0, size >> 3);
194 		list_del(&pt->head);
195 		ptc->refs--;
196 		mutex_unlock(&mmu->ptc.mutex);
197 		return pt;
198 	}
199 	mutex_unlock(&mmu->ptc.mutex);
200 
201 	/* No such luck, we need to allocate. */
202 	if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL)))
203 		return NULL;
204 	pt->ptc = ptc;
205 	pt->sub = false;
206 
207 	ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST,
208 			      size, align, zero, &pt->memory);
209 	if (ret) {
210 		kfree(pt);
211 		return NULL;
212 	}
213 
214 	pt->base = 0;
215 	pt->addr = nvkm_memory_addr(pt->memory);
216 	return pt;
217 }
218 
219 void
220 nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu)
221 {
222 	struct nvkm_mmu_ptc *ptc;
223 	list_for_each_entry(ptc, &mmu->ptc.list, head) {
224 		struct nvkm_mmu_pt *pt, *tt;
225 		list_for_each_entry_safe(pt, tt, &ptc->item, head) {
226 			nvkm_memory_unref(&pt->memory);
227 			list_del(&pt->head);
228 			kfree(pt);
229 		}
230 	}
231 }
232 
233 static void
234 nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu)
235 {
236 	struct nvkm_mmu_ptc *ptc, *ptct;
237 
238 	list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) {
239 		WARN_ON(!list_empty(&ptc->item));
240 		list_del(&ptc->head);
241 		kfree(ptc);
242 	}
243 }
244 
245 static void
246 nvkm_mmu_ptc_init(struct nvkm_mmu *mmu)
247 {
248 	mutex_init(&mmu->ptc.mutex);
249 	INIT_LIST_HEAD(&mmu->ptc.list);
250 	mutex_init(&mmu->ptp.mutex);
251 	INIT_LIST_HEAD(&mmu->ptp.list);
252 }
253 
254 static void
255 nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type)
256 {
257 	if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) {
258 		mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type;
259 		mmu->type[mmu->type_nr].heap = heap;
260 		mmu->type_nr++;
261 	}
262 }
263 
264 static int
265 nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size)
266 {
267 	if (size) {
268 		if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) {
269 			mmu->heap[mmu->heap_nr].type = type;
270 			mmu->heap[mmu->heap_nr].size = size;
271 			return mmu->heap_nr++;
272 		}
273 	}
274 	return -EINVAL;
275 }
276 
277 static void
278 nvkm_mmu_host(struct nvkm_mmu *mmu)
279 {
280 	struct nvkm_device *device = mmu->subdev.device;
281 	u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys;
282 	int heap;
283 
284 	/* Non-mappable system memory. */
285 	heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL);
286 	nvkm_mmu_type(mmu, heap, type);
287 
288 	/* Non-coherent, cached, system memory.
289 	 *
290 	 * Block-linear mappings of system memory must be done through
291 	 * BAR1, and cannot be supported on systems where we're unable
292 	 * to map BAR1 with write-combining.
293 	 */
294 	type |= NVKM_MEM_MAPPABLE;
295 	if (!device->bar || device->bar->iomap_uncached)
296 		nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
297 	else
298 		nvkm_mmu_type(mmu, heap, type);
299 
300 	/* Coherent, cached, system memory.
301 	 *
302 	 * Unsupported on systems that aren't able to support snooped
303 	 * mappings, and also for block-linear mappings which must be
304 	 * done through BAR1.
305 	 */
306 	type |= NVKM_MEM_COHERENT;
307 	if (device->func->cpu_coherent)
308 		nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND);
309 
310 	/* Uncached system memory. */
311 	nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED);
312 }
313 
314 static void
315 nvkm_mmu_vram(struct nvkm_mmu *mmu)
316 {
317 	struct nvkm_device *device = mmu->subdev.device;
318 	struct nvkm_mm *mm = &device->fb->ram->vram;
319 	const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL);
320 	const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP);
321 	const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED);
322 	u8 type = NVKM_MEM_KIND * !!mmu->func->kind;
323 	u8 heap = NVKM_MEM_VRAM;
324 	int heapM, heapN, heapU;
325 
326 	/* Mixed-memory doesn't support compression or display. */
327 	heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT);
328 
329 	heap |= NVKM_MEM_COMP;
330 	heap |= NVKM_MEM_DISP;
331 	heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT);
332 	heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT);
333 
334 	/* Add non-mappable VRAM types first so that they're preferred
335 	 * over anything else.  Mixed-memory will be slower than other
336 	 * heaps, it's prioritised last.
337 	 */
338 	nvkm_mmu_type(mmu, heapU, type);
339 	nvkm_mmu_type(mmu, heapN, type);
340 	nvkm_mmu_type(mmu, heapM, type);
341 
342 	/* Add host memory types next, under the assumption that users
343 	 * wanting mappable memory want to use them as staging buffers
344 	 * or the like.
345 	 */
346 	nvkm_mmu_host(mmu);
347 
348 	/* Mappable VRAM types go last, as they're basically the worst
349 	 * possible type to ask for unless there's no other choice.
350 	 */
351 	if (device->bar) {
352 		/* Write-combined BAR1 access. */
353 		type |= NVKM_MEM_MAPPABLE;
354 		if (!device->bar->iomap_uncached) {
355 			nvkm_mmu_type(mmu, heapN, type);
356 			nvkm_mmu_type(mmu, heapM, type);
357 		}
358 
359 		/* Uncached BAR1 access. */
360 		type |= NVKM_MEM_COHERENT;
361 		type |= NVKM_MEM_UNCACHED;
362 		nvkm_mmu_type(mmu, heapN, type);
363 		nvkm_mmu_type(mmu, heapM, type);
364 	}
365 }
366 
367 static int
368 nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
369 {
370 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
371 
372 	/* Determine available memory types. */
373 	if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram)
374 		nvkm_mmu_vram(mmu);
375 	else
376 		nvkm_mmu_host(mmu);
377 
378 	if (mmu->func->vmm.global) {
379 		int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL,
380 				       "gart", &mmu->vmm);
381 		if (ret)
382 			return ret;
383 	}
384 
385 	return 0;
386 }
387 
388 static int
389 nvkm_mmu_init(struct nvkm_subdev *subdev)
390 {
391 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
392 	if (mmu->func->init)
393 		mmu->func->init(mmu);
394 	return 0;
395 }
396 
397 static void *
398 nvkm_mmu_dtor(struct nvkm_subdev *subdev)
399 {
400 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
401 
402 	nvkm_vmm_unref(&mmu->vmm);
403 
404 	nvkm_mmu_ptc_fini(mmu);
405 	return mmu;
406 }
407 
408 static const struct nvkm_subdev_func
409 nvkm_mmu = {
410 	.dtor = nvkm_mmu_dtor,
411 	.oneinit = nvkm_mmu_oneinit,
412 	.init = nvkm_mmu_init,
413 };
414 
415 void
416 nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device,
417 	      int index, struct nvkm_mmu *mmu)
418 {
419 	nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev);
420 	mmu->func = func;
421 	mmu->dma_bits = func->dma_bits;
422 	nvkm_mmu_ptc_init(mmu);
423 	mmu->user.ctor = nvkm_ummu_new;
424 	mmu->user.base = func->mmu.user;
425 }
426 
427 int
428 nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device,
429 	      int index, struct nvkm_mmu **pmmu)
430 {
431 	if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL)))
432 		return -ENOMEM;
433 	nvkm_mmu_ctor(func, device, index, *pmmu);
434 	return 0;
435 }
436