1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <drm/drmP.h>
29 #include <drm/radeon_drm.h>
30 #include "radeon.h"
31 
32 /*
33  * GART
34  * The GART (Graphics Aperture Remapping Table) is an aperture
35  * in the GPU's address space.  System pages can be mapped into
36  * the aperture and look like contiguous pages from the GPU's
37  * perspective.  A page table maps the pages in the aperture
38  * to the actual backing pages in system memory.
39  *
40  * Radeon GPUs support both an internal GART, as described above,
41  * and AGP.  AGP works similarly, but the GART table is configured
42  * and maintained by the northbridge rather than the driver.
43  * Radeon hw has a separate AGP aperture that is programmed to
44  * point to the AGP aperture provided by the northbridge and the
45  * requests are passed through to the northbridge aperture.
46  * Both AGP and internal GART can be used at the same time, however
47  * that is not currently supported by the driver.
48  *
49  * This file handles the common internal GART management.
50  */
51 
52 /*
53  * Common GART table functions.
54  */
55 /**
56  * radeon_gart_table_ram_alloc - allocate system ram for gart page table
57  *
58  * @rdev: radeon_device pointer
59  *
60  * Allocate system memory for GART page table
61  * (r1xx-r3xx, non-pcie r4xx, rs400).  These asics require the
62  * gart table to be in system memory.
63  * Returns 0 for success, -ENOMEM for failure.
64  */
65 int radeon_gart_table_ram_alloc(struct radeon_device *rdev)
66 {
67 	void *ptr;
68 
69 	ptr = pci_alloc_consistent(rdev->pdev, rdev->gart.table_size,
70 				   &rdev->gart.table_addr);
71 	if (ptr == NULL) {
72 		return -ENOMEM;
73 	}
74 #ifdef CONFIG_X86
75 	if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
76 	    rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
77 		set_memory_uc((unsigned long)ptr,
78 			      rdev->gart.table_size >> PAGE_SHIFT);
79 	}
80 #endif
81 	rdev->gart.ptr = ptr;
82 	memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size);
83 	return 0;
84 }
85 
86 /**
87  * radeon_gart_table_ram_free - free system ram for gart page table
88  *
89  * @rdev: radeon_device pointer
90  *
91  * Free system memory for GART page table
92  * (r1xx-r3xx, non-pcie r4xx, rs400).  These asics require the
93  * gart table to be in system memory.
94  */
95 void radeon_gart_table_ram_free(struct radeon_device *rdev)
96 {
97 	if (rdev->gart.ptr == NULL) {
98 		return;
99 	}
100 #ifdef CONFIG_X86
101 	if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
102 	    rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
103 		set_memory_wb((unsigned long)rdev->gart.ptr,
104 			      rdev->gart.table_size >> PAGE_SHIFT);
105 	}
106 #endif
107 	pci_free_consistent(rdev->pdev, rdev->gart.table_size,
108 			    (void *)rdev->gart.ptr,
109 			    rdev->gart.table_addr);
110 	rdev->gart.ptr = NULL;
111 	rdev->gart.table_addr = 0;
112 }
113 
114 /**
115  * radeon_gart_table_vram_alloc - allocate vram for gart page table
116  *
117  * @rdev: radeon_device pointer
118  *
119  * Allocate video memory for GART page table
120  * (pcie r4xx, r5xx+).  These asics require the
121  * gart table to be in video memory.
122  * Returns 0 for success, error for failure.
123  */
124 int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
125 {
126 	int r;
127 
128 	if (rdev->gart.robj == NULL) {
129 		r = radeon_bo_create(rdev, rdev->gart.table_size,
130 				     PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
131 				     0, NULL, NULL, &rdev->gart.robj);
132 		if (r) {
133 			return r;
134 		}
135 	}
136 	return 0;
137 }
138 
139 /**
140  * radeon_gart_table_vram_pin - pin gart page table in vram
141  *
142  * @rdev: radeon_device pointer
143  *
144  * Pin the GART page table in vram so it will not be moved
145  * by the memory manager (pcie r4xx, r5xx+).  These asics require the
146  * gart table to be in video memory.
147  * Returns 0 for success, error for failure.
148  */
149 int radeon_gart_table_vram_pin(struct radeon_device *rdev)
150 {
151 	uint64_t gpu_addr;
152 	int r;
153 
154 	r = radeon_bo_reserve(rdev->gart.robj, false);
155 	if (unlikely(r != 0))
156 		return r;
157 	r = radeon_bo_pin(rdev->gart.robj,
158 				RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
159 	if (r) {
160 		radeon_bo_unreserve(rdev->gart.robj);
161 		return r;
162 	}
163 	r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr);
164 	if (r)
165 		radeon_bo_unpin(rdev->gart.robj);
166 	radeon_bo_unreserve(rdev->gart.robj);
167 	rdev->gart.table_addr = gpu_addr;
168 
169 	if (!r) {
170 		int i;
171 
172 		/* We might have dropped some GART table updates while it wasn't
173 		 * mapped, restore all entries
174 		 */
175 		for (i = 0; i < rdev->gart.num_gpu_pages; i++)
176 			radeon_gart_set_page(rdev, i, rdev->gart.pages_entry[i]);
177 		mb();
178 		radeon_gart_tlb_flush(rdev);
179 	}
180 
181 	return r;
182 }
183 
184 /**
185  * radeon_gart_table_vram_unpin - unpin gart page table in vram
186  *
187  * @rdev: radeon_device pointer
188  *
189  * Unpin the GART page table in vram (pcie r4xx, r5xx+).
190  * These asics require the gart table to be in video memory.
191  */
192 void radeon_gart_table_vram_unpin(struct radeon_device *rdev)
193 {
194 	int r;
195 
196 	if (rdev->gart.robj == NULL) {
197 		return;
198 	}
199 	r = radeon_bo_reserve(rdev->gart.robj, false);
200 	if (likely(r == 0)) {
201 		radeon_bo_kunmap(rdev->gart.robj);
202 		radeon_bo_unpin(rdev->gart.robj);
203 		radeon_bo_unreserve(rdev->gart.robj);
204 		rdev->gart.ptr = NULL;
205 	}
206 }
207 
208 /**
209  * radeon_gart_table_vram_free - free gart page table vram
210  *
211  * @rdev: radeon_device pointer
212  *
213  * Free the video memory used for the GART page table
214  * (pcie r4xx, r5xx+).  These asics require the gart table to
215  * be in video memory.
216  */
217 void radeon_gart_table_vram_free(struct radeon_device *rdev)
218 {
219 	if (rdev->gart.robj == NULL) {
220 		return;
221 	}
222 	radeon_bo_unref(&rdev->gart.robj);
223 }
224 
225 /*
226  * Common gart functions.
227  */
228 /**
229  * radeon_gart_unbind - unbind pages from the gart page table
230  *
231  * @rdev: radeon_device pointer
232  * @offset: offset into the GPU's gart aperture
233  * @pages: number of pages to unbind
234  *
235  * Unbinds the requested pages from the gart page table and
236  * replaces them with the dummy page (all asics).
237  */
238 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
239 			int pages)
240 {
241 	unsigned t;
242 	unsigned p;
243 	int i, j;
244 
245 	if (!rdev->gart.ready) {
246 		WARN(1, "trying to unbind memory from uninitialized GART !\n");
247 		return;
248 	}
249 	t = offset / RADEON_GPU_PAGE_SIZE;
250 	p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
251 	for (i = 0; i < pages; i++, p++) {
252 		if (rdev->gart.pages[p]) {
253 			rdev->gart.pages[p] = NULL;
254 			for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
255 				rdev->gart.pages_entry[t] = rdev->dummy_page.entry;
256 				if (rdev->gart.ptr) {
257 					radeon_gart_set_page(rdev, t,
258 							     rdev->dummy_page.entry);
259 				}
260 			}
261 		}
262 	}
263 	if (rdev->gart.ptr) {
264 		mb();
265 		radeon_gart_tlb_flush(rdev);
266 	}
267 }
268 
269 /**
270  * radeon_gart_bind - bind pages into the gart page table
271  *
272  * @rdev: radeon_device pointer
273  * @offset: offset into the GPU's gart aperture
274  * @pages: number of pages to bind
275  * @pagelist: pages to bind
276  * @dma_addr: DMA addresses of pages
277  * @flags: RADEON_GART_PAGE_* flags
278  *
279  * Binds the requested pages to the gart page table
280  * (all asics).
281  * Returns 0 for success, -EINVAL for failure.
282  */
283 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
284 		     int pages, struct page **pagelist, dma_addr_t *dma_addr,
285 		     uint32_t flags)
286 {
287 	unsigned t;
288 	unsigned p;
289 	uint64_t page_base, page_entry;
290 	int i, j;
291 
292 	if (!rdev->gart.ready) {
293 		WARN(1, "trying to bind memory to uninitialized GART !\n");
294 		return -EINVAL;
295 	}
296 	t = offset / RADEON_GPU_PAGE_SIZE;
297 	p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
298 
299 	for (i = 0; i < pages; i++, p++) {
300 		rdev->gart.pages[p] = pagelist[i];
301 		page_base = dma_addr[i];
302 		for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
303 			page_entry = radeon_gart_get_page_entry(page_base, flags);
304 			rdev->gart.pages_entry[t] = page_entry;
305 			if (rdev->gart.ptr) {
306 				radeon_gart_set_page(rdev, t, page_entry);
307 			}
308 			page_base += RADEON_GPU_PAGE_SIZE;
309 		}
310 	}
311 	if (rdev->gart.ptr) {
312 		mb();
313 		radeon_gart_tlb_flush(rdev);
314 	}
315 	return 0;
316 }
317 
318 /**
319  * radeon_gart_init - init the driver info for managing the gart
320  *
321  * @rdev: radeon_device pointer
322  *
323  * Allocate the dummy page and init the gart driver info (all asics).
324  * Returns 0 for success, error for failure.
325  */
326 int radeon_gart_init(struct radeon_device *rdev)
327 {
328 	int r, i;
329 
330 	if (rdev->gart.pages) {
331 		return 0;
332 	}
333 	/* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */
334 	if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) {
335 		DRM_ERROR("Page size is smaller than GPU page size!\n");
336 		return -EINVAL;
337 	}
338 	r = radeon_dummy_page_init(rdev);
339 	if (r)
340 		return r;
341 	/* Compute table size */
342 	rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE;
343 	rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE;
344 	DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
345 		 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages);
346 	/* Allocate pages table */
347 	rdev->gart.pages = vzalloc(sizeof(void *) * rdev->gart.num_cpu_pages);
348 	if (rdev->gart.pages == NULL) {
349 		radeon_gart_fini(rdev);
350 		return -ENOMEM;
351 	}
352 	rdev->gart.pages_entry = vmalloc(sizeof(uint64_t) *
353 					 rdev->gart.num_gpu_pages);
354 	if (rdev->gart.pages_entry == NULL) {
355 		radeon_gart_fini(rdev);
356 		return -ENOMEM;
357 	}
358 	/* set GART entry to point to the dummy page by default */
359 	for (i = 0; i < rdev->gart.num_gpu_pages; i++)
360 		rdev->gart.pages_entry[i] = rdev->dummy_page.entry;
361 	return 0;
362 }
363 
364 /**
365  * radeon_gart_fini - tear down the driver info for managing the gart
366  *
367  * @rdev: radeon_device pointer
368  *
369  * Tear down the gart driver info and free the dummy page (all asics).
370  */
371 void radeon_gart_fini(struct radeon_device *rdev)
372 {
373 	if (rdev->gart.ready) {
374 		/* unbind pages */
375 		radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages);
376 	}
377 	rdev->gart.ready = false;
378 	vfree(rdev->gart.pages);
379 	vfree(rdev->gart.pages_entry);
380 	rdev->gart.pages = NULL;
381 	rdev->gart.pages_entry = NULL;
382 
383 	radeon_dummy_page_fini(rdev);
384 }
385