1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <drm/drmP.h>
29 #include "radeon.h"
30 #include <drm/radeon_drm.h>
31 #include "radeon_asic.h"
32 
33 #include <linux/vga_switcheroo.h>
34 #include <linux/slab.h>
35 #include <linux/pm_runtime.h>
36 
37 #if defined(CONFIG_VGA_SWITCHEROO)
38 bool radeon_is_px(void);
39 #else
40 static inline bool radeon_is_px(void) { return false; }
41 #endif
42 
43 /**
44  * radeon_driver_unload_kms - Main unload function for KMS.
45  *
46  * @dev: drm dev pointer
47  *
48  * This is the main unload function for KMS (all asics).
49  * It calls radeon_modeset_fini() to tear down the
50  * displays, and radeon_device_fini() to tear down
51  * the rest of the device (CP, writeback, etc.).
52  * Returns 0 on success.
53  */
54 int radeon_driver_unload_kms(struct drm_device *dev)
55 {
56 	struct radeon_device *rdev = dev->dev_private;
57 
58 	if (rdev == NULL)
59 		return 0;
60 
61 	if (rdev->rmmio == NULL)
62 		goto done_free;
63 
64 	pm_runtime_get_sync(dev->dev);
65 
66 	radeon_acpi_fini(rdev);
67 
68 	radeon_modeset_fini(rdev);
69 	radeon_device_fini(rdev);
70 
71 done_free:
72 	kfree(rdev);
73 	dev->dev_private = NULL;
74 	return 0;
75 }
76 
77 /**
78  * radeon_driver_load_kms - Main load function for KMS.
79  *
80  * @dev: drm dev pointer
81  * @flags: device flags
82  *
83  * This is the main load function for KMS (all asics).
84  * It calls radeon_device_init() to set up the non-display
85  * parts of the chip (asic init, CP, writeback, etc.), and
86  * radeon_modeset_init() to set up the display parts
87  * (crtcs, encoders, hotplug detect, etc.).
88  * Returns 0 on success, error on failure.
89  */
90 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
91 {
92 	struct radeon_device *rdev;
93 	int r, acpi_status;
94 
95 	rdev = kzalloc(sizeof(struct radeon_device), GFP_KERNEL);
96 	if (rdev == NULL) {
97 		return -ENOMEM;
98 	}
99 	dev->dev_private = (void *)rdev;
100 
101 	/* update BUS flag */
102 	if (drm_pci_device_is_agp(dev)) {
103 		flags |= RADEON_IS_AGP;
104 	} else if (pci_is_pcie(dev->pdev)) {
105 		flags |= RADEON_IS_PCIE;
106 	} else {
107 		flags |= RADEON_IS_PCI;
108 	}
109 
110 	/* radeon_device_init should report only fatal error
111 	 * like memory allocation failure or iomapping failure,
112 	 * or memory manager initialization failure, it must
113 	 * properly initialize the GPU MC controller and permit
114 	 * VRAM allocation
115 	 */
116 	r = radeon_device_init(rdev, dev, dev->pdev, flags);
117 	if (r) {
118 		dev_err(&dev->pdev->dev, "Fatal error during GPU init\n");
119 		goto out;
120 	}
121 
122 	/* Again modeset_init should fail only on fatal error
123 	 * otherwise it should provide enough functionalities
124 	 * for shadowfb to run
125 	 */
126 	r = radeon_modeset_init(rdev);
127 	if (r)
128 		dev_err(&dev->pdev->dev, "Fatal error during modeset init\n");
129 
130 	/* Call ACPI methods: require modeset init
131 	 * but failure is not fatal
132 	 */
133 	if (!r) {
134 		acpi_status = radeon_acpi_init(rdev);
135 		if (acpi_status)
136 		dev_dbg(&dev->pdev->dev,
137 				"Error during ACPI methods call\n");
138 	}
139 
140 	if ((radeon_runtime_pm == 1) ||
141 	    ((radeon_runtime_pm == -1) && radeon_is_px())) {
142 		pm_runtime_use_autosuspend(dev->dev);
143 		pm_runtime_set_autosuspend_delay(dev->dev, 5000);
144 		pm_runtime_set_active(dev->dev);
145 		pm_runtime_allow(dev->dev);
146 		pm_runtime_mark_last_busy(dev->dev);
147 		pm_runtime_put_autosuspend(dev->dev);
148 	}
149 
150 out:
151 	if (r)
152 		radeon_driver_unload_kms(dev);
153 
154 
155 	return r;
156 }
157 
158 /**
159  * radeon_set_filp_rights - Set filp right.
160  *
161  * @dev: drm dev pointer
162  * @owner: drm file
163  * @applier: drm file
164  * @value: value
165  *
166  * Sets the filp rights for the device (all asics).
167  */
168 static void radeon_set_filp_rights(struct drm_device *dev,
169 				   struct drm_file **owner,
170 				   struct drm_file *applier,
171 				   uint32_t *value)
172 {
173 	mutex_lock(&dev->struct_mutex);
174 	if (*value == 1) {
175 		/* wants rights */
176 		if (!*owner)
177 			*owner = applier;
178 	} else if (*value == 0) {
179 		/* revokes rights */
180 		if (*owner == applier)
181 			*owner = NULL;
182 	}
183 	*value = *owner == applier ? 1 : 0;
184 	mutex_unlock(&dev->struct_mutex);
185 }
186 
187 /*
188  * Userspace get information ioctl
189  */
190 /**
191  * radeon_info_ioctl - answer a device specific request.
192  *
193  * @rdev: radeon device pointer
194  * @data: request object
195  * @filp: drm filp
196  *
197  * This function is used to pass device specific parameters to the userspace
198  * drivers.  Examples include: pci device id, pipeline parms, tiling params,
199  * etc. (all asics).
200  * Returns 0 on success, -EINVAL on failure.
201  */
202 static int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
203 {
204 	struct radeon_device *rdev = dev->dev_private;
205 	struct drm_radeon_info *info = data;
206 	struct radeon_mode_info *minfo = &rdev->mode_info;
207 	uint32_t *value, value_tmp, *value_ptr, value_size;
208 	uint64_t value64;
209 	struct drm_crtc *crtc;
210 	int i, found;
211 
212 	value_ptr = (uint32_t *)((unsigned long)info->value);
213 	value = &value_tmp;
214 	value_size = sizeof(uint32_t);
215 
216 	switch (info->request) {
217 	case RADEON_INFO_DEVICE_ID:
218 		*value = dev->pdev->device;
219 		break;
220 	case RADEON_INFO_NUM_GB_PIPES:
221 		*value = rdev->num_gb_pipes;
222 		break;
223 	case RADEON_INFO_NUM_Z_PIPES:
224 		*value = rdev->num_z_pipes;
225 		break;
226 	case RADEON_INFO_ACCEL_WORKING:
227 		/* xf86-video-ati 6.13.0 relies on this being false for evergreen */
228 		if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK))
229 			*value = false;
230 		else
231 			*value = rdev->accel_working;
232 		break;
233 	case RADEON_INFO_CRTC_FROM_ID:
234 		if (copy_from_user(value, value_ptr, sizeof(uint32_t))) {
235 			DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__);
236 			return -EFAULT;
237 		}
238 		for (i = 0, found = 0; i < rdev->num_crtc; i++) {
239 			crtc = (struct drm_crtc *)minfo->crtcs[i];
240 			if (crtc && crtc->base.id == *value) {
241 				struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
242 				*value = radeon_crtc->crtc_id;
243 				found = 1;
244 				break;
245 			}
246 		}
247 		if (!found) {
248 			DRM_DEBUG_KMS("unknown crtc id %d\n", *value);
249 			return -EINVAL;
250 		}
251 		break;
252 	case RADEON_INFO_ACCEL_WORKING2:
253 		*value = rdev->accel_working;
254 		break;
255 	case RADEON_INFO_TILING_CONFIG:
256 		if (rdev->family >= CHIP_BONAIRE)
257 			*value = rdev->config.cik.tile_config;
258 		else if (rdev->family >= CHIP_TAHITI)
259 			*value = rdev->config.si.tile_config;
260 		else if (rdev->family >= CHIP_CAYMAN)
261 			*value = rdev->config.cayman.tile_config;
262 		else if (rdev->family >= CHIP_CEDAR)
263 			*value = rdev->config.evergreen.tile_config;
264 		else if (rdev->family >= CHIP_RV770)
265 			*value = rdev->config.rv770.tile_config;
266 		else if (rdev->family >= CHIP_R600)
267 			*value = rdev->config.r600.tile_config;
268 		else {
269 			DRM_DEBUG_KMS("tiling config is r6xx+ only!\n");
270 			return -EINVAL;
271 		}
272 		break;
273 	case RADEON_INFO_WANT_HYPERZ:
274 		/* The "value" here is both an input and output parameter.
275 		 * If the input value is 1, filp requests hyper-z access.
276 		 * If the input value is 0, filp revokes its hyper-z access.
277 		 *
278 		 * When returning, the value is 1 if filp owns hyper-z access,
279 		 * 0 otherwise. */
280 		if (copy_from_user(value, value_ptr, sizeof(uint32_t))) {
281 			DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__);
282 			return -EFAULT;
283 		}
284 		if (*value >= 2) {
285 			DRM_DEBUG_KMS("WANT_HYPERZ: invalid value %d\n", *value);
286 			return -EINVAL;
287 		}
288 		radeon_set_filp_rights(dev, &rdev->hyperz_filp, filp, value);
289 		break;
290 	case RADEON_INFO_WANT_CMASK:
291 		/* The same logic as Hyper-Z. */
292 		if (copy_from_user(value, value_ptr, sizeof(uint32_t))) {
293 			DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__);
294 			return -EFAULT;
295 		}
296 		if (*value >= 2) {
297 			DRM_DEBUG_KMS("WANT_CMASK: invalid value %d\n", *value);
298 			return -EINVAL;
299 		}
300 		radeon_set_filp_rights(dev, &rdev->cmask_filp, filp, value);
301 		break;
302 	case RADEON_INFO_CLOCK_CRYSTAL_FREQ:
303 		/* return clock value in KHz */
304 		if (rdev->asic->get_xclk)
305 			*value = radeon_get_xclk(rdev) * 10;
306 		else
307 			*value = rdev->clock.spll.reference_freq * 10;
308 		break;
309 	case RADEON_INFO_NUM_BACKENDS:
310 		if (rdev->family >= CHIP_BONAIRE)
311 			*value = rdev->config.cik.max_backends_per_se *
312 				rdev->config.cik.max_shader_engines;
313 		else if (rdev->family >= CHIP_TAHITI)
314 			*value = rdev->config.si.max_backends_per_se *
315 				rdev->config.si.max_shader_engines;
316 		else if (rdev->family >= CHIP_CAYMAN)
317 			*value = rdev->config.cayman.max_backends_per_se *
318 				rdev->config.cayman.max_shader_engines;
319 		else if (rdev->family >= CHIP_CEDAR)
320 			*value = rdev->config.evergreen.max_backends;
321 		else if (rdev->family >= CHIP_RV770)
322 			*value = rdev->config.rv770.max_backends;
323 		else if (rdev->family >= CHIP_R600)
324 			*value = rdev->config.r600.max_backends;
325 		else {
326 			return -EINVAL;
327 		}
328 		break;
329 	case RADEON_INFO_NUM_TILE_PIPES:
330 		if (rdev->family >= CHIP_BONAIRE)
331 			*value = rdev->config.cik.max_tile_pipes;
332 		else if (rdev->family >= CHIP_TAHITI)
333 			*value = rdev->config.si.max_tile_pipes;
334 		else if (rdev->family >= CHIP_CAYMAN)
335 			*value = rdev->config.cayman.max_tile_pipes;
336 		else if (rdev->family >= CHIP_CEDAR)
337 			*value = rdev->config.evergreen.max_tile_pipes;
338 		else if (rdev->family >= CHIP_RV770)
339 			*value = rdev->config.rv770.max_tile_pipes;
340 		else if (rdev->family >= CHIP_R600)
341 			*value = rdev->config.r600.max_tile_pipes;
342 		else {
343 			return -EINVAL;
344 		}
345 		break;
346 	case RADEON_INFO_FUSION_GART_WORKING:
347 		*value = 1;
348 		break;
349 	case RADEON_INFO_BACKEND_MAP:
350 		if (rdev->family >= CHIP_BONAIRE)
351 			*value = rdev->config.cik.backend_map;
352 		else if (rdev->family >= CHIP_TAHITI)
353 			*value = rdev->config.si.backend_map;
354 		else if (rdev->family >= CHIP_CAYMAN)
355 			*value = rdev->config.cayman.backend_map;
356 		else if (rdev->family >= CHIP_CEDAR)
357 			*value = rdev->config.evergreen.backend_map;
358 		else if (rdev->family >= CHIP_RV770)
359 			*value = rdev->config.rv770.backend_map;
360 		else if (rdev->family >= CHIP_R600)
361 			*value = rdev->config.r600.backend_map;
362 		else {
363 			return -EINVAL;
364 		}
365 		break;
366 	case RADEON_INFO_VA_START:
367 		/* this is where we report if vm is supported or not */
368 		if (rdev->family < CHIP_CAYMAN)
369 			return -EINVAL;
370 		*value = RADEON_VA_RESERVED_SIZE;
371 		break;
372 	case RADEON_INFO_IB_VM_MAX_SIZE:
373 		/* this is where we report if vm is supported or not */
374 		if (rdev->family < CHIP_CAYMAN)
375 			return -EINVAL;
376 		*value = RADEON_IB_VM_MAX_SIZE;
377 		break;
378 	case RADEON_INFO_MAX_PIPES:
379 		if (rdev->family >= CHIP_BONAIRE)
380 			*value = rdev->config.cik.max_cu_per_sh;
381 		else if (rdev->family >= CHIP_TAHITI)
382 			*value = rdev->config.si.max_cu_per_sh;
383 		else if (rdev->family >= CHIP_CAYMAN)
384 			*value = rdev->config.cayman.max_pipes_per_simd;
385 		else if (rdev->family >= CHIP_CEDAR)
386 			*value = rdev->config.evergreen.max_pipes;
387 		else if (rdev->family >= CHIP_RV770)
388 			*value = rdev->config.rv770.max_pipes;
389 		else if (rdev->family >= CHIP_R600)
390 			*value = rdev->config.r600.max_pipes;
391 		else {
392 			return -EINVAL;
393 		}
394 		break;
395 	case RADEON_INFO_TIMESTAMP:
396 		if (rdev->family < CHIP_R600) {
397 			DRM_DEBUG_KMS("timestamp is r6xx+ only!\n");
398 			return -EINVAL;
399 		}
400 		value = (uint32_t*)&value64;
401 		value_size = sizeof(uint64_t);
402 		value64 = radeon_get_gpu_clock_counter(rdev);
403 		break;
404 	case RADEON_INFO_MAX_SE:
405 		if (rdev->family >= CHIP_BONAIRE)
406 			*value = rdev->config.cik.max_shader_engines;
407 		else if (rdev->family >= CHIP_TAHITI)
408 			*value = rdev->config.si.max_shader_engines;
409 		else if (rdev->family >= CHIP_CAYMAN)
410 			*value = rdev->config.cayman.max_shader_engines;
411 		else if (rdev->family >= CHIP_CEDAR)
412 			*value = rdev->config.evergreen.num_ses;
413 		else
414 			*value = 1;
415 		break;
416 	case RADEON_INFO_MAX_SH_PER_SE:
417 		if (rdev->family >= CHIP_BONAIRE)
418 			*value = rdev->config.cik.max_sh_per_se;
419 		else if (rdev->family >= CHIP_TAHITI)
420 			*value = rdev->config.si.max_sh_per_se;
421 		else
422 			return -EINVAL;
423 		break;
424 	case RADEON_INFO_FASTFB_WORKING:
425 		*value = rdev->fastfb_working;
426 		break;
427 	case RADEON_INFO_RING_WORKING:
428 		if (copy_from_user(value, value_ptr, sizeof(uint32_t))) {
429 			DRM_ERROR("copy_from_user %s:%u\n", __func__, __LINE__);
430 			return -EFAULT;
431 		}
432 		switch (*value) {
433 		case RADEON_CS_RING_GFX:
434 		case RADEON_CS_RING_COMPUTE:
435 			*value = rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready;
436 			break;
437 		case RADEON_CS_RING_DMA:
438 			*value = rdev->ring[R600_RING_TYPE_DMA_INDEX].ready;
439 			*value |= rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready;
440 			break;
441 		case RADEON_CS_RING_UVD:
442 			*value = rdev->ring[R600_RING_TYPE_UVD_INDEX].ready;
443 			break;
444 		default:
445 			return -EINVAL;
446 		}
447 		break;
448 	case RADEON_INFO_SI_TILE_MODE_ARRAY:
449 		if (rdev->family >= CHIP_BONAIRE) {
450 			value = rdev->config.cik.tile_mode_array;
451 			value_size = sizeof(uint32_t)*32;
452 		} else if (rdev->family >= CHIP_TAHITI) {
453 			value = rdev->config.si.tile_mode_array;
454 			value_size = sizeof(uint32_t)*32;
455 		} else {
456 			DRM_DEBUG_KMS("tile mode array is si+ only!\n");
457 			return -EINVAL;
458 		}
459 		break;
460 	case RADEON_INFO_CIK_MACROTILE_MODE_ARRAY:
461 		if (rdev->family >= CHIP_BONAIRE) {
462 			value = rdev->config.cik.macrotile_mode_array;
463 			value_size = sizeof(uint32_t)*16;
464 		} else {
465 			DRM_DEBUG_KMS("macrotile mode array is cik+ only!\n");
466 			return -EINVAL;
467 		}
468 		break;
469 	case RADEON_INFO_SI_CP_DMA_COMPUTE:
470 		*value = 1;
471 		break;
472 	case RADEON_INFO_SI_BACKEND_ENABLED_MASK:
473 		if (rdev->family >= CHIP_BONAIRE) {
474 			*value = rdev->config.cik.backend_enable_mask;
475 		} else if (rdev->family >= CHIP_TAHITI) {
476 			*value = rdev->config.si.backend_enable_mask;
477 		} else {
478 			DRM_DEBUG_KMS("BACKEND_ENABLED_MASK is si+ only!\n");
479 		}
480 		break;
481 	case RADEON_INFO_MAX_SCLK:
482 		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
483 		    rdev->pm.dpm_enabled)
484 			*value = rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk * 10;
485 		else
486 			*value = rdev->pm.default_sclk * 10;
487 		break;
488 	default:
489 		DRM_DEBUG_KMS("Invalid request %d\n", info->request);
490 		return -EINVAL;
491 	}
492 	if (copy_to_user(value_ptr, (char*)value, value_size)) {
493 		DRM_ERROR("copy_to_user %s:%u\n", __func__, __LINE__);
494 		return -EFAULT;
495 	}
496 	return 0;
497 }
498 
499 
500 /*
501  * Outdated mess for old drm with Xorg being in charge (void function now).
502  */
503 /**
504  * radeon_driver_firstopen_kms - drm callback for last close
505  *
506  * @dev: drm dev pointer
507  *
508  * Switch vga switcheroo state after last close (all asics).
509  */
510 void radeon_driver_lastclose_kms(struct drm_device *dev)
511 {
512 	vga_switcheroo_process_delayed_switch();
513 }
514 
515 /**
516  * radeon_driver_open_kms - drm callback for open
517  *
518  * @dev: drm dev pointer
519  * @file_priv: drm file
520  *
521  * On device open, init vm on cayman+ (all asics).
522  * Returns 0 on success, error on failure.
523  */
524 int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
525 {
526 	struct radeon_device *rdev = dev->dev_private;
527 	int r;
528 
529 	file_priv->driver_priv = NULL;
530 
531 	r = pm_runtime_get_sync(dev->dev);
532 	if (r < 0)
533 		return r;
534 
535 	/* new gpu have virtual address space support */
536 	if (rdev->family >= CHIP_CAYMAN) {
537 		struct radeon_fpriv *fpriv;
538 		struct radeon_bo_va *bo_va;
539 		int r;
540 
541 		fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
542 		if (unlikely(!fpriv)) {
543 			return -ENOMEM;
544 		}
545 
546 		radeon_vm_init(rdev, &fpriv->vm);
547 
548 		r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false);
549 		if (r)
550 			return r;
551 
552 		/* map the ib pool buffer read only into
553 		 * virtual address space */
554 		bo_va = radeon_vm_bo_add(rdev, &fpriv->vm,
555 					 rdev->ring_tmp_bo.bo);
556 		r = radeon_vm_bo_set_addr(rdev, bo_va, RADEON_VA_IB_OFFSET,
557 					  RADEON_VM_PAGE_READABLE |
558 					  RADEON_VM_PAGE_SNOOPED);
559 
560 		radeon_bo_unreserve(rdev->ring_tmp_bo.bo);
561 		if (r) {
562 			radeon_vm_fini(rdev, &fpriv->vm);
563 			kfree(fpriv);
564 			return r;
565 		}
566 
567 		file_priv->driver_priv = fpriv;
568 	}
569 
570 	pm_runtime_mark_last_busy(dev->dev);
571 	pm_runtime_put_autosuspend(dev->dev);
572 	return 0;
573 }
574 
575 /**
576  * radeon_driver_postclose_kms - drm callback for post close
577  *
578  * @dev: drm dev pointer
579  * @file_priv: drm file
580  *
581  * On device post close, tear down vm on cayman+ (all asics).
582  */
583 void radeon_driver_postclose_kms(struct drm_device *dev,
584 				 struct drm_file *file_priv)
585 {
586 	struct radeon_device *rdev = dev->dev_private;
587 
588 	/* new gpu have virtual address space support */
589 	if (rdev->family >= CHIP_CAYMAN && file_priv->driver_priv) {
590 		struct radeon_fpriv *fpriv = file_priv->driver_priv;
591 		struct radeon_bo_va *bo_va;
592 		int r;
593 
594 		r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false);
595 		if (!r) {
596 			bo_va = radeon_vm_bo_find(&fpriv->vm,
597 						  rdev->ring_tmp_bo.bo);
598 			if (bo_va)
599 				radeon_vm_bo_rmv(rdev, bo_va);
600 			radeon_bo_unreserve(rdev->ring_tmp_bo.bo);
601 		}
602 
603 		radeon_vm_fini(rdev, &fpriv->vm);
604 		kfree(fpriv);
605 		file_priv->driver_priv = NULL;
606 	}
607 }
608 
609 /**
610  * radeon_driver_preclose_kms - drm callback for pre close
611  *
612  * @dev: drm dev pointer
613  * @file_priv: drm file
614  *
615  * On device pre close, tear down hyperz and cmask filps on r1xx-r5xx
616  * (all asics).
617  */
618 void radeon_driver_preclose_kms(struct drm_device *dev,
619 				struct drm_file *file_priv)
620 {
621 	struct radeon_device *rdev = dev->dev_private;
622 	if (rdev->hyperz_filp == file_priv)
623 		rdev->hyperz_filp = NULL;
624 	if (rdev->cmask_filp == file_priv)
625 		rdev->cmask_filp = NULL;
626 	radeon_uvd_free_handles(rdev, file_priv);
627 }
628 
629 /*
630  * VBlank related functions.
631  */
632 /**
633  * radeon_get_vblank_counter_kms - get frame count
634  *
635  * @dev: drm dev pointer
636  * @crtc: crtc to get the frame count from
637  *
638  * Gets the frame count on the requested crtc (all asics).
639  * Returns frame count on success, -EINVAL on failure.
640  */
641 u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc)
642 {
643 	struct radeon_device *rdev = dev->dev_private;
644 
645 	if (crtc < 0 || crtc >= rdev->num_crtc) {
646 		DRM_ERROR("Invalid crtc %d\n", crtc);
647 		return -EINVAL;
648 	}
649 
650 	return radeon_get_vblank_counter(rdev, crtc);
651 }
652 
653 /**
654  * radeon_enable_vblank_kms - enable vblank interrupt
655  *
656  * @dev: drm dev pointer
657  * @crtc: crtc to enable vblank interrupt for
658  *
659  * Enable the interrupt on the requested crtc (all asics).
660  * Returns 0 on success, -EINVAL on failure.
661  */
662 int radeon_enable_vblank_kms(struct drm_device *dev, int crtc)
663 {
664 	struct radeon_device *rdev = dev->dev_private;
665 	unsigned long irqflags;
666 	int r;
667 
668 	if (crtc < 0 || crtc >= rdev->num_crtc) {
669 		DRM_ERROR("Invalid crtc %d\n", crtc);
670 		return -EINVAL;
671 	}
672 
673 	spin_lock_irqsave(&rdev->irq.lock, irqflags);
674 	rdev->irq.crtc_vblank_int[crtc] = true;
675 	r = radeon_irq_set(rdev);
676 	spin_unlock_irqrestore(&rdev->irq.lock, irqflags);
677 	return r;
678 }
679 
680 /**
681  * radeon_disable_vblank_kms - disable vblank interrupt
682  *
683  * @dev: drm dev pointer
684  * @crtc: crtc to disable vblank interrupt for
685  *
686  * Disable the interrupt on the requested crtc (all asics).
687  */
688 void radeon_disable_vblank_kms(struct drm_device *dev, int crtc)
689 {
690 	struct radeon_device *rdev = dev->dev_private;
691 	unsigned long irqflags;
692 
693 	if (crtc < 0 || crtc >= rdev->num_crtc) {
694 		DRM_ERROR("Invalid crtc %d\n", crtc);
695 		return;
696 	}
697 
698 	spin_lock_irqsave(&rdev->irq.lock, irqflags);
699 	rdev->irq.crtc_vblank_int[crtc] = false;
700 	radeon_irq_set(rdev);
701 	spin_unlock_irqrestore(&rdev->irq.lock, irqflags);
702 }
703 
704 /**
705  * radeon_get_vblank_timestamp_kms - get vblank timestamp
706  *
707  * @dev: drm dev pointer
708  * @crtc: crtc to get the timestamp for
709  * @max_error: max error
710  * @vblank_time: time value
711  * @flags: flags passed to the driver
712  *
713  * Gets the timestamp on the requested crtc based on the
714  * scanout position.  (all asics).
715  * Returns postive status flags on success, negative error on failure.
716  */
717 int radeon_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
718 				    int *max_error,
719 				    struct timeval *vblank_time,
720 				    unsigned flags)
721 {
722 	struct drm_crtc *drmcrtc;
723 	struct radeon_device *rdev = dev->dev_private;
724 
725 	if (crtc < 0 || crtc >= dev->num_crtcs) {
726 		DRM_ERROR("Invalid crtc %d\n", crtc);
727 		return -EINVAL;
728 	}
729 
730 	/* Get associated drm_crtc: */
731 	drmcrtc = &rdev->mode_info.crtcs[crtc]->base;
732 
733 	/* Helper routine in DRM core does all the work: */
734 	return drm_calc_vbltimestamp_from_scanoutpos(dev, crtc, max_error,
735 						     vblank_time, flags,
736 						     drmcrtc, &drmcrtc->hwmode);
737 }
738 
739 #define KMS_INVALID_IOCTL(name)						\
740 static int name(struct drm_device *dev, void *data, struct drm_file	\
741 		*file_priv)						\
742 {									\
743 	DRM_ERROR("invalid ioctl with kms %s\n", __func__);		\
744 	return -EINVAL;							\
745 }
746 
747 /*
748  * All these ioctls are invalid in kms world.
749  */
750 KMS_INVALID_IOCTL(radeon_cp_init_kms)
751 KMS_INVALID_IOCTL(radeon_cp_start_kms)
752 KMS_INVALID_IOCTL(radeon_cp_stop_kms)
753 KMS_INVALID_IOCTL(radeon_cp_reset_kms)
754 KMS_INVALID_IOCTL(radeon_cp_idle_kms)
755 KMS_INVALID_IOCTL(radeon_cp_resume_kms)
756 KMS_INVALID_IOCTL(radeon_engine_reset_kms)
757 KMS_INVALID_IOCTL(radeon_fullscreen_kms)
758 KMS_INVALID_IOCTL(radeon_cp_swap_kms)
759 KMS_INVALID_IOCTL(radeon_cp_clear_kms)
760 KMS_INVALID_IOCTL(radeon_cp_vertex_kms)
761 KMS_INVALID_IOCTL(radeon_cp_indices_kms)
762 KMS_INVALID_IOCTL(radeon_cp_texture_kms)
763 KMS_INVALID_IOCTL(radeon_cp_stipple_kms)
764 KMS_INVALID_IOCTL(radeon_cp_indirect_kms)
765 KMS_INVALID_IOCTL(radeon_cp_vertex2_kms)
766 KMS_INVALID_IOCTL(radeon_cp_cmdbuf_kms)
767 KMS_INVALID_IOCTL(radeon_cp_getparam_kms)
768 KMS_INVALID_IOCTL(radeon_cp_flip_kms)
769 KMS_INVALID_IOCTL(radeon_mem_alloc_kms)
770 KMS_INVALID_IOCTL(radeon_mem_free_kms)
771 KMS_INVALID_IOCTL(radeon_mem_init_heap_kms)
772 KMS_INVALID_IOCTL(radeon_irq_emit_kms)
773 KMS_INVALID_IOCTL(radeon_irq_wait_kms)
774 KMS_INVALID_IOCTL(radeon_cp_setparam_kms)
775 KMS_INVALID_IOCTL(radeon_surface_alloc_kms)
776 KMS_INVALID_IOCTL(radeon_surface_free_kms)
777 
778 
779 const struct drm_ioctl_desc radeon_ioctls_kms[] = {
780 	DRM_IOCTL_DEF_DRV(RADEON_CP_INIT, radeon_cp_init_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
781 	DRM_IOCTL_DEF_DRV(RADEON_CP_START, radeon_cp_start_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
782 	DRM_IOCTL_DEF_DRV(RADEON_CP_STOP, radeon_cp_stop_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
783 	DRM_IOCTL_DEF_DRV(RADEON_CP_RESET, radeon_cp_reset_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
784 	DRM_IOCTL_DEF_DRV(RADEON_CP_IDLE, radeon_cp_idle_kms, DRM_AUTH),
785 	DRM_IOCTL_DEF_DRV(RADEON_CP_RESUME, radeon_cp_resume_kms, DRM_AUTH),
786 	DRM_IOCTL_DEF_DRV(RADEON_RESET, radeon_engine_reset_kms, DRM_AUTH),
787 	DRM_IOCTL_DEF_DRV(RADEON_FULLSCREEN, radeon_fullscreen_kms, DRM_AUTH),
788 	DRM_IOCTL_DEF_DRV(RADEON_SWAP, radeon_cp_swap_kms, DRM_AUTH),
789 	DRM_IOCTL_DEF_DRV(RADEON_CLEAR, radeon_cp_clear_kms, DRM_AUTH),
790 	DRM_IOCTL_DEF_DRV(RADEON_VERTEX, radeon_cp_vertex_kms, DRM_AUTH),
791 	DRM_IOCTL_DEF_DRV(RADEON_INDICES, radeon_cp_indices_kms, DRM_AUTH),
792 	DRM_IOCTL_DEF_DRV(RADEON_TEXTURE, radeon_cp_texture_kms, DRM_AUTH),
793 	DRM_IOCTL_DEF_DRV(RADEON_STIPPLE, radeon_cp_stipple_kms, DRM_AUTH),
794 	DRM_IOCTL_DEF_DRV(RADEON_INDIRECT, radeon_cp_indirect_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
795 	DRM_IOCTL_DEF_DRV(RADEON_VERTEX2, radeon_cp_vertex2_kms, DRM_AUTH),
796 	DRM_IOCTL_DEF_DRV(RADEON_CMDBUF, radeon_cp_cmdbuf_kms, DRM_AUTH),
797 	DRM_IOCTL_DEF_DRV(RADEON_GETPARAM, radeon_cp_getparam_kms, DRM_AUTH),
798 	DRM_IOCTL_DEF_DRV(RADEON_FLIP, radeon_cp_flip_kms, DRM_AUTH),
799 	DRM_IOCTL_DEF_DRV(RADEON_ALLOC, radeon_mem_alloc_kms, DRM_AUTH),
800 	DRM_IOCTL_DEF_DRV(RADEON_FREE, radeon_mem_free_kms, DRM_AUTH),
801 	DRM_IOCTL_DEF_DRV(RADEON_INIT_HEAP, radeon_mem_init_heap_kms, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
802 	DRM_IOCTL_DEF_DRV(RADEON_IRQ_EMIT, radeon_irq_emit_kms, DRM_AUTH),
803 	DRM_IOCTL_DEF_DRV(RADEON_IRQ_WAIT, radeon_irq_wait_kms, DRM_AUTH),
804 	DRM_IOCTL_DEF_DRV(RADEON_SETPARAM, radeon_cp_setparam_kms, DRM_AUTH),
805 	DRM_IOCTL_DEF_DRV(RADEON_SURF_ALLOC, radeon_surface_alloc_kms, DRM_AUTH),
806 	DRM_IOCTL_DEF_DRV(RADEON_SURF_FREE, radeon_surface_free_kms, DRM_AUTH),
807 	/* KMS */
808 	DRM_IOCTL_DEF_DRV(RADEON_GEM_INFO, radeon_gem_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
809 	DRM_IOCTL_DEF_DRV(RADEON_GEM_CREATE, radeon_gem_create_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
810 	DRM_IOCTL_DEF_DRV(RADEON_GEM_MMAP, radeon_gem_mmap_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
811 	DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_DOMAIN, radeon_gem_set_domain_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
812 	DRM_IOCTL_DEF_DRV(RADEON_GEM_PREAD, radeon_gem_pread_ioctl, DRM_AUTH|DRM_UNLOCKED),
813 	DRM_IOCTL_DEF_DRV(RADEON_GEM_PWRITE, radeon_gem_pwrite_ioctl, DRM_AUTH|DRM_UNLOCKED),
814 	DRM_IOCTL_DEF_DRV(RADEON_GEM_WAIT_IDLE, radeon_gem_wait_idle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
815 	DRM_IOCTL_DEF_DRV(RADEON_CS, radeon_cs_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
816 	DRM_IOCTL_DEF_DRV(RADEON_INFO, radeon_info_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
817 	DRM_IOCTL_DEF_DRV(RADEON_GEM_SET_TILING, radeon_gem_set_tiling_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
818 	DRM_IOCTL_DEF_DRV(RADEON_GEM_GET_TILING, radeon_gem_get_tiling_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
819 	DRM_IOCTL_DEF_DRV(RADEON_GEM_BUSY, radeon_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
820 	DRM_IOCTL_DEF_DRV(RADEON_GEM_VA, radeon_gem_va_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
821 };
822 int radeon_max_kms_ioctl = DRM_ARRAY_SIZE(radeon_ioctls_kms);
823