1 /*
2  * Copyright 2012 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Ben Skeggs
23  */
24 
25 #include <linux/delay.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/vga_switcheroo.h>
30 #include <linux/mmu_notifier.h>
31 #include <linux/dynamic_debug.h>
32 
33 #include <drm/drm_aperture.h>
34 #include <drm/drm_drv.h>
35 #include <drm/drm_fbdev_generic.h>
36 #include <drm/drm_gem_ttm_helper.h>
37 #include <drm/drm_ioctl.h>
38 #include <drm/drm_vblank.h>
39 
40 #include <core/gpuobj.h>
41 #include <core/option.h>
42 #include <core/pci.h>
43 #include <core/tegra.h>
44 
45 #include <nvif/driver.h>
46 #include <nvif/fifo.h>
47 #include <nvif/push006c.h>
48 #include <nvif/user.h>
49 
50 #include <nvif/class.h>
51 #include <nvif/cl0002.h>
52 
53 #include "nouveau_drv.h"
54 #include "nouveau_dma.h"
55 #include "nouveau_ttm.h"
56 #include "nouveau_gem.h"
57 #include "nouveau_vga.h"
58 #include "nouveau_led.h"
59 #include "nouveau_hwmon.h"
60 #include "nouveau_acpi.h"
61 #include "nouveau_bios.h"
62 #include "nouveau_ioctl.h"
63 #include "nouveau_abi16.h"
64 #include "nouveau_fence.h"
65 #include "nouveau_debugfs.h"
66 #include "nouveau_usif.h"
67 #include "nouveau_connector.h"
68 #include "nouveau_platform.h"
69 #include "nouveau_svm.h"
70 #include "nouveau_dmem.h"
71 
72 DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0,
73 			"DRM_UT_CORE",
74 			"DRM_UT_DRIVER",
75 			"DRM_UT_KMS",
76 			"DRM_UT_PRIME",
77 			"DRM_UT_ATOMIC",
78 			"DRM_UT_VBL",
79 			"DRM_UT_STATE",
80 			"DRM_UT_LEASE",
81 			"DRM_UT_DP",
82 			"DRM_UT_DRMRES");
83 
84 MODULE_PARM_DESC(config, "option string to pass to driver core");
85 static char *nouveau_config;
86 module_param_named(config, nouveau_config, charp, 0400);
87 
88 MODULE_PARM_DESC(debug, "debug string to pass to driver core");
89 static char *nouveau_debug;
90 module_param_named(debug, nouveau_debug, charp, 0400);
91 
92 MODULE_PARM_DESC(noaccel, "disable kernel/abi16 acceleration");
93 static int nouveau_noaccel = 0;
94 module_param_named(noaccel, nouveau_noaccel, int, 0400);
95 
96 MODULE_PARM_DESC(modeset, "enable driver (default: auto, "
97 		          "0 = disabled, 1 = enabled, 2 = headless)");
98 int nouveau_modeset = -1;
99 module_param_named(modeset, nouveau_modeset, int, 0400);
100 
101 MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)");
102 static int nouveau_atomic = 0;
103 module_param_named(atomic, nouveau_atomic, int, 0400);
104 
105 MODULE_PARM_DESC(runpm, "disable (0), force enable (1), optimus only default (-1)");
106 static int nouveau_runtime_pm = -1;
107 module_param_named(runpm, nouveau_runtime_pm, int, 0400);
108 
109 static struct drm_driver driver_stub;
110 static struct drm_driver driver_pci;
111 static struct drm_driver driver_platform;
112 
113 static u64
114 nouveau_pci_name(struct pci_dev *pdev)
115 {
116 	u64 name = (u64)pci_domain_nr(pdev->bus) << 32;
117 	name |= pdev->bus->number << 16;
118 	name |= PCI_SLOT(pdev->devfn) << 8;
119 	return name | PCI_FUNC(pdev->devfn);
120 }
121 
122 static u64
123 nouveau_platform_name(struct platform_device *platformdev)
124 {
125 	return platformdev->id;
126 }
127 
128 static u64
129 nouveau_name(struct drm_device *dev)
130 {
131 	if (dev_is_pci(dev->dev))
132 		return nouveau_pci_name(to_pci_dev(dev->dev));
133 	else
134 		return nouveau_platform_name(to_platform_device(dev->dev));
135 }
136 
137 static inline bool
138 nouveau_cli_work_ready(struct dma_fence *fence)
139 {
140 	bool ret = true;
141 
142 	spin_lock_irq(fence->lock);
143 	if (!dma_fence_is_signaled_locked(fence))
144 		ret = false;
145 	spin_unlock_irq(fence->lock);
146 
147 	if (ret == true)
148 		dma_fence_put(fence);
149 	return ret;
150 }
151 
152 static void
153 nouveau_cli_work(struct work_struct *w)
154 {
155 	struct nouveau_cli *cli = container_of(w, typeof(*cli), work);
156 	struct nouveau_cli_work *work, *wtmp;
157 	mutex_lock(&cli->lock);
158 	list_for_each_entry_safe(work, wtmp, &cli->worker, head) {
159 		if (!work->fence || nouveau_cli_work_ready(work->fence)) {
160 			list_del(&work->head);
161 			work->func(work);
162 		}
163 	}
164 	mutex_unlock(&cli->lock);
165 }
166 
167 static void
168 nouveau_cli_work_fence(struct dma_fence *fence, struct dma_fence_cb *cb)
169 {
170 	struct nouveau_cli_work *work = container_of(cb, typeof(*work), cb);
171 	schedule_work(&work->cli->work);
172 }
173 
174 void
175 nouveau_cli_work_queue(struct nouveau_cli *cli, struct dma_fence *fence,
176 		       struct nouveau_cli_work *work)
177 {
178 	work->fence = dma_fence_get(fence);
179 	work->cli = cli;
180 	mutex_lock(&cli->lock);
181 	list_add_tail(&work->head, &cli->worker);
182 	if (dma_fence_add_callback(fence, &work->cb, nouveau_cli_work_fence))
183 		nouveau_cli_work_fence(fence, &work->cb);
184 	mutex_unlock(&cli->lock);
185 }
186 
187 static void
188 nouveau_cli_fini(struct nouveau_cli *cli)
189 {
190 	/* All our channels are dead now, which means all the fences they
191 	 * own are signalled, and all callback functions have been called.
192 	 *
193 	 * So, after flushing the workqueue, there should be nothing left.
194 	 */
195 	flush_work(&cli->work);
196 	WARN_ON(!list_empty(&cli->worker));
197 
198 	usif_client_fini(cli);
199 	nouveau_vmm_fini(&cli->svm);
200 	nouveau_vmm_fini(&cli->vmm);
201 	nvif_mmu_dtor(&cli->mmu);
202 	nvif_device_dtor(&cli->device);
203 	mutex_lock(&cli->drm->master.lock);
204 	nvif_client_dtor(&cli->base);
205 	mutex_unlock(&cli->drm->master.lock);
206 }
207 
208 static int
209 nouveau_cli_init(struct nouveau_drm *drm, const char *sname,
210 		 struct nouveau_cli *cli)
211 {
212 	static const struct nvif_mclass
213 	mems[] = {
214 		{ NVIF_CLASS_MEM_GF100, -1 },
215 		{ NVIF_CLASS_MEM_NV50 , -1 },
216 		{ NVIF_CLASS_MEM_NV04 , -1 },
217 		{}
218 	};
219 	static const struct nvif_mclass
220 	mmus[] = {
221 		{ NVIF_CLASS_MMU_GF100, -1 },
222 		{ NVIF_CLASS_MMU_NV50 , -1 },
223 		{ NVIF_CLASS_MMU_NV04 , -1 },
224 		{}
225 	};
226 	static const struct nvif_mclass
227 	vmms[] = {
228 		{ NVIF_CLASS_VMM_GP100, -1 },
229 		{ NVIF_CLASS_VMM_GM200, -1 },
230 		{ NVIF_CLASS_VMM_GF100, -1 },
231 		{ NVIF_CLASS_VMM_NV50 , -1 },
232 		{ NVIF_CLASS_VMM_NV04 , -1 },
233 		{}
234 	};
235 	u64 device = nouveau_name(drm->dev);
236 	int ret;
237 
238 	snprintf(cli->name, sizeof(cli->name), "%s", sname);
239 	cli->drm = drm;
240 	mutex_init(&cli->mutex);
241 	usif_client_init(cli);
242 
243 	INIT_WORK(&cli->work, nouveau_cli_work);
244 	INIT_LIST_HEAD(&cli->worker);
245 	mutex_init(&cli->lock);
246 
247 	if (cli == &drm->master) {
248 		ret = nvif_driver_init(NULL, nouveau_config, nouveau_debug,
249 				       cli->name, device, &cli->base);
250 	} else {
251 		mutex_lock(&drm->master.lock);
252 		ret = nvif_client_ctor(&drm->master.base, cli->name, device,
253 				       &cli->base);
254 		mutex_unlock(&drm->master.lock);
255 	}
256 	if (ret) {
257 		NV_PRINTK(err, cli, "Client allocation failed: %d\n", ret);
258 		goto done;
259 	}
260 
261 	ret = nvif_device_ctor(&cli->base.object, "drmDevice", 0, NV_DEVICE,
262 			       &(struct nv_device_v0) {
263 					.device = ~0,
264 					.priv = true,
265 			       }, sizeof(struct nv_device_v0),
266 			       &cli->device);
267 	if (ret) {
268 		NV_PRINTK(err, cli, "Device allocation failed: %d\n", ret);
269 		goto done;
270 	}
271 
272 	ret = nvif_mclass(&cli->device.object, mmus);
273 	if (ret < 0) {
274 		NV_PRINTK(err, cli, "No supported MMU class\n");
275 		goto done;
276 	}
277 
278 	ret = nvif_mmu_ctor(&cli->device.object, "drmMmu", mmus[ret].oclass,
279 			    &cli->mmu);
280 	if (ret) {
281 		NV_PRINTK(err, cli, "MMU allocation failed: %d\n", ret);
282 		goto done;
283 	}
284 
285 	ret = nvif_mclass(&cli->mmu.object, vmms);
286 	if (ret < 0) {
287 		NV_PRINTK(err, cli, "No supported VMM class\n");
288 		goto done;
289 	}
290 
291 	ret = nouveau_vmm_init(cli, vmms[ret].oclass, &cli->vmm);
292 	if (ret) {
293 		NV_PRINTK(err, cli, "VMM allocation failed: %d\n", ret);
294 		goto done;
295 	}
296 
297 	ret = nvif_mclass(&cli->mmu.object, mems);
298 	if (ret < 0) {
299 		NV_PRINTK(err, cli, "No supported MEM class\n");
300 		goto done;
301 	}
302 
303 	cli->mem = &mems[ret];
304 	return 0;
305 done:
306 	if (ret)
307 		nouveau_cli_fini(cli);
308 	return ret;
309 }
310 
311 static void
312 nouveau_accel_ce_fini(struct nouveau_drm *drm)
313 {
314 	nouveau_channel_idle(drm->cechan);
315 	nvif_object_dtor(&drm->ttm.copy);
316 	nouveau_channel_del(&drm->cechan);
317 }
318 
319 static void
320 nouveau_accel_ce_init(struct nouveau_drm *drm)
321 {
322 	struct nvif_device *device = &drm->client.device;
323 	u64 runm;
324 	int ret = 0;
325 
326 	/* Allocate channel that has access to a (preferably async) copy
327 	 * engine, to use for TTM buffer moves.
328 	 */
329 	runm = nvif_fifo_runlist_ce(device);
330 	if (!runm) {
331 		NV_DEBUG(drm, "no ce runlist\n");
332 		return;
333 	}
334 
335 	ret = nouveau_channel_new(drm, device, false, runm, NvDmaFB, NvDmaTT, &drm->cechan);
336 	if (ret)
337 		NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
338 }
339 
340 static void
341 nouveau_accel_gr_fini(struct nouveau_drm *drm)
342 {
343 	nouveau_channel_idle(drm->channel);
344 	nvif_object_dtor(&drm->ntfy);
345 	nvkm_gpuobj_del(&drm->notify);
346 	nouveau_channel_del(&drm->channel);
347 }
348 
349 static void
350 nouveau_accel_gr_init(struct nouveau_drm *drm)
351 {
352 	struct nvif_device *device = &drm->client.device;
353 	u64 runm;
354 	int ret;
355 
356 	/* Allocate channel that has access to the graphics engine. */
357 	runm = nvif_fifo_runlist(device, NV_DEVICE_HOST_RUNLIST_ENGINES_GR);
358 	if (!runm) {
359 		NV_DEBUG(drm, "no gr runlist\n");
360 		return;
361 	}
362 
363 	ret = nouveau_channel_new(drm, device, false, runm, NvDmaFB, NvDmaTT, &drm->channel);
364 	if (ret) {
365 		NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
366 		nouveau_accel_gr_fini(drm);
367 		return;
368 	}
369 
370 	/* A SW class is used on pre-NV50 HW to assist with handling the
371 	 * synchronisation of page flips, as well as to implement fences
372 	 * on TNT/TNT2 HW that lacks any kind of support in host.
373 	 */
374 	if (!drm->channel->nvsw.client && device->info.family < NV_DEVICE_INFO_V0_TESLA) {
375 		ret = nvif_object_ctor(&drm->channel->user, "drmNvsw",
376 				       NVDRM_NVSW, nouveau_abi16_swclass(drm),
377 				       NULL, 0, &drm->channel->nvsw);
378 		if (ret == 0) {
379 			struct nvif_push *push = drm->channel->chan.push;
380 			ret = PUSH_WAIT(push, 2);
381 			if (ret == 0)
382 				PUSH_NVSQ(push, NV_SW, 0x0000, drm->channel->nvsw.handle);
383 		}
384 
385 		if (ret) {
386 			NV_ERROR(drm, "failed to allocate sw class, %d\n", ret);
387 			nouveau_accel_gr_fini(drm);
388 			return;
389 		}
390 	}
391 
392 	/* NvMemoryToMemoryFormat requires a notifier ctxdma for some reason,
393 	 * even if notification is never requested, so, allocate a ctxdma on
394 	 * any GPU where it's possible we'll end up using M2MF for BO moves.
395 	 */
396 	if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
397 		ret = nvkm_gpuobj_new(nvxx_device(device), 32, 0, false, NULL,
398 				      &drm->notify);
399 		if (ret) {
400 			NV_ERROR(drm, "failed to allocate notifier, %d\n", ret);
401 			nouveau_accel_gr_fini(drm);
402 			return;
403 		}
404 
405 		ret = nvif_object_ctor(&drm->channel->user, "drmM2mfNtfy",
406 				       NvNotify0, NV_DMA_IN_MEMORY,
407 				       &(struct nv_dma_v0) {
408 						.target = NV_DMA_V0_TARGET_VRAM,
409 						.access = NV_DMA_V0_ACCESS_RDWR,
410 						.start = drm->notify->addr,
411 						.limit = drm->notify->addr + 31
412 				       }, sizeof(struct nv_dma_v0),
413 				       &drm->ntfy);
414 		if (ret) {
415 			nouveau_accel_gr_fini(drm);
416 			return;
417 		}
418 	}
419 }
420 
421 static void
422 nouveau_accel_fini(struct nouveau_drm *drm)
423 {
424 	nouveau_accel_ce_fini(drm);
425 	nouveau_accel_gr_fini(drm);
426 	if (drm->fence)
427 		nouveau_fence(drm)->dtor(drm);
428 	nouveau_channels_fini(drm);
429 }
430 
431 static void
432 nouveau_accel_init(struct nouveau_drm *drm)
433 {
434 	struct nvif_device *device = &drm->client.device;
435 	struct nvif_sclass *sclass;
436 	int ret, i, n;
437 
438 	if (nouveau_noaccel)
439 		return;
440 
441 	/* Initialise global support for channels, and synchronisation. */
442 	ret = nouveau_channels_init(drm);
443 	if (ret)
444 		return;
445 
446 	/*XXX: this is crap, but the fence/channel stuff is a little
447 	 *     backwards in some places.  this will be fixed.
448 	 */
449 	ret = n = nvif_object_sclass_get(&device->object, &sclass);
450 	if (ret < 0)
451 		return;
452 
453 	for (ret = -ENOSYS, i = 0; i < n; i++) {
454 		switch (sclass[i].oclass) {
455 		case NV03_CHANNEL_DMA:
456 			ret = nv04_fence_create(drm);
457 			break;
458 		case NV10_CHANNEL_DMA:
459 			ret = nv10_fence_create(drm);
460 			break;
461 		case NV17_CHANNEL_DMA:
462 		case NV40_CHANNEL_DMA:
463 			ret = nv17_fence_create(drm);
464 			break;
465 		case NV50_CHANNEL_GPFIFO:
466 			ret = nv50_fence_create(drm);
467 			break;
468 		case G82_CHANNEL_GPFIFO:
469 			ret = nv84_fence_create(drm);
470 			break;
471 		case FERMI_CHANNEL_GPFIFO:
472 		case KEPLER_CHANNEL_GPFIFO_A:
473 		case KEPLER_CHANNEL_GPFIFO_B:
474 		case MAXWELL_CHANNEL_GPFIFO_A:
475 		case PASCAL_CHANNEL_GPFIFO_A:
476 		case VOLTA_CHANNEL_GPFIFO_A:
477 		case TURING_CHANNEL_GPFIFO_A:
478 		case AMPERE_CHANNEL_GPFIFO_A:
479 		case AMPERE_CHANNEL_GPFIFO_B:
480 			ret = nvc0_fence_create(drm);
481 			break;
482 		default:
483 			break;
484 		}
485 	}
486 
487 	nvif_object_sclass_put(&sclass);
488 	if (ret) {
489 		NV_ERROR(drm, "failed to initialise sync subsystem, %d\n", ret);
490 		nouveau_accel_fini(drm);
491 		return;
492 	}
493 
494 	/* Volta requires access to a doorbell register for kickoff. */
495 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) {
496 		ret = nvif_user_ctor(device, "drmUsermode");
497 		if (ret)
498 			return;
499 	}
500 
501 	/* Allocate channels we need to support various functions. */
502 	nouveau_accel_gr_init(drm);
503 	nouveau_accel_ce_init(drm);
504 
505 	/* Initialise accelerated TTM buffer moves. */
506 	nouveau_bo_move_init(drm);
507 }
508 
509 static void __printf(2, 3)
510 nouveau_drm_errorf(struct nvif_object *object, const char *fmt, ...)
511 {
512 	struct nouveau_drm *drm = container_of(object->parent, typeof(*drm), parent);
513 	struct va_format vaf;
514 	va_list va;
515 
516 	va_start(va, fmt);
517 	vaf.fmt = fmt;
518 	vaf.va = &va;
519 	NV_ERROR(drm, "%pV", &vaf);
520 	va_end(va);
521 }
522 
523 static void __printf(2, 3)
524 nouveau_drm_debugf(struct nvif_object *object, const char *fmt, ...)
525 {
526 	struct nouveau_drm *drm = container_of(object->parent, typeof(*drm), parent);
527 	struct va_format vaf;
528 	va_list va;
529 
530 	va_start(va, fmt);
531 	vaf.fmt = fmt;
532 	vaf.va = &va;
533 	NV_DEBUG(drm, "%pV", &vaf);
534 	va_end(va);
535 }
536 
537 static const struct nvif_parent_func
538 nouveau_parent = {
539 	.debugf = nouveau_drm_debugf,
540 	.errorf = nouveau_drm_errorf,
541 };
542 
543 static int
544 nouveau_drm_device_init(struct drm_device *dev)
545 {
546 	struct nouveau_drm *drm;
547 	int ret;
548 
549 	if (!(drm = kzalloc(sizeof(*drm), GFP_KERNEL)))
550 		return -ENOMEM;
551 	dev->dev_private = drm;
552 	drm->dev = dev;
553 
554 	nvif_parent_ctor(&nouveau_parent, &drm->parent);
555 	drm->master.base.object.parent = &drm->parent;
556 
557 	ret = nouveau_cli_init(drm, "DRM-master", &drm->master);
558 	if (ret)
559 		goto fail_alloc;
560 
561 	ret = nouveau_cli_init(drm, "DRM", &drm->client);
562 	if (ret)
563 		goto fail_master;
564 
565 	nvxx_client(&drm->client.base)->debug =
566 		nvkm_dbgopt(nouveau_debug, "DRM");
567 
568 	INIT_LIST_HEAD(&drm->clients);
569 	mutex_init(&drm->clients_lock);
570 	spin_lock_init(&drm->tile.lock);
571 
572 	/* workaround an odd issue on nvc1 by disabling the device's
573 	 * nosnoop capability.  hopefully won't cause issues until a
574 	 * better fix is found - assuming there is one...
575 	 */
576 	if (drm->client.device.info.chipset == 0xc1)
577 		nvif_mask(&drm->client.device.object, 0x00088080, 0x00000800, 0x00000000);
578 
579 	nouveau_vga_init(drm);
580 
581 	ret = nouveau_ttm_init(drm);
582 	if (ret)
583 		goto fail_ttm;
584 
585 	ret = nouveau_bios_init(dev);
586 	if (ret)
587 		goto fail_bios;
588 
589 	nouveau_accel_init(drm);
590 
591 	ret = nouveau_display_create(dev);
592 	if (ret)
593 		goto fail_dispctor;
594 
595 	if (dev->mode_config.num_crtc) {
596 		ret = nouveau_display_init(dev, false, false);
597 		if (ret)
598 			goto fail_dispinit;
599 	}
600 
601 	nouveau_debugfs_init(drm);
602 	nouveau_hwmon_init(dev);
603 	nouveau_svm_init(drm);
604 	nouveau_dmem_init(drm);
605 	nouveau_led_init(dev);
606 
607 	if (nouveau_pmops_runtime()) {
608 		pm_runtime_use_autosuspend(dev->dev);
609 		pm_runtime_set_autosuspend_delay(dev->dev, 5000);
610 		pm_runtime_set_active(dev->dev);
611 		pm_runtime_allow(dev->dev);
612 		pm_runtime_mark_last_busy(dev->dev);
613 		pm_runtime_put(dev->dev);
614 	}
615 
616 	return 0;
617 
618 fail_dispinit:
619 	nouveau_display_destroy(dev);
620 fail_dispctor:
621 	nouveau_accel_fini(drm);
622 	nouveau_bios_takedown(dev);
623 fail_bios:
624 	nouveau_ttm_fini(drm);
625 fail_ttm:
626 	nouveau_vga_fini(drm);
627 	nouveau_cli_fini(&drm->client);
628 fail_master:
629 	nouveau_cli_fini(&drm->master);
630 fail_alloc:
631 	nvif_parent_dtor(&drm->parent);
632 	kfree(drm);
633 	return ret;
634 }
635 
636 static void
637 nouveau_drm_device_fini(struct drm_device *dev)
638 {
639 	struct nouveau_cli *cli, *temp_cli;
640 	struct nouveau_drm *drm = nouveau_drm(dev);
641 
642 	if (nouveau_pmops_runtime()) {
643 		pm_runtime_get_sync(dev->dev);
644 		pm_runtime_forbid(dev->dev);
645 	}
646 
647 	nouveau_led_fini(dev);
648 	nouveau_dmem_fini(drm);
649 	nouveau_svm_fini(drm);
650 	nouveau_hwmon_fini(dev);
651 	nouveau_debugfs_fini(drm);
652 
653 	if (dev->mode_config.num_crtc)
654 		nouveau_display_fini(dev, false, false);
655 	nouveau_display_destroy(dev);
656 
657 	nouveau_accel_fini(drm);
658 	nouveau_bios_takedown(dev);
659 
660 	nouveau_ttm_fini(drm);
661 	nouveau_vga_fini(drm);
662 
663 	/*
664 	 * There may be existing clients from as-yet unclosed files. For now,
665 	 * clean them up here rather than deferring until the file is closed,
666 	 * but this likely not correct if we want to support hot-unplugging
667 	 * properly.
668 	 */
669 	mutex_lock(&drm->clients_lock);
670 	list_for_each_entry_safe(cli, temp_cli, &drm->clients, head) {
671 		list_del(&cli->head);
672 		mutex_lock(&cli->mutex);
673 		if (cli->abi16)
674 			nouveau_abi16_fini(cli->abi16);
675 		mutex_unlock(&cli->mutex);
676 		nouveau_cli_fini(cli);
677 		kfree(cli);
678 	}
679 	mutex_unlock(&drm->clients_lock);
680 
681 	nouveau_cli_fini(&drm->client);
682 	nouveau_cli_fini(&drm->master);
683 	nvif_parent_dtor(&drm->parent);
684 	mutex_destroy(&drm->clients_lock);
685 	kfree(drm);
686 }
687 
688 /*
689  * On some Intel PCIe bridge controllers doing a
690  * D0 -> D3hot -> D3cold -> D0 sequence causes Nvidia GPUs to not reappear.
691  * Skipping the intermediate D3hot step seems to make it work again. This is
692  * probably caused by not meeting the expectation the involved AML code has
693  * when the GPU is put into D3hot state before invoking it.
694  *
695  * This leads to various manifestations of this issue:
696  *  - AML code execution to power on the GPU hits an infinite loop (as the
697  *    code waits on device memory to change).
698  *  - kernel crashes, as all PCI reads return -1, which most code isn't able
699  *    to handle well enough.
700  *
701  * In all cases dmesg will contain at least one line like this:
702  * 'nouveau 0000:01:00.0: Refused to change power state, currently in D3'
703  * followed by a lot of nouveau timeouts.
704  *
705  * In the \_SB.PCI0.PEG0.PG00._OFF code deeper down writes bit 0x80 to the not
706  * documented PCI config space register 0x248 of the Intel PCIe bridge
707  * controller (0x1901) in order to change the state of the PCIe link between
708  * the PCIe port and the GPU. There are alternative code paths using other
709  * registers, which seem to work fine (executed pre Windows 8):
710  *  - 0xbc bit 0x20 (publicly available documentation claims 'reserved')
711  *  - 0xb0 bit 0x10 (link disable)
712  * Changing the conditions inside the firmware by poking into the relevant
713  * addresses does resolve the issue, but it seemed to be ACPI private memory
714  * and not any device accessible memory at all, so there is no portable way of
715  * changing the conditions.
716  * On a XPS 9560 that means bits [0,3] on \CPEX need to be cleared.
717  *
718  * The only systems where this behavior can be seen are hybrid graphics laptops
719  * with a secondary Nvidia Maxwell, Pascal or Turing GPU. It's unclear whether
720  * this issue only occurs in combination with listed Intel PCIe bridge
721  * controllers and the mentioned GPUs or other devices as well.
722  *
723  * documentation on the PCIe bridge controller can be found in the
724  * "7th Generation Intel® Processor Families for H Platforms Datasheet Volume 2"
725  * Section "12 PCI Express* Controller (x16) Registers"
726  */
727 
728 static void quirk_broken_nv_runpm(struct pci_dev *pdev)
729 {
730 	struct drm_device *dev = pci_get_drvdata(pdev);
731 	struct nouveau_drm *drm = nouveau_drm(dev);
732 	struct pci_dev *bridge = pci_upstream_bridge(pdev);
733 
734 	if (!bridge || bridge->vendor != PCI_VENDOR_ID_INTEL)
735 		return;
736 
737 	switch (bridge->device) {
738 	case 0x1901:
739 		drm->old_pm_cap = pdev->pm_cap;
740 		pdev->pm_cap = 0;
741 		NV_INFO(drm, "Disabling PCI power management to avoid bug\n");
742 		break;
743 	}
744 }
745 
746 static int nouveau_drm_probe(struct pci_dev *pdev,
747 			     const struct pci_device_id *pent)
748 {
749 	struct nvkm_device *device;
750 	struct drm_device *drm_dev;
751 	int ret;
752 
753 	if (vga_switcheroo_client_probe_defer(pdev))
754 		return -EPROBE_DEFER;
755 
756 	/* We need to check that the chipset is supported before booting
757 	 * fbdev off the hardware, as there's no way to put it back.
758 	 */
759 	ret = nvkm_device_pci_new(pdev, nouveau_config, "error",
760 				  true, false, 0, &device);
761 	if (ret)
762 		return ret;
763 
764 	nvkm_device_del(&device);
765 
766 	/* Remove conflicting drivers (vesafb, efifb etc). */
767 	ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &driver_pci);
768 	if (ret)
769 		return ret;
770 
771 	ret = nvkm_device_pci_new(pdev, nouveau_config, nouveau_debug,
772 				  true, true, ~0ULL, &device);
773 	if (ret)
774 		return ret;
775 
776 	pci_set_master(pdev);
777 
778 	if (nouveau_atomic)
779 		driver_pci.driver_features |= DRIVER_ATOMIC;
780 
781 	drm_dev = drm_dev_alloc(&driver_pci, &pdev->dev);
782 	if (IS_ERR(drm_dev)) {
783 		ret = PTR_ERR(drm_dev);
784 		goto fail_nvkm;
785 	}
786 
787 	ret = pci_enable_device(pdev);
788 	if (ret)
789 		goto fail_drm;
790 
791 	pci_set_drvdata(pdev, drm_dev);
792 
793 	ret = nouveau_drm_device_init(drm_dev);
794 	if (ret)
795 		goto fail_pci;
796 
797 	ret = drm_dev_register(drm_dev, pent->driver_data);
798 	if (ret)
799 		goto fail_drm_dev_init;
800 
801 	if (nouveau_drm(drm_dev)->client.device.info.ram_size <= 32 * 1024 * 1024)
802 		drm_fbdev_generic_setup(drm_dev, 8);
803 	else
804 		drm_fbdev_generic_setup(drm_dev, 32);
805 
806 	quirk_broken_nv_runpm(pdev);
807 	return 0;
808 
809 fail_drm_dev_init:
810 	nouveau_drm_device_fini(drm_dev);
811 fail_pci:
812 	pci_disable_device(pdev);
813 fail_drm:
814 	drm_dev_put(drm_dev);
815 fail_nvkm:
816 	nvkm_device_del(&device);
817 	return ret;
818 }
819 
820 void
821 nouveau_drm_device_remove(struct drm_device *dev)
822 {
823 	struct nouveau_drm *drm = nouveau_drm(dev);
824 	struct nvkm_client *client;
825 	struct nvkm_device *device;
826 
827 	drm_dev_unplug(dev);
828 
829 	client = nvxx_client(&drm->client.base);
830 	device = nvkm_device_find(client->device);
831 
832 	nouveau_drm_device_fini(dev);
833 	drm_dev_put(dev);
834 	nvkm_device_del(&device);
835 }
836 
837 static void
838 nouveau_drm_remove(struct pci_dev *pdev)
839 {
840 	struct drm_device *dev = pci_get_drvdata(pdev);
841 	struct nouveau_drm *drm = nouveau_drm(dev);
842 
843 	/* revert our workaround */
844 	if (drm->old_pm_cap)
845 		pdev->pm_cap = drm->old_pm_cap;
846 	nouveau_drm_device_remove(dev);
847 	pci_disable_device(pdev);
848 }
849 
850 static int
851 nouveau_do_suspend(struct drm_device *dev, bool runtime)
852 {
853 	struct nouveau_drm *drm = nouveau_drm(dev);
854 	struct ttm_resource_manager *man;
855 	int ret;
856 
857 	nouveau_svm_suspend(drm);
858 	nouveau_dmem_suspend(drm);
859 	nouveau_led_suspend(dev);
860 
861 	if (dev->mode_config.num_crtc) {
862 		NV_DEBUG(drm, "suspending display...\n");
863 		ret = nouveau_display_suspend(dev, runtime);
864 		if (ret)
865 			return ret;
866 	}
867 
868 	NV_DEBUG(drm, "evicting buffers...\n");
869 
870 	man = ttm_manager_type(&drm->ttm.bdev, TTM_PL_VRAM);
871 	ttm_resource_manager_evict_all(&drm->ttm.bdev, man);
872 
873 	NV_DEBUG(drm, "waiting for kernel channels to go idle...\n");
874 	if (drm->cechan) {
875 		ret = nouveau_channel_idle(drm->cechan);
876 		if (ret)
877 			goto fail_display;
878 	}
879 
880 	if (drm->channel) {
881 		ret = nouveau_channel_idle(drm->channel);
882 		if (ret)
883 			goto fail_display;
884 	}
885 
886 	NV_DEBUG(drm, "suspending fence...\n");
887 	if (drm->fence && nouveau_fence(drm)->suspend) {
888 		if (!nouveau_fence(drm)->suspend(drm)) {
889 			ret = -ENOMEM;
890 			goto fail_display;
891 		}
892 	}
893 
894 	NV_DEBUG(drm, "suspending object tree...\n");
895 	ret = nvif_client_suspend(&drm->master.base);
896 	if (ret)
897 		goto fail_client;
898 
899 	return 0;
900 
901 fail_client:
902 	if (drm->fence && nouveau_fence(drm)->resume)
903 		nouveau_fence(drm)->resume(drm);
904 
905 fail_display:
906 	if (dev->mode_config.num_crtc) {
907 		NV_DEBUG(drm, "resuming display...\n");
908 		nouveau_display_resume(dev, runtime);
909 	}
910 	return ret;
911 }
912 
913 static int
914 nouveau_do_resume(struct drm_device *dev, bool runtime)
915 {
916 	int ret = 0;
917 	struct nouveau_drm *drm = nouveau_drm(dev);
918 
919 	NV_DEBUG(drm, "resuming object tree...\n");
920 	ret = nvif_client_resume(&drm->master.base);
921 	if (ret) {
922 		NV_ERROR(drm, "Client resume failed with error: %d\n", ret);
923 		return ret;
924 	}
925 
926 	NV_DEBUG(drm, "resuming fence...\n");
927 	if (drm->fence && nouveau_fence(drm)->resume)
928 		nouveau_fence(drm)->resume(drm);
929 
930 	nouveau_run_vbios_init(dev);
931 
932 	if (dev->mode_config.num_crtc) {
933 		NV_DEBUG(drm, "resuming display...\n");
934 		nouveau_display_resume(dev, runtime);
935 	}
936 
937 	nouveau_led_resume(dev);
938 	nouveau_dmem_resume(drm);
939 	nouveau_svm_resume(drm);
940 	return 0;
941 }
942 
943 int
944 nouveau_pmops_suspend(struct device *dev)
945 {
946 	struct pci_dev *pdev = to_pci_dev(dev);
947 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
948 	int ret;
949 
950 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF ||
951 	    drm_dev->switch_power_state == DRM_SWITCH_POWER_DYNAMIC_OFF)
952 		return 0;
953 
954 	ret = nouveau_do_suspend(drm_dev, false);
955 	if (ret)
956 		return ret;
957 
958 	pci_save_state(pdev);
959 	pci_disable_device(pdev);
960 	pci_set_power_state(pdev, PCI_D3hot);
961 	udelay(200);
962 	return 0;
963 }
964 
965 int
966 nouveau_pmops_resume(struct device *dev)
967 {
968 	struct pci_dev *pdev = to_pci_dev(dev);
969 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
970 	int ret;
971 
972 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF ||
973 	    drm_dev->switch_power_state == DRM_SWITCH_POWER_DYNAMIC_OFF)
974 		return 0;
975 
976 	pci_set_power_state(pdev, PCI_D0);
977 	pci_restore_state(pdev);
978 	ret = pci_enable_device(pdev);
979 	if (ret)
980 		return ret;
981 	pci_set_master(pdev);
982 
983 	ret = nouveau_do_resume(drm_dev, false);
984 
985 	/* Monitors may have been connected / disconnected during suspend */
986 	nouveau_display_hpd_resume(drm_dev);
987 
988 	return ret;
989 }
990 
991 static int
992 nouveau_pmops_freeze(struct device *dev)
993 {
994 	struct pci_dev *pdev = to_pci_dev(dev);
995 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
996 	return nouveau_do_suspend(drm_dev, false);
997 }
998 
999 static int
1000 nouveau_pmops_thaw(struct device *dev)
1001 {
1002 	struct pci_dev *pdev = to_pci_dev(dev);
1003 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
1004 	return nouveau_do_resume(drm_dev, false);
1005 }
1006 
1007 bool
1008 nouveau_pmops_runtime(void)
1009 {
1010 	if (nouveau_runtime_pm == -1)
1011 		return nouveau_is_optimus() || nouveau_is_v1_dsm();
1012 	return nouveau_runtime_pm == 1;
1013 }
1014 
1015 static int
1016 nouveau_pmops_runtime_suspend(struct device *dev)
1017 {
1018 	struct pci_dev *pdev = to_pci_dev(dev);
1019 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
1020 	int ret;
1021 
1022 	if (!nouveau_pmops_runtime()) {
1023 		pm_runtime_forbid(dev);
1024 		return -EBUSY;
1025 	}
1026 
1027 	nouveau_switcheroo_optimus_dsm();
1028 	ret = nouveau_do_suspend(drm_dev, true);
1029 	pci_save_state(pdev);
1030 	pci_disable_device(pdev);
1031 	pci_ignore_hotplug(pdev);
1032 	pci_set_power_state(pdev, PCI_D3cold);
1033 	drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF;
1034 	return ret;
1035 }
1036 
1037 static int
1038 nouveau_pmops_runtime_resume(struct device *dev)
1039 {
1040 	struct pci_dev *pdev = to_pci_dev(dev);
1041 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
1042 	struct nouveau_drm *drm = nouveau_drm(drm_dev);
1043 	struct nvif_device *device = &nouveau_drm(drm_dev)->client.device;
1044 	int ret;
1045 
1046 	if (!nouveau_pmops_runtime()) {
1047 		pm_runtime_forbid(dev);
1048 		return -EBUSY;
1049 	}
1050 
1051 	pci_set_power_state(pdev, PCI_D0);
1052 	pci_restore_state(pdev);
1053 	ret = pci_enable_device(pdev);
1054 	if (ret)
1055 		return ret;
1056 	pci_set_master(pdev);
1057 
1058 	ret = nouveau_do_resume(drm_dev, true);
1059 	if (ret) {
1060 		NV_ERROR(drm, "resume failed with: %d\n", ret);
1061 		return ret;
1062 	}
1063 
1064 	/* do magic */
1065 	nvif_mask(&device->object, 0x088488, (1 << 25), (1 << 25));
1066 	drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
1067 
1068 	/* Monitors may have been connected / disconnected during suspend */
1069 	nouveau_display_hpd_resume(drm_dev);
1070 
1071 	return ret;
1072 }
1073 
1074 static int
1075 nouveau_pmops_runtime_idle(struct device *dev)
1076 {
1077 	if (!nouveau_pmops_runtime()) {
1078 		pm_runtime_forbid(dev);
1079 		return -EBUSY;
1080 	}
1081 
1082 	pm_runtime_mark_last_busy(dev);
1083 	pm_runtime_autosuspend(dev);
1084 	/* we don't want the main rpm_idle to call suspend - we want to autosuspend */
1085 	return 1;
1086 }
1087 
1088 static int
1089 nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
1090 {
1091 	struct nouveau_drm *drm = nouveau_drm(dev);
1092 	struct nouveau_cli *cli;
1093 	char name[32], tmpname[TASK_COMM_LEN];
1094 	int ret;
1095 
1096 	/* need to bring up power immediately if opening device */
1097 	ret = pm_runtime_get_sync(dev->dev);
1098 	if (ret < 0 && ret != -EACCES) {
1099 		pm_runtime_put_autosuspend(dev->dev);
1100 		return ret;
1101 	}
1102 
1103 	get_task_comm(tmpname, current);
1104 	snprintf(name, sizeof(name), "%s[%d]", tmpname, pid_nr(fpriv->pid));
1105 
1106 	if (!(cli = kzalloc(sizeof(*cli), GFP_KERNEL))) {
1107 		ret = -ENOMEM;
1108 		goto done;
1109 	}
1110 
1111 	ret = nouveau_cli_init(drm, name, cli);
1112 	if (ret)
1113 		goto done;
1114 
1115 	fpriv->driver_priv = cli;
1116 
1117 	mutex_lock(&drm->clients_lock);
1118 	list_add(&cli->head, &drm->clients);
1119 	mutex_unlock(&drm->clients_lock);
1120 
1121 done:
1122 	if (ret && cli) {
1123 		nouveau_cli_fini(cli);
1124 		kfree(cli);
1125 	}
1126 
1127 	pm_runtime_mark_last_busy(dev->dev);
1128 	pm_runtime_put_autosuspend(dev->dev);
1129 	return ret;
1130 }
1131 
1132 static void
1133 nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv)
1134 {
1135 	struct nouveau_cli *cli = nouveau_cli(fpriv);
1136 	struct nouveau_drm *drm = nouveau_drm(dev);
1137 	int dev_index;
1138 
1139 	/*
1140 	 * The device is gone, and as it currently stands all clients are
1141 	 * cleaned up in the removal codepath. In the future this may change
1142 	 * so that we can support hot-unplugging, but for now we immediately
1143 	 * return to avoid a double-free situation.
1144 	 */
1145 	if (!drm_dev_enter(dev, &dev_index))
1146 		return;
1147 
1148 	pm_runtime_get_sync(dev->dev);
1149 
1150 	mutex_lock(&cli->mutex);
1151 	if (cli->abi16)
1152 		nouveau_abi16_fini(cli->abi16);
1153 	mutex_unlock(&cli->mutex);
1154 
1155 	mutex_lock(&drm->clients_lock);
1156 	list_del(&cli->head);
1157 	mutex_unlock(&drm->clients_lock);
1158 
1159 	nouveau_cli_fini(cli);
1160 	kfree(cli);
1161 	pm_runtime_mark_last_busy(dev->dev);
1162 	pm_runtime_put_autosuspend(dev->dev);
1163 	drm_dev_exit(dev_index);
1164 }
1165 
1166 static const struct drm_ioctl_desc
1167 nouveau_ioctls[] = {
1168 	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_RENDER_ALLOW),
1169 	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1170 	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_RENDER_ALLOW),
1171 	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_RENDER_ALLOW),
1172 	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_RENDER_ALLOW),
1173 	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_RENDER_ALLOW),
1174 	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_RENDER_ALLOW),
1175 	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_INIT, nouveau_svmm_init, DRM_RENDER_ALLOW),
1176 	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_BIND, nouveau_svmm_bind, DRM_RENDER_ALLOW),
1177 	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_RENDER_ALLOW),
1178 	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_RENDER_ALLOW),
1179 	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_RENDER_ALLOW),
1180 	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_RENDER_ALLOW),
1181 	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_RENDER_ALLOW),
1182 };
1183 
1184 long
1185 nouveau_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1186 {
1187 	struct drm_file *filp = file->private_data;
1188 	struct drm_device *dev = filp->minor->dev;
1189 	long ret;
1190 
1191 	ret = pm_runtime_get_sync(dev->dev);
1192 	if (ret < 0 && ret != -EACCES) {
1193 		pm_runtime_put_autosuspend(dev->dev);
1194 		return ret;
1195 	}
1196 
1197 	switch (_IOC_NR(cmd) - DRM_COMMAND_BASE) {
1198 	case DRM_NOUVEAU_NVIF:
1199 		ret = usif_ioctl(filp, (void __user *)arg, _IOC_SIZE(cmd));
1200 		break;
1201 	default:
1202 		ret = drm_ioctl(file, cmd, arg);
1203 		break;
1204 	}
1205 
1206 	pm_runtime_mark_last_busy(dev->dev);
1207 	pm_runtime_put_autosuspend(dev->dev);
1208 	return ret;
1209 }
1210 
1211 static const struct file_operations
1212 nouveau_driver_fops = {
1213 	.owner = THIS_MODULE,
1214 	.open = drm_open,
1215 	.release = drm_release,
1216 	.unlocked_ioctl = nouveau_drm_ioctl,
1217 	.mmap = drm_gem_mmap,
1218 	.poll = drm_poll,
1219 	.read = drm_read,
1220 #if defined(CONFIG_COMPAT)
1221 	.compat_ioctl = nouveau_compat_ioctl,
1222 #endif
1223 	.llseek = noop_llseek,
1224 };
1225 
1226 static struct drm_driver
1227 driver_stub = {
1228 	.driver_features = DRIVER_GEM |
1229 			   DRIVER_MODESET |
1230 			   DRIVER_RENDER,
1231 	.open = nouveau_drm_open,
1232 	.postclose = nouveau_drm_postclose,
1233 	.lastclose = nouveau_vga_lastclose,
1234 
1235 #if defined(CONFIG_DEBUG_FS)
1236 	.debugfs_init = nouveau_drm_debugfs_init,
1237 #endif
1238 
1239 	.ioctls = nouveau_ioctls,
1240 	.num_ioctls = ARRAY_SIZE(nouveau_ioctls),
1241 	.fops = &nouveau_driver_fops,
1242 
1243 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
1244 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
1245 	.gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table,
1246 	.gem_prime_mmap = drm_gem_prime_mmap,
1247 
1248 	.dumb_create = nouveau_display_dumb_create,
1249 	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
1250 
1251 	.name = DRIVER_NAME,
1252 	.desc = DRIVER_DESC,
1253 #ifdef GIT_REVISION
1254 	.date = GIT_REVISION,
1255 #else
1256 	.date = DRIVER_DATE,
1257 #endif
1258 	.major = DRIVER_MAJOR,
1259 	.minor = DRIVER_MINOR,
1260 	.patchlevel = DRIVER_PATCHLEVEL,
1261 };
1262 
1263 static struct pci_device_id
1264 nouveau_drm_pci_table[] = {
1265 	{
1266 		PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID),
1267 		.class = PCI_BASE_CLASS_DISPLAY << 16,
1268 		.class_mask  = 0xff << 16,
1269 	},
1270 	{
1271 		PCI_DEVICE(PCI_VENDOR_ID_NVIDIA_SGS, PCI_ANY_ID),
1272 		.class = PCI_BASE_CLASS_DISPLAY << 16,
1273 		.class_mask  = 0xff << 16,
1274 	},
1275 	{}
1276 };
1277 
1278 static void nouveau_display_options(void)
1279 {
1280 	DRM_DEBUG_DRIVER("Loading Nouveau with parameters:\n");
1281 
1282 	DRM_DEBUG_DRIVER("... tv_disable   : %d\n", nouveau_tv_disable);
1283 	DRM_DEBUG_DRIVER("... ignorelid    : %d\n", nouveau_ignorelid);
1284 	DRM_DEBUG_DRIVER("... duallink     : %d\n", nouveau_duallink);
1285 	DRM_DEBUG_DRIVER("... config       : %s\n", nouveau_config);
1286 	DRM_DEBUG_DRIVER("... debug        : %s\n", nouveau_debug);
1287 	DRM_DEBUG_DRIVER("... noaccel      : %d\n", nouveau_noaccel);
1288 	DRM_DEBUG_DRIVER("... modeset      : %d\n", nouveau_modeset);
1289 	DRM_DEBUG_DRIVER("... runpm        : %d\n", nouveau_runtime_pm);
1290 	DRM_DEBUG_DRIVER("... vram_pushbuf : %d\n", nouveau_vram_pushbuf);
1291 	DRM_DEBUG_DRIVER("... hdmimhz      : %d\n", nouveau_hdmimhz);
1292 }
1293 
1294 static const struct dev_pm_ops nouveau_pm_ops = {
1295 	.suspend = nouveau_pmops_suspend,
1296 	.resume = nouveau_pmops_resume,
1297 	.freeze = nouveau_pmops_freeze,
1298 	.thaw = nouveau_pmops_thaw,
1299 	.poweroff = nouveau_pmops_freeze,
1300 	.restore = nouveau_pmops_resume,
1301 	.runtime_suspend = nouveau_pmops_runtime_suspend,
1302 	.runtime_resume = nouveau_pmops_runtime_resume,
1303 	.runtime_idle = nouveau_pmops_runtime_idle,
1304 };
1305 
1306 static struct pci_driver
1307 nouveau_drm_pci_driver = {
1308 	.name = "nouveau",
1309 	.id_table = nouveau_drm_pci_table,
1310 	.probe = nouveau_drm_probe,
1311 	.remove = nouveau_drm_remove,
1312 	.driver.pm = &nouveau_pm_ops,
1313 };
1314 
1315 struct drm_device *
1316 nouveau_platform_device_create(const struct nvkm_device_tegra_func *func,
1317 			       struct platform_device *pdev,
1318 			       struct nvkm_device **pdevice)
1319 {
1320 	struct drm_device *drm;
1321 	int err;
1322 
1323 	err = nvkm_device_tegra_new(func, pdev, nouveau_config, nouveau_debug,
1324 				    true, true, ~0ULL, pdevice);
1325 	if (err)
1326 		goto err_free;
1327 
1328 	drm = drm_dev_alloc(&driver_platform, &pdev->dev);
1329 	if (IS_ERR(drm)) {
1330 		err = PTR_ERR(drm);
1331 		goto err_free;
1332 	}
1333 
1334 	err = nouveau_drm_device_init(drm);
1335 	if (err)
1336 		goto err_put;
1337 
1338 	platform_set_drvdata(pdev, drm);
1339 
1340 	return drm;
1341 
1342 err_put:
1343 	drm_dev_put(drm);
1344 err_free:
1345 	nvkm_device_del(pdevice);
1346 
1347 	return ERR_PTR(err);
1348 }
1349 
1350 static int __init
1351 nouveau_drm_init(void)
1352 {
1353 	driver_pci = driver_stub;
1354 	driver_platform = driver_stub;
1355 
1356 	nouveau_display_options();
1357 
1358 	if (nouveau_modeset == -1) {
1359 		if (drm_firmware_drivers_only())
1360 			nouveau_modeset = 0;
1361 	}
1362 
1363 	if (!nouveau_modeset)
1364 		return 0;
1365 
1366 #ifdef CONFIG_NOUVEAU_PLATFORM_DRIVER
1367 	platform_driver_register(&nouveau_platform_driver);
1368 #endif
1369 
1370 	nouveau_register_dsm_handler();
1371 	nouveau_backlight_ctor();
1372 
1373 #ifdef CONFIG_PCI
1374 	return pci_register_driver(&nouveau_drm_pci_driver);
1375 #else
1376 	return 0;
1377 #endif
1378 }
1379 
1380 static void __exit
1381 nouveau_drm_exit(void)
1382 {
1383 	if (!nouveau_modeset)
1384 		return;
1385 
1386 #ifdef CONFIG_PCI
1387 	pci_unregister_driver(&nouveau_drm_pci_driver);
1388 #endif
1389 	nouveau_backlight_dtor();
1390 	nouveau_unregister_dsm_handler();
1391 
1392 #ifdef CONFIG_NOUVEAU_PLATFORM_DRIVER
1393 	platform_driver_unregister(&nouveau_platform_driver);
1394 #endif
1395 	if (IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM))
1396 		mmu_notifier_synchronize();
1397 }
1398 
1399 module_init(nouveau_drm_init);
1400 module_exit(nouveau_drm_exit);
1401 
1402 MODULE_DEVICE_TABLE(pci, nouveau_drm_pci_table);
1403 MODULE_AUTHOR(DRIVER_AUTHOR);
1404 MODULE_DESCRIPTION(DRIVER_DESC);
1405 MODULE_LICENSE("GPL and additional rights");
1406