1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/irq.h>
29 #include <drm/drmP.h>
30 #include <drm/drm_crtc_helper.h>
31 #include <drm/amdgpu_drm.h>
32 #include "amdgpu.h"
33 #include "amdgpu_ih.h"
34 #include "atom.h"
35 #include "amdgpu_connectors.h"
36 #include "amdgpu_trace.h"
37 
38 #include <linux/pm_runtime.h>
39 
40 #ifdef CONFIG_DRM_AMD_DC
41 #include "amdgpu_dm_irq.h"
42 #endif
43 
44 #define AMDGPU_WAIT_IDLE_TIMEOUT 200
45 
46 /*
47  * Handle hotplug events outside the interrupt handler proper.
48  */
49 /**
50  * amdgpu_hotplug_work_func - display hotplug work handler
51  *
52  * @work: work struct
53  *
54  * This is the hot plug event work handler (all asics).
55  * The work gets scheduled from the irq handler if there
56  * was a hot plug interrupt.  It walks the connector table
57  * and calls the hotplug handler for each one, then sends
58  * a drm hotplug event to alert userspace.
59  */
60 static void amdgpu_hotplug_work_func(struct work_struct *work)
61 {
62 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
63 						  hotplug_work);
64 	struct drm_device *dev = adev->ddev;
65 	struct drm_mode_config *mode_config = &dev->mode_config;
66 	struct drm_connector *connector;
67 
68 	mutex_lock(&mode_config->mutex);
69 	list_for_each_entry(connector, &mode_config->connector_list, head)
70 		amdgpu_connector_hotplug(connector);
71 	mutex_unlock(&mode_config->mutex);
72 	/* Just fire off a uevent and let userspace tell us what to do */
73 	drm_helper_hpd_irq_event(dev);
74 }
75 
76 /**
77  * amdgpu_irq_reset_work_func - execute gpu reset
78  *
79  * @work: work struct
80  *
81  * Execute scheduled gpu reset (cayman+).
82  * This function is called when the irq handler
83  * thinks we need a gpu reset.
84  */
85 static void amdgpu_irq_reset_work_func(struct work_struct *work)
86 {
87 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
88 						  reset_work);
89 
90 	if (!amdgpu_sriov_vf(adev))
91 		amdgpu_gpu_reset(adev);
92 }
93 
94 /* Disable *all* interrupts */
95 static void amdgpu_irq_disable_all(struct amdgpu_device *adev)
96 {
97 	unsigned long irqflags;
98 	unsigned i, j, k;
99 	int r;
100 
101 	spin_lock_irqsave(&adev->irq.lock, irqflags);
102 	for (i = 0; i < AMDGPU_IH_CLIENTID_MAX; ++i) {
103 		if (!adev->irq.client[i].sources)
104 			continue;
105 
106 		for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
107 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
108 
109 			if (!src || !src->funcs->set || !src->num_types)
110 				continue;
111 
112 			for (k = 0; k < src->num_types; ++k) {
113 				atomic_set(&src->enabled_types[k], 0);
114 				r = src->funcs->set(adev, src, k,
115 						    AMDGPU_IRQ_STATE_DISABLE);
116 				if (r)
117 					DRM_ERROR("error disabling interrupt (%d)\n",
118 						  r);
119 			}
120 		}
121 	}
122 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);
123 }
124 
125 /**
126  * amdgpu_irq_preinstall - drm irq preinstall callback
127  *
128  * @dev: drm dev pointer
129  *
130  * Gets the hw ready to enable irqs (all asics).
131  * This function disables all interrupt sources on the GPU.
132  */
133 void amdgpu_irq_preinstall(struct drm_device *dev)
134 {
135 	struct amdgpu_device *adev = dev->dev_private;
136 
137 	/* Disable *all* interrupts */
138 	amdgpu_irq_disable_all(adev);
139 	/* Clear bits */
140 	amdgpu_ih_process(adev);
141 }
142 
143 /**
144  * amdgpu_irq_postinstall - drm irq preinstall callback
145  *
146  * @dev: drm dev pointer
147  *
148  * Handles stuff to be done after enabling irqs (all asics).
149  * Returns 0 on success.
150  */
151 int amdgpu_irq_postinstall(struct drm_device *dev)
152 {
153 	dev->max_vblank_count = 0x00ffffff;
154 	return 0;
155 }
156 
157 /**
158  * amdgpu_irq_uninstall - drm irq uninstall callback
159  *
160  * @dev: drm dev pointer
161  *
162  * This function disables all interrupt sources on the GPU (all asics).
163  */
164 void amdgpu_irq_uninstall(struct drm_device *dev)
165 {
166 	struct amdgpu_device *adev = dev->dev_private;
167 
168 	if (adev == NULL) {
169 		return;
170 	}
171 	amdgpu_irq_disable_all(adev);
172 }
173 
174 /**
175  * amdgpu_irq_handler - irq handler
176  *
177  * @int irq, void *arg: args
178  *
179  * This is the irq handler for the amdgpu driver (all asics).
180  */
181 irqreturn_t amdgpu_irq_handler(int irq, void *arg)
182 {
183 	struct drm_device *dev = (struct drm_device *) arg;
184 	struct amdgpu_device *adev = dev->dev_private;
185 	irqreturn_t ret;
186 
187 	ret = amdgpu_ih_process(adev);
188 	if (ret == IRQ_HANDLED)
189 		pm_runtime_mark_last_busy(dev->dev);
190 	return ret;
191 }
192 
193 /**
194  * amdgpu_msi_ok - asic specific msi checks
195  *
196  * @adev: amdgpu device pointer
197  *
198  * Handles asic specific MSI checks to determine if
199  * MSIs should be enabled on a particular chip (all asics).
200  * Returns true if MSIs should be enabled, false if MSIs
201  * should not be enabled.
202  */
203 static bool amdgpu_msi_ok(struct amdgpu_device *adev)
204 {
205 	/* force MSI on */
206 	if (amdgpu_msi == 1)
207 		return true;
208 	else if (amdgpu_msi == 0)
209 		return false;
210 
211 	return true;
212 }
213 
214 /**
215  * amdgpu_irq_init - init driver interrupt info
216  *
217  * @adev: amdgpu device pointer
218  *
219  * Sets up the work irq handlers, vblank init, MSIs, etc. (all asics).
220  * Returns 0 for success, error for failure.
221  */
222 int amdgpu_irq_init(struct amdgpu_device *adev)
223 {
224 	int r = 0;
225 
226 	spin_lock_init(&adev->irq.lock);
227 
228 	/* enable msi */
229 	adev->irq.msi_enabled = false;
230 
231 	if (amdgpu_msi_ok(adev)) {
232 		int ret = pci_enable_msi(adev->pdev);
233 		if (!ret) {
234 			adev->irq.msi_enabled = true;
235 			dev_info(adev->dev, "amdgpu: using MSI.\n");
236 		}
237 	}
238 
239 	if (!amdgpu_device_has_dc_support(adev)) {
240 		if (!adev->enable_virtual_display)
241 			/* Disable vblank irqs aggressively for power-saving */
242 			/* XXX: can this be enabled for DC? */
243 			adev->ddev->vblank_disable_immediate = true;
244 
245 		r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc);
246 		if (r)
247 			return r;
248 
249 		/* pre DCE11 */
250 		INIT_WORK(&adev->hotplug_work,
251 				amdgpu_hotplug_work_func);
252 	}
253 
254 	INIT_WORK(&adev->reset_work, amdgpu_irq_reset_work_func);
255 
256 	adev->irq.installed = true;
257 	r = drm_irq_install(adev->ddev, adev->ddev->pdev->irq);
258 	if (r) {
259 		adev->irq.installed = false;
260 		flush_work(&adev->hotplug_work);
261 		cancel_work_sync(&adev->reset_work);
262 		return r;
263 	}
264 
265 	DRM_INFO("amdgpu: irq initialized.\n");
266 	return 0;
267 }
268 
269 /**
270  * amdgpu_irq_fini - tear down driver interrupt info
271  *
272  * @adev: amdgpu device pointer
273  *
274  * Tears down the work irq handlers, vblank handlers, MSIs, etc. (all asics).
275  */
276 void amdgpu_irq_fini(struct amdgpu_device *adev)
277 {
278 	unsigned i, j;
279 
280 	if (adev->irq.installed) {
281 		drm_irq_uninstall(adev->ddev);
282 		adev->irq.installed = false;
283 		if (adev->irq.msi_enabled)
284 			pci_disable_msi(adev->pdev);
285 		flush_work(&adev->hotplug_work);
286 		cancel_work_sync(&adev->reset_work);
287 	}
288 
289 	for (i = 0; i < AMDGPU_IH_CLIENTID_MAX; ++i) {
290 		if (!adev->irq.client[i].sources)
291 			continue;
292 
293 		for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
294 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
295 
296 			if (!src)
297 				continue;
298 
299 			kfree(src->enabled_types);
300 			src->enabled_types = NULL;
301 			if (src->data) {
302 				kfree(src->data);
303 				kfree(src);
304 				adev->irq.client[i].sources[j] = NULL;
305 			}
306 		}
307 		kfree(adev->irq.client[i].sources);
308 	}
309 }
310 
311 /**
312  * amdgpu_irq_add_id - register irq source
313  *
314  * @adev: amdgpu device pointer
315  * @src_id: source id for this source
316  * @source: irq source
317  *
318  */
319 int amdgpu_irq_add_id(struct amdgpu_device *adev,
320 		      unsigned client_id, unsigned src_id,
321 		      struct amdgpu_irq_src *source)
322 {
323 	if (client_id >= AMDGPU_IH_CLIENTID_MAX)
324 		return -EINVAL;
325 
326 	if (src_id >= AMDGPU_MAX_IRQ_SRC_ID)
327 		return -EINVAL;
328 
329 	if (!source->funcs)
330 		return -EINVAL;
331 
332 	if (!adev->irq.client[client_id].sources) {
333 		adev->irq.client[client_id].sources =
334 			kcalloc(AMDGPU_MAX_IRQ_SRC_ID,
335 				sizeof(struct amdgpu_irq_src *),
336 				GFP_KERNEL);
337 		if (!adev->irq.client[client_id].sources)
338 			return -ENOMEM;
339 	}
340 
341 	if (adev->irq.client[client_id].sources[src_id] != NULL)
342 		return -EINVAL;
343 
344 	if (source->num_types && !source->enabled_types) {
345 		atomic_t *types;
346 
347 		types = kcalloc(source->num_types, sizeof(atomic_t),
348 				GFP_KERNEL);
349 		if (!types)
350 			return -ENOMEM;
351 
352 		source->enabled_types = types;
353 	}
354 
355 	adev->irq.client[client_id].sources[src_id] = source;
356 	return 0;
357 }
358 
359 /**
360  * amdgpu_irq_dispatch - dispatch irq to IP blocks
361  *
362  * @adev: amdgpu device pointer
363  * @entry: interrupt vector
364  *
365  * Dispatches the irq to the different IP blocks
366  */
367 void amdgpu_irq_dispatch(struct amdgpu_device *adev,
368 			 struct amdgpu_iv_entry *entry)
369 {
370 	unsigned client_id = entry->client_id;
371 	unsigned src_id = entry->src_id;
372 	struct amdgpu_irq_src *src;
373 	int r;
374 
375 	trace_amdgpu_iv(entry);
376 
377 	if (client_id >= AMDGPU_IH_CLIENTID_MAX) {
378 		DRM_DEBUG("Invalid client_id in IV: %d\n", client_id);
379 		return;
380 	}
381 
382 	if (src_id >= AMDGPU_MAX_IRQ_SRC_ID) {
383 		DRM_DEBUG("Invalid src_id in IV: %d\n", src_id);
384 		return;
385 	}
386 
387 	if (adev->irq.virq[src_id]) {
388 		generic_handle_irq(irq_find_mapping(adev->irq.domain, src_id));
389 	} else {
390 		if (!adev->irq.client[client_id].sources) {
391 			DRM_DEBUG("Unregistered interrupt client_id: %d src_id: %d\n",
392 				  client_id, src_id);
393 			return;
394 		}
395 
396 		src = adev->irq.client[client_id].sources[src_id];
397 		if (!src) {
398 			DRM_DEBUG("Unhandled interrupt src_id: %d\n", src_id);
399 			return;
400 		}
401 
402 		r = src->funcs->process(adev, src, entry);
403 		if (r)
404 			DRM_ERROR("error processing interrupt (%d)\n", r);
405 	}
406 }
407 
408 /**
409  * amdgpu_irq_update - update hw interrupt state
410  *
411  * @adev: amdgpu device pointer
412  * @src: interrupt src you want to enable
413  * @type: type of interrupt you want to update
414  *
415  * Updates the interrupt state for a specific src (all asics).
416  */
417 int amdgpu_irq_update(struct amdgpu_device *adev,
418 			     struct amdgpu_irq_src *src, unsigned type)
419 {
420 	unsigned long irqflags;
421 	enum amdgpu_interrupt_state state;
422 	int r;
423 
424 	spin_lock_irqsave(&adev->irq.lock, irqflags);
425 
426 	/* we need to determine after taking the lock, otherwise
427 	   we might disable just enabled interrupts again */
428 	if (amdgpu_irq_enabled(adev, src, type))
429 		state = AMDGPU_IRQ_STATE_ENABLE;
430 	else
431 		state = AMDGPU_IRQ_STATE_DISABLE;
432 
433 	r = src->funcs->set(adev, src, type, state);
434 	spin_unlock_irqrestore(&adev->irq.lock, irqflags);
435 	return r;
436 }
437 
438 void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev)
439 {
440 	int i, j, k;
441 
442 	for (i = 0; i < AMDGPU_IH_CLIENTID_MAX; ++i) {
443 		if (!adev->irq.client[i].sources)
444 			continue;
445 
446 		for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
447 			struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
448 
449 			if (!src)
450 				continue;
451 			for (k = 0; k < src->num_types; k++)
452 				amdgpu_irq_update(adev, src, k);
453 		}
454 	}
455 }
456 
457 /**
458  * amdgpu_irq_get - enable interrupt
459  *
460  * @adev: amdgpu device pointer
461  * @src: interrupt src you want to enable
462  * @type: type of interrupt you want to enable
463  *
464  * Enables the interrupt type for a specific src (all asics).
465  */
466 int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
467 		   unsigned type)
468 {
469 	if (!adev->ddev->irq_enabled)
470 		return -ENOENT;
471 
472 	if (type >= src->num_types)
473 		return -EINVAL;
474 
475 	if (!src->enabled_types || !src->funcs->set)
476 		return -EINVAL;
477 
478 	if (atomic_inc_return(&src->enabled_types[type]) == 1)
479 		return amdgpu_irq_update(adev, src, type);
480 
481 	return 0;
482 }
483 
484 /**
485  * amdgpu_irq_put - disable interrupt
486  *
487  * @adev: amdgpu device pointer
488  * @src: interrupt src you want to disable
489  * @type: type of interrupt you want to disable
490  *
491  * Disables the interrupt type for a specific src (all asics).
492  */
493 int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
494 		   unsigned type)
495 {
496 	if (!adev->ddev->irq_enabled)
497 		return -ENOENT;
498 
499 	if (type >= src->num_types)
500 		return -EINVAL;
501 
502 	if (!src->enabled_types || !src->funcs->set)
503 		return -EINVAL;
504 
505 	if (atomic_dec_and_test(&src->enabled_types[type]))
506 		return amdgpu_irq_update(adev, src, type);
507 
508 	return 0;
509 }
510 
511 /**
512  * amdgpu_irq_enabled - test if irq is enabled or not
513  *
514  * @adev: amdgpu device pointer
515  * @idx: interrupt src you want to test
516  *
517  * Tests if the given interrupt source is enabled or not
518  */
519 bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
520 			unsigned type)
521 {
522 	if (!adev->ddev->irq_enabled)
523 		return false;
524 
525 	if (type >= src->num_types)
526 		return false;
527 
528 	if (!src->enabled_types || !src->funcs->set)
529 		return false;
530 
531 	return !!atomic_read(&src->enabled_types[type]);
532 }
533 
534 /* gen irq */
535 static void amdgpu_irq_mask(struct irq_data *irqd)
536 {
537 	/* XXX */
538 }
539 
540 static void amdgpu_irq_unmask(struct irq_data *irqd)
541 {
542 	/* XXX */
543 }
544 
545 static struct irq_chip amdgpu_irq_chip = {
546 	.name = "amdgpu-ih",
547 	.irq_mask = amdgpu_irq_mask,
548 	.irq_unmask = amdgpu_irq_unmask,
549 };
550 
551 static int amdgpu_irqdomain_map(struct irq_domain *d,
552 				unsigned int irq, irq_hw_number_t hwirq)
553 {
554 	if (hwirq >= AMDGPU_MAX_IRQ_SRC_ID)
555 		return -EPERM;
556 
557 	irq_set_chip_and_handler(irq,
558 				 &amdgpu_irq_chip, handle_simple_irq);
559 	return 0;
560 }
561 
562 static const struct irq_domain_ops amdgpu_hw_irqdomain_ops = {
563 	.map = amdgpu_irqdomain_map,
564 };
565 
566 /**
567  * amdgpu_irq_add_domain - create a linear irq domain
568  *
569  * @adev: amdgpu device pointer
570  *
571  * Create an irq domain for GPU interrupt sources
572  * that may be driven by another driver (e.g., ACP).
573  */
574 int amdgpu_irq_add_domain(struct amdgpu_device *adev)
575 {
576 	adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID,
577 						 &amdgpu_hw_irqdomain_ops, adev);
578 	if (!adev->irq.domain) {
579 		DRM_ERROR("GPU irq add domain failed\n");
580 		return -ENODEV;
581 	}
582 
583 	return 0;
584 }
585 
586 /**
587  * amdgpu_irq_remove_domain - remove the irq domain
588  *
589  * @adev: amdgpu device pointer
590  *
591  * Remove the irq domain for GPU interrupt sources
592  * that may be driven by another driver (e.g., ACP).
593  */
594 void amdgpu_irq_remove_domain(struct amdgpu_device *adev)
595 {
596 	if (adev->irq.domain) {
597 		irq_domain_remove(adev->irq.domain);
598 		adev->irq.domain = NULL;
599 	}
600 }
601 
602 /**
603  * amdgpu_irq_create_mapping - create a mapping between a domain irq and a
604  *                             Linux irq
605  *
606  * @adev: amdgpu device pointer
607  * @src_id: IH source id
608  *
609  * Create a mapping between a domain irq (GPU IH src id) and a Linux irq
610  * Use this for components that generate a GPU interrupt, but are driven
611  * by a different driver (e.g., ACP).
612  * Returns the Linux irq.
613  */
614 unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id)
615 {
616 	adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id);
617 
618 	return adev->irq.virq[src_id];
619 }
620