Loading...
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/irq.h>
29#include <drm/drmP.h>
30#include <drm/drm_crtc_helper.h>
31#include <drm/amdgpu_drm.h>
32#include "amdgpu.h"
33#include "amdgpu_ih.h"
34#include "atom.h"
35#include "amdgpu_connectors.h"
36#include "amdgpu_trace.h"
37
38#include <linux/pm_runtime.h>
39
40#ifdef CONFIG_DRM_AMD_DC
41#include "amdgpu_dm_irq.h"
42#endif
43
44#define AMDGPU_WAIT_IDLE_TIMEOUT 200
45
46/*
47 * Handle hotplug events outside the interrupt handler proper.
48 */
49/**
50 * amdgpu_hotplug_work_func - display hotplug work handler
51 *
52 * @work: work struct
53 *
54 * This is the hot plug event work handler (all asics).
55 * The work gets scheduled from the irq handler if there
56 * was a hot plug interrupt. It walks the connector table
57 * and calls the hotplug handler for each one, then sends
58 * a drm hotplug event to alert userspace.
59 */
60static void amdgpu_hotplug_work_func(struct work_struct *work)
61{
62 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
63 hotplug_work);
64 struct drm_device *dev = adev->ddev;
65 struct drm_mode_config *mode_config = &dev->mode_config;
66 struct drm_connector *connector;
67
68 mutex_lock(&mode_config->mutex);
69 list_for_each_entry(connector, &mode_config->connector_list, head)
70 amdgpu_connector_hotplug(connector);
71 mutex_unlock(&mode_config->mutex);
72 /* Just fire off a uevent and let userspace tell us what to do */
73 drm_helper_hpd_irq_event(dev);
74}
75
76/**
77 * amdgpu_irq_reset_work_func - execute gpu reset
78 *
79 * @work: work struct
80 *
81 * Execute scheduled gpu reset (cayman+).
82 * This function is called when the irq handler
83 * thinks we need a gpu reset.
84 */
85static void amdgpu_irq_reset_work_func(struct work_struct *work)
86{
87 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
88 reset_work);
89
90 if (!amdgpu_sriov_vf(adev))
91 amdgpu_device_gpu_recover(adev, NULL, false);
92}
93
94/* Disable *all* interrupts */
95void amdgpu_irq_disable_all(struct amdgpu_device *adev)
96{
97 unsigned long irqflags;
98 unsigned i, j, k;
99 int r;
100
101 spin_lock_irqsave(&adev->irq.lock, irqflags);
102 for (i = 0; i < AMDGPU_IH_CLIENTID_MAX; ++i) {
103 if (!adev->irq.client[i].sources)
104 continue;
105
106 for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
107 struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
108
109 if (!src || !src->funcs->set || !src->num_types)
110 continue;
111
112 for (k = 0; k < src->num_types; ++k) {
113 atomic_set(&src->enabled_types[k], 0);
114 r = src->funcs->set(adev, src, k,
115 AMDGPU_IRQ_STATE_DISABLE);
116 if (r)
117 DRM_ERROR("error disabling interrupt (%d)\n",
118 r);
119 }
120 }
121 }
122 spin_unlock_irqrestore(&adev->irq.lock, irqflags);
123}
124
125/**
126 * amdgpu_irq_handler - irq handler
127 *
128 * @int irq, void *arg: args
129 *
130 * This is the irq handler for the amdgpu driver (all asics).
131 */
132irqreturn_t amdgpu_irq_handler(int irq, void *arg)
133{
134 struct drm_device *dev = (struct drm_device *) arg;
135 struct amdgpu_device *adev = dev->dev_private;
136 irqreturn_t ret;
137
138 ret = amdgpu_ih_process(adev);
139 if (ret == IRQ_HANDLED)
140 pm_runtime_mark_last_busy(dev->dev);
141 return ret;
142}
143
144/**
145 * amdgpu_msi_ok - asic specific msi checks
146 *
147 * @adev: amdgpu device pointer
148 *
149 * Handles asic specific MSI checks to determine if
150 * MSIs should be enabled on a particular chip (all asics).
151 * Returns true if MSIs should be enabled, false if MSIs
152 * should not be enabled.
153 */
154static bool amdgpu_msi_ok(struct amdgpu_device *adev)
155{
156 /* force MSI on */
157 if (amdgpu_msi == 1)
158 return true;
159 else if (amdgpu_msi == 0)
160 return false;
161
162 return true;
163}
164
165/**
166 * amdgpu_irq_init - init driver interrupt info
167 *
168 * @adev: amdgpu device pointer
169 *
170 * Sets up the work irq handlers, vblank init, MSIs, etc. (all asics).
171 * Returns 0 for success, error for failure.
172 */
173int amdgpu_irq_init(struct amdgpu_device *adev)
174{
175 int r = 0;
176
177 spin_lock_init(&adev->irq.lock);
178
179 /* enable msi */
180 adev->irq.msi_enabled = false;
181
182 if (amdgpu_msi_ok(adev)) {
183 int ret = pci_enable_msi(adev->pdev);
184 if (!ret) {
185 adev->irq.msi_enabled = true;
186 dev_dbg(adev->dev, "amdgpu: using MSI.\n");
187 }
188 }
189
190 if (!amdgpu_device_has_dc_support(adev)) {
191 if (!adev->enable_virtual_display)
192 /* Disable vblank irqs aggressively for power-saving */
193 /* XXX: can this be enabled for DC? */
194 adev->ddev->vblank_disable_immediate = true;
195
196 r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc);
197 if (r)
198 return r;
199
200 /* pre DCE11 */
201 INIT_WORK(&adev->hotplug_work,
202 amdgpu_hotplug_work_func);
203 }
204
205 INIT_WORK(&adev->reset_work, amdgpu_irq_reset_work_func);
206
207 adev->irq.installed = true;
208 r = drm_irq_install(adev->ddev, adev->ddev->pdev->irq);
209 if (r) {
210 adev->irq.installed = false;
211 if (!amdgpu_device_has_dc_support(adev))
212 flush_work(&adev->hotplug_work);
213 cancel_work_sync(&adev->reset_work);
214 return r;
215 }
216 adev->ddev->max_vblank_count = 0x00ffffff;
217
218 DRM_DEBUG("amdgpu: irq initialized.\n");
219 return 0;
220}
221
222/**
223 * amdgpu_irq_fini - tear down driver interrupt info
224 *
225 * @adev: amdgpu device pointer
226 *
227 * Tears down the work irq handlers, vblank handlers, MSIs, etc. (all asics).
228 */
229void amdgpu_irq_fini(struct amdgpu_device *adev)
230{
231 unsigned i, j;
232
233 if (adev->irq.installed) {
234 drm_irq_uninstall(adev->ddev);
235 adev->irq.installed = false;
236 if (adev->irq.msi_enabled)
237 pci_disable_msi(adev->pdev);
238 if (!amdgpu_device_has_dc_support(adev))
239 flush_work(&adev->hotplug_work);
240 cancel_work_sync(&adev->reset_work);
241 }
242
243 for (i = 0; i < AMDGPU_IH_CLIENTID_MAX; ++i) {
244 if (!adev->irq.client[i].sources)
245 continue;
246
247 for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
248 struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
249
250 if (!src)
251 continue;
252
253 kfree(src->enabled_types);
254 src->enabled_types = NULL;
255 if (src->data) {
256 kfree(src->data);
257 kfree(src);
258 adev->irq.client[i].sources[j] = NULL;
259 }
260 }
261 kfree(adev->irq.client[i].sources);
262 adev->irq.client[i].sources = NULL;
263 }
264}
265
266/**
267 * amdgpu_irq_add_id - register irq source
268 *
269 * @adev: amdgpu device pointer
270 * @src_id: source id for this source
271 * @source: irq source
272 *
273 */
274int amdgpu_irq_add_id(struct amdgpu_device *adev,
275 unsigned client_id, unsigned src_id,
276 struct amdgpu_irq_src *source)
277{
278 if (client_id >= AMDGPU_IH_CLIENTID_MAX)
279 return -EINVAL;
280
281 if (src_id >= AMDGPU_MAX_IRQ_SRC_ID)
282 return -EINVAL;
283
284 if (!source->funcs)
285 return -EINVAL;
286
287 if (!adev->irq.client[client_id].sources) {
288 adev->irq.client[client_id].sources =
289 kcalloc(AMDGPU_MAX_IRQ_SRC_ID,
290 sizeof(struct amdgpu_irq_src *),
291 GFP_KERNEL);
292 if (!adev->irq.client[client_id].sources)
293 return -ENOMEM;
294 }
295
296 if (adev->irq.client[client_id].sources[src_id] != NULL)
297 return -EINVAL;
298
299 if (source->num_types && !source->enabled_types) {
300 atomic_t *types;
301
302 types = kcalloc(source->num_types, sizeof(atomic_t),
303 GFP_KERNEL);
304 if (!types)
305 return -ENOMEM;
306
307 source->enabled_types = types;
308 }
309
310 adev->irq.client[client_id].sources[src_id] = source;
311 return 0;
312}
313
314/**
315 * amdgpu_irq_dispatch - dispatch irq to IP blocks
316 *
317 * @adev: amdgpu device pointer
318 * @entry: interrupt vector
319 *
320 * Dispatches the irq to the different IP blocks
321 */
322void amdgpu_irq_dispatch(struct amdgpu_device *adev,
323 struct amdgpu_iv_entry *entry)
324{
325 unsigned client_id = entry->client_id;
326 unsigned src_id = entry->src_id;
327 struct amdgpu_irq_src *src;
328 int r;
329
330 trace_amdgpu_iv(entry);
331
332 if (client_id >= AMDGPU_IH_CLIENTID_MAX) {
333 DRM_DEBUG("Invalid client_id in IV: %d\n", client_id);
334 return;
335 }
336
337 if (src_id >= AMDGPU_MAX_IRQ_SRC_ID) {
338 DRM_DEBUG("Invalid src_id in IV: %d\n", src_id);
339 return;
340 }
341
342 if (adev->irq.virq[src_id]) {
343 generic_handle_irq(irq_find_mapping(adev->irq.domain, src_id));
344 } else {
345 if (!adev->irq.client[client_id].sources) {
346 DRM_DEBUG("Unregistered interrupt client_id: %d src_id: %d\n",
347 client_id, src_id);
348 return;
349 }
350
351 src = adev->irq.client[client_id].sources[src_id];
352 if (!src) {
353 DRM_DEBUG("Unhandled interrupt src_id: %d\n", src_id);
354 return;
355 }
356
357 r = src->funcs->process(adev, src, entry);
358 if (r)
359 DRM_ERROR("error processing interrupt (%d)\n", r);
360 }
361}
362
363/**
364 * amdgpu_irq_update - update hw interrupt state
365 *
366 * @adev: amdgpu device pointer
367 * @src: interrupt src you want to enable
368 * @type: type of interrupt you want to update
369 *
370 * Updates the interrupt state for a specific src (all asics).
371 */
372int amdgpu_irq_update(struct amdgpu_device *adev,
373 struct amdgpu_irq_src *src, unsigned type)
374{
375 unsigned long irqflags;
376 enum amdgpu_interrupt_state state;
377 int r;
378
379 spin_lock_irqsave(&adev->irq.lock, irqflags);
380
381 /* we need to determine after taking the lock, otherwise
382 we might disable just enabled interrupts again */
383 if (amdgpu_irq_enabled(adev, src, type))
384 state = AMDGPU_IRQ_STATE_ENABLE;
385 else
386 state = AMDGPU_IRQ_STATE_DISABLE;
387
388 r = src->funcs->set(adev, src, type, state);
389 spin_unlock_irqrestore(&adev->irq.lock, irqflags);
390 return r;
391}
392
393void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev)
394{
395 int i, j, k;
396
397 for (i = 0; i < AMDGPU_IH_CLIENTID_MAX; ++i) {
398 if (!adev->irq.client[i].sources)
399 continue;
400
401 for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
402 struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
403
404 if (!src)
405 continue;
406 for (k = 0; k < src->num_types; k++)
407 amdgpu_irq_update(adev, src, k);
408 }
409 }
410}
411
412/**
413 * amdgpu_irq_get - enable interrupt
414 *
415 * @adev: amdgpu device pointer
416 * @src: interrupt src you want to enable
417 * @type: type of interrupt you want to enable
418 *
419 * Enables the interrupt type for a specific src (all asics).
420 */
421int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
422 unsigned type)
423{
424 if (!adev->ddev->irq_enabled)
425 return -ENOENT;
426
427 if (type >= src->num_types)
428 return -EINVAL;
429
430 if (!src->enabled_types || !src->funcs->set)
431 return -EINVAL;
432
433 if (atomic_inc_return(&src->enabled_types[type]) == 1)
434 return amdgpu_irq_update(adev, src, type);
435
436 return 0;
437}
438
439/**
440 * amdgpu_irq_put - disable interrupt
441 *
442 * @adev: amdgpu device pointer
443 * @src: interrupt src you want to disable
444 * @type: type of interrupt you want to disable
445 *
446 * Disables the interrupt type for a specific src (all asics).
447 */
448int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
449 unsigned type)
450{
451 if (!adev->ddev->irq_enabled)
452 return -ENOENT;
453
454 if (type >= src->num_types)
455 return -EINVAL;
456
457 if (!src->enabled_types || !src->funcs->set)
458 return -EINVAL;
459
460 if (atomic_dec_and_test(&src->enabled_types[type]))
461 return amdgpu_irq_update(adev, src, type);
462
463 return 0;
464}
465
466/**
467 * amdgpu_irq_enabled - test if irq is enabled or not
468 *
469 * @adev: amdgpu device pointer
470 * @idx: interrupt src you want to test
471 *
472 * Tests if the given interrupt source is enabled or not
473 */
474bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
475 unsigned type)
476{
477 if (!adev->ddev->irq_enabled)
478 return false;
479
480 if (type >= src->num_types)
481 return false;
482
483 if (!src->enabled_types || !src->funcs->set)
484 return false;
485
486 return !!atomic_read(&src->enabled_types[type]);
487}
488
489/* gen irq */
490static void amdgpu_irq_mask(struct irq_data *irqd)
491{
492 /* XXX */
493}
494
495static void amdgpu_irq_unmask(struct irq_data *irqd)
496{
497 /* XXX */
498}
499
500static struct irq_chip amdgpu_irq_chip = {
501 .name = "amdgpu-ih",
502 .irq_mask = amdgpu_irq_mask,
503 .irq_unmask = amdgpu_irq_unmask,
504};
505
506static int amdgpu_irqdomain_map(struct irq_domain *d,
507 unsigned int irq, irq_hw_number_t hwirq)
508{
509 if (hwirq >= AMDGPU_MAX_IRQ_SRC_ID)
510 return -EPERM;
511
512 irq_set_chip_and_handler(irq,
513 &amdgpu_irq_chip, handle_simple_irq);
514 return 0;
515}
516
517static const struct irq_domain_ops amdgpu_hw_irqdomain_ops = {
518 .map = amdgpu_irqdomain_map,
519};
520
521/**
522 * amdgpu_irq_add_domain - create a linear irq domain
523 *
524 * @adev: amdgpu device pointer
525 *
526 * Create an irq domain for GPU interrupt sources
527 * that may be driven by another driver (e.g., ACP).
528 */
529int amdgpu_irq_add_domain(struct amdgpu_device *adev)
530{
531 adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID,
532 &amdgpu_hw_irqdomain_ops, adev);
533 if (!adev->irq.domain) {
534 DRM_ERROR("GPU irq add domain failed\n");
535 return -ENODEV;
536 }
537
538 return 0;
539}
540
541/**
542 * amdgpu_irq_remove_domain - remove the irq domain
543 *
544 * @adev: amdgpu device pointer
545 *
546 * Remove the irq domain for GPU interrupt sources
547 * that may be driven by another driver (e.g., ACP).
548 */
549void amdgpu_irq_remove_domain(struct amdgpu_device *adev)
550{
551 if (adev->irq.domain) {
552 irq_domain_remove(adev->irq.domain);
553 adev->irq.domain = NULL;
554 }
555}
556
557/**
558 * amdgpu_irq_create_mapping - create a mapping between a domain irq and a
559 * Linux irq
560 *
561 * @adev: amdgpu device pointer
562 * @src_id: IH source id
563 *
564 * Create a mapping between a domain irq (GPU IH src id) and a Linux irq
565 * Use this for components that generate a GPU interrupt, but are driven
566 * by a different driver (e.g., ACP).
567 * Returns the Linux irq.
568 */
569unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id)
570{
571 adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id);
572
573 return adev->irq.virq[src_id];
574}
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28
29/**
30 * DOC: Interrupt Handling
31 *
32 * Interrupts generated within GPU hardware raise interrupt requests that are
33 * passed to amdgpu IRQ handler which is responsible for detecting source and
34 * type of the interrupt and dispatching matching handlers. If handling an
35 * interrupt requires calling kernel functions that may sleep processing is
36 * dispatched to work handlers.
37 *
38 * If MSI functionality is not disabled by module parameter then MSI
39 * support will be enabled.
40 *
41 * For GPU interrupt sources that may be driven by another driver, IRQ domain
42 * support is used (with mapping between virtual and hardware IRQs).
43 */
44
45#include <linux/irq.h>
46#include <linux/pci.h>
47
48#include <drm/drm_crtc_helper.h>
49#include <drm/drm_vblank.h>
50#include <drm/amdgpu_drm.h>
51#include <drm/drm_drv.h>
52#include "amdgpu.h"
53#include "amdgpu_ih.h"
54#include "atom.h"
55#include "amdgpu_connectors.h"
56#include "amdgpu_trace.h"
57#include "amdgpu_amdkfd.h"
58#include "amdgpu_ras.h"
59
60#include <linux/pm_runtime.h>
61
62#ifdef CONFIG_DRM_AMD_DC
63#include "amdgpu_dm_irq.h"
64#endif
65
66#define AMDGPU_WAIT_IDLE_TIMEOUT 200
67
68const char *soc15_ih_clientid_name[] = {
69 "IH",
70 "SDMA2 or ACP",
71 "ATHUB",
72 "BIF",
73 "SDMA3 or DCE",
74 "SDMA4 or ISP",
75 "VMC1 or PCIE0",
76 "RLC",
77 "SDMA0",
78 "SDMA1",
79 "SE0SH",
80 "SE1SH",
81 "SE2SH",
82 "SE3SH",
83 "VCN1 or UVD1",
84 "THM",
85 "VCN or UVD",
86 "SDMA5 or VCE0",
87 "VMC",
88 "SDMA6 or XDMA",
89 "GRBM_CP",
90 "ATS",
91 "ROM_SMUIO",
92 "DF",
93 "SDMA7 or VCE1",
94 "PWR",
95 "reserved",
96 "UTCL2",
97 "EA",
98 "UTCL2LOG",
99 "MP0",
100 "MP1"
101};
102
103/**
104 * amdgpu_irq_disable_all - disable *all* interrupts
105 *
106 * @adev: amdgpu device pointer
107 *
108 * Disable all types of interrupts from all sources.
109 */
110void amdgpu_irq_disable_all(struct amdgpu_device *adev)
111{
112 unsigned long irqflags;
113 unsigned i, j, k;
114 int r;
115
116 spin_lock_irqsave(&adev->irq.lock, irqflags);
117 for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; ++i) {
118 if (!adev->irq.client[i].sources)
119 continue;
120
121 for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
122 struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
123
124 if (!src || !src->funcs->set || !src->num_types)
125 continue;
126
127 for (k = 0; k < src->num_types; ++k) {
128 atomic_set(&src->enabled_types[k], 0);
129 r = src->funcs->set(adev, src, k,
130 AMDGPU_IRQ_STATE_DISABLE);
131 if (r)
132 DRM_ERROR("error disabling interrupt (%d)\n",
133 r);
134 }
135 }
136 }
137 spin_unlock_irqrestore(&adev->irq.lock, irqflags);
138}
139
140/**
141 * amdgpu_irq_handler - IRQ handler
142 *
143 * @irq: IRQ number (unused)
144 * @arg: pointer to DRM device
145 *
146 * IRQ handler for amdgpu driver (all ASICs).
147 *
148 * Returns:
149 * result of handling the IRQ, as defined by &irqreturn_t
150 */
151static irqreturn_t amdgpu_irq_handler(int irq, void *arg)
152{
153 struct drm_device *dev = (struct drm_device *) arg;
154 struct amdgpu_device *adev = drm_to_adev(dev);
155 irqreturn_t ret;
156
157 ret = amdgpu_ih_process(adev, &adev->irq.ih);
158 if (ret == IRQ_HANDLED)
159 pm_runtime_mark_last_busy(dev->dev);
160
161 amdgpu_ras_interrupt_fatal_error_handler(adev);
162
163 return ret;
164}
165
166/**
167 * amdgpu_irq_handle_ih1 - kick of processing for IH1
168 *
169 * @work: work structure in struct amdgpu_irq
170 *
171 * Kick of processing IH ring 1.
172 */
173static void amdgpu_irq_handle_ih1(struct work_struct *work)
174{
175 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
176 irq.ih1_work);
177
178 amdgpu_ih_process(adev, &adev->irq.ih1);
179}
180
181/**
182 * amdgpu_irq_handle_ih2 - kick of processing for IH2
183 *
184 * @work: work structure in struct amdgpu_irq
185 *
186 * Kick of processing IH ring 2.
187 */
188static void amdgpu_irq_handle_ih2(struct work_struct *work)
189{
190 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
191 irq.ih2_work);
192
193 amdgpu_ih_process(adev, &adev->irq.ih2);
194}
195
196/**
197 * amdgpu_irq_handle_ih_soft - kick of processing for ih_soft
198 *
199 * @work: work structure in struct amdgpu_irq
200 *
201 * Kick of processing IH soft ring.
202 */
203static void amdgpu_irq_handle_ih_soft(struct work_struct *work)
204{
205 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
206 irq.ih_soft_work);
207
208 amdgpu_ih_process(adev, &adev->irq.ih_soft);
209}
210
211/**
212 * amdgpu_msi_ok - check whether MSI functionality is enabled
213 *
214 * @adev: amdgpu device pointer (unused)
215 *
216 * Checks whether MSI functionality has been disabled via module parameter
217 * (all ASICs).
218 *
219 * Returns:
220 * *true* if MSIs are allowed to be enabled or *false* otherwise
221 */
222static bool amdgpu_msi_ok(struct amdgpu_device *adev)
223{
224 if (amdgpu_msi == 1)
225 return true;
226 else if (amdgpu_msi == 0)
227 return false;
228
229 return true;
230}
231
232static void amdgpu_restore_msix(struct amdgpu_device *adev)
233{
234 u16 ctrl;
235
236 pci_read_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
237 if (!(ctrl & PCI_MSIX_FLAGS_ENABLE))
238 return;
239
240 /* VF FLR */
241 ctrl &= ~PCI_MSIX_FLAGS_ENABLE;
242 pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);
243 ctrl |= PCI_MSIX_FLAGS_ENABLE;
244 pci_write_config_word(adev->pdev, adev->pdev->msix_cap + PCI_MSIX_FLAGS, ctrl);
245}
246
247/**
248 * amdgpu_irq_init - initialize interrupt handling
249 *
250 * @adev: amdgpu device pointer
251 *
252 * Sets up work functions for hotplug and reset interrupts, enables MSI
253 * functionality, initializes vblank, hotplug and reset interrupt handling.
254 *
255 * Returns:
256 * 0 on success or error code on failure
257 */
258int amdgpu_irq_init(struct amdgpu_device *adev)
259{
260 int r = 0;
261 unsigned int irq;
262
263 spin_lock_init(&adev->irq.lock);
264
265 /* Enable MSI if not disabled by module parameter */
266 adev->irq.msi_enabled = false;
267
268 if (amdgpu_msi_ok(adev)) {
269 int nvec = pci_msix_vec_count(adev->pdev);
270 unsigned int flags;
271
272 if (nvec <= 0) {
273 flags = PCI_IRQ_MSI;
274 } else {
275 flags = PCI_IRQ_MSI | PCI_IRQ_MSIX;
276 }
277 /* we only need one vector */
278 nvec = pci_alloc_irq_vectors(adev->pdev, 1, 1, flags);
279 if (nvec > 0) {
280 adev->irq.msi_enabled = true;
281 dev_dbg(adev->dev, "using MSI/MSI-X.\n");
282 }
283 }
284
285 INIT_WORK(&adev->irq.ih1_work, amdgpu_irq_handle_ih1);
286 INIT_WORK(&adev->irq.ih2_work, amdgpu_irq_handle_ih2);
287 INIT_WORK(&adev->irq.ih_soft_work, amdgpu_irq_handle_ih_soft);
288
289 /* Use vector 0 for MSI-X. */
290 r = pci_irq_vector(adev->pdev, 0);
291 if (r < 0)
292 return r;
293 irq = r;
294
295 /* PCI devices require shared interrupts. */
296 r = request_irq(irq, amdgpu_irq_handler, IRQF_SHARED, adev_to_drm(adev)->driver->name,
297 adev_to_drm(adev));
298 if (r)
299 return r;
300 adev->irq.installed = true;
301 adev->irq.irq = irq;
302 adev_to_drm(adev)->max_vblank_count = 0x00ffffff;
303
304 DRM_DEBUG("amdgpu: irq initialized.\n");
305 return 0;
306}
307
308
309void amdgpu_irq_fini_hw(struct amdgpu_device *adev)
310{
311 if (adev->irq.installed) {
312 free_irq(adev->irq.irq, adev_to_drm(adev));
313 adev->irq.installed = false;
314 if (adev->irq.msi_enabled)
315 pci_free_irq_vectors(adev->pdev);
316 }
317
318 amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft);
319 amdgpu_ih_ring_fini(adev, &adev->irq.ih);
320 amdgpu_ih_ring_fini(adev, &adev->irq.ih1);
321 amdgpu_ih_ring_fini(adev, &adev->irq.ih2);
322}
323
324/**
325 * amdgpu_irq_fini_sw - shut down interrupt handling
326 *
327 * @adev: amdgpu device pointer
328 *
329 * Tears down work functions for hotplug and reset interrupts, disables MSI
330 * functionality, shuts down vblank, hotplug and reset interrupt handling,
331 * turns off interrupts from all sources (all ASICs).
332 */
333void amdgpu_irq_fini_sw(struct amdgpu_device *adev)
334{
335 unsigned i, j;
336
337 for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; ++i) {
338 if (!adev->irq.client[i].sources)
339 continue;
340
341 for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
342 struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
343
344 if (!src)
345 continue;
346
347 kfree(src->enabled_types);
348 src->enabled_types = NULL;
349 }
350 kfree(adev->irq.client[i].sources);
351 adev->irq.client[i].sources = NULL;
352 }
353}
354
355/**
356 * amdgpu_irq_add_id - register IRQ source
357 *
358 * @adev: amdgpu device pointer
359 * @client_id: client id
360 * @src_id: source id
361 * @source: IRQ source pointer
362 *
363 * Registers IRQ source on a client.
364 *
365 * Returns:
366 * 0 on success or error code otherwise
367 */
368int amdgpu_irq_add_id(struct amdgpu_device *adev,
369 unsigned client_id, unsigned src_id,
370 struct amdgpu_irq_src *source)
371{
372 if (client_id >= AMDGPU_IRQ_CLIENTID_MAX)
373 return -EINVAL;
374
375 if (src_id >= AMDGPU_MAX_IRQ_SRC_ID)
376 return -EINVAL;
377
378 if (!source->funcs)
379 return -EINVAL;
380
381 if (!adev->irq.client[client_id].sources) {
382 adev->irq.client[client_id].sources =
383 kcalloc(AMDGPU_MAX_IRQ_SRC_ID,
384 sizeof(struct amdgpu_irq_src *),
385 GFP_KERNEL);
386 if (!adev->irq.client[client_id].sources)
387 return -ENOMEM;
388 }
389
390 if (adev->irq.client[client_id].sources[src_id] != NULL)
391 return -EINVAL;
392
393 if (source->num_types && !source->enabled_types) {
394 atomic_t *types;
395
396 types = kcalloc(source->num_types, sizeof(atomic_t),
397 GFP_KERNEL);
398 if (!types)
399 return -ENOMEM;
400
401 source->enabled_types = types;
402 }
403
404 adev->irq.client[client_id].sources[src_id] = source;
405 return 0;
406}
407
408/**
409 * amdgpu_irq_dispatch - dispatch IRQ to IP blocks
410 *
411 * @adev: amdgpu device pointer
412 * @ih: interrupt ring instance
413 *
414 * Dispatches IRQ to IP blocks.
415 */
416void amdgpu_irq_dispatch(struct amdgpu_device *adev,
417 struct amdgpu_ih_ring *ih)
418{
419 u32 ring_index = ih->rptr >> 2;
420 struct amdgpu_iv_entry entry;
421 unsigned client_id, src_id;
422 struct amdgpu_irq_src *src;
423 bool handled = false;
424 int r;
425
426 entry.ih = ih;
427 entry.iv_entry = (const uint32_t *)&ih->ring[ring_index];
428 amdgpu_ih_decode_iv(adev, &entry);
429
430 trace_amdgpu_iv(ih - &adev->irq.ih, &entry);
431
432 client_id = entry.client_id;
433 src_id = entry.src_id;
434
435 if (client_id >= AMDGPU_IRQ_CLIENTID_MAX) {
436 DRM_DEBUG("Invalid client_id in IV: %d\n", client_id);
437
438 } else if (src_id >= AMDGPU_MAX_IRQ_SRC_ID) {
439 DRM_DEBUG("Invalid src_id in IV: %d\n", src_id);
440
441 } else if ((client_id == AMDGPU_IRQ_CLIENTID_LEGACY) &&
442 adev->irq.virq[src_id]) {
443 generic_handle_domain_irq(adev->irq.domain, src_id);
444
445 } else if (!adev->irq.client[client_id].sources) {
446 DRM_DEBUG("Unregistered interrupt client_id: %d src_id: %d\n",
447 client_id, src_id);
448
449 } else if ((src = adev->irq.client[client_id].sources[src_id])) {
450 r = src->funcs->process(adev, src, &entry);
451 if (r < 0)
452 DRM_ERROR("error processing interrupt (%d)\n", r);
453 else if (r)
454 handled = true;
455
456 } else {
457 DRM_DEBUG("Unhandled interrupt src_id: %d\n", src_id);
458 }
459
460 /* Send it to amdkfd as well if it isn't already handled */
461 if (!handled)
462 amdgpu_amdkfd_interrupt(adev, entry.iv_entry);
463
464 if (amdgpu_ih_ts_after(ih->processed_timestamp, entry.timestamp))
465 ih->processed_timestamp = entry.timestamp;
466}
467
468/**
469 * amdgpu_irq_delegate - delegate IV to soft IH ring
470 *
471 * @adev: amdgpu device pointer
472 * @entry: IV entry
473 * @num_dw: size of IV
474 *
475 * Delegate the IV to the soft IH ring and schedule processing of it. Used
476 * if the hardware delegation to IH1 or IH2 doesn't work for some reason.
477 */
478void amdgpu_irq_delegate(struct amdgpu_device *adev,
479 struct amdgpu_iv_entry *entry,
480 unsigned int num_dw)
481{
482 amdgpu_ih_ring_write(&adev->irq.ih_soft, entry->iv_entry, num_dw);
483 schedule_work(&adev->irq.ih_soft_work);
484}
485
486/**
487 * amdgpu_irq_update - update hardware interrupt state
488 *
489 * @adev: amdgpu device pointer
490 * @src: interrupt source pointer
491 * @type: type of interrupt
492 *
493 * Updates interrupt state for the specific source (all ASICs).
494 */
495int amdgpu_irq_update(struct amdgpu_device *adev,
496 struct amdgpu_irq_src *src, unsigned type)
497{
498 unsigned long irqflags;
499 enum amdgpu_interrupt_state state;
500 int r;
501
502 spin_lock_irqsave(&adev->irq.lock, irqflags);
503
504 /* We need to determine after taking the lock, otherwise
505 we might disable just enabled interrupts again */
506 if (amdgpu_irq_enabled(adev, src, type))
507 state = AMDGPU_IRQ_STATE_ENABLE;
508 else
509 state = AMDGPU_IRQ_STATE_DISABLE;
510
511 r = src->funcs->set(adev, src, type, state);
512 spin_unlock_irqrestore(&adev->irq.lock, irqflags);
513 return r;
514}
515
516/**
517 * amdgpu_irq_gpu_reset_resume_helper - update interrupt states on all sources
518 *
519 * @adev: amdgpu device pointer
520 *
521 * Updates state of all types of interrupts on all sources on resume after
522 * reset.
523 */
524void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev)
525{
526 int i, j, k;
527
528 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
529 amdgpu_restore_msix(adev);
530
531 for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; ++i) {
532 if (!adev->irq.client[i].sources)
533 continue;
534
535 for (j = 0; j < AMDGPU_MAX_IRQ_SRC_ID; ++j) {
536 struct amdgpu_irq_src *src = adev->irq.client[i].sources[j];
537
538 if (!src || !src->funcs || !src->funcs->set)
539 continue;
540 for (k = 0; k < src->num_types; k++)
541 amdgpu_irq_update(adev, src, k);
542 }
543 }
544}
545
546/**
547 * amdgpu_irq_get - enable interrupt
548 *
549 * @adev: amdgpu device pointer
550 * @src: interrupt source pointer
551 * @type: type of interrupt
552 *
553 * Enables specified type of interrupt on the specified source (all ASICs).
554 *
555 * Returns:
556 * 0 on success or error code otherwise
557 */
558int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
559 unsigned type)
560{
561 if (!adev->irq.installed)
562 return -ENOENT;
563
564 if (type >= src->num_types)
565 return -EINVAL;
566
567 if (!src->enabled_types || !src->funcs->set)
568 return -EINVAL;
569
570 if (atomic_inc_return(&src->enabled_types[type]) == 1)
571 return amdgpu_irq_update(adev, src, type);
572
573 return 0;
574}
575
576/**
577 * amdgpu_irq_put - disable interrupt
578 *
579 * @adev: amdgpu device pointer
580 * @src: interrupt source pointer
581 * @type: type of interrupt
582 *
583 * Enables specified type of interrupt on the specified source (all ASICs).
584 *
585 * Returns:
586 * 0 on success or error code otherwise
587 */
588int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
589 unsigned type)
590{
591 if (!adev->irq.installed)
592 return -ENOENT;
593
594 if (type >= src->num_types)
595 return -EINVAL;
596
597 if (!src->enabled_types || !src->funcs->set)
598 return -EINVAL;
599
600 if (atomic_dec_and_test(&src->enabled_types[type]))
601 return amdgpu_irq_update(adev, src, type);
602
603 return 0;
604}
605
606/**
607 * amdgpu_irq_enabled - check whether interrupt is enabled or not
608 *
609 * @adev: amdgpu device pointer
610 * @src: interrupt source pointer
611 * @type: type of interrupt
612 *
613 * Checks whether the given type of interrupt is enabled on the given source.
614 *
615 * Returns:
616 * *true* if interrupt is enabled, *false* if interrupt is disabled or on
617 * invalid parameters
618 */
619bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
620 unsigned type)
621{
622 if (!adev->irq.installed)
623 return false;
624
625 if (type >= src->num_types)
626 return false;
627
628 if (!src->enabled_types || !src->funcs->set)
629 return false;
630
631 return !!atomic_read(&src->enabled_types[type]);
632}
633
634/* XXX: Generic IRQ handling */
635static void amdgpu_irq_mask(struct irq_data *irqd)
636{
637 /* XXX */
638}
639
640static void amdgpu_irq_unmask(struct irq_data *irqd)
641{
642 /* XXX */
643}
644
645/* amdgpu hardware interrupt chip descriptor */
646static struct irq_chip amdgpu_irq_chip = {
647 .name = "amdgpu-ih",
648 .irq_mask = amdgpu_irq_mask,
649 .irq_unmask = amdgpu_irq_unmask,
650};
651
652/**
653 * amdgpu_irqdomain_map - create mapping between virtual and hardware IRQ numbers
654 *
655 * @d: amdgpu IRQ domain pointer (unused)
656 * @irq: virtual IRQ number
657 * @hwirq: hardware irq number
658 *
659 * Current implementation assigns simple interrupt handler to the given virtual
660 * IRQ.
661 *
662 * Returns:
663 * 0 on success or error code otherwise
664 */
665static int amdgpu_irqdomain_map(struct irq_domain *d,
666 unsigned int irq, irq_hw_number_t hwirq)
667{
668 if (hwirq >= AMDGPU_MAX_IRQ_SRC_ID)
669 return -EPERM;
670
671 irq_set_chip_and_handler(irq,
672 &amdgpu_irq_chip, handle_simple_irq);
673 return 0;
674}
675
676/* Implementation of methods for amdgpu IRQ domain */
677static const struct irq_domain_ops amdgpu_hw_irqdomain_ops = {
678 .map = amdgpu_irqdomain_map,
679};
680
681/**
682 * amdgpu_irq_add_domain - create a linear IRQ domain
683 *
684 * @adev: amdgpu device pointer
685 *
686 * Creates an IRQ domain for GPU interrupt sources
687 * that may be driven by another driver (e.g., ACP).
688 *
689 * Returns:
690 * 0 on success or error code otherwise
691 */
692int amdgpu_irq_add_domain(struct amdgpu_device *adev)
693{
694 adev->irq.domain = irq_domain_add_linear(NULL, AMDGPU_MAX_IRQ_SRC_ID,
695 &amdgpu_hw_irqdomain_ops, adev);
696 if (!adev->irq.domain) {
697 DRM_ERROR("GPU irq add domain failed\n");
698 return -ENODEV;
699 }
700
701 return 0;
702}
703
704/**
705 * amdgpu_irq_remove_domain - remove the IRQ domain
706 *
707 * @adev: amdgpu device pointer
708 *
709 * Removes the IRQ domain for GPU interrupt sources
710 * that may be driven by another driver (e.g., ACP).
711 */
712void amdgpu_irq_remove_domain(struct amdgpu_device *adev)
713{
714 if (adev->irq.domain) {
715 irq_domain_remove(adev->irq.domain);
716 adev->irq.domain = NULL;
717 }
718}
719
720/**
721 * amdgpu_irq_create_mapping - create mapping between domain Linux IRQs
722 *
723 * @adev: amdgpu device pointer
724 * @src_id: IH source id
725 *
726 * Creates mapping between a domain IRQ (GPU IH src id) and a Linux IRQ
727 * Use this for components that generate a GPU interrupt, but are driven
728 * by a different driver (e.g., ACP).
729 *
730 * Returns:
731 * Linux IRQ
732 */
733unsigned amdgpu_irq_create_mapping(struct amdgpu_device *adev, unsigned src_id)
734{
735 adev->irq.virq[src_id] = irq_create_mapping(adev->irq.domain, src_id);
736
737 return adev->irq.virq[src_id];
738}