Loading...
Note: File does not exist in v6.13.7.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2013 Intel Corporation.
6 *
7 * Intel MIC Host driver.
8 */
9#include <linux/pci.h>
10#include <linux/interrupt.h>
11
12#include "../common/mic_dev.h"
13#include "mic_device.h"
14
15static irqreturn_t mic_thread_fn(int irq, void *dev)
16{
17 struct mic_device *mdev = dev;
18 struct mic_intr_info *intr_info = mdev->intr_info;
19 struct mic_irq_info *irq_info = &mdev->irq_info;
20 struct mic_intr_cb *intr_cb;
21 struct pci_dev *pdev = mdev->pdev;
22 int i;
23
24 spin_lock(&irq_info->mic_thread_lock);
25 for (i = intr_info->intr_start_idx[MIC_INTR_DB];
26 i < intr_info->intr_len[MIC_INTR_DB]; i++)
27 if (test_and_clear_bit(i, &irq_info->mask)) {
28 list_for_each_entry(intr_cb, &irq_info->cb_list[i],
29 list)
30 if (intr_cb->thread_fn)
31 intr_cb->thread_fn(pdev->irq,
32 intr_cb->data);
33 }
34 spin_unlock(&irq_info->mic_thread_lock);
35 return IRQ_HANDLED;
36}
37/**
38 * mic_interrupt - Generic interrupt handler for
39 * MSI and INTx based interrupts.
40 */
41static irqreturn_t mic_interrupt(int irq, void *dev)
42{
43 struct mic_device *mdev = dev;
44 struct mic_intr_info *intr_info = mdev->intr_info;
45 struct mic_irq_info *irq_info = &mdev->irq_info;
46 struct mic_intr_cb *intr_cb;
47 struct pci_dev *pdev = mdev->pdev;
48 u32 mask;
49 int i;
50
51 mask = mdev->ops->ack_interrupt(mdev);
52 if (!mask)
53 return IRQ_NONE;
54
55 spin_lock(&irq_info->mic_intr_lock);
56 for (i = intr_info->intr_start_idx[MIC_INTR_DB];
57 i < intr_info->intr_len[MIC_INTR_DB]; i++)
58 if (mask & BIT(i)) {
59 list_for_each_entry(intr_cb, &irq_info->cb_list[i],
60 list)
61 if (intr_cb->handler)
62 intr_cb->handler(pdev->irq,
63 intr_cb->data);
64 set_bit(i, &irq_info->mask);
65 }
66 spin_unlock(&irq_info->mic_intr_lock);
67 return IRQ_WAKE_THREAD;
68}
69
70/* Return the interrupt offset from the index. Index is 0 based. */
71static u16 mic_map_src_to_offset(struct mic_device *mdev,
72 int intr_src, enum mic_intr_type type)
73{
74 if (type >= MIC_NUM_INTR_TYPES)
75 return MIC_NUM_OFFSETS;
76 if (intr_src >= mdev->intr_info->intr_len[type])
77 return MIC_NUM_OFFSETS;
78
79 return mdev->intr_info->intr_start_idx[type] + intr_src;
80}
81
82/* Return next available msix_entry. */
83static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)
84{
85 int i;
86 struct mic_irq_info *info = &mdev->irq_info;
87
88 for (i = 0; i < info->num_vectors; i++)
89 if (!info->mic_msi_map[i])
90 return &info->msix_entries[i];
91 return NULL;
92}
93
94/**
95 * mic_register_intr_callback - Register a callback handler for the
96 * given source id.
97 *
98 * @mdev: pointer to the mic_device instance
99 * @idx: The source id to be registered.
100 * @handler: The function to be called when the source id receives
101 * the interrupt.
102 * @thread_fn: thread fn. corresponding to the handler
103 * @data: Private data of the requester.
104 * Return the callback structure that was registered or an
105 * appropriate error on failure.
106 */
107static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,
108 u8 idx, irq_handler_t handler, irq_handler_t thread_fn,
109 void *data)
110{
111 struct mic_intr_cb *intr_cb;
112 unsigned long flags;
113 int rc;
114 intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL);
115
116 if (!intr_cb)
117 return ERR_PTR(-ENOMEM);
118
119 intr_cb->handler = handler;
120 intr_cb->thread_fn = thread_fn;
121 intr_cb->data = data;
122 intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,
123 0, 0, GFP_KERNEL);
124 if (intr_cb->cb_id < 0) {
125 rc = intr_cb->cb_id;
126 goto ida_fail;
127 }
128
129 spin_lock(&mdev->irq_info.mic_thread_lock);
130 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
131 list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);
132 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
133 spin_unlock(&mdev->irq_info.mic_thread_lock);
134
135 return intr_cb;
136ida_fail:
137 kfree(intr_cb);
138 return ERR_PTR(rc);
139}
140
141/**
142 * mic_unregister_intr_callback - Unregister the callback handler
143 * identified by its callback id.
144 *
145 * @mdev: pointer to the mic_device instance
146 * @idx: The callback structure id to be unregistered.
147 * Return the source id that was unregistered or MIC_NUM_OFFSETS if no
148 * such callback handler was found.
149 */
150static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)
151{
152 struct list_head *pos, *tmp;
153 struct mic_intr_cb *intr_cb;
154 unsigned long flags;
155 int i;
156
157 spin_lock(&mdev->irq_info.mic_thread_lock);
158 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
159 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
160 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
161 intr_cb = list_entry(pos, struct mic_intr_cb, list);
162 if (intr_cb->cb_id == idx) {
163 list_del(pos);
164 ida_simple_remove(&mdev->irq_info.cb_ida,
165 intr_cb->cb_id);
166 kfree(intr_cb);
167 spin_unlock_irqrestore(
168 &mdev->irq_info.mic_intr_lock, flags);
169 spin_unlock(&mdev->irq_info.mic_thread_lock);
170 return i;
171 }
172 }
173 }
174 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
175 spin_unlock(&mdev->irq_info.mic_thread_lock);
176 return MIC_NUM_OFFSETS;
177}
178
179/**
180 * mic_setup_msix - Initializes MSIx interrupts.
181 *
182 * @mdev: pointer to mic_device instance
183 *
184 *
185 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
186 */
187static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)
188{
189 int rc, i;
190 int entry_size = sizeof(*mdev->irq_info.msix_entries);
191
192 mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,
193 entry_size, GFP_KERNEL);
194 if (!mdev->irq_info.msix_entries) {
195 rc = -ENOMEM;
196 goto err_nomem1;
197 }
198
199 for (i = 0; i < MIC_MIN_MSIX; i++)
200 mdev->irq_info.msix_entries[i].entry = i;
201
202 rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries,
203 MIC_MIN_MSIX);
204 if (rc) {
205 dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc);
206 goto err_enable_msix;
207 }
208
209 mdev->irq_info.num_vectors = MIC_MIN_MSIX;
210 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
211 mdev->irq_info.num_vectors), GFP_KERNEL);
212
213 if (!mdev->irq_info.mic_msi_map) {
214 rc = -ENOMEM;
215 goto err_nomem2;
216 }
217
218 dev_dbg(&mdev->pdev->dev,
219 "%d MSIx irqs setup\n", mdev->irq_info.num_vectors);
220 return 0;
221err_nomem2:
222 pci_disable_msix(pdev);
223err_enable_msix:
224 kfree(mdev->irq_info.msix_entries);
225err_nomem1:
226 mdev->irq_info.num_vectors = 0;
227 return rc;
228}
229
230/**
231 * mic_setup_callbacks - Initialize data structures needed
232 * to handle callbacks.
233 *
234 * @mdev: pointer to mic_device instance
235 */
236static int mic_setup_callbacks(struct mic_device *mdev)
237{
238 int i;
239
240 mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,
241 sizeof(*mdev->irq_info.cb_list),
242 GFP_KERNEL);
243 if (!mdev->irq_info.cb_list)
244 return -ENOMEM;
245
246 for (i = 0; i < MIC_NUM_OFFSETS; i++)
247 INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);
248 ida_init(&mdev->irq_info.cb_ida);
249 spin_lock_init(&mdev->irq_info.mic_intr_lock);
250 spin_lock_init(&mdev->irq_info.mic_thread_lock);
251 return 0;
252}
253
254/**
255 * mic_release_callbacks - Uninitialize data structures needed
256 * to handle callbacks.
257 *
258 * @mdev: pointer to mic_device instance
259 */
260static void mic_release_callbacks(struct mic_device *mdev)
261{
262 unsigned long flags;
263 struct list_head *pos, *tmp;
264 struct mic_intr_cb *intr_cb;
265 int i;
266
267 spin_lock(&mdev->irq_info.mic_thread_lock);
268 spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
269 for (i = 0; i < MIC_NUM_OFFSETS; i++) {
270 if (list_empty(&mdev->irq_info.cb_list[i]))
271 break;
272
273 list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
274 intr_cb = list_entry(pos, struct mic_intr_cb, list);
275 list_del(pos);
276 ida_simple_remove(&mdev->irq_info.cb_ida,
277 intr_cb->cb_id);
278 kfree(intr_cb);
279 }
280 }
281 spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
282 spin_unlock(&mdev->irq_info.mic_thread_lock);
283 ida_destroy(&mdev->irq_info.cb_ida);
284 kfree(mdev->irq_info.cb_list);
285}
286
287/**
288 * mic_setup_msi - Initializes MSI interrupts.
289 *
290 * @mdev: pointer to mic_device instance
291 * @pdev: PCI device structure
292 *
293 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
294 */
295static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)
296{
297 int rc;
298
299 rc = pci_enable_msi(pdev);
300 if (rc) {
301 dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc);
302 return rc;
303 }
304
305 mdev->irq_info.num_vectors = 1;
306 mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
307 mdev->irq_info.num_vectors), GFP_KERNEL);
308
309 if (!mdev->irq_info.mic_msi_map) {
310 rc = -ENOMEM;
311 goto err_nomem1;
312 }
313
314 rc = mic_setup_callbacks(mdev);
315 if (rc) {
316 dev_err(&pdev->dev, "Error setting up callbacks\n");
317 goto err_nomem2;
318 }
319
320 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
321 0, "mic-msi", mdev);
322 if (rc) {
323 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
324 goto err_irq_req_fail;
325 }
326
327 dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);
328 return 0;
329err_irq_req_fail:
330 mic_release_callbacks(mdev);
331err_nomem2:
332 kfree(mdev->irq_info.mic_msi_map);
333err_nomem1:
334 pci_disable_msi(pdev);
335 mdev->irq_info.num_vectors = 0;
336 return rc;
337}
338
339/**
340 * mic_setup_intx - Initializes legacy interrupts.
341 *
342 * @mdev: pointer to mic_device instance
343 * @pdev: PCI device structure
344 *
345 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
346 */
347static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)
348{
349 int rc;
350
351 /* Enable intx */
352 pci_intx(pdev, 1);
353 rc = mic_setup_callbacks(mdev);
354 if (rc) {
355 dev_err(&pdev->dev, "Error setting up callbacks\n");
356 goto err_nomem;
357 }
358
359 rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
360 IRQF_SHARED, "mic-intx", mdev);
361 if (rc)
362 goto err;
363
364 dev_dbg(&pdev->dev, "intx irq setup\n");
365 return 0;
366err:
367 mic_release_callbacks(mdev);
368err_nomem:
369 return rc;
370}
371
372/**
373 * mic_next_db - Retrieve the next doorbell interrupt source id.
374 * The id is picked sequentially from the available pool of
375 * doorlbell ids.
376 *
377 * @mdev: pointer to the mic_device instance.
378 *
379 * Returns the next doorbell interrupt source.
380 */
381int mic_next_db(struct mic_device *mdev)
382{
383 int next_db;
384
385 next_db = mdev->irq_info.next_avail_src %
386 mdev->intr_info->intr_len[MIC_INTR_DB];
387 mdev->irq_info.next_avail_src++;
388 return next_db;
389}
390
391#define COOKIE_ID_SHIFT 16
392#define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
393#define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
394#define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT)
395
396/**
397 * mic_request_threaded_irq - request an irq. mic_mutex needs
398 * to be held before calling this function.
399 *
400 * @mdev: pointer to mic_device instance
401 * @handler: The callback function that handles the interrupt.
402 * The function needs to call ack_interrupts
403 * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts.
404 * @thread_fn: thread fn required by request_threaded_irq.
405 * @name: The ASCII name of the callee requesting the irq.
406 * @data: private data that is returned back when calling the
407 * function handler.
408 * @intr_src: The source id of the requester. Its the doorbell id
409 * for Doorbell interrupts and DMA channel id for DMA interrupts.
410 * @type: The type of interrupt. Values defined in mic_intr_type
411 *
412 * returns: The cookie that is transparent to the caller. Passed
413 * back when calling mic_free_irq. An appropriate error code
414 * is returned on failure. Caller needs to use IS_ERR(return_val)
415 * to check for failure and PTR_ERR(return_val) to obtained the
416 * error code.
417 *
418 */
419struct mic_irq *
420mic_request_threaded_irq(struct mic_device *mdev,
421 irq_handler_t handler, irq_handler_t thread_fn,
422 const char *name, void *data, int intr_src,
423 enum mic_intr_type type)
424{
425 u16 offset;
426 int rc = 0;
427 struct msix_entry *msix = NULL;
428 unsigned long cookie = 0;
429 u16 entry;
430 struct mic_intr_cb *intr_cb;
431 struct pci_dev *pdev = mdev->pdev;
432
433 offset = mic_map_src_to_offset(mdev, intr_src, type);
434 if (offset >= MIC_NUM_OFFSETS) {
435 dev_err(&mdev->pdev->dev,
436 "Error mapping index %d to a valid source id.\n",
437 intr_src);
438 rc = -EINVAL;
439 goto err;
440 }
441
442 if (mdev->irq_info.num_vectors > 1) {
443 msix = mic_get_available_vector(mdev);
444 if (!msix) {
445 dev_err(&mdev->pdev->dev,
446 "No MSIx vectors available for use.\n");
447 rc = -ENOSPC;
448 goto err;
449 }
450
451 rc = request_threaded_irq(msix->vector, handler, thread_fn,
452 0, name, data);
453 if (rc) {
454 dev_dbg(&mdev->pdev->dev,
455 "request irq failed rc = %d\n", rc);
456 goto err;
457 }
458 entry = msix->entry;
459 mdev->irq_info.mic_msi_map[entry] |= BIT(offset);
460 mdev->intr_ops->program_msi_to_src_map(mdev,
461 entry, offset, true);
462 cookie = MK_COOKIE(entry, offset);
463 dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n",
464 msix->vector, intr_src);
465 } else {
466 intr_cb = mic_register_intr_callback(mdev, offset, handler,
467 thread_fn, data);
468 if (IS_ERR(intr_cb)) {
469 dev_err(&mdev->pdev->dev,
470 "No available callback entries for use\n");
471 rc = PTR_ERR(intr_cb);
472 goto err;
473 }
474
475 entry = 0;
476 if (pci_dev_msi_enabled(pdev)) {
477 mdev->irq_info.mic_msi_map[entry] |= (1 << offset);
478 mdev->intr_ops->program_msi_to_src_map(mdev,
479 entry, offset, true);
480 }
481 cookie = MK_COOKIE(entry, intr_cb->cb_id);
482 dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n",
483 intr_cb->cb_id, intr_src);
484 }
485 return (struct mic_irq *)cookie;
486err:
487 return ERR_PTR(rc);
488}
489
490/**
491 * mic_free_irq - free irq. mic_mutex
492 * needs to be held before calling this function.
493 *
494 * @mdev: pointer to mic_device instance
495 * @cookie: cookie obtained during a successful call to mic_request_threaded_irq
496 * @data: private data specified by the calling function during the
497 * mic_request_threaded_irq
498 *
499 * returns: none.
500 */
501void mic_free_irq(struct mic_device *mdev,
502 struct mic_irq *cookie, void *data)
503{
504 u32 offset;
505 u32 entry;
506 u8 src_id;
507 unsigned int irq;
508 struct pci_dev *pdev = mdev->pdev;
509
510 entry = GET_ENTRY((unsigned long)cookie);
511 offset = GET_OFFSET((unsigned long)cookie);
512 if (mdev->irq_info.num_vectors > 1) {
513 if (entry >= mdev->irq_info.num_vectors) {
514 dev_warn(&mdev->pdev->dev,
515 "entry %d should be < num_irq %d\n",
516 entry, mdev->irq_info.num_vectors);
517 return;
518 }
519 irq = mdev->irq_info.msix_entries[entry].vector;
520 free_irq(irq, data);
521 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));
522 mdev->intr_ops->program_msi_to_src_map(mdev,
523 entry, offset, false);
524
525 dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq);
526 } else {
527 irq = pdev->irq;
528 src_id = mic_unregister_intr_callback(mdev, offset);
529 if (src_id >= MIC_NUM_OFFSETS) {
530 dev_warn(&mdev->pdev->dev, "Error unregistering callback\n");
531 return;
532 }
533 if (pci_dev_msi_enabled(pdev)) {
534 mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));
535 mdev->intr_ops->program_msi_to_src_map(mdev,
536 entry, src_id, false);
537 }
538 dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n",
539 offset, src_id);
540 }
541}
542
543/**
544 * mic_setup_interrupts - Initializes interrupts.
545 *
546 * @mdev: pointer to mic_device instance
547 * @pdev: PCI device structure
548 *
549 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
550 */
551int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
552{
553 int rc;
554
555 rc = mic_setup_msix(mdev, pdev);
556 if (!rc)
557 goto done;
558
559 rc = mic_setup_msi(mdev, pdev);
560 if (!rc)
561 goto done;
562
563 rc = mic_setup_intx(mdev, pdev);
564 if (rc) {
565 dev_err(&mdev->pdev->dev, "no usable interrupts\n");
566 return rc;
567 }
568done:
569 mdev->intr_ops->enable_interrupts(mdev);
570 return 0;
571}
572
573/**
574 * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts
575 *
576 * @mdev: pointer to mic_device instance
577 * @pdev: PCI device structure
578 *
579 * returns none.
580 */
581void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
582{
583 int i;
584
585 mdev->intr_ops->disable_interrupts(mdev);
586 if (mdev->irq_info.num_vectors > 1) {
587 for (i = 0; i < mdev->irq_info.num_vectors; i++) {
588 if (mdev->irq_info.mic_msi_map[i])
589 dev_warn(&pdev->dev, "irq %d may still be in use.\n",
590 mdev->irq_info.msix_entries[i].vector);
591 }
592 kfree(mdev->irq_info.mic_msi_map);
593 kfree(mdev->irq_info.msix_entries);
594 pci_disable_msix(pdev);
595 } else {
596 if (pci_dev_msi_enabled(pdev)) {
597 free_irq(pdev->irq, mdev);
598 kfree(mdev->irq_info.mic_msi_map);
599 pci_disable_msi(pdev);
600 } else {
601 free_irq(pdev->irq, mdev);
602 }
603 mic_release_callbacks(mdev);
604 }
605}
606
607/**
608 * mic_intr_restore - Restore MIC interrupt registers.
609 *
610 * @mdev: pointer to mic_device instance.
611 *
612 * Restore the interrupt registers to values previously
613 * stored in the SW data structures. mic_mutex needs to
614 * be held before calling this function.
615 *
616 * returns None.
617 */
618void mic_intr_restore(struct mic_device *mdev)
619{
620 int entry, offset;
621 struct pci_dev *pdev = mdev->pdev;
622
623 if (!pci_dev_msi_enabled(pdev))
624 return;
625
626 for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {
627 for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) {
628 if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))
629 mdev->intr_ops->program_msi_to_src_map(mdev,
630 entry, offset, true);
631 }
632 }
633}