1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Intel MIC Platform Software Stack (MPSS)
  4 *
  5 * Copyright(c) 2013 Intel Corporation.
  6 *
  7 * Disclaimer: The codes contained in these modules may be specific to
  8 * the Intel Software Development Platform codenamed: Knights Ferry, and
  9 * the Intel product codenamed: Knights Corner, and are not backward
 10 * compatible with other Intel products. Additionally, Intel will NOT
 11 * support the codes or instruction set in future products.
 12 *
 13 * Intel MIC Card driver.
 14 */
 15#include <linux/module.h>
 16#include <linux/pci.h>
 17#include <linux/interrupt.h>
 18#include <linux/reboot.h>
 19#include <linux/dmaengine.h>
 20#include <linux/kmod.h>
 21
 22#include <linux/mic_common.h>
 23#include "../common/mic_dev.h"
 24#include "mic_device.h"
 25
 26static struct mic_driver *g_drv;
 27
 28static int __init mic_dp_init(void)
 29{
 30	struct mic_driver *mdrv = g_drv;
 31	struct mic_device *mdev = &mdrv->mdev;
 32	struct mic_bootparam __iomem *bootparam;
 33	u64 lo, hi, dp_dma_addr;
 34	u32 magic;
 35
 36	lo = mic_read_spad(&mdrv->mdev, MIC_DPLO_SPAD);
 37	hi = mic_read_spad(&mdrv->mdev, MIC_DPHI_SPAD);
 38
 39	dp_dma_addr = lo | (hi << 32);
 40	mdrv->dp = mic_card_map(mdev, dp_dma_addr, MIC_DP_SIZE);
 41	if (!mdrv->dp) {
 42		dev_err(mdrv->dev, "Cannot remap Aperture BAR\n");
 43		return -ENOMEM;
 44	}
 45	bootparam = mdrv->dp;
 46	magic = ioread32(&bootparam->magic);
 47	if (MIC_MAGIC != magic) {
 48		dev_err(mdrv->dev, "bootparam magic mismatch 0x%x\n", magic);
 49		return -EIO;
 50	}
 51	return 0;
 52}
 53
 54/* Uninitialize the device page */
 55static void mic_dp_uninit(void)
 56{
 57	mic_card_unmap(&g_drv->mdev, g_drv->dp);
 58}
 59
 60/**
 61 * mic_request_card_irq - request an irq.
 62 *
 63 * @handler: interrupt handler passed to request_threaded_irq.
 64 * @thread_fn: thread fn. passed to request_threaded_irq.
 65 * @name: The ASCII name of the callee requesting the irq.
 66 * @data: private data that is returned back when calling the
 67 * function handler.
 68 * @index: The doorbell index of the requester.
 69 *
 70 * returns: The cookie that is transparent to the caller. Passed
 71 * back when calling mic_free_irq. An appropriate error code
 72 * is returned on failure. Caller needs to use IS_ERR(return_val)
 73 * to check for failure and PTR_ERR(return_val) to obtained the
 74 * error code.
 75 *
 76 */
 77struct mic_irq *
 78mic_request_card_irq(irq_handler_t handler,
 79		     irq_handler_t thread_fn, const char *name,
 80		     void *data, int index)
 81{
 82	int rc = 0;
 83	unsigned long cookie;
 84	struct mic_driver *mdrv = g_drv;
 85
 86	rc  = request_threaded_irq(mic_db_to_irq(mdrv, index), handler,
 87				   thread_fn, 0, name, data);
 88	if (rc) {
 89		dev_err(mdrv->dev, "request_threaded_irq failed rc = %d\n", rc);
 90		goto err;
 91	}
 92	mdrv->irq_info.irq_usage_count[index]++;
 93	cookie = index;
 94	return (struct mic_irq *)cookie;
 95err:
 96	return ERR_PTR(rc);
 97}
 98
 99/**
100 * mic_free_card_irq - free irq.
101 *
102 * @cookie: cookie obtained during a successful call to mic_request_threaded_irq
103 * @data: private data specified by the calling function during the
104 * mic_request_threaded_irq
105 *
106 * returns: none.
107 */
108void mic_free_card_irq(struct mic_irq *cookie, void *data)
109{
110	int index;
111	struct mic_driver *mdrv = g_drv;
112
113	index = (unsigned long)cookie & 0xFFFFU;
114	free_irq(mic_db_to_irq(mdrv, index), data);
115	mdrv->irq_info.irq_usage_count[index]--;
116}
117
118/**
119 * mic_next_card_db - Get the doorbell with minimum usage count.
120 *
121 * Returns the irq index.
122 */
123int mic_next_card_db(void)
124{
125	int i;
126	int index = 0;
127	struct mic_driver *mdrv = g_drv;
128
129	for (i = 0; i < mdrv->intr_info.num_intr; i++) {
130		if (mdrv->irq_info.irq_usage_count[i] <
131			mdrv->irq_info.irq_usage_count[index])
132			index = i;
133	}
134
135	return index;
136}
137
138/**
139 * mic_init_irq - Initialize irq information.
140 *
141 * Returns 0 in success. Appropriate error code on failure.
142 */
143static int mic_init_irq(void)
144{
145	struct mic_driver *mdrv = g_drv;
146
147	mdrv->irq_info.irq_usage_count = kzalloc((sizeof(u32) *
148			mdrv->intr_info.num_intr),
149			GFP_KERNEL);
150	if (!mdrv->irq_info.irq_usage_count)
151		return -ENOMEM;
152	return 0;
153}
154
155/**
156 * mic_uninit_irq - Uninitialize irq information.
157 *
158 * None.
159 */
160static void mic_uninit_irq(void)
161{
162	struct mic_driver *mdrv = g_drv;
163
164	kfree(mdrv->irq_info.irq_usage_count);
165}
166
167static inline struct mic_driver *scdev_to_mdrv(struct scif_hw_dev *scdev)
168{
169	return dev_get_drvdata(scdev->dev.parent);
170}
171
172static struct mic_irq *
173___mic_request_irq(struct scif_hw_dev *scdev,
174		   irqreturn_t (*func)(int irq, void *data),
175				       const char *name, void *data,
176				       int db)
177{
178	return mic_request_card_irq(func, NULL, name, data, db);
179}
180
181static void
182___mic_free_irq(struct scif_hw_dev *scdev,
183		struct mic_irq *cookie, void *data)
184{
185	return mic_free_card_irq(cookie, data);
186}
187
188static void ___mic_ack_interrupt(struct scif_hw_dev *scdev, int num)
189{
190	struct mic_driver *mdrv = scdev_to_mdrv(scdev);
191
192	mic_ack_interrupt(&mdrv->mdev);
193}
194
195static int ___mic_next_db(struct scif_hw_dev *scdev)
196{
197	return mic_next_card_db();
198}
199
200static void ___mic_send_intr(struct scif_hw_dev *scdev, int db)
201{
202	struct mic_driver *mdrv = scdev_to_mdrv(scdev);
203
204	mic_send_intr(&mdrv->mdev, db);
205}
206
207static void ___mic_send_p2p_intr(struct scif_hw_dev *scdev, int db,
208				 struct mic_mw *mw)
209{
210	mic_send_p2p_intr(db, mw);
211}
212
213static void __iomem *
214___mic_ioremap(struct scif_hw_dev *scdev,
215	       phys_addr_t pa, size_t len)
216{
217	struct mic_driver *mdrv = scdev_to_mdrv(scdev);
218
219	return mic_card_map(&mdrv->mdev, pa, len);
220}
221
222static void ___mic_iounmap(struct scif_hw_dev *scdev, void __iomem *va)
223{
224	struct mic_driver *mdrv = scdev_to_mdrv(scdev);
225
226	mic_card_unmap(&mdrv->mdev, va);
227}
228
229static struct scif_hw_ops scif_hw_ops = {
230	.request_irq = ___mic_request_irq,
231	.free_irq = ___mic_free_irq,
232	.ack_interrupt = ___mic_ack_interrupt,
233	.next_db = ___mic_next_db,
234	.send_intr = ___mic_send_intr,
235	.send_p2p_intr = ___mic_send_p2p_intr,
236	.remap = ___mic_ioremap,
237	.unmap = ___mic_iounmap,
238};
239
240static inline struct mic_driver *vpdev_to_mdrv(struct vop_device *vpdev)
241{
242	return dev_get_drvdata(vpdev->dev.parent);
243}
244
245static struct mic_irq *
246__mic_request_irq(struct vop_device *vpdev,
247		  irqreturn_t (*func)(int irq, void *data),
248		   const char *name, void *data, int intr_src)
249{
250	return mic_request_card_irq(func, NULL, name, data, intr_src);
251}
252
253static void __mic_free_irq(struct vop_device *vpdev,
254			   struct mic_irq *cookie, void *data)
255{
256	return mic_free_card_irq(cookie, data);
257}
258
259static void __mic_ack_interrupt(struct vop_device *vpdev, int num)
260{
261	struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
262
263	mic_ack_interrupt(&mdrv->mdev);
264}
265
266static int __mic_next_db(struct vop_device *vpdev)
267{
268	return mic_next_card_db();
269}
270
271static void __iomem *__mic_get_remote_dp(struct vop_device *vpdev)
272{
273	struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
274
275	return mdrv->dp;
276}
277
278static void __mic_send_intr(struct vop_device *vpdev, int db)
279{
280	struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
281
282	mic_send_intr(&mdrv->mdev, db);
283}
284
285static void __iomem *__mic_ioremap(struct vop_device *vpdev,
286				   dma_addr_t pa, size_t len)
287{
288	struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
289
290	return mic_card_map(&mdrv->mdev, pa, len);
291}
292
293static void __mic_iounmap(struct vop_device *vpdev, void __iomem *va)
294{
295	struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
296
297	mic_card_unmap(&mdrv->mdev, va);
298}
299
300static struct vop_hw_ops vop_hw_ops = {
301	.request_irq = __mic_request_irq,
302	.free_irq = __mic_free_irq,
303	.ack_interrupt = __mic_ack_interrupt,
304	.next_db = __mic_next_db,
305	.get_remote_dp = __mic_get_remote_dp,
306	.send_intr = __mic_send_intr,
307	.remap = __mic_ioremap,
308	.unmap = __mic_iounmap,
309};
310
311static int mic_request_dma_chans(struct mic_driver *mdrv)
312{
313	dma_cap_mask_t mask;
314	struct dma_chan *chan;
315
316	dma_cap_zero(mask);
317	dma_cap_set(DMA_MEMCPY, mask);
318
319	do {
320		chan = dma_request_channel(mask, NULL, NULL);
321		if (chan) {
322			mdrv->dma_ch[mdrv->num_dma_ch++] = chan;
323			if (mdrv->num_dma_ch >= MIC_MAX_DMA_CHAN)
324				break;
325		}
326	} while (chan);
327	dev_info(mdrv->dev, "DMA channels # %d\n", mdrv->num_dma_ch);
328	return mdrv->num_dma_ch;
329}
330
331static void mic_free_dma_chans(struct mic_driver *mdrv)
332{
333	int i = 0;
334
335	for (i = 0; i < mdrv->num_dma_ch; i++) {
336		dma_release_channel(mdrv->dma_ch[i]);
337		mdrv->dma_ch[i] = NULL;
338	}
339	mdrv->num_dma_ch = 0;
340}
341
342/*
343 * mic_driver_init - MIC driver initialization tasks.
344 *
345 * Returns 0 in success. Appropriate error code on failure.
346 */
347int __init mic_driver_init(struct mic_driver *mdrv)
348{
349	int rc;
350	struct mic_bootparam __iomem *bootparam;
351	u8 node_id;
352
353	g_drv = mdrv;
354	/* Unloading the card module is not supported. */
355	if (!try_module_get(mdrv->dev->driver->owner)) {
356		rc = -ENODEV;
357		goto done;
358	}
359	rc = mic_dp_init();
360	if (rc)
361		goto put;
362	rc = mic_init_irq();
363	if (rc)
364		goto dp_uninit;
365	if (!mic_request_dma_chans(mdrv)) {
366		rc = -ENODEV;
367		goto irq_uninit;
368	}
369	mdrv->vpdev = vop_register_device(mdrv->dev, VOP_DEV_TRNSP,
370					  NULL, &vop_hw_ops, 0,
371					  NULL, mdrv->dma_ch[0]);
372	if (IS_ERR(mdrv->vpdev)) {
373		rc = PTR_ERR(mdrv->vpdev);
374		goto dma_free;
375	}
376	bootparam = mdrv->dp;
377	node_id = ioread8(&bootparam->node_id);
378	mdrv->scdev = scif_register_device(mdrv->dev, MIC_SCIF_DEV,
379					   NULL, &scif_hw_ops,
380					   0, node_id, &mdrv->mdev.mmio, NULL,
381					   NULL, mdrv->dp, mdrv->dma_ch,
382					   mdrv->num_dma_ch, true);
383	if (IS_ERR(mdrv->scdev)) {
384		rc = PTR_ERR(mdrv->scdev);
385		goto vop_remove;
386	}
387	mic_create_card_debug_dir(mdrv);
388done:
389	return rc;
390vop_remove:
391	vop_unregister_device(mdrv->vpdev);
392dma_free:
393	mic_free_dma_chans(mdrv);
394irq_uninit:
395	mic_uninit_irq();
396dp_uninit:
397	mic_dp_uninit();
398put:
399	module_put(mdrv->dev->driver->owner);
400	return rc;
401}
402
403/*
404 * mic_driver_uninit - MIC driver uninitialization tasks.
405 *
406 * Returns None
407 */
408void mic_driver_uninit(struct mic_driver *mdrv)
409{
410	mic_delete_card_debug_dir(mdrv);
411	scif_unregister_device(mdrv->scdev);
412	vop_unregister_device(mdrv->vpdev);
413	mic_free_dma_chans(mdrv);
414	mic_uninit_irq();
415	mic_dp_uninit();
416	module_put(mdrv->dev->driver->owner);
417}