1/*
  2 * Intel MIC Platform Software Stack (MPSS)
  3 *
  4 * Copyright(c) 2014 Intel Corporation.
  5 *
  6 * This program is free software; you can redistribute it and/or modify
  7 * it under the terms of the GNU General Public License, version 2, as
  8 * published by the Free Software Foundation.
  9 *
 10 * This program is distributed in the hope that it will be useful, but
 11 * WITHOUT ANY WARRANTY; without even the implied warranty of
 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 13 * General Public License for more details.
 14 *
 15 * Intel SCIF driver.
 16 *
 17 */
 18#include <linux/module.h>
 19#include <linux/idr.h>
 20
 21#include <linux/mic_common.h>
 22#include "../common/mic_dev.h"
 23#include "../bus/scif_bus.h"
 24#include "scif_peer_bus.h"
 25#include "scif_main.h"
 26#include "scif_map.h"
 27
 28struct scif_info scif_info = {
 29	.mdev = {
 30		.minor = MISC_DYNAMIC_MINOR,
 31		.name = "scif",
 32		.fops = &scif_fops,
 33	}
 34};
 35
 36struct scif_dev *scif_dev;
 37struct kmem_cache *unaligned_cache;
 38static atomic_t g_loopb_cnt;
 39
 40/* Runs in the context of intr_wq */
 41static void scif_intr_bh_handler(struct work_struct *work)
 42{
 43	struct scif_dev *scifdev =
 44			container_of(work, struct scif_dev, intr_bh);
 45
 46	if (scifdev_self(scifdev))
 47		scif_loopb_msg_handler(scifdev, scifdev->qpairs);
 48	else
 49		scif_nodeqp_intrhandler(scifdev, scifdev->qpairs);
 50}
 51
 52int scif_setup_intr_wq(struct scif_dev *scifdev)
 53{
 54	if (!scifdev->intr_wq) {
 55		snprintf(scifdev->intr_wqname, sizeof(scifdev->intr_wqname),
 56			 "SCIF INTR %d", scifdev->node);
 57		scifdev->intr_wq =
 58			alloc_ordered_workqueue(scifdev->intr_wqname, 0);
 59		if (!scifdev->intr_wq)
 60			return -ENOMEM;
 61		INIT_WORK(&scifdev->intr_bh, scif_intr_bh_handler);
 62	}
 63	return 0;
 64}
 65
 66void scif_destroy_intr_wq(struct scif_dev *scifdev)
 67{
 68	if (scifdev->intr_wq) {
 69		destroy_workqueue(scifdev->intr_wq);
 70		scifdev->intr_wq = NULL;
 71	}
 72}
 73
 74irqreturn_t scif_intr_handler(int irq, void *data)
 75{
 76	struct scif_dev *scifdev = data;
 77	struct scif_hw_dev *sdev = scifdev->sdev;
 78
 79	sdev->hw_ops->ack_interrupt(sdev, scifdev->db);
 80	queue_work(scifdev->intr_wq, &scifdev->intr_bh);
 81	return IRQ_HANDLED;
 82}
 83
 84static void scif_qp_setup_handler(struct work_struct *work)
 85{
 86	struct scif_dev *scifdev = container_of(work, struct scif_dev,
 87						qp_dwork.work);
 88	struct scif_hw_dev *sdev = scifdev->sdev;
 89	dma_addr_t da = 0;
 90	int err;
 91
 92	if (scif_is_mgmt_node()) {
 93		struct mic_bootparam *bp = sdev->dp;
 94
 95		da = bp->scif_card_dma_addr;
 96		scifdev->rdb = bp->h2c_scif_db;
 97	} else {
 98		struct mic_bootparam __iomem *bp = sdev->rdp;
 99
100		da = readq(&bp->scif_host_dma_addr);
101		scifdev->rdb = ioread8(&bp->c2h_scif_db);
102	}
103	if (da) {
104		err = scif_qp_response(da, scifdev);
105		if (err)
106			dev_err(&scifdev->sdev->dev,
107				"scif_qp_response err %d\n", err);
108	} else {
109		schedule_delayed_work(&scifdev->qp_dwork,
110				      msecs_to_jiffies(1000));
111	}
112}
113
114static int scif_setup_scifdev(void)
115{
116	/* We support a maximum of 129 SCIF nodes including the mgmt node */
117#define MAX_SCIF_NODES 129
118	int i;
119	u8 num_nodes = MAX_SCIF_NODES;
120
121	scif_dev = kcalloc(num_nodes, sizeof(*scif_dev), GFP_KERNEL);
122	if (!scif_dev)
123		return -ENOMEM;
124	for (i = 0; i < num_nodes; i++) {
125		struct scif_dev *scifdev = &scif_dev[i];
126
127		scifdev->node = i;
128		scifdev->exit = OP_IDLE;
129		init_waitqueue_head(&scifdev->disconn_wq);
130		mutex_init(&scifdev->lock);
131		INIT_WORK(&scifdev->peer_add_work, scif_add_peer_device);
132		INIT_DELAYED_WORK(&scifdev->p2p_dwork,
133				  scif_poll_qp_state);
134		INIT_DELAYED_WORK(&scifdev->qp_dwork,
135				  scif_qp_setup_handler);
136		INIT_LIST_HEAD(&scifdev->p2p);
137		RCU_INIT_POINTER(scifdev->spdev, NULL);
138	}
139	return 0;
140}
141
142static void scif_destroy_scifdev(void)
143{
144	kfree(scif_dev);
145}
146
147static int scif_probe(struct scif_hw_dev *sdev)
148{
149	struct scif_dev *scifdev = &scif_dev[sdev->dnode];
150	int rc;
151
152	dev_set_drvdata(&sdev->dev, sdev);
153	scifdev->sdev = sdev;
154
155	if (1 == atomic_add_return(1, &g_loopb_cnt)) {
156		struct scif_dev *loopb_dev = &scif_dev[sdev->snode];
157
158		loopb_dev->sdev = sdev;
159		rc = scif_setup_loopback_qp(loopb_dev);
160		if (rc)
161			goto exit;
162	}
163
164	rc = scif_setup_intr_wq(scifdev);
165	if (rc)
166		goto destroy_loopb;
167	rc = scif_setup_qp(scifdev);
168	if (rc)
169		goto destroy_intr;
170	scifdev->db = sdev->hw_ops->next_db(sdev);
171	scifdev->cookie = sdev->hw_ops->request_irq(sdev, scif_intr_handler,
172						    "SCIF_INTR", scifdev,
173						    scifdev->db);
174	if (IS_ERR(scifdev->cookie)) {
175		rc = PTR_ERR(scifdev->cookie);
176		goto free_qp;
177	}
178	if (scif_is_mgmt_node()) {
179		struct mic_bootparam *bp = sdev->dp;
180
181		bp->c2h_scif_db = scifdev->db;
182		bp->scif_host_dma_addr = scifdev->qp_dma_addr;
183	} else {
184		struct mic_bootparam __iomem *bp = sdev->rdp;
185
186		iowrite8(scifdev->db, &bp->h2c_scif_db);
187		writeq(scifdev->qp_dma_addr, &bp->scif_card_dma_addr);
188	}
189	schedule_delayed_work(&scifdev->qp_dwork,
190			      msecs_to_jiffies(1000));
191	return rc;
192free_qp:
193	scif_free_qp(scifdev);
194destroy_intr:
195	scif_destroy_intr_wq(scifdev);
196destroy_loopb:
197	if (atomic_dec_and_test(&g_loopb_cnt))
198		scif_destroy_loopback_qp(&scif_dev[sdev->snode]);
199exit:
200	return rc;
201}
202
203void scif_stop(struct scif_dev *scifdev)
204{
205	struct scif_dev *dev;
206	int i;
207
208	for (i = scif_info.maxid; i >= 0; i--) {
209		dev = &scif_dev[i];
210		if (scifdev_self(dev))
211			continue;
212		scif_handle_remove_node(i);
213	}
214}
215
216static void scif_remove(struct scif_hw_dev *sdev)
217{
218	struct scif_dev *scifdev = &scif_dev[sdev->dnode];
219
220	if (scif_is_mgmt_node()) {
221		struct mic_bootparam *bp = sdev->dp;
222
223		bp->c2h_scif_db = -1;
224		bp->scif_host_dma_addr = 0x0;
225	} else {
226		struct mic_bootparam __iomem *bp = sdev->rdp;
227
228		iowrite8(-1, &bp->h2c_scif_db);
229		writeq(0x0, &bp->scif_card_dma_addr);
230	}
231	if (scif_is_mgmt_node()) {
232		scif_disconnect_node(scifdev->node, true);
233	} else {
234		scif_info.card_initiated_exit = true;
235		scif_stop(scifdev);
236	}
237	if (atomic_dec_and_test(&g_loopb_cnt))
238		scif_destroy_loopback_qp(&scif_dev[sdev->snode]);
239	if (scifdev->cookie) {
240		sdev->hw_ops->free_irq(sdev, scifdev->cookie, scifdev);
241		scifdev->cookie = NULL;
242	}
243	scif_destroy_intr_wq(scifdev);
244	cancel_delayed_work(&scifdev->qp_dwork);
245	scif_free_qp(scifdev);
246	scifdev->rdb = -1;
247	scifdev->sdev = NULL;
248}
249
250static struct scif_hw_dev_id id_table[] = {
251	{ MIC_SCIF_DEV, SCIF_DEV_ANY_ID },
252	{ 0 },
253};
254
255static struct scif_driver scif_driver = {
256	.driver.name =	KBUILD_MODNAME,
257	.driver.owner =	THIS_MODULE,
258	.id_table = id_table,
259	.probe = scif_probe,
260	.remove = scif_remove,
261};
262
263static int _scif_init(void)
264{
265	int rc;
266
267	mutex_init(&scif_info.eplock);
268	spin_lock_init(&scif_info.rmalock);
269	spin_lock_init(&scif_info.nb_connect_lock);
270	spin_lock_init(&scif_info.port_lock);
271	mutex_init(&scif_info.conflock);
272	mutex_init(&scif_info.connlock);
273	mutex_init(&scif_info.fencelock);
274	INIT_LIST_HEAD(&scif_info.uaccept);
275	INIT_LIST_HEAD(&scif_info.listen);
276	INIT_LIST_HEAD(&scif_info.zombie);
277	INIT_LIST_HEAD(&scif_info.connected);
278	INIT_LIST_HEAD(&scif_info.disconnected);
279	INIT_LIST_HEAD(&scif_info.rma);
280	INIT_LIST_HEAD(&scif_info.rma_tc);
281	INIT_LIST_HEAD(&scif_info.mmu_notif_cleanup);
282	INIT_LIST_HEAD(&scif_info.fence);
283	INIT_LIST_HEAD(&scif_info.nb_connect_list);
284	init_waitqueue_head(&scif_info.exitwq);
285	scif_info.rma_tc_limit = SCIF_RMA_TEMP_CACHE_LIMIT;
286	scif_info.en_msg_log = 0;
287	scif_info.p2p_enable = 1;
288	rc = scif_setup_scifdev();
289	if (rc)
290		goto error;
291	unaligned_cache = kmem_cache_create("Unaligned_DMA",
292					    SCIF_KMEM_UNALIGNED_BUF_SIZE,
293					    0, SLAB_HWCACHE_ALIGN, NULL);
294	if (!unaligned_cache) {
295		rc = -ENOMEM;
296		goto free_sdev;
297	}
298	INIT_WORK(&scif_info.misc_work, scif_misc_handler);
299	INIT_WORK(&scif_info.mmu_notif_work, scif_mmu_notif_handler);
300	INIT_WORK(&scif_info.conn_work, scif_conn_handler);
301	idr_init(&scif_ports);
302	return 0;
303free_sdev:
304	scif_destroy_scifdev();
305error:
306	return rc;
307}
308
309static void _scif_exit(void)
310{
311	idr_destroy(&scif_ports);
312	kmem_cache_destroy(unaligned_cache);
313	scif_destroy_scifdev();
314}
315
316static int __init scif_init(void)
317{
318	struct miscdevice *mdev = &scif_info.mdev;
319	int rc;
320
321	_scif_init();
322	iova_cache_get();
323	rc = scif_peer_bus_init();
324	if (rc)
325		goto exit;
326	rc = scif_register_driver(&scif_driver);
327	if (rc)
328		goto peer_bus_exit;
329	rc = misc_register(mdev);
330	if (rc)
331		goto unreg_scif;
332	scif_init_debugfs();
333	return 0;
334unreg_scif:
335	scif_unregister_driver(&scif_driver);
336peer_bus_exit:
337	scif_peer_bus_exit();
338exit:
339	_scif_exit();
340	return rc;
341}
342
343static void __exit scif_exit(void)
344{
345	scif_exit_debugfs();
346	misc_deregister(&scif_info.mdev);
347	scif_unregister_driver(&scif_driver);
348	scif_peer_bus_exit();
349	iova_cache_put();
350	_scif_exit();
351}
352
353module_init(scif_init);
354module_exit(scif_exit);
355
356MODULE_DEVICE_TABLE(scif, id_table);
357MODULE_AUTHOR("Intel Corporation");
358MODULE_DESCRIPTION("Intel(R) SCIF driver");
359MODULE_LICENSE("GPL v2");