Loading...
Note: File does not exist in v6.13.7.
1/*
2 * Intel MIC Platform Software Stack (MPSS)
3 *
4 * Copyright(c) 2013 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
17 *
18 * Disclaimer: The codes contained in these modules may be specific to
19 * the Intel Software Development Platform codenamed: Knights Ferry, and
20 * the Intel product codenamed: Knights Corner, and are not backward
21 * compatible with other Intel products. Additionally, Intel will NOT
22 * support the codes or instruction set in future products.
23 *
24 * Adapted from:
25 *
26 * virtio for kvm on s390
27 *
28 * Copyright IBM Corp. 2008
29 *
30 * This program is free software; you can redistribute it and/or modify
31 * it under the terms of the GNU General Public License (version 2 only)
32 * as published by the Free Software Foundation.
33 *
34 * Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
35 *
36 * Intel MIC Card driver.
37 *
38 */
39#include <linux/delay.h>
40#include <linux/slab.h>
41#include <linux/virtio_config.h>
42
43#include "../common/mic_dev.h"
44#include "mic_virtio.h"
45
46#define VIRTIO_SUBCODE_64 0x0D00
47
48#define MIC_MAX_VRINGS 4
49struct mic_vdev {
50 struct virtio_device vdev;
51 struct mic_device_desc __iomem *desc;
52 struct mic_device_ctrl __iomem *dc;
53 struct mic_device *mdev;
54 void __iomem *vr[MIC_MAX_VRINGS];
55 int used_size[MIC_MAX_VRINGS];
56 struct completion reset_done;
57 struct mic_irq *virtio_cookie;
58 int c2h_vdev_db;
59};
60
61static struct mic_irq *virtio_config_cookie;
62#define to_micvdev(vd) container_of(vd, struct mic_vdev, vdev)
63
64/* Helper API to obtain the parent of the virtio device */
65static inline struct device *mic_dev(struct mic_vdev *mvdev)
66{
67 return mvdev->vdev.dev.parent;
68}
69
70/* This gets the device's feature bits. */
71static u32 mic_get_features(struct virtio_device *vdev)
72{
73 unsigned int i, bits;
74 u32 features = 0;
75 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
76 u8 __iomem *in_features = mic_vq_features(desc);
77 int feature_len = ioread8(&desc->feature_len);
78
79 bits = min_t(unsigned, feature_len,
80 sizeof(vdev->features)) * 8;
81 for (i = 0; i < bits; i++)
82 if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
83 features |= BIT(i);
84
85 return features;
86}
87
88static void mic_finalize_features(struct virtio_device *vdev)
89{
90 unsigned int i, bits;
91 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
92 u8 feature_len = ioread8(&desc->feature_len);
93 /* Second half of bitmap is features we accept. */
94 u8 __iomem *out_features =
95 mic_vq_features(desc) + feature_len;
96
97 /* Give virtio_ring a chance to accept features. */
98 vring_transport_features(vdev);
99
100 memset_io(out_features, 0, feature_len);
101 bits = min_t(unsigned, feature_len,
102 sizeof(vdev->features)) * 8;
103 for (i = 0; i < bits; i++) {
104 if (test_bit(i, vdev->features))
105 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
106 &out_features[i / 8]);
107 }
108}
109
110/*
111 * Reading and writing elements in config space
112 */
113static void mic_get(struct virtio_device *vdev, unsigned int offset,
114 void *buf, unsigned len)
115{
116 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
117
118 if (offset + len > ioread8(&desc->config_len))
119 return;
120 memcpy_fromio(buf, mic_vq_configspace(desc) + offset, len);
121}
122
123static void mic_set(struct virtio_device *vdev, unsigned int offset,
124 const void *buf, unsigned len)
125{
126 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
127
128 if (offset + len > ioread8(&desc->config_len))
129 return;
130 memcpy_toio(mic_vq_configspace(desc) + offset, buf, len);
131}
132
133/*
134 * The operations to get and set the status word just access the status
135 * field of the device descriptor. set_status also interrupts the host
136 * to tell about status changes.
137 */
138static u8 mic_get_status(struct virtio_device *vdev)
139{
140 return ioread8(&to_micvdev(vdev)->desc->status);
141}
142
143static void mic_set_status(struct virtio_device *vdev, u8 status)
144{
145 struct mic_vdev *mvdev = to_micvdev(vdev);
146 if (!status)
147 return;
148 iowrite8(status, &mvdev->desc->status);
149 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
150}
151
152/* Inform host on a virtio device reset and wait for ack from host */
153static void mic_reset_inform_host(struct virtio_device *vdev)
154{
155 struct mic_vdev *mvdev = to_micvdev(vdev);
156 struct mic_device_ctrl __iomem *dc = mvdev->dc;
157 int retry;
158
159 iowrite8(0, &dc->host_ack);
160 iowrite8(1, &dc->vdev_reset);
161 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
162
163 /* Wait till host completes all card accesses and acks the reset */
164 for (retry = 100; retry--;) {
165 if (ioread8(&dc->host_ack))
166 break;
167 msleep(100);
168 };
169
170 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry);
171
172 /* Reset status to 0 in case we timed out */
173 iowrite8(0, &mvdev->desc->status);
174}
175
176static void mic_reset(struct virtio_device *vdev)
177{
178 struct mic_vdev *mvdev = to_micvdev(vdev);
179
180 dev_dbg(mic_dev(mvdev), "%s: virtio id %d\n",
181 __func__, vdev->id.device);
182
183 mic_reset_inform_host(vdev);
184 complete_all(&mvdev->reset_done);
185}
186
187/*
188 * The virtio_ring code calls this API when it wants to notify the Host.
189 */
190static bool mic_notify(struct virtqueue *vq)
191{
192 struct mic_vdev *mvdev = vq->priv;
193
194 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
195 return true;
196}
197
198static void mic_del_vq(struct virtqueue *vq, int n)
199{
200 struct mic_vdev *mvdev = to_micvdev(vq->vdev);
201 struct vring *vr = (struct vring *)(vq + 1);
202
203 free_pages((unsigned long) vr->used, get_order(mvdev->used_size[n]));
204 vring_del_virtqueue(vq);
205 mic_card_unmap(mvdev->mdev, mvdev->vr[n]);
206 mvdev->vr[n] = NULL;
207}
208
209static void mic_del_vqs(struct virtio_device *vdev)
210{
211 struct mic_vdev *mvdev = to_micvdev(vdev);
212 struct virtqueue *vq, *n;
213 int idx = 0;
214
215 dev_dbg(mic_dev(mvdev), "%s\n", __func__);
216
217 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
218 mic_del_vq(vq, idx++);
219}
220
221/*
222 * This routine will assign vring's allocated in host/io memory. Code in
223 * virtio_ring.c however continues to access this io memory as if it were local
224 * memory without io accessors.
225 */
226static struct virtqueue *mic_find_vq(struct virtio_device *vdev,
227 unsigned index,
228 void (*callback)(struct virtqueue *vq),
229 const char *name)
230{
231 struct mic_vdev *mvdev = to_micvdev(vdev);
232 struct mic_vqconfig __iomem *vqconfig;
233 struct mic_vqconfig config;
234 struct virtqueue *vq;
235 void __iomem *va;
236 struct _mic_vring_info __iomem *info;
237 void *used;
238 int vr_size, _vr_size, err, magic;
239 struct vring *vr;
240 u8 type = ioread8(&mvdev->desc->type);
241
242 if (index >= ioread8(&mvdev->desc->num_vq))
243 return ERR_PTR(-ENOENT);
244
245 if (!name)
246 return ERR_PTR(-ENOENT);
247
248 /* First assign the vring's allocated in host memory */
249 vqconfig = mic_vq_config(mvdev->desc) + index;
250 memcpy_fromio(&config, vqconfig, sizeof(config));
251 _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
252 vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
253 va = mic_card_map(mvdev->mdev, le64_to_cpu(config.address), vr_size);
254 if (!va)
255 return ERR_PTR(-ENOMEM);
256 mvdev->vr[index] = va;
257 memset_io(va, 0x0, _vr_size);
258 vq = vring_new_virtqueue(index, le16_to_cpu(config.num),
259 MIC_VIRTIO_RING_ALIGN, vdev, false,
260 (void __force *)va, mic_notify, callback,
261 name);
262 if (!vq) {
263 err = -ENOMEM;
264 goto unmap;
265 }
266 info = va + _vr_size;
267 magic = ioread32(&info->magic);
268
269 if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
270 err = -EIO;
271 goto unmap;
272 }
273
274 /* Allocate and reassign used ring now */
275 mvdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
276 sizeof(struct vring_used_elem) *
277 le16_to_cpu(config.num));
278 used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
279 get_order(mvdev->used_size[index]));
280 if (!used) {
281 err = -ENOMEM;
282 dev_err(mic_dev(mvdev), "%s %d err %d\n",
283 __func__, __LINE__, err);
284 goto del_vq;
285 }
286 iowrite64(virt_to_phys(used), &vqconfig->used_address);
287
288 /*
289 * To reassign the used ring here we are directly accessing
290 * struct vring_virtqueue which is a private data structure
291 * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in
292 * vring_new_virtqueue() would ensure that
293 * (&vq->vring == (struct vring *) (&vq->vq + 1));
294 */
295 vr = (struct vring *)(vq + 1);
296 vr->used = used;
297
298 vq->priv = mvdev;
299 return vq;
300del_vq:
301 vring_del_virtqueue(vq);
302unmap:
303 mic_card_unmap(mvdev->mdev, mvdev->vr[index]);
304 return ERR_PTR(err);
305}
306
307static int mic_find_vqs(struct virtio_device *vdev, unsigned nvqs,
308 struct virtqueue *vqs[],
309 vq_callback_t *callbacks[],
310 const char *names[])
311{
312 struct mic_vdev *mvdev = to_micvdev(vdev);
313 struct mic_device_ctrl __iomem *dc = mvdev->dc;
314 int i, err, retry;
315
316 /* We must have this many virtqueues. */
317 if (nvqs > ioread8(&mvdev->desc->num_vq))
318 return -ENOENT;
319
320 for (i = 0; i < nvqs; ++i) {
321 dev_dbg(mic_dev(mvdev), "%s: %d: %s\n",
322 __func__, i, names[i]);
323 vqs[i] = mic_find_vq(vdev, i, callbacks[i], names[i]);
324 if (IS_ERR(vqs[i])) {
325 err = PTR_ERR(vqs[i]);
326 goto error;
327 }
328 }
329
330 iowrite8(1, &dc->used_address_updated);
331 /*
332 * Send an interrupt to the host to inform it that used
333 * rings have been re-assigned.
334 */
335 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db);
336 for (retry = 100; retry--;) {
337 if (!ioread8(&dc->used_address_updated))
338 break;
339 msleep(100);
340 };
341
342 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry);
343 if (!retry) {
344 err = -ENODEV;
345 goto error;
346 }
347
348 return 0;
349error:
350 mic_del_vqs(vdev);
351 return err;
352}
353
354/*
355 * The config ops structure as defined by virtio config
356 */
357static struct virtio_config_ops mic_vq_config_ops = {
358 .get_features = mic_get_features,
359 .finalize_features = mic_finalize_features,
360 .get = mic_get,
361 .set = mic_set,
362 .get_status = mic_get_status,
363 .set_status = mic_set_status,
364 .reset = mic_reset,
365 .find_vqs = mic_find_vqs,
366 .del_vqs = mic_del_vqs,
367};
368
369static irqreturn_t
370mic_virtio_intr_handler(int irq, void *data)
371{
372 struct mic_vdev *mvdev = data;
373 struct virtqueue *vq;
374
375 mic_ack_interrupt(mvdev->mdev);
376 list_for_each_entry(vq, &mvdev->vdev.vqs, list)
377 vring_interrupt(0, vq);
378
379 return IRQ_HANDLED;
380}
381
382static void mic_virtio_release_dev(struct device *_d)
383{
384 /*
385 * No need for a release method similar to virtio PCI.
386 * Provide an empty one to avoid getting a warning from core.
387 */
388}
389
390/*
391 * adds a new device and register it with virtio
392 * appropriate drivers are loaded by the device model
393 */
394static int mic_add_device(struct mic_device_desc __iomem *d,
395 unsigned int offset, struct mic_driver *mdrv)
396{
397 struct mic_vdev *mvdev;
398 int ret;
399 int virtio_db;
400 u8 type = ioread8(&d->type);
401
402 mvdev = kzalloc(sizeof(*mvdev), GFP_KERNEL);
403 if (!mvdev) {
404 dev_err(mdrv->dev, "Cannot allocate mic dev %u type %u\n",
405 offset, type);
406 return -ENOMEM;
407 }
408
409 mvdev->mdev = &mdrv->mdev;
410 mvdev->vdev.dev.parent = mdrv->dev;
411 mvdev->vdev.dev.release = mic_virtio_release_dev;
412 mvdev->vdev.id.device = type;
413 mvdev->vdev.config = &mic_vq_config_ops;
414 mvdev->desc = d;
415 mvdev->dc = (void __iomem *)d + mic_aligned_desc_size(d);
416 init_completion(&mvdev->reset_done);
417
418 virtio_db = mic_next_card_db();
419 mvdev->virtio_cookie = mic_request_card_irq(mic_virtio_intr_handler,
420 "virtio intr", mvdev, virtio_db);
421 if (IS_ERR(mvdev->virtio_cookie)) {
422 ret = PTR_ERR(mvdev->virtio_cookie);
423 goto kfree;
424 }
425 iowrite8((u8)virtio_db, &mvdev->dc->h2c_vdev_db);
426 mvdev->c2h_vdev_db = ioread8(&mvdev->dc->c2h_vdev_db);
427
428 ret = register_virtio_device(&mvdev->vdev);
429 if (ret) {
430 dev_err(mic_dev(mvdev),
431 "Failed to register mic device %u type %u\n",
432 offset, type);
433 goto free_irq;
434 }
435 iowrite64((u64)mvdev, &mvdev->dc->vdev);
436 dev_dbg(mic_dev(mvdev), "%s: registered mic device %u type %u mvdev %p\n",
437 __func__, offset, type, mvdev);
438
439 return 0;
440
441free_irq:
442 mic_free_card_irq(mvdev->virtio_cookie, mvdev);
443kfree:
444 kfree(mvdev);
445 return ret;
446}
447
448/*
449 * match for a mic device with a specific desc pointer
450 */
451static int mic_match_desc(struct device *dev, void *data)
452{
453 struct virtio_device *vdev = dev_to_virtio(dev);
454 struct mic_vdev *mvdev = to_micvdev(vdev);
455
456 return mvdev->desc == (void __iomem *)data;
457}
458
459static void mic_handle_config_change(struct mic_device_desc __iomem *d,
460 unsigned int offset, struct mic_driver *mdrv)
461{
462 struct mic_device_ctrl __iomem *dc
463 = (void __iomem *)d + mic_aligned_desc_size(d);
464 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev);
465 struct virtio_driver *drv;
466
467 if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
468 return;
469
470 dev_dbg(mdrv->dev, "%s %d\n", __func__, __LINE__);
471 drv = container_of(mvdev->vdev.dev.driver,
472 struct virtio_driver, driver);
473 if (drv->config_changed)
474 drv->config_changed(&mvdev->vdev);
475 iowrite8(1, &dc->guest_ack);
476}
477
478/*
479 * removes a virtio device if a hot remove event has been
480 * requested by the host.
481 */
482static int mic_remove_device(struct mic_device_desc __iomem *d,
483 unsigned int offset, struct mic_driver *mdrv)
484{
485 struct mic_device_ctrl __iomem *dc
486 = (void __iomem *)d + mic_aligned_desc_size(d);
487 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev);
488 u8 status;
489 int ret = -1;
490
491 if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) {
492 dev_dbg(mdrv->dev,
493 "%s %d config_change %d type %d mvdev %p\n",
494 __func__, __LINE__,
495 ioread8(&dc->config_change), ioread8(&d->type), mvdev);
496
497 status = ioread8(&d->status);
498 reinit_completion(&mvdev->reset_done);
499 unregister_virtio_device(&mvdev->vdev);
500 mic_free_card_irq(mvdev->virtio_cookie, mvdev);
501 if (status & VIRTIO_CONFIG_S_DRIVER_OK)
502 wait_for_completion(&mvdev->reset_done);
503 kfree(mvdev);
504 iowrite8(1, &dc->guest_ack);
505 dev_dbg(mdrv->dev, "%s %d guest_ack %d\n",
506 __func__, __LINE__, ioread8(&dc->guest_ack));
507 ret = 0;
508 }
509
510 return ret;
511}
512
513#define REMOVE_DEVICES true
514
515static void mic_scan_devices(struct mic_driver *mdrv, bool remove)
516{
517 s8 type;
518 unsigned int i;
519 struct mic_device_desc __iomem *d;
520 struct mic_device_ctrl __iomem *dc;
521 struct device *dev;
522 int ret;
523
524 for (i = sizeof(struct mic_bootparam); i < MIC_DP_SIZE;
525 i += mic_total_desc_size(d)) {
526 d = mdrv->dp + i;
527 dc = (void __iomem *)d + mic_aligned_desc_size(d);
528 /*
529 * This read barrier is paired with the corresponding write
530 * barrier on the host which is inserted before adding or
531 * removing a virtio device descriptor, by updating the type.
532 */
533 rmb();
534 type = ioread8(&d->type);
535
536 /* end of list */
537 if (type == 0)
538 break;
539
540 if (type == -1)
541 continue;
542
543 /* device already exists */
544 dev = device_find_child(mdrv->dev, (void __force *)d,
545 mic_match_desc);
546 if (dev) {
547 if (remove)
548 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE,
549 &dc->config_change);
550 put_device(dev);
551 mic_handle_config_change(d, i, mdrv);
552 ret = mic_remove_device(d, i, mdrv);
553 if (!ret && !remove)
554 iowrite8(-1, &d->type);
555 if (remove) {
556 iowrite8(0, &dc->config_change);
557 iowrite8(0, &dc->guest_ack);
558 }
559 continue;
560 }
561
562 /* new device */
563 dev_dbg(mdrv->dev, "%s %d Adding new virtio device %p\n",
564 __func__, __LINE__, d);
565 if (!remove)
566 mic_add_device(d, i, mdrv);
567 }
568}
569
570/*
571 * mic_hotplug_device tries to find changes in the device page.
572 */
573static void mic_hotplug_devices(struct work_struct *work)
574{
575 struct mic_driver *mdrv = container_of(work,
576 struct mic_driver, hotplug_work);
577
578 mic_scan_devices(mdrv, !REMOVE_DEVICES);
579}
580
581/*
582 * Interrupt handler for hot plug/config changes etc.
583 */
584static irqreturn_t
585mic_extint_handler(int irq, void *data)
586{
587 struct mic_driver *mdrv = (struct mic_driver *)data;
588
589 dev_dbg(mdrv->dev, "%s %d hotplug work\n",
590 __func__, __LINE__);
591 mic_ack_interrupt(&mdrv->mdev);
592 schedule_work(&mdrv->hotplug_work);
593 return IRQ_HANDLED;
594}
595
596/*
597 * Init function for virtio
598 */
599int mic_devices_init(struct mic_driver *mdrv)
600{
601 int rc;
602 struct mic_bootparam __iomem *bootparam;
603 int config_db;
604
605 INIT_WORK(&mdrv->hotplug_work, mic_hotplug_devices);
606 mic_scan_devices(mdrv, !REMOVE_DEVICES);
607
608 config_db = mic_next_card_db();
609 virtio_config_cookie = mic_request_card_irq(mic_extint_handler,
610 "virtio_config_intr", mdrv, config_db);
611 if (IS_ERR(virtio_config_cookie)) {
612 rc = PTR_ERR(virtio_config_cookie);
613 goto exit;
614 }
615
616 bootparam = mdrv->dp;
617 iowrite8(config_db, &bootparam->h2c_config_db);
618 return 0;
619exit:
620 return rc;
621}
622
623/*
624 * Uninit function for virtio
625 */
626void mic_devices_uninit(struct mic_driver *mdrv)
627{
628 struct mic_bootparam __iomem *bootparam = mdrv->dp;
629 iowrite8(-1, &bootparam->h2c_config_db);
630 mic_free_card_irq(virtio_config_cookie, mdrv);
631 flush_work(&mdrv->hotplug_work);
632 mic_scan_devices(mdrv, REMOVE_DEVICES);
633}