Loading...
1/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19#include <linux/virtio.h>
20#include <linux/virtio_ring.h>
21#include <linux/virtio_config.h>
22#include <linux/device.h>
23#include <linux/slab.h>
24
25/* virtio guest is communicating with a virtual "device" that actually runs on
26 * a host processor. Memory barriers are used to control SMP effects. */
27#ifdef CONFIG_SMP
28/* Where possible, use SMP barriers which are more lightweight than mandatory
29 * barriers, because mandatory barriers control MMIO effects on accesses
30 * through relaxed memory I/O windows (which virtio does not use). */
31#define virtio_mb() smp_mb()
32#define virtio_rmb() smp_rmb()
33#define virtio_wmb() smp_wmb()
34#else
35/* We must force memory ordering even if guest is UP since host could be
36 * running on another CPU, but SMP barriers are defined to barrier() in that
37 * configuration. So fall back to mandatory barriers instead. */
38#define virtio_mb() mb()
39#define virtio_rmb() rmb()
40#define virtio_wmb() wmb()
41#endif
42
43#ifdef DEBUG
44/* For development, we want to crash whenever the ring is screwed. */
45#define BAD_RING(_vq, fmt, args...) \
46 do { \
47 dev_err(&(_vq)->vq.vdev->dev, \
48 "%s:"fmt, (_vq)->vq.name, ##args); \
49 BUG(); \
50 } while (0)
51/* Caller is supposed to guarantee no reentry. */
52#define START_USE(_vq) \
53 do { \
54 if ((_vq)->in_use) \
55 panic("%s:in_use = %i\n", \
56 (_vq)->vq.name, (_vq)->in_use); \
57 (_vq)->in_use = __LINE__; \
58 } while (0)
59#define END_USE(_vq) \
60 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
61#else
62#define BAD_RING(_vq, fmt, args...) \
63 do { \
64 dev_err(&_vq->vq.vdev->dev, \
65 "%s:"fmt, (_vq)->vq.name, ##args); \
66 (_vq)->broken = true; \
67 } while (0)
68#define START_USE(vq)
69#define END_USE(vq)
70#endif
71
72struct vring_virtqueue
73{
74 struct virtqueue vq;
75
76 /* Actual memory layout for this queue */
77 struct vring vring;
78
79 /* Other side has made a mess, don't try any more. */
80 bool broken;
81
82 /* Host supports indirect buffers */
83 bool indirect;
84
85 /* Host publishes avail event idx */
86 bool event;
87
88 /* Number of free buffers */
89 unsigned int num_free;
90 /* Head of free buffer list. */
91 unsigned int free_head;
92 /* Number we've added since last sync. */
93 unsigned int num_added;
94
95 /* Last used index we've seen. */
96 u16 last_used_idx;
97
98 /* How to notify other side. FIXME: commonalize hcalls! */
99 void (*notify)(struct virtqueue *vq);
100
101#ifdef DEBUG
102 /* They're supposed to lock for us. */
103 unsigned int in_use;
104#endif
105
106 /* Tokens for callbacks. */
107 void *data[];
108};
109
110#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
111
112/* Set up an indirect table of descriptors and add it to the queue. */
113static int vring_add_indirect(struct vring_virtqueue *vq,
114 struct scatterlist sg[],
115 unsigned int out,
116 unsigned int in,
117 gfp_t gfp)
118{
119 struct vring_desc *desc;
120 unsigned head;
121 int i;
122
123 desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
124 if (!desc)
125 return -ENOMEM;
126
127 /* Transfer entries from the sg list into the indirect page */
128 for (i = 0; i < out; i++) {
129 desc[i].flags = VRING_DESC_F_NEXT;
130 desc[i].addr = sg_phys(sg);
131 desc[i].len = sg->length;
132 desc[i].next = i+1;
133 sg++;
134 }
135 for (; i < (out + in); i++) {
136 desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
137 desc[i].addr = sg_phys(sg);
138 desc[i].len = sg->length;
139 desc[i].next = i+1;
140 sg++;
141 }
142
143 /* Last one doesn't continue. */
144 desc[i-1].flags &= ~VRING_DESC_F_NEXT;
145 desc[i-1].next = 0;
146
147 /* We're about to use a buffer */
148 vq->num_free--;
149
150 /* Use a single buffer which doesn't continue */
151 head = vq->free_head;
152 vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT;
153 vq->vring.desc[head].addr = virt_to_phys(desc);
154 vq->vring.desc[head].len = i * sizeof(struct vring_desc);
155
156 /* Update free pointer */
157 vq->free_head = vq->vring.desc[head].next;
158
159 return head;
160}
161
162int virtqueue_add_buf_gfp(struct virtqueue *_vq,
163 struct scatterlist sg[],
164 unsigned int out,
165 unsigned int in,
166 void *data,
167 gfp_t gfp)
168{
169 struct vring_virtqueue *vq = to_vvq(_vq);
170 unsigned int i, avail, uninitialized_var(prev);
171 int head;
172
173 START_USE(vq);
174
175 BUG_ON(data == NULL);
176
177 /* If the host supports indirect descriptor tables, and we have multiple
178 * buffers, then go indirect. FIXME: tune this threshold */
179 if (vq->indirect && (out + in) > 1 && vq->num_free) {
180 head = vring_add_indirect(vq, sg, out, in, gfp);
181 if (likely(head >= 0))
182 goto add_head;
183 }
184
185 BUG_ON(out + in > vq->vring.num);
186 BUG_ON(out + in == 0);
187
188 if (vq->num_free < out + in) {
189 pr_debug("Can't add buf len %i - avail = %i\n",
190 out + in, vq->num_free);
191 /* FIXME: for historical reasons, we force a notify here if
192 * there are outgoing parts to the buffer. Presumably the
193 * host should service the ring ASAP. */
194 if (out)
195 vq->notify(&vq->vq);
196 END_USE(vq);
197 return -ENOSPC;
198 }
199
200 /* We're about to use some buffers from the free list. */
201 vq->num_free -= out + in;
202
203 head = vq->free_head;
204 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) {
205 vq->vring.desc[i].flags = VRING_DESC_F_NEXT;
206 vq->vring.desc[i].addr = sg_phys(sg);
207 vq->vring.desc[i].len = sg->length;
208 prev = i;
209 sg++;
210 }
211 for (; in; i = vq->vring.desc[i].next, in--) {
212 vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
213 vq->vring.desc[i].addr = sg_phys(sg);
214 vq->vring.desc[i].len = sg->length;
215 prev = i;
216 sg++;
217 }
218 /* Last one doesn't continue. */
219 vq->vring.desc[prev].flags &= ~VRING_DESC_F_NEXT;
220
221 /* Update free pointer */
222 vq->free_head = i;
223
224add_head:
225 /* Set token. */
226 vq->data[head] = data;
227
228 /* Put entry in available array (but don't update avail->idx until they
229 * do sync). FIXME: avoid modulus here? */
230 avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num;
231 vq->vring.avail->ring[avail] = head;
232
233 pr_debug("Added buffer head %i to %p\n", head, vq);
234 END_USE(vq);
235
236 return vq->num_free;
237}
238EXPORT_SYMBOL_GPL(virtqueue_add_buf_gfp);
239
240void virtqueue_kick(struct virtqueue *_vq)
241{
242 struct vring_virtqueue *vq = to_vvq(_vq);
243 u16 new, old;
244 START_USE(vq);
245 /* Descriptors and available array need to be set before we expose the
246 * new available array entries. */
247 virtio_wmb();
248
249 old = vq->vring.avail->idx;
250 new = vq->vring.avail->idx = old + vq->num_added;
251 vq->num_added = 0;
252
253 /* Need to update avail index before checking if we should notify */
254 virtio_mb();
255
256 if (vq->event ?
257 vring_need_event(vring_avail_event(&vq->vring), new, old) :
258 !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
259 /* Prod other side to tell it about changes. */
260 vq->notify(&vq->vq);
261
262 END_USE(vq);
263}
264EXPORT_SYMBOL_GPL(virtqueue_kick);
265
266static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
267{
268 unsigned int i;
269
270 /* Clear data ptr. */
271 vq->data[head] = NULL;
272
273 /* Put back on free list: find end */
274 i = head;
275
276 /* Free the indirect table */
277 if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
278 kfree(phys_to_virt(vq->vring.desc[i].addr));
279
280 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
281 i = vq->vring.desc[i].next;
282 vq->num_free++;
283 }
284
285 vq->vring.desc[i].next = vq->free_head;
286 vq->free_head = head;
287 /* Plus final descriptor */
288 vq->num_free++;
289}
290
291static inline bool more_used(const struct vring_virtqueue *vq)
292{
293 return vq->last_used_idx != vq->vring.used->idx;
294}
295
296void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
297{
298 struct vring_virtqueue *vq = to_vvq(_vq);
299 void *ret;
300 unsigned int i;
301
302 START_USE(vq);
303
304 if (unlikely(vq->broken)) {
305 END_USE(vq);
306 return NULL;
307 }
308
309 if (!more_used(vq)) {
310 pr_debug("No more buffers in queue\n");
311 END_USE(vq);
312 return NULL;
313 }
314
315 /* Only get used array entries after they have been exposed by host. */
316 virtio_rmb();
317
318 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
319 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
320
321 if (unlikely(i >= vq->vring.num)) {
322 BAD_RING(vq, "id %u out of range\n", i);
323 return NULL;
324 }
325 if (unlikely(!vq->data[i])) {
326 BAD_RING(vq, "id %u is not a head!\n", i);
327 return NULL;
328 }
329
330 /* detach_buf clears data, so grab it now. */
331 ret = vq->data[i];
332 detach_buf(vq, i);
333 vq->last_used_idx++;
334 /* If we expect an interrupt for the next entry, tell host
335 * by writing event index and flush out the write before
336 * the read in the next get_buf call. */
337 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
338 vring_used_event(&vq->vring) = vq->last_used_idx;
339 virtio_mb();
340 }
341
342 END_USE(vq);
343 return ret;
344}
345EXPORT_SYMBOL_GPL(virtqueue_get_buf);
346
347void virtqueue_disable_cb(struct virtqueue *_vq)
348{
349 struct vring_virtqueue *vq = to_vvq(_vq);
350
351 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
352}
353EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
354
355bool virtqueue_enable_cb(struct virtqueue *_vq)
356{
357 struct vring_virtqueue *vq = to_vvq(_vq);
358
359 START_USE(vq);
360
361 /* We optimistically turn back on interrupts, then check if there was
362 * more to do. */
363 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
364 * either clear the flags bit or point the event index at the next
365 * entry. Always do both to keep code simple. */
366 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
367 vring_used_event(&vq->vring) = vq->last_used_idx;
368 virtio_mb();
369 if (unlikely(more_used(vq))) {
370 END_USE(vq);
371 return false;
372 }
373
374 END_USE(vq);
375 return true;
376}
377EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
378
379bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
380{
381 struct vring_virtqueue *vq = to_vvq(_vq);
382 u16 bufs;
383
384 START_USE(vq);
385
386 /* We optimistically turn back on interrupts, then check if there was
387 * more to do. */
388 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
389 * either clear the flags bit or point the event index at the next
390 * entry. Always do both to keep code simple. */
391 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
392 /* TODO: tune this threshold */
393 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
394 vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
395 virtio_mb();
396 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
397 END_USE(vq);
398 return false;
399 }
400
401 END_USE(vq);
402 return true;
403}
404EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
405
406void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
407{
408 struct vring_virtqueue *vq = to_vvq(_vq);
409 unsigned int i;
410 void *buf;
411
412 START_USE(vq);
413
414 for (i = 0; i < vq->vring.num; i++) {
415 if (!vq->data[i])
416 continue;
417 /* detach_buf clears data, so grab it now. */
418 buf = vq->data[i];
419 detach_buf(vq, i);
420 vq->vring.avail->idx--;
421 END_USE(vq);
422 return buf;
423 }
424 /* That should have freed everything. */
425 BUG_ON(vq->num_free != vq->vring.num);
426
427 END_USE(vq);
428 return NULL;
429}
430EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
431
432irqreturn_t vring_interrupt(int irq, void *_vq)
433{
434 struct vring_virtqueue *vq = to_vvq(_vq);
435
436 if (!more_used(vq)) {
437 pr_debug("virtqueue interrupt with no work for %p\n", vq);
438 return IRQ_NONE;
439 }
440
441 if (unlikely(vq->broken))
442 return IRQ_HANDLED;
443
444 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
445 if (vq->vq.callback)
446 vq->vq.callback(&vq->vq);
447
448 return IRQ_HANDLED;
449}
450EXPORT_SYMBOL_GPL(vring_interrupt);
451
452struct virtqueue *vring_new_virtqueue(unsigned int num,
453 unsigned int vring_align,
454 struct virtio_device *vdev,
455 void *pages,
456 void (*notify)(struct virtqueue *),
457 void (*callback)(struct virtqueue *),
458 const char *name)
459{
460 struct vring_virtqueue *vq;
461 unsigned int i;
462
463 /* We assume num is a power of 2. */
464 if (num & (num - 1)) {
465 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
466 return NULL;
467 }
468
469 vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);
470 if (!vq)
471 return NULL;
472
473 vring_init(&vq->vring, num, pages, vring_align);
474 vq->vq.callback = callback;
475 vq->vq.vdev = vdev;
476 vq->vq.name = name;
477 vq->notify = notify;
478 vq->broken = false;
479 vq->last_used_idx = 0;
480 vq->num_added = 0;
481 list_add_tail(&vq->vq.list, &vdev->vqs);
482#ifdef DEBUG
483 vq->in_use = false;
484#endif
485
486 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
487 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
488
489 /* No callback? Tell other side not to bother us. */
490 if (!callback)
491 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
492
493 /* Put everything in free lists. */
494 vq->num_free = num;
495 vq->free_head = 0;
496 for (i = 0; i < num-1; i++) {
497 vq->vring.desc[i].next = i+1;
498 vq->data[i] = NULL;
499 }
500 vq->data[i] = NULL;
501
502 return &vq->vq;
503}
504EXPORT_SYMBOL_GPL(vring_new_virtqueue);
505
506void vring_del_virtqueue(struct virtqueue *vq)
507{
508 list_del(&vq->list);
509 kfree(to_vvq(vq));
510}
511EXPORT_SYMBOL_GPL(vring_del_virtqueue);
512
513/* Manipulates transport-specific feature bits. */
514void vring_transport_features(struct virtio_device *vdev)
515{
516 unsigned int i;
517
518 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
519 switch (i) {
520 case VIRTIO_RING_F_INDIRECT_DESC:
521 break;
522 case VIRTIO_RING_F_EVENT_IDX:
523 break;
524 default:
525 /* We don't understand this bit. */
526 clear_bit(i, vdev->features);
527 }
528 }
529}
530EXPORT_SYMBOL_GPL(vring_transport_features);
531
532MODULE_LICENSE("GPL");
1/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19#include <linux/virtio.h>
20#include <linux/virtio_ring.h>
21#include <linux/virtio_config.h>
22#include <linux/device.h>
23#include <linux/slab.h>
24#include <linux/module.h>
25#include <linux/hrtimer.h>
26#include <linux/kmemleak.h>
27#include <linux/dma-mapping.h>
28#include <xen/xen.h>
29
30#ifdef DEBUG
31/* For development, we want to crash whenever the ring is screwed. */
32#define BAD_RING(_vq, fmt, args...) \
33 do { \
34 dev_err(&(_vq)->vq.vdev->dev, \
35 "%s:"fmt, (_vq)->vq.name, ##args); \
36 BUG(); \
37 } while (0)
38/* Caller is supposed to guarantee no reentry. */
39#define START_USE(_vq) \
40 do { \
41 if ((_vq)->in_use) \
42 panic("%s:in_use = %i\n", \
43 (_vq)->vq.name, (_vq)->in_use); \
44 (_vq)->in_use = __LINE__; \
45 } while (0)
46#define END_USE(_vq) \
47 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
48#else
49#define BAD_RING(_vq, fmt, args...) \
50 do { \
51 dev_err(&_vq->vq.vdev->dev, \
52 "%s:"fmt, (_vq)->vq.name, ##args); \
53 (_vq)->broken = true; \
54 } while (0)
55#define START_USE(vq)
56#define END_USE(vq)
57#endif
58
59struct vring_desc_state {
60 void *data; /* Data for callback. */
61 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
62};
63
64struct vring_virtqueue {
65 struct virtqueue vq;
66
67 /* Actual memory layout for this queue */
68 struct vring vring;
69
70 /* Can we use weak barriers? */
71 bool weak_barriers;
72
73 /* Other side has made a mess, don't try any more. */
74 bool broken;
75
76 /* Host supports indirect buffers */
77 bool indirect;
78
79 /* Host publishes avail event idx */
80 bool event;
81
82 /* Head of free buffer list. */
83 unsigned int free_head;
84 /* Number we've added since last sync. */
85 unsigned int num_added;
86
87 /* Last used index we've seen. */
88 u16 last_used_idx;
89
90 /* Last written value to avail->flags */
91 u16 avail_flags_shadow;
92
93 /* Last written value to avail->idx in guest byte order */
94 u16 avail_idx_shadow;
95
96 /* How to notify other side. FIXME: commonalize hcalls! */
97 bool (*notify)(struct virtqueue *vq);
98
99 /* DMA, allocation, and size information */
100 bool we_own_ring;
101 size_t queue_size_in_bytes;
102 dma_addr_t queue_dma_addr;
103
104#ifdef DEBUG
105 /* They're supposed to lock for us. */
106 unsigned int in_use;
107
108 /* Figure out if their kicks are too delayed. */
109 bool last_add_time_valid;
110 ktime_t last_add_time;
111#endif
112
113 /* Per-descriptor state. */
114 struct vring_desc_state desc_state[];
115};
116
117#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
118
119/*
120 * Modern virtio devices have feature bits to specify whether they need a
121 * quirk and bypass the IOMMU. If not there, just use the DMA API.
122 *
123 * If there, the interaction between virtio and DMA API is messy.
124 *
125 * On most systems with virtio, physical addresses match bus addresses,
126 * and it doesn't particularly matter whether we use the DMA API.
127 *
128 * On some systems, including Xen and any system with a physical device
129 * that speaks virtio behind a physical IOMMU, we must use the DMA API
130 * for virtio DMA to work at all.
131 *
132 * On other systems, including SPARC and PPC64, virtio-pci devices are
133 * enumerated as though they are behind an IOMMU, but the virtio host
134 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
135 * there or somehow map everything as the identity.
136 *
137 * For the time being, we preserve historic behavior and bypass the DMA
138 * API.
139 *
140 * TODO: install a per-device DMA ops structure that does the right thing
141 * taking into account all the above quirks, and use the DMA API
142 * unconditionally on data path.
143 */
144
145static bool vring_use_dma_api(struct virtio_device *vdev)
146{
147 if (!virtio_has_iommu_quirk(vdev))
148 return true;
149
150 /* Otherwise, we are left to guess. */
151 /*
152 * In theory, it's possible to have a buggy QEMU-supposed
153 * emulated Q35 IOMMU and Xen enabled at the same time. On
154 * such a configuration, virtio has never worked and will
155 * not work without an even larger kludge. Instead, enable
156 * the DMA API if we're a Xen guest, which at least allows
157 * all of the sensible Xen configurations to work correctly.
158 */
159 if (xen_domain())
160 return true;
161
162 return false;
163}
164
165/*
166 * The DMA ops on various arches are rather gnarly right now, and
167 * making all of the arch DMA ops work on the vring device itself
168 * is a mess. For now, we use the parent device for DMA ops.
169 */
170static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
171{
172 return vq->vq.vdev->dev.parent;
173}
174
175/* Map one sg entry. */
176static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
177 struct scatterlist *sg,
178 enum dma_data_direction direction)
179{
180 if (!vring_use_dma_api(vq->vq.vdev))
181 return (dma_addr_t)sg_phys(sg);
182
183 /*
184 * We can't use dma_map_sg, because we don't use scatterlists in
185 * the way it expects (we don't guarantee that the scatterlist
186 * will exist for the lifetime of the mapping).
187 */
188 return dma_map_page(vring_dma_dev(vq),
189 sg_page(sg), sg->offset, sg->length,
190 direction);
191}
192
193static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
194 void *cpu_addr, size_t size,
195 enum dma_data_direction direction)
196{
197 if (!vring_use_dma_api(vq->vq.vdev))
198 return (dma_addr_t)virt_to_phys(cpu_addr);
199
200 return dma_map_single(vring_dma_dev(vq),
201 cpu_addr, size, direction);
202}
203
204static void vring_unmap_one(const struct vring_virtqueue *vq,
205 struct vring_desc *desc)
206{
207 u16 flags;
208
209 if (!vring_use_dma_api(vq->vq.vdev))
210 return;
211
212 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
213
214 if (flags & VRING_DESC_F_INDIRECT) {
215 dma_unmap_single(vring_dma_dev(vq),
216 virtio64_to_cpu(vq->vq.vdev, desc->addr),
217 virtio32_to_cpu(vq->vq.vdev, desc->len),
218 (flags & VRING_DESC_F_WRITE) ?
219 DMA_FROM_DEVICE : DMA_TO_DEVICE);
220 } else {
221 dma_unmap_page(vring_dma_dev(vq),
222 virtio64_to_cpu(vq->vq.vdev, desc->addr),
223 virtio32_to_cpu(vq->vq.vdev, desc->len),
224 (flags & VRING_DESC_F_WRITE) ?
225 DMA_FROM_DEVICE : DMA_TO_DEVICE);
226 }
227}
228
229static int vring_mapping_error(const struct vring_virtqueue *vq,
230 dma_addr_t addr)
231{
232 if (!vring_use_dma_api(vq->vq.vdev))
233 return 0;
234
235 return dma_mapping_error(vring_dma_dev(vq), addr);
236}
237
238static struct vring_desc *alloc_indirect(struct virtqueue *_vq,
239 unsigned int total_sg, gfp_t gfp)
240{
241 struct vring_desc *desc;
242 unsigned int i;
243
244 /*
245 * We require lowmem mappings for the descriptors because
246 * otherwise virt_to_phys will give us bogus addresses in the
247 * virtqueue.
248 */
249 gfp &= ~__GFP_HIGHMEM;
250
251 desc = kmalloc(total_sg * sizeof(struct vring_desc), gfp);
252 if (!desc)
253 return NULL;
254
255 for (i = 0; i < total_sg; i++)
256 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
257 return desc;
258}
259
260static inline int virtqueue_add(struct virtqueue *_vq,
261 struct scatterlist *sgs[],
262 unsigned int total_sg,
263 unsigned int out_sgs,
264 unsigned int in_sgs,
265 void *data,
266 gfp_t gfp)
267{
268 struct vring_virtqueue *vq = to_vvq(_vq);
269 struct scatterlist *sg;
270 struct vring_desc *desc;
271 unsigned int i, n, avail, descs_used, uninitialized_var(prev), err_idx;
272 int head;
273 bool indirect;
274
275 START_USE(vq);
276
277 BUG_ON(data == NULL);
278
279 if (unlikely(vq->broken)) {
280 END_USE(vq);
281 return -EIO;
282 }
283
284#ifdef DEBUG
285 {
286 ktime_t now = ktime_get();
287
288 /* No kick or get, with .1 second between? Warn. */
289 if (vq->last_add_time_valid)
290 WARN_ON(ktime_to_ms(ktime_sub(now, vq->last_add_time))
291 > 100);
292 vq->last_add_time = now;
293 vq->last_add_time_valid = true;
294 }
295#endif
296
297 BUG_ON(total_sg > vq->vring.num);
298 BUG_ON(total_sg == 0);
299
300 head = vq->free_head;
301
302 /* If the host supports indirect descriptor tables, and we have multiple
303 * buffers, then go indirect. FIXME: tune this threshold */
304 if (vq->indirect && total_sg > 1 && vq->vq.num_free)
305 desc = alloc_indirect(_vq, total_sg, gfp);
306 else
307 desc = NULL;
308
309 if (desc) {
310 /* Use a single buffer which doesn't continue */
311 indirect = true;
312 /* Set up rest to use this indirect table. */
313 i = 0;
314 descs_used = 1;
315 } else {
316 indirect = false;
317 desc = vq->vring.desc;
318 i = head;
319 descs_used = total_sg;
320 }
321
322 if (vq->vq.num_free < descs_used) {
323 pr_debug("Can't add buf len %i - avail = %i\n",
324 descs_used, vq->vq.num_free);
325 /* FIXME: for historical reasons, we force a notify here if
326 * there are outgoing parts to the buffer. Presumably the
327 * host should service the ring ASAP. */
328 if (out_sgs)
329 vq->notify(&vq->vq);
330 if (indirect)
331 kfree(desc);
332 END_USE(vq);
333 return -ENOSPC;
334 }
335
336 for (n = 0; n < out_sgs; n++) {
337 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
338 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
339 if (vring_mapping_error(vq, addr))
340 goto unmap_release;
341
342 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);
343 desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
344 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
345 prev = i;
346 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
347 }
348 }
349 for (; n < (out_sgs + in_sgs); n++) {
350 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
351 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
352 if (vring_mapping_error(vq, addr))
353 goto unmap_release;
354
355 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);
356 desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
357 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
358 prev = i;
359 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
360 }
361 }
362 /* Last one doesn't continue. */
363 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
364
365 if (indirect) {
366 /* Now that the indirect table is filled in, map it. */
367 dma_addr_t addr = vring_map_single(
368 vq, desc, total_sg * sizeof(struct vring_desc),
369 DMA_TO_DEVICE);
370 if (vring_mapping_error(vq, addr))
371 goto unmap_release;
372
373 vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);
374 vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, addr);
375
376 vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));
377 }
378
379 /* We're using some buffers from the free list. */
380 vq->vq.num_free -= descs_used;
381
382 /* Update free pointer */
383 if (indirect)
384 vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);
385 else
386 vq->free_head = i;
387
388 /* Store token and indirect buffer state. */
389 vq->desc_state[head].data = data;
390 if (indirect)
391 vq->desc_state[head].indir_desc = desc;
392
393 /* Put entry in available array (but don't update avail->idx until they
394 * do sync). */
395 avail = vq->avail_idx_shadow & (vq->vring.num - 1);
396 vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
397
398 /* Descriptors and available array need to be set before we expose the
399 * new available array entries. */
400 virtio_wmb(vq->weak_barriers);
401 vq->avail_idx_shadow++;
402 vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);
403 vq->num_added++;
404
405 pr_debug("Added buffer head %i to %p\n", head, vq);
406 END_USE(vq);
407
408 /* This is very unlikely, but theoretically possible. Kick
409 * just in case. */
410 if (unlikely(vq->num_added == (1 << 16) - 1))
411 virtqueue_kick(_vq);
412
413 return 0;
414
415unmap_release:
416 err_idx = i;
417 i = head;
418
419 for (n = 0; n < total_sg; n++) {
420 if (i == err_idx)
421 break;
422 vring_unmap_one(vq, &desc[i]);
423 i = virtio16_to_cpu(_vq->vdev, vq->vring.desc[i].next);
424 }
425
426 vq->vq.num_free += total_sg;
427
428 if (indirect)
429 kfree(desc);
430
431 END_USE(vq);
432 return -EIO;
433}
434
435/**
436 * virtqueue_add_sgs - expose buffers to other end
437 * @vq: the struct virtqueue we're talking about.
438 * @sgs: array of terminated scatterlists.
439 * @out_num: the number of scatterlists readable by other side
440 * @in_num: the number of scatterlists which are writable (after readable ones)
441 * @data: the token identifying the buffer.
442 * @gfp: how to do memory allocations (if necessary).
443 *
444 * Caller must ensure we don't call this with other virtqueue operations
445 * at the same time (except where noted).
446 *
447 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
448 */
449int virtqueue_add_sgs(struct virtqueue *_vq,
450 struct scatterlist *sgs[],
451 unsigned int out_sgs,
452 unsigned int in_sgs,
453 void *data,
454 gfp_t gfp)
455{
456 unsigned int i, total_sg = 0;
457
458 /* Count them first. */
459 for (i = 0; i < out_sgs + in_sgs; i++) {
460 struct scatterlist *sg;
461 for (sg = sgs[i]; sg; sg = sg_next(sg))
462 total_sg++;
463 }
464 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs, data, gfp);
465}
466EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
467
468/**
469 * virtqueue_add_outbuf - expose output buffers to other end
470 * @vq: the struct virtqueue we're talking about.
471 * @sg: scatterlist (must be well-formed and terminated!)
472 * @num: the number of entries in @sg readable by other side
473 * @data: the token identifying the buffer.
474 * @gfp: how to do memory allocations (if necessary).
475 *
476 * Caller must ensure we don't call this with other virtqueue operations
477 * at the same time (except where noted).
478 *
479 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
480 */
481int virtqueue_add_outbuf(struct virtqueue *vq,
482 struct scatterlist *sg, unsigned int num,
483 void *data,
484 gfp_t gfp)
485{
486 return virtqueue_add(vq, &sg, num, 1, 0, data, gfp);
487}
488EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
489
490/**
491 * virtqueue_add_inbuf - expose input buffers to other end
492 * @vq: the struct virtqueue we're talking about.
493 * @sg: scatterlist (must be well-formed and terminated!)
494 * @num: the number of entries in @sg writable by other side
495 * @data: the token identifying the buffer.
496 * @gfp: how to do memory allocations (if necessary).
497 *
498 * Caller must ensure we don't call this with other virtqueue operations
499 * at the same time (except where noted).
500 *
501 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
502 */
503int virtqueue_add_inbuf(struct virtqueue *vq,
504 struct scatterlist *sg, unsigned int num,
505 void *data,
506 gfp_t gfp)
507{
508 return virtqueue_add(vq, &sg, num, 0, 1, data, gfp);
509}
510EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
511
512/**
513 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
514 * @vq: the struct virtqueue
515 *
516 * Instead of virtqueue_kick(), you can do:
517 * if (virtqueue_kick_prepare(vq))
518 * virtqueue_notify(vq);
519 *
520 * This is sometimes useful because the virtqueue_kick_prepare() needs
521 * to be serialized, but the actual virtqueue_notify() call does not.
522 */
523bool virtqueue_kick_prepare(struct virtqueue *_vq)
524{
525 struct vring_virtqueue *vq = to_vvq(_vq);
526 u16 new, old;
527 bool needs_kick;
528
529 START_USE(vq);
530 /* We need to expose available array entries before checking avail
531 * event. */
532 virtio_mb(vq->weak_barriers);
533
534 old = vq->avail_idx_shadow - vq->num_added;
535 new = vq->avail_idx_shadow;
536 vq->num_added = 0;
537
538#ifdef DEBUG
539 if (vq->last_add_time_valid) {
540 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(),
541 vq->last_add_time)) > 100);
542 }
543 vq->last_add_time_valid = false;
544#endif
545
546 if (vq->event) {
547 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),
548 new, old);
549 } else {
550 needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));
551 }
552 END_USE(vq);
553 return needs_kick;
554}
555EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
556
557/**
558 * virtqueue_notify - second half of split virtqueue_kick call.
559 * @vq: the struct virtqueue
560 *
561 * This does not need to be serialized.
562 *
563 * Returns false if host notify failed or queue is broken, otherwise true.
564 */
565bool virtqueue_notify(struct virtqueue *_vq)
566{
567 struct vring_virtqueue *vq = to_vvq(_vq);
568
569 if (unlikely(vq->broken))
570 return false;
571
572 /* Prod other side to tell it about changes. */
573 if (!vq->notify(_vq)) {
574 vq->broken = true;
575 return false;
576 }
577 return true;
578}
579EXPORT_SYMBOL_GPL(virtqueue_notify);
580
581/**
582 * virtqueue_kick - update after add_buf
583 * @vq: the struct virtqueue
584 *
585 * After one or more virtqueue_add_* calls, invoke this to kick
586 * the other side.
587 *
588 * Caller must ensure we don't call this with other virtqueue
589 * operations at the same time (except where noted).
590 *
591 * Returns false if kick failed, otherwise true.
592 */
593bool virtqueue_kick(struct virtqueue *vq)
594{
595 if (virtqueue_kick_prepare(vq))
596 return virtqueue_notify(vq);
597 return true;
598}
599EXPORT_SYMBOL_GPL(virtqueue_kick);
600
601static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
602{
603 unsigned int i, j;
604 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
605
606 /* Clear data ptr. */
607 vq->desc_state[head].data = NULL;
608
609 /* Put back on free list: unmap first-level descriptors and find end */
610 i = head;
611
612 while (vq->vring.desc[i].flags & nextflag) {
613 vring_unmap_one(vq, &vq->vring.desc[i]);
614 i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);
615 vq->vq.num_free++;
616 }
617
618 vring_unmap_one(vq, &vq->vring.desc[i]);
619 vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);
620 vq->free_head = head;
621
622 /* Plus final descriptor */
623 vq->vq.num_free++;
624
625 /* Free the indirect table, if any, now that it's unmapped. */
626 if (vq->desc_state[head].indir_desc) {
627 struct vring_desc *indir_desc = vq->desc_state[head].indir_desc;
628 u32 len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len);
629
630 BUG_ON(!(vq->vring.desc[head].flags &
631 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
632 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
633
634 for (j = 0; j < len / sizeof(struct vring_desc); j++)
635 vring_unmap_one(vq, &indir_desc[j]);
636
637 kfree(vq->desc_state[head].indir_desc);
638 vq->desc_state[head].indir_desc = NULL;
639 }
640}
641
642static inline bool more_used(const struct vring_virtqueue *vq)
643{
644 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);
645}
646
647/**
648 * virtqueue_get_buf - get the next used buffer
649 * @vq: the struct virtqueue we're talking about.
650 * @len: the length written into the buffer
651 *
652 * If the device wrote data into the buffer, @len will be set to the
653 * amount written. This means you don't need to clear the buffer
654 * beforehand to ensure there's no data leakage in the case of short
655 * writes.
656 *
657 * Caller must ensure we don't call this with other virtqueue
658 * operations at the same time (except where noted).
659 *
660 * Returns NULL if there are no used buffers, or the "data" token
661 * handed to virtqueue_add_*().
662 */
663void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
664{
665 struct vring_virtqueue *vq = to_vvq(_vq);
666 void *ret;
667 unsigned int i;
668 u16 last_used;
669
670 START_USE(vq);
671
672 if (unlikely(vq->broken)) {
673 END_USE(vq);
674 return NULL;
675 }
676
677 if (!more_used(vq)) {
678 pr_debug("No more buffers in queue\n");
679 END_USE(vq);
680 return NULL;
681 }
682
683 /* Only get used array entries after they have been exposed by host. */
684 virtio_rmb(vq->weak_barriers);
685
686 last_used = (vq->last_used_idx & (vq->vring.num - 1));
687 i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);
688 *len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);
689
690 if (unlikely(i >= vq->vring.num)) {
691 BAD_RING(vq, "id %u out of range\n", i);
692 return NULL;
693 }
694 if (unlikely(!vq->desc_state[i].data)) {
695 BAD_RING(vq, "id %u is not a head!\n", i);
696 return NULL;
697 }
698
699 /* detach_buf clears data, so grab it now. */
700 ret = vq->desc_state[i].data;
701 detach_buf(vq, i);
702 vq->last_used_idx++;
703 /* If we expect an interrupt for the next entry, tell host
704 * by writing event index and flush out the write before
705 * the read in the next get_buf call. */
706 if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
707 virtio_store_mb(vq->weak_barriers,
708 &vring_used_event(&vq->vring),
709 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
710
711#ifdef DEBUG
712 vq->last_add_time_valid = false;
713#endif
714
715 END_USE(vq);
716 return ret;
717}
718EXPORT_SYMBOL_GPL(virtqueue_get_buf);
719
720/**
721 * virtqueue_disable_cb - disable callbacks
722 * @vq: the struct virtqueue we're talking about.
723 *
724 * Note that this is not necessarily synchronous, hence unreliable and only
725 * useful as an optimization.
726 *
727 * Unlike other operations, this need not be serialized.
728 */
729void virtqueue_disable_cb(struct virtqueue *_vq)
730{
731 struct vring_virtqueue *vq = to_vvq(_vq);
732
733 if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
734 vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
735 if (!vq->event)
736 vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
737 }
738
739}
740EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
741
742/**
743 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
744 * @vq: the struct virtqueue we're talking about.
745 *
746 * This re-enables callbacks; it returns current queue state
747 * in an opaque unsigned value. This value should be later tested by
748 * virtqueue_poll, to detect a possible race between the driver checking for
749 * more work, and enabling callbacks.
750 *
751 * Caller must ensure we don't call this with other virtqueue
752 * operations at the same time (except where noted).
753 */
754unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
755{
756 struct vring_virtqueue *vq = to_vvq(_vq);
757 u16 last_used_idx;
758
759 START_USE(vq);
760
761 /* We optimistically turn back on interrupts, then check if there was
762 * more to do. */
763 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
764 * either clear the flags bit or point the event index at the next
765 * entry. Always do both to keep code simple. */
766 if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
767 vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
768 if (!vq->event)
769 vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
770 }
771 vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);
772 END_USE(vq);
773 return last_used_idx;
774}
775EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
776
777/**
778 * virtqueue_poll - query pending used buffers
779 * @vq: the struct virtqueue we're talking about.
780 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
781 *
782 * Returns "true" if there are pending used buffers in the queue.
783 *
784 * This does not need to be serialized.
785 */
786bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
787{
788 struct vring_virtqueue *vq = to_vvq(_vq);
789
790 virtio_mb(vq->weak_barriers);
791 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);
792}
793EXPORT_SYMBOL_GPL(virtqueue_poll);
794
795/**
796 * virtqueue_enable_cb - restart callbacks after disable_cb.
797 * @vq: the struct virtqueue we're talking about.
798 *
799 * This re-enables callbacks; it returns "false" if there are pending
800 * buffers in the queue, to detect a possible race between the driver
801 * checking for more work, and enabling callbacks.
802 *
803 * Caller must ensure we don't call this with other virtqueue
804 * operations at the same time (except where noted).
805 */
806bool virtqueue_enable_cb(struct virtqueue *_vq)
807{
808 unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);
809 return !virtqueue_poll(_vq, last_used_idx);
810}
811EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
812
813/**
814 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
815 * @vq: the struct virtqueue we're talking about.
816 *
817 * This re-enables callbacks but hints to the other side to delay
818 * interrupts until most of the available buffers have been processed;
819 * it returns "false" if there are many pending buffers in the queue,
820 * to detect a possible race between the driver checking for more work,
821 * and enabling callbacks.
822 *
823 * Caller must ensure we don't call this with other virtqueue
824 * operations at the same time (except where noted).
825 */
826bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
827{
828 struct vring_virtqueue *vq = to_vvq(_vq);
829 u16 bufs;
830
831 START_USE(vq);
832
833 /* We optimistically turn back on interrupts, then check if there was
834 * more to do. */
835 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
836 * either clear the flags bit or point the event index at the next
837 * entry. Always update the event index to keep code simple. */
838 if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
839 vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
840 if (!vq->event)
841 vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
842 }
843 /* TODO: tune this threshold */
844 bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4;
845
846 virtio_store_mb(vq->weak_barriers,
847 &vring_used_event(&vq->vring),
848 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
849
850 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {
851 END_USE(vq);
852 return false;
853 }
854
855 END_USE(vq);
856 return true;
857}
858EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
859
860/**
861 * virtqueue_detach_unused_buf - detach first unused buffer
862 * @vq: the struct virtqueue we're talking about.
863 *
864 * Returns NULL or the "data" token handed to virtqueue_add_*().
865 * This is not valid on an active queue; it is useful only for device
866 * shutdown.
867 */
868void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
869{
870 struct vring_virtqueue *vq = to_vvq(_vq);
871 unsigned int i;
872 void *buf;
873
874 START_USE(vq);
875
876 for (i = 0; i < vq->vring.num; i++) {
877 if (!vq->desc_state[i].data)
878 continue;
879 /* detach_buf clears data, so grab it now. */
880 buf = vq->desc_state[i].data;
881 detach_buf(vq, i);
882 vq->avail_idx_shadow--;
883 vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);
884 END_USE(vq);
885 return buf;
886 }
887 /* That should have freed everything. */
888 BUG_ON(vq->vq.num_free != vq->vring.num);
889
890 END_USE(vq);
891 return NULL;
892}
893EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
894
895irqreturn_t vring_interrupt(int irq, void *_vq)
896{
897 struct vring_virtqueue *vq = to_vvq(_vq);
898
899 if (!more_used(vq)) {
900 pr_debug("virtqueue interrupt with no work for %p\n", vq);
901 return IRQ_NONE;
902 }
903
904 if (unlikely(vq->broken))
905 return IRQ_HANDLED;
906
907 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
908 if (vq->vq.callback)
909 vq->vq.callback(&vq->vq);
910
911 return IRQ_HANDLED;
912}
913EXPORT_SYMBOL_GPL(vring_interrupt);
914
915struct virtqueue *__vring_new_virtqueue(unsigned int index,
916 struct vring vring,
917 struct virtio_device *vdev,
918 bool weak_barriers,
919 bool (*notify)(struct virtqueue *),
920 void (*callback)(struct virtqueue *),
921 const char *name)
922{
923 unsigned int i;
924 struct vring_virtqueue *vq;
925
926 vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state),
927 GFP_KERNEL);
928 if (!vq)
929 return NULL;
930
931 vq->vring = vring;
932 vq->vq.callback = callback;
933 vq->vq.vdev = vdev;
934 vq->vq.name = name;
935 vq->vq.num_free = vring.num;
936 vq->vq.index = index;
937 vq->we_own_ring = false;
938 vq->queue_dma_addr = 0;
939 vq->queue_size_in_bytes = 0;
940 vq->notify = notify;
941 vq->weak_barriers = weak_barriers;
942 vq->broken = false;
943 vq->last_used_idx = 0;
944 vq->avail_flags_shadow = 0;
945 vq->avail_idx_shadow = 0;
946 vq->num_added = 0;
947 list_add_tail(&vq->vq.list, &vdev->vqs);
948#ifdef DEBUG
949 vq->in_use = false;
950 vq->last_add_time_valid = false;
951#endif
952
953 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
954 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
955
956 /* No callback? Tell other side not to bother us. */
957 if (!callback) {
958 vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
959 if (!vq->event)
960 vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);
961 }
962
963 /* Put everything in free lists. */
964 vq->free_head = 0;
965 for (i = 0; i < vring.num-1; i++)
966 vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
967 memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state));
968
969 return &vq->vq;
970}
971EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
972
973static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
974 dma_addr_t *dma_handle, gfp_t flag)
975{
976 if (vring_use_dma_api(vdev)) {
977 return dma_alloc_coherent(vdev->dev.parent, size,
978 dma_handle, flag);
979 } else {
980 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
981 if (queue) {
982 phys_addr_t phys_addr = virt_to_phys(queue);
983 *dma_handle = (dma_addr_t)phys_addr;
984
985 /*
986 * Sanity check: make sure we dind't truncate
987 * the address. The only arches I can find that
988 * have 64-bit phys_addr_t but 32-bit dma_addr_t
989 * are certain non-highmem MIPS and x86
990 * configurations, but these configurations
991 * should never allocate physical pages above 32
992 * bits, so this is fine. Just in case, throw a
993 * warning and abort if we end up with an
994 * unrepresentable address.
995 */
996 if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
997 free_pages_exact(queue, PAGE_ALIGN(size));
998 return NULL;
999 }
1000 }
1001 return queue;
1002 }
1003}
1004
1005static void vring_free_queue(struct virtio_device *vdev, size_t size,
1006 void *queue, dma_addr_t dma_handle)
1007{
1008 if (vring_use_dma_api(vdev)) {
1009 dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
1010 } else {
1011 free_pages_exact(queue, PAGE_ALIGN(size));
1012 }
1013}
1014
1015struct virtqueue *vring_create_virtqueue(
1016 unsigned int index,
1017 unsigned int num,
1018 unsigned int vring_align,
1019 struct virtio_device *vdev,
1020 bool weak_barriers,
1021 bool may_reduce_num,
1022 bool (*notify)(struct virtqueue *),
1023 void (*callback)(struct virtqueue *),
1024 const char *name)
1025{
1026 struct virtqueue *vq;
1027 void *queue = NULL;
1028 dma_addr_t dma_addr;
1029 size_t queue_size_in_bytes;
1030 struct vring vring;
1031
1032 /* We assume num is a power of 2. */
1033 if (num & (num - 1)) {
1034 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
1035 return NULL;
1036 }
1037
1038 /* TODO: allocate each queue chunk individually */
1039 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
1040 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1041 &dma_addr,
1042 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1043 if (queue)
1044 break;
1045 }
1046
1047 if (!num)
1048 return NULL;
1049
1050 if (!queue) {
1051 /* Try to get a single page. You are my only hope! */
1052 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1053 &dma_addr, GFP_KERNEL|__GFP_ZERO);
1054 }
1055 if (!queue)
1056 return NULL;
1057
1058 queue_size_in_bytes = vring_size(num, vring_align);
1059 vring_init(&vring, num, queue, vring_align);
1060
1061 vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers,
1062 notify, callback, name);
1063 if (!vq) {
1064 vring_free_queue(vdev, queue_size_in_bytes, queue,
1065 dma_addr);
1066 return NULL;
1067 }
1068
1069 to_vvq(vq)->queue_dma_addr = dma_addr;
1070 to_vvq(vq)->queue_size_in_bytes = queue_size_in_bytes;
1071 to_vvq(vq)->we_own_ring = true;
1072
1073 return vq;
1074}
1075EXPORT_SYMBOL_GPL(vring_create_virtqueue);
1076
1077struct virtqueue *vring_new_virtqueue(unsigned int index,
1078 unsigned int num,
1079 unsigned int vring_align,
1080 struct virtio_device *vdev,
1081 bool weak_barriers,
1082 void *pages,
1083 bool (*notify)(struct virtqueue *vq),
1084 void (*callback)(struct virtqueue *vq),
1085 const char *name)
1086{
1087 struct vring vring;
1088 vring_init(&vring, num, pages, vring_align);
1089 return __vring_new_virtqueue(index, vring, vdev, weak_barriers,
1090 notify, callback, name);
1091}
1092EXPORT_SYMBOL_GPL(vring_new_virtqueue);
1093
1094void vring_del_virtqueue(struct virtqueue *_vq)
1095{
1096 struct vring_virtqueue *vq = to_vvq(_vq);
1097
1098 if (vq->we_own_ring) {
1099 vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes,
1100 vq->vring.desc, vq->queue_dma_addr);
1101 }
1102 list_del(&_vq->list);
1103 kfree(vq);
1104}
1105EXPORT_SYMBOL_GPL(vring_del_virtqueue);
1106
1107/* Manipulates transport-specific feature bits. */
1108void vring_transport_features(struct virtio_device *vdev)
1109{
1110 unsigned int i;
1111
1112 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
1113 switch (i) {
1114 case VIRTIO_RING_F_INDIRECT_DESC:
1115 break;
1116 case VIRTIO_RING_F_EVENT_IDX:
1117 break;
1118 case VIRTIO_F_VERSION_1:
1119 break;
1120 case VIRTIO_F_IOMMU_PLATFORM:
1121 break;
1122 default:
1123 /* We don't understand this bit. */
1124 __virtio_clear_bit(vdev, i);
1125 }
1126 }
1127}
1128EXPORT_SYMBOL_GPL(vring_transport_features);
1129
1130/**
1131 * virtqueue_get_vring_size - return the size of the virtqueue's vring
1132 * @vq: the struct virtqueue containing the vring of interest.
1133 *
1134 * Returns the size of the vring. This is mainly used for boasting to
1135 * userspace. Unlike other operations, this need not be serialized.
1136 */
1137unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
1138{
1139
1140 struct vring_virtqueue *vq = to_vvq(_vq);
1141
1142 return vq->vring.num;
1143}
1144EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
1145
1146bool virtqueue_is_broken(struct virtqueue *_vq)
1147{
1148 struct vring_virtqueue *vq = to_vvq(_vq);
1149
1150 return vq->broken;
1151}
1152EXPORT_SYMBOL_GPL(virtqueue_is_broken);
1153
1154/*
1155 * This should prevent the device from being used, allowing drivers to
1156 * recover. You may need to grab appropriate locks to flush.
1157 */
1158void virtio_break_device(struct virtio_device *dev)
1159{
1160 struct virtqueue *_vq;
1161
1162 list_for_each_entry(_vq, &dev->vqs, list) {
1163 struct vring_virtqueue *vq = to_vvq(_vq);
1164 vq->broken = true;
1165 }
1166}
1167EXPORT_SYMBOL_GPL(virtio_break_device);
1168
1169dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
1170{
1171 struct vring_virtqueue *vq = to_vvq(_vq);
1172
1173 BUG_ON(!vq->we_own_ring);
1174
1175 return vq->queue_dma_addr;
1176}
1177EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
1178
1179dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
1180{
1181 struct vring_virtqueue *vq = to_vvq(_vq);
1182
1183 BUG_ON(!vq->we_own_ring);
1184
1185 return vq->queue_dma_addr +
1186 ((char *)vq->vring.avail - (char *)vq->vring.desc);
1187}
1188EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
1189
1190dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
1191{
1192 struct vring_virtqueue *vq = to_vvq(_vq);
1193
1194 BUG_ON(!vq->we_own_ring);
1195
1196 return vq->queue_dma_addr +
1197 ((char *)vq->vring.used - (char *)vq->vring.desc);
1198}
1199EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
1200
1201const struct vring *virtqueue_get_vring(struct virtqueue *vq)
1202{
1203 return &to_vvq(vq)->vring;
1204}
1205EXPORT_SYMBOL_GPL(virtqueue_get_vring);
1206
1207MODULE_LICENSE("GPL");