Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Virtio ring implementation.
3 *
4 * Copyright 2007 Rusty Russell IBM Corporation
5 */
6#include <linux/virtio.h>
7#include <linux/virtio_ring.h>
8#include <linux/virtio_config.h>
9#include <linux/device.h>
10#include <linux/slab.h>
11#include <linux/module.h>
12#include <linux/hrtimer.h>
13#include <linux/dma-mapping.h>
14#include <linux/kmsan.h>
15#include <linux/spinlock.h>
16#include <xen/xen.h>
17
18#ifdef DEBUG
19/* For development, we want to crash whenever the ring is screwed. */
20#define BAD_RING(_vq, fmt, args...) \
21 do { \
22 dev_err(&(_vq)->vq.vdev->dev, \
23 "%s:"fmt, (_vq)->vq.name, ##args); \
24 BUG(); \
25 } while (0)
26/* Caller is supposed to guarantee no reentry. */
27#define START_USE(_vq) \
28 do { \
29 if ((_vq)->in_use) \
30 panic("%s:in_use = %i\n", \
31 (_vq)->vq.name, (_vq)->in_use); \
32 (_vq)->in_use = __LINE__; \
33 } while (0)
34#define END_USE(_vq) \
35 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36#define LAST_ADD_TIME_UPDATE(_vq) \
37 do { \
38 ktime_t now = ktime_get(); \
39 \
40 /* No kick or get, with .1 second between? Warn. */ \
41 if ((_vq)->last_add_time_valid) \
42 WARN_ON(ktime_to_ms(ktime_sub(now, \
43 (_vq)->last_add_time)) > 100); \
44 (_vq)->last_add_time = now; \
45 (_vq)->last_add_time_valid = true; \
46 } while (0)
47#define LAST_ADD_TIME_CHECK(_vq) \
48 do { \
49 if ((_vq)->last_add_time_valid) { \
50 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
51 (_vq)->last_add_time)) > 100); \
52 } \
53 } while (0)
54#define LAST_ADD_TIME_INVALID(_vq) \
55 ((_vq)->last_add_time_valid = false)
56#else
57#define BAD_RING(_vq, fmt, args...) \
58 do { \
59 dev_err(&_vq->vq.vdev->dev, \
60 "%s:"fmt, (_vq)->vq.name, ##args); \
61 (_vq)->broken = true; \
62 } while (0)
63#define START_USE(vq)
64#define END_USE(vq)
65#define LAST_ADD_TIME_UPDATE(vq)
66#define LAST_ADD_TIME_CHECK(vq)
67#define LAST_ADD_TIME_INVALID(vq)
68#endif
69
70struct vring_desc_state_split {
71 void *data; /* Data for callback. */
72 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
73};
74
75struct vring_desc_state_packed {
76 void *data; /* Data for callback. */
77 struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
78 u16 num; /* Descriptor list length. */
79 u16 last; /* The last desc state in a list. */
80};
81
82struct vring_desc_extra {
83 dma_addr_t addr; /* Descriptor DMA addr. */
84 u32 len; /* Descriptor length. */
85 u16 flags; /* Descriptor flags. */
86 u16 next; /* The next desc state in a list. */
87};
88
89struct vring_virtqueue_split {
90 /* Actual memory layout for this queue. */
91 struct vring vring;
92
93 /* Last written value to avail->flags */
94 u16 avail_flags_shadow;
95
96 /*
97 * Last written value to avail->idx in
98 * guest byte order.
99 */
100 u16 avail_idx_shadow;
101
102 /* Per-descriptor state. */
103 struct vring_desc_state_split *desc_state;
104 struct vring_desc_extra *desc_extra;
105
106 /* DMA address and size information */
107 dma_addr_t queue_dma_addr;
108 size_t queue_size_in_bytes;
109
110 /*
111 * The parameters for creating vrings are reserved for creating new
112 * vring.
113 */
114 u32 vring_align;
115 bool may_reduce_num;
116};
117
118struct vring_virtqueue_packed {
119 /* Actual memory layout for this queue. */
120 struct {
121 unsigned int num;
122 struct vring_packed_desc *desc;
123 struct vring_packed_desc_event *driver;
124 struct vring_packed_desc_event *device;
125 } vring;
126
127 /* Driver ring wrap counter. */
128 bool avail_wrap_counter;
129
130 /* Avail used flags. */
131 u16 avail_used_flags;
132
133 /* Index of the next avail descriptor. */
134 u16 next_avail_idx;
135
136 /*
137 * Last written value to driver->flags in
138 * guest byte order.
139 */
140 u16 event_flags_shadow;
141
142 /* Per-descriptor state. */
143 struct vring_desc_state_packed *desc_state;
144 struct vring_desc_extra *desc_extra;
145
146 /* DMA address and size information */
147 dma_addr_t ring_dma_addr;
148 dma_addr_t driver_event_dma_addr;
149 dma_addr_t device_event_dma_addr;
150 size_t ring_size_in_bytes;
151 size_t event_size_in_bytes;
152};
153
154struct vring_virtqueue {
155 struct virtqueue vq;
156
157 /* Is this a packed ring? */
158 bool packed_ring;
159
160 /* Is DMA API used? */
161 bool use_dma_api;
162
163 /* Can we use weak barriers? */
164 bool weak_barriers;
165
166 /* Other side has made a mess, don't try any more. */
167 bool broken;
168
169 /* Host supports indirect buffers */
170 bool indirect;
171
172 /* Host publishes avail event idx */
173 bool event;
174
175 /* Do DMA mapping by driver */
176 bool premapped;
177
178 /* Do unmap or not for desc. Just when premapped is False and
179 * use_dma_api is true, this is true.
180 */
181 bool do_unmap;
182
183 /* Head of free buffer list. */
184 unsigned int free_head;
185 /* Number we've added since last sync. */
186 unsigned int num_added;
187
188 /* Last used index we've seen.
189 * for split ring, it just contains last used index
190 * for packed ring:
191 * bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index.
192 * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter.
193 */
194 u16 last_used_idx;
195
196 /* Hint for event idx: already triggered no need to disable. */
197 bool event_triggered;
198
199 union {
200 /* Available for split ring */
201 struct vring_virtqueue_split split;
202
203 /* Available for packed ring */
204 struct vring_virtqueue_packed packed;
205 };
206
207 /* How to notify other side. FIXME: commonalize hcalls! */
208 bool (*notify)(struct virtqueue *vq);
209
210 /* DMA, allocation, and size information */
211 bool we_own_ring;
212
213 /* Device used for doing DMA */
214 struct device *dma_dev;
215
216#ifdef DEBUG
217 /* They're supposed to lock for us. */
218 unsigned int in_use;
219
220 /* Figure out if their kicks are too delayed. */
221 bool last_add_time_valid;
222 ktime_t last_add_time;
223#endif
224};
225
226static struct virtqueue *__vring_new_virtqueue(unsigned int index,
227 struct vring_virtqueue_split *vring_split,
228 struct virtio_device *vdev,
229 bool weak_barriers,
230 bool context,
231 bool (*notify)(struct virtqueue *),
232 void (*callback)(struct virtqueue *),
233 const char *name,
234 struct device *dma_dev);
235static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
236static void vring_free(struct virtqueue *_vq);
237
238/*
239 * Helpers.
240 */
241
242#define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
243
244static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
245 unsigned int total_sg)
246{
247 /*
248 * If the host supports indirect descriptor tables, and we have multiple
249 * buffers, then go indirect. FIXME: tune this threshold
250 */
251 return (vq->indirect && total_sg > 1 && vq->vq.num_free);
252}
253
254/*
255 * Modern virtio devices have feature bits to specify whether they need a
256 * quirk and bypass the IOMMU. If not there, just use the DMA API.
257 *
258 * If there, the interaction between virtio and DMA API is messy.
259 *
260 * On most systems with virtio, physical addresses match bus addresses,
261 * and it doesn't particularly matter whether we use the DMA API.
262 *
263 * On some systems, including Xen and any system with a physical device
264 * that speaks virtio behind a physical IOMMU, we must use the DMA API
265 * for virtio DMA to work at all.
266 *
267 * On other systems, including SPARC and PPC64, virtio-pci devices are
268 * enumerated as though they are behind an IOMMU, but the virtio host
269 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
270 * there or somehow map everything as the identity.
271 *
272 * For the time being, we preserve historic behavior and bypass the DMA
273 * API.
274 *
275 * TODO: install a per-device DMA ops structure that does the right thing
276 * taking into account all the above quirks, and use the DMA API
277 * unconditionally on data path.
278 */
279
280static bool vring_use_dma_api(const struct virtio_device *vdev)
281{
282 if (!virtio_has_dma_quirk(vdev))
283 return true;
284
285 /* Otherwise, we are left to guess. */
286 /*
287 * In theory, it's possible to have a buggy QEMU-supposed
288 * emulated Q35 IOMMU and Xen enabled at the same time. On
289 * such a configuration, virtio has never worked and will
290 * not work without an even larger kludge. Instead, enable
291 * the DMA API if we're a Xen guest, which at least allows
292 * all of the sensible Xen configurations to work correctly.
293 */
294 if (xen_domain())
295 return true;
296
297 return false;
298}
299
300size_t virtio_max_dma_size(const struct virtio_device *vdev)
301{
302 size_t max_segment_size = SIZE_MAX;
303
304 if (vring_use_dma_api(vdev))
305 max_segment_size = dma_max_mapping_size(vdev->dev.parent);
306
307 return max_segment_size;
308}
309EXPORT_SYMBOL_GPL(virtio_max_dma_size);
310
311static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
312 dma_addr_t *dma_handle, gfp_t flag,
313 struct device *dma_dev)
314{
315 if (vring_use_dma_api(vdev)) {
316 return dma_alloc_coherent(dma_dev, size,
317 dma_handle, flag);
318 } else {
319 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
320
321 if (queue) {
322 phys_addr_t phys_addr = virt_to_phys(queue);
323 *dma_handle = (dma_addr_t)phys_addr;
324
325 /*
326 * Sanity check: make sure we dind't truncate
327 * the address. The only arches I can find that
328 * have 64-bit phys_addr_t but 32-bit dma_addr_t
329 * are certain non-highmem MIPS and x86
330 * configurations, but these configurations
331 * should never allocate physical pages above 32
332 * bits, so this is fine. Just in case, throw a
333 * warning and abort if we end up with an
334 * unrepresentable address.
335 */
336 if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
337 free_pages_exact(queue, PAGE_ALIGN(size));
338 return NULL;
339 }
340 }
341 return queue;
342 }
343}
344
345static void vring_free_queue(struct virtio_device *vdev, size_t size,
346 void *queue, dma_addr_t dma_handle,
347 struct device *dma_dev)
348{
349 if (vring_use_dma_api(vdev))
350 dma_free_coherent(dma_dev, size, queue, dma_handle);
351 else
352 free_pages_exact(queue, PAGE_ALIGN(size));
353}
354
355/*
356 * The DMA ops on various arches are rather gnarly right now, and
357 * making all of the arch DMA ops work on the vring device itself
358 * is a mess.
359 */
360static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
361{
362 return vq->dma_dev;
363}
364
365/* Map one sg entry. */
366static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg,
367 enum dma_data_direction direction, dma_addr_t *addr)
368{
369 if (vq->premapped) {
370 *addr = sg_dma_address(sg);
371 return 0;
372 }
373
374 if (!vq->use_dma_api) {
375 /*
376 * If DMA is not used, KMSAN doesn't know that the scatterlist
377 * is initialized by the hardware. Explicitly check/unpoison it
378 * depending on the direction.
379 */
380 kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction);
381 *addr = (dma_addr_t)sg_phys(sg);
382 return 0;
383 }
384
385 /*
386 * We can't use dma_map_sg, because we don't use scatterlists in
387 * the way it expects (we don't guarantee that the scatterlist
388 * will exist for the lifetime of the mapping).
389 */
390 *addr = dma_map_page(vring_dma_dev(vq),
391 sg_page(sg), sg->offset, sg->length,
392 direction);
393
394 if (dma_mapping_error(vring_dma_dev(vq), *addr))
395 return -ENOMEM;
396
397 return 0;
398}
399
400static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
401 void *cpu_addr, size_t size,
402 enum dma_data_direction direction)
403{
404 if (!vq->use_dma_api)
405 return (dma_addr_t)virt_to_phys(cpu_addr);
406
407 return dma_map_single(vring_dma_dev(vq),
408 cpu_addr, size, direction);
409}
410
411static int vring_mapping_error(const struct vring_virtqueue *vq,
412 dma_addr_t addr)
413{
414 if (!vq->use_dma_api)
415 return 0;
416
417 return dma_mapping_error(vring_dma_dev(vq), addr);
418}
419
420static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
421{
422 vq->vq.num_free = num;
423
424 if (vq->packed_ring)
425 vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
426 else
427 vq->last_used_idx = 0;
428
429 vq->event_triggered = false;
430 vq->num_added = 0;
431
432#ifdef DEBUG
433 vq->in_use = false;
434 vq->last_add_time_valid = false;
435#endif
436}
437
438
439/*
440 * Split ring specific functions - *_split().
441 */
442
443static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
444 const struct vring_desc *desc)
445{
446 u16 flags;
447
448 if (!vq->do_unmap)
449 return;
450
451 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
452
453 dma_unmap_page(vring_dma_dev(vq),
454 virtio64_to_cpu(vq->vq.vdev, desc->addr),
455 virtio32_to_cpu(vq->vq.vdev, desc->len),
456 (flags & VRING_DESC_F_WRITE) ?
457 DMA_FROM_DEVICE : DMA_TO_DEVICE);
458}
459
460static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
461 unsigned int i)
462{
463 struct vring_desc_extra *extra = vq->split.desc_extra;
464 u16 flags;
465
466 flags = extra[i].flags;
467
468 if (flags & VRING_DESC_F_INDIRECT) {
469 if (!vq->use_dma_api)
470 goto out;
471
472 dma_unmap_single(vring_dma_dev(vq),
473 extra[i].addr,
474 extra[i].len,
475 (flags & VRING_DESC_F_WRITE) ?
476 DMA_FROM_DEVICE : DMA_TO_DEVICE);
477 } else {
478 if (!vq->do_unmap)
479 goto out;
480
481 dma_unmap_page(vring_dma_dev(vq),
482 extra[i].addr,
483 extra[i].len,
484 (flags & VRING_DESC_F_WRITE) ?
485 DMA_FROM_DEVICE : DMA_TO_DEVICE);
486 }
487
488out:
489 return extra[i].next;
490}
491
492static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
493 unsigned int total_sg,
494 gfp_t gfp)
495{
496 struct vring_desc *desc;
497 unsigned int i;
498
499 /*
500 * We require lowmem mappings for the descriptors because
501 * otherwise virt_to_phys will give us bogus addresses in the
502 * virtqueue.
503 */
504 gfp &= ~__GFP_HIGHMEM;
505
506 desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
507 if (!desc)
508 return NULL;
509
510 for (i = 0; i < total_sg; i++)
511 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
512 return desc;
513}
514
515static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
516 struct vring_desc *desc,
517 unsigned int i,
518 dma_addr_t addr,
519 unsigned int len,
520 u16 flags,
521 bool indirect)
522{
523 struct vring_virtqueue *vring = to_vvq(vq);
524 struct vring_desc_extra *extra = vring->split.desc_extra;
525 u16 next;
526
527 desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
528 desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
529 desc[i].len = cpu_to_virtio32(vq->vdev, len);
530
531 if (!indirect) {
532 next = extra[i].next;
533 desc[i].next = cpu_to_virtio16(vq->vdev, next);
534
535 extra[i].addr = addr;
536 extra[i].len = len;
537 extra[i].flags = flags;
538 } else
539 next = virtio16_to_cpu(vq->vdev, desc[i].next);
540
541 return next;
542}
543
544static inline int virtqueue_add_split(struct virtqueue *_vq,
545 struct scatterlist *sgs[],
546 unsigned int total_sg,
547 unsigned int out_sgs,
548 unsigned int in_sgs,
549 void *data,
550 void *ctx,
551 gfp_t gfp)
552{
553 struct vring_virtqueue *vq = to_vvq(_vq);
554 struct scatterlist *sg;
555 struct vring_desc *desc;
556 unsigned int i, n, avail, descs_used, prev, err_idx;
557 int head;
558 bool indirect;
559
560 START_USE(vq);
561
562 BUG_ON(data == NULL);
563 BUG_ON(ctx && vq->indirect);
564
565 if (unlikely(vq->broken)) {
566 END_USE(vq);
567 return -EIO;
568 }
569
570 LAST_ADD_TIME_UPDATE(vq);
571
572 BUG_ON(total_sg == 0);
573
574 head = vq->free_head;
575
576 if (virtqueue_use_indirect(vq, total_sg))
577 desc = alloc_indirect_split(_vq, total_sg, gfp);
578 else {
579 desc = NULL;
580 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
581 }
582
583 if (desc) {
584 /* Use a single buffer which doesn't continue */
585 indirect = true;
586 /* Set up rest to use this indirect table. */
587 i = 0;
588 descs_used = 1;
589 } else {
590 indirect = false;
591 desc = vq->split.vring.desc;
592 i = head;
593 descs_used = total_sg;
594 }
595
596 if (unlikely(vq->vq.num_free < descs_used)) {
597 pr_debug("Can't add buf len %i - avail = %i\n",
598 descs_used, vq->vq.num_free);
599 /* FIXME: for historical reasons, we force a notify here if
600 * there are outgoing parts to the buffer. Presumably the
601 * host should service the ring ASAP. */
602 if (out_sgs)
603 vq->notify(&vq->vq);
604 if (indirect)
605 kfree(desc);
606 END_USE(vq);
607 return -ENOSPC;
608 }
609
610 for (n = 0; n < out_sgs; n++) {
611 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
612 dma_addr_t addr;
613
614 if (vring_map_one_sg(vq, sg, DMA_TO_DEVICE, &addr))
615 goto unmap_release;
616
617 prev = i;
618 /* Note that we trust indirect descriptor
619 * table since it use stream DMA mapping.
620 */
621 i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,
622 VRING_DESC_F_NEXT,
623 indirect);
624 }
625 }
626 for (; n < (out_sgs + in_sgs); n++) {
627 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
628 dma_addr_t addr;
629
630 if (vring_map_one_sg(vq, sg, DMA_FROM_DEVICE, &addr))
631 goto unmap_release;
632
633 prev = i;
634 /* Note that we trust indirect descriptor
635 * table since it use stream DMA mapping.
636 */
637 i = virtqueue_add_desc_split(_vq, desc, i, addr,
638 sg->length,
639 VRING_DESC_F_NEXT |
640 VRING_DESC_F_WRITE,
641 indirect);
642 }
643 }
644 /* Last one doesn't continue. */
645 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
646 if (!indirect && vq->do_unmap)
647 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
648 ~VRING_DESC_F_NEXT;
649
650 if (indirect) {
651 /* Now that the indirect table is filled in, map it. */
652 dma_addr_t addr = vring_map_single(
653 vq, desc, total_sg * sizeof(struct vring_desc),
654 DMA_TO_DEVICE);
655 if (vring_mapping_error(vq, addr)) {
656 if (vq->premapped)
657 goto free_indirect;
658
659 goto unmap_release;
660 }
661
662 virtqueue_add_desc_split(_vq, vq->split.vring.desc,
663 head, addr,
664 total_sg * sizeof(struct vring_desc),
665 VRING_DESC_F_INDIRECT,
666 false);
667 }
668
669 /* We're using some buffers from the free list. */
670 vq->vq.num_free -= descs_used;
671
672 /* Update free pointer */
673 if (indirect)
674 vq->free_head = vq->split.desc_extra[head].next;
675 else
676 vq->free_head = i;
677
678 /* Store token and indirect buffer state. */
679 vq->split.desc_state[head].data = data;
680 if (indirect)
681 vq->split.desc_state[head].indir_desc = desc;
682 else
683 vq->split.desc_state[head].indir_desc = ctx;
684
685 /* Put entry in available array (but don't update avail->idx until they
686 * do sync). */
687 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
688 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
689
690 /* Descriptors and available array need to be set before we expose the
691 * new available array entries. */
692 virtio_wmb(vq->weak_barriers);
693 vq->split.avail_idx_shadow++;
694 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
695 vq->split.avail_idx_shadow);
696 vq->num_added++;
697
698 pr_debug("Added buffer head %i to %p\n", head, vq);
699 END_USE(vq);
700
701 /* This is very unlikely, but theoretically possible. Kick
702 * just in case. */
703 if (unlikely(vq->num_added == (1 << 16) - 1))
704 virtqueue_kick(_vq);
705
706 return 0;
707
708unmap_release:
709 err_idx = i;
710
711 if (indirect)
712 i = 0;
713 else
714 i = head;
715
716 for (n = 0; n < total_sg; n++) {
717 if (i == err_idx)
718 break;
719 if (indirect) {
720 vring_unmap_one_split_indirect(vq, &desc[i]);
721 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
722 } else
723 i = vring_unmap_one_split(vq, i);
724 }
725
726free_indirect:
727 if (indirect)
728 kfree(desc);
729
730 END_USE(vq);
731 return -ENOMEM;
732}
733
734static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
735{
736 struct vring_virtqueue *vq = to_vvq(_vq);
737 u16 new, old;
738 bool needs_kick;
739
740 START_USE(vq);
741 /* We need to expose available array entries before checking avail
742 * event. */
743 virtio_mb(vq->weak_barriers);
744
745 old = vq->split.avail_idx_shadow - vq->num_added;
746 new = vq->split.avail_idx_shadow;
747 vq->num_added = 0;
748
749 LAST_ADD_TIME_CHECK(vq);
750 LAST_ADD_TIME_INVALID(vq);
751
752 if (vq->event) {
753 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
754 vring_avail_event(&vq->split.vring)),
755 new, old);
756 } else {
757 needs_kick = !(vq->split.vring.used->flags &
758 cpu_to_virtio16(_vq->vdev,
759 VRING_USED_F_NO_NOTIFY));
760 }
761 END_USE(vq);
762 return needs_kick;
763}
764
765static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
766 void **ctx)
767{
768 unsigned int i, j;
769 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
770
771 /* Clear data ptr. */
772 vq->split.desc_state[head].data = NULL;
773
774 /* Put back on free list: unmap first-level descriptors and find end */
775 i = head;
776
777 while (vq->split.vring.desc[i].flags & nextflag) {
778 vring_unmap_one_split(vq, i);
779 i = vq->split.desc_extra[i].next;
780 vq->vq.num_free++;
781 }
782
783 vring_unmap_one_split(vq, i);
784 vq->split.desc_extra[i].next = vq->free_head;
785 vq->free_head = head;
786
787 /* Plus final descriptor */
788 vq->vq.num_free++;
789
790 if (vq->indirect) {
791 struct vring_desc *indir_desc =
792 vq->split.desc_state[head].indir_desc;
793 u32 len;
794
795 /* Free the indirect table, if any, now that it's unmapped. */
796 if (!indir_desc)
797 return;
798
799 len = vq->split.desc_extra[head].len;
800
801 BUG_ON(!(vq->split.desc_extra[head].flags &
802 VRING_DESC_F_INDIRECT));
803 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
804
805 if (vq->do_unmap) {
806 for (j = 0; j < len / sizeof(struct vring_desc); j++)
807 vring_unmap_one_split_indirect(vq, &indir_desc[j]);
808 }
809
810 kfree(indir_desc);
811 vq->split.desc_state[head].indir_desc = NULL;
812 } else if (ctx) {
813 *ctx = vq->split.desc_state[head].indir_desc;
814 }
815}
816
817static bool more_used_split(const struct vring_virtqueue *vq)
818{
819 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
820 vq->split.vring.used->idx);
821}
822
823static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
824 unsigned int *len,
825 void **ctx)
826{
827 struct vring_virtqueue *vq = to_vvq(_vq);
828 void *ret;
829 unsigned int i;
830 u16 last_used;
831
832 START_USE(vq);
833
834 if (unlikely(vq->broken)) {
835 END_USE(vq);
836 return NULL;
837 }
838
839 if (!more_used_split(vq)) {
840 pr_debug("No more buffers in queue\n");
841 END_USE(vq);
842 return NULL;
843 }
844
845 /* Only get used array entries after they have been exposed by host. */
846 virtio_rmb(vq->weak_barriers);
847
848 last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
849 i = virtio32_to_cpu(_vq->vdev,
850 vq->split.vring.used->ring[last_used].id);
851 *len = virtio32_to_cpu(_vq->vdev,
852 vq->split.vring.used->ring[last_used].len);
853
854 if (unlikely(i >= vq->split.vring.num)) {
855 BAD_RING(vq, "id %u out of range\n", i);
856 return NULL;
857 }
858 if (unlikely(!vq->split.desc_state[i].data)) {
859 BAD_RING(vq, "id %u is not a head!\n", i);
860 return NULL;
861 }
862
863 /* detach_buf_split clears data, so grab it now. */
864 ret = vq->split.desc_state[i].data;
865 detach_buf_split(vq, i, ctx);
866 vq->last_used_idx++;
867 /* If we expect an interrupt for the next entry, tell host
868 * by writing event index and flush out the write before
869 * the read in the next get_buf call. */
870 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
871 virtio_store_mb(vq->weak_barriers,
872 &vring_used_event(&vq->split.vring),
873 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
874
875 LAST_ADD_TIME_INVALID(vq);
876
877 END_USE(vq);
878 return ret;
879}
880
881static void virtqueue_disable_cb_split(struct virtqueue *_vq)
882{
883 struct vring_virtqueue *vq = to_vvq(_vq);
884
885 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
886 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
887
888 /*
889 * If device triggered an event already it won't trigger one again:
890 * no need to disable.
891 */
892 if (vq->event_triggered)
893 return;
894
895 if (vq->event)
896 /* TODO: this is a hack. Figure out a cleaner value to write. */
897 vring_used_event(&vq->split.vring) = 0x0;
898 else
899 vq->split.vring.avail->flags =
900 cpu_to_virtio16(_vq->vdev,
901 vq->split.avail_flags_shadow);
902 }
903}
904
905static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
906{
907 struct vring_virtqueue *vq = to_vvq(_vq);
908 u16 last_used_idx;
909
910 START_USE(vq);
911
912 /* We optimistically turn back on interrupts, then check if there was
913 * more to do. */
914 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
915 * either clear the flags bit or point the event index at the next
916 * entry. Always do both to keep code simple. */
917 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
918 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
919 if (!vq->event)
920 vq->split.vring.avail->flags =
921 cpu_to_virtio16(_vq->vdev,
922 vq->split.avail_flags_shadow);
923 }
924 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
925 last_used_idx = vq->last_used_idx);
926 END_USE(vq);
927 return last_used_idx;
928}
929
930static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
931{
932 struct vring_virtqueue *vq = to_vvq(_vq);
933
934 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
935 vq->split.vring.used->idx);
936}
937
938static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
939{
940 struct vring_virtqueue *vq = to_vvq(_vq);
941 u16 bufs;
942
943 START_USE(vq);
944
945 /* We optimistically turn back on interrupts, then check if there was
946 * more to do. */
947 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
948 * either clear the flags bit or point the event index at the next
949 * entry. Always update the event index to keep code simple. */
950 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
951 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
952 if (!vq->event)
953 vq->split.vring.avail->flags =
954 cpu_to_virtio16(_vq->vdev,
955 vq->split.avail_flags_shadow);
956 }
957 /* TODO: tune this threshold */
958 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
959
960 virtio_store_mb(vq->weak_barriers,
961 &vring_used_event(&vq->split.vring),
962 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
963
964 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
965 - vq->last_used_idx) > bufs)) {
966 END_USE(vq);
967 return false;
968 }
969
970 END_USE(vq);
971 return true;
972}
973
974static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
975{
976 struct vring_virtqueue *vq = to_vvq(_vq);
977 unsigned int i;
978 void *buf;
979
980 START_USE(vq);
981
982 for (i = 0; i < vq->split.vring.num; i++) {
983 if (!vq->split.desc_state[i].data)
984 continue;
985 /* detach_buf_split clears data, so grab it now. */
986 buf = vq->split.desc_state[i].data;
987 detach_buf_split(vq, i, NULL);
988 vq->split.avail_idx_shadow--;
989 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
990 vq->split.avail_idx_shadow);
991 END_USE(vq);
992 return buf;
993 }
994 /* That should have freed everything. */
995 BUG_ON(vq->vq.num_free != vq->split.vring.num);
996
997 END_USE(vq);
998 return NULL;
999}
1000
1001static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split,
1002 struct vring_virtqueue *vq)
1003{
1004 struct virtio_device *vdev;
1005
1006 vdev = vq->vq.vdev;
1007
1008 vring_split->avail_flags_shadow = 0;
1009 vring_split->avail_idx_shadow = 0;
1010
1011 /* No callback? Tell other side not to bother us. */
1012 if (!vq->vq.callback) {
1013 vring_split->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
1014 if (!vq->event)
1015 vring_split->vring.avail->flags = cpu_to_virtio16(vdev,
1016 vring_split->avail_flags_shadow);
1017 }
1018}
1019
1020static void virtqueue_reinit_split(struct vring_virtqueue *vq)
1021{
1022 int num;
1023
1024 num = vq->split.vring.num;
1025
1026 vq->split.vring.avail->flags = 0;
1027 vq->split.vring.avail->idx = 0;
1028
1029 /* reset avail event */
1030 vq->split.vring.avail->ring[num] = 0;
1031
1032 vq->split.vring.used->flags = 0;
1033 vq->split.vring.used->idx = 0;
1034
1035 /* reset used event */
1036 *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;
1037
1038 virtqueue_init(vq, num);
1039
1040 virtqueue_vring_init_split(&vq->split, vq);
1041}
1042
1043static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
1044 struct vring_virtqueue_split *vring_split)
1045{
1046 vq->split = *vring_split;
1047
1048 /* Put everything in free lists. */
1049 vq->free_head = 0;
1050}
1051
1052static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
1053{
1054 struct vring_desc_state_split *state;
1055 struct vring_desc_extra *extra;
1056 u32 num = vring_split->vring.num;
1057
1058 state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL);
1059 if (!state)
1060 goto err_state;
1061
1062 extra = vring_alloc_desc_extra(num);
1063 if (!extra)
1064 goto err_extra;
1065
1066 memset(state, 0, num * sizeof(struct vring_desc_state_split));
1067
1068 vring_split->desc_state = state;
1069 vring_split->desc_extra = extra;
1070 return 0;
1071
1072err_extra:
1073 kfree(state);
1074err_state:
1075 return -ENOMEM;
1076}
1077
1078static void vring_free_split(struct vring_virtqueue_split *vring_split,
1079 struct virtio_device *vdev, struct device *dma_dev)
1080{
1081 vring_free_queue(vdev, vring_split->queue_size_in_bytes,
1082 vring_split->vring.desc,
1083 vring_split->queue_dma_addr,
1084 dma_dev);
1085
1086 kfree(vring_split->desc_state);
1087 kfree(vring_split->desc_extra);
1088}
1089
1090static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
1091 struct virtio_device *vdev,
1092 u32 num,
1093 unsigned int vring_align,
1094 bool may_reduce_num,
1095 struct device *dma_dev)
1096{
1097 void *queue = NULL;
1098 dma_addr_t dma_addr;
1099
1100 /* We assume num is a power of 2. */
1101 if (!is_power_of_2(num)) {
1102 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
1103 return -EINVAL;
1104 }
1105
1106 /* TODO: allocate each queue chunk individually */
1107 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
1108 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1109 &dma_addr,
1110 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1111 dma_dev);
1112 if (queue)
1113 break;
1114 if (!may_reduce_num)
1115 return -ENOMEM;
1116 }
1117
1118 if (!num)
1119 return -ENOMEM;
1120
1121 if (!queue) {
1122 /* Try to get a single page. You are my only hope! */
1123 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1124 &dma_addr, GFP_KERNEL | __GFP_ZERO,
1125 dma_dev);
1126 }
1127 if (!queue)
1128 return -ENOMEM;
1129
1130 vring_init(&vring_split->vring, num, queue, vring_align);
1131
1132 vring_split->queue_dma_addr = dma_addr;
1133 vring_split->queue_size_in_bytes = vring_size(num, vring_align);
1134
1135 vring_split->vring_align = vring_align;
1136 vring_split->may_reduce_num = may_reduce_num;
1137
1138 return 0;
1139}
1140
1141static struct virtqueue *vring_create_virtqueue_split(
1142 unsigned int index,
1143 unsigned int num,
1144 unsigned int vring_align,
1145 struct virtio_device *vdev,
1146 bool weak_barriers,
1147 bool may_reduce_num,
1148 bool context,
1149 bool (*notify)(struct virtqueue *),
1150 void (*callback)(struct virtqueue *),
1151 const char *name,
1152 struct device *dma_dev)
1153{
1154 struct vring_virtqueue_split vring_split = {};
1155 struct virtqueue *vq;
1156 int err;
1157
1158 err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align,
1159 may_reduce_num, dma_dev);
1160 if (err)
1161 return NULL;
1162
1163 vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
1164 context, notify, callback, name, dma_dev);
1165 if (!vq) {
1166 vring_free_split(&vring_split, vdev, dma_dev);
1167 return NULL;
1168 }
1169
1170 to_vvq(vq)->we_own_ring = true;
1171
1172 return vq;
1173}
1174
1175static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
1176{
1177 struct vring_virtqueue_split vring_split = {};
1178 struct vring_virtqueue *vq = to_vvq(_vq);
1179 struct virtio_device *vdev = _vq->vdev;
1180 int err;
1181
1182 err = vring_alloc_queue_split(&vring_split, vdev, num,
1183 vq->split.vring_align,
1184 vq->split.may_reduce_num,
1185 vring_dma_dev(vq));
1186 if (err)
1187 goto err;
1188
1189 err = vring_alloc_state_extra_split(&vring_split);
1190 if (err)
1191 goto err_state_extra;
1192
1193 vring_free(&vq->vq);
1194
1195 virtqueue_vring_init_split(&vring_split, vq);
1196
1197 virtqueue_init(vq, vring_split.vring.num);
1198 virtqueue_vring_attach_split(vq, &vring_split);
1199
1200 return 0;
1201
1202err_state_extra:
1203 vring_free_split(&vring_split, vdev, vring_dma_dev(vq));
1204err:
1205 virtqueue_reinit_split(vq);
1206 return -ENOMEM;
1207}
1208
1209
1210/*
1211 * Packed ring specific functions - *_packed().
1212 */
1213static bool packed_used_wrap_counter(u16 last_used_idx)
1214{
1215 return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1216}
1217
1218static u16 packed_last_used(u16 last_used_idx)
1219{
1220 return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1221}
1222
1223static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
1224 const struct vring_desc_extra *extra)
1225{
1226 u16 flags;
1227
1228 flags = extra->flags;
1229
1230 if (flags & VRING_DESC_F_INDIRECT) {
1231 if (!vq->use_dma_api)
1232 return;
1233
1234 dma_unmap_single(vring_dma_dev(vq),
1235 extra->addr, extra->len,
1236 (flags & VRING_DESC_F_WRITE) ?
1237 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1238 } else {
1239 if (!vq->do_unmap)
1240 return;
1241
1242 dma_unmap_page(vring_dma_dev(vq),
1243 extra->addr, extra->len,
1244 (flags & VRING_DESC_F_WRITE) ?
1245 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1246 }
1247}
1248
1249static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
1250 const struct vring_packed_desc *desc)
1251{
1252 u16 flags;
1253
1254 if (!vq->do_unmap)
1255 return;
1256
1257 flags = le16_to_cpu(desc->flags);
1258
1259 dma_unmap_page(vring_dma_dev(vq),
1260 le64_to_cpu(desc->addr),
1261 le32_to_cpu(desc->len),
1262 (flags & VRING_DESC_F_WRITE) ?
1263 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1264}
1265
1266static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
1267 gfp_t gfp)
1268{
1269 struct vring_packed_desc *desc;
1270
1271 /*
1272 * We require lowmem mappings for the descriptors because
1273 * otherwise virt_to_phys will give us bogus addresses in the
1274 * virtqueue.
1275 */
1276 gfp &= ~__GFP_HIGHMEM;
1277
1278 desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
1279
1280 return desc;
1281}
1282
1283static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
1284 struct scatterlist *sgs[],
1285 unsigned int total_sg,
1286 unsigned int out_sgs,
1287 unsigned int in_sgs,
1288 void *data,
1289 gfp_t gfp)
1290{
1291 struct vring_packed_desc *desc;
1292 struct scatterlist *sg;
1293 unsigned int i, n, err_idx;
1294 u16 head, id;
1295 dma_addr_t addr;
1296
1297 head = vq->packed.next_avail_idx;
1298 desc = alloc_indirect_packed(total_sg, gfp);
1299 if (!desc)
1300 return -ENOMEM;
1301
1302 if (unlikely(vq->vq.num_free < 1)) {
1303 pr_debug("Can't add buf len 1 - avail = 0\n");
1304 kfree(desc);
1305 END_USE(vq);
1306 return -ENOSPC;
1307 }
1308
1309 i = 0;
1310 id = vq->free_head;
1311 BUG_ON(id == vq->packed.vring.num);
1312
1313 for (n = 0; n < out_sgs + in_sgs; n++) {
1314 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1315 if (vring_map_one_sg(vq, sg, n < out_sgs ?
1316 DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr))
1317 goto unmap_release;
1318
1319 desc[i].flags = cpu_to_le16(n < out_sgs ?
1320 0 : VRING_DESC_F_WRITE);
1321 desc[i].addr = cpu_to_le64(addr);
1322 desc[i].len = cpu_to_le32(sg->length);
1323 i++;
1324 }
1325 }
1326
1327 /* Now that the indirect table is filled in, map it. */
1328 addr = vring_map_single(vq, desc,
1329 total_sg * sizeof(struct vring_packed_desc),
1330 DMA_TO_DEVICE);
1331 if (vring_mapping_error(vq, addr)) {
1332 if (vq->premapped)
1333 goto free_desc;
1334
1335 goto unmap_release;
1336 }
1337
1338 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1339 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1340 sizeof(struct vring_packed_desc));
1341 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1342
1343 if (vq->do_unmap) {
1344 vq->packed.desc_extra[id].addr = addr;
1345 vq->packed.desc_extra[id].len = total_sg *
1346 sizeof(struct vring_packed_desc);
1347 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1348 vq->packed.avail_used_flags;
1349 }
1350
1351 /*
1352 * A driver MUST NOT make the first descriptor in the list
1353 * available before all subsequent descriptors comprising
1354 * the list are made available.
1355 */
1356 virtio_wmb(vq->weak_barriers);
1357 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1358 vq->packed.avail_used_flags);
1359
1360 /* We're using some buffers from the free list. */
1361 vq->vq.num_free -= 1;
1362
1363 /* Update free pointer */
1364 n = head + 1;
1365 if (n >= vq->packed.vring.num) {
1366 n = 0;
1367 vq->packed.avail_wrap_counter ^= 1;
1368 vq->packed.avail_used_flags ^=
1369 1 << VRING_PACKED_DESC_F_AVAIL |
1370 1 << VRING_PACKED_DESC_F_USED;
1371 }
1372 vq->packed.next_avail_idx = n;
1373 vq->free_head = vq->packed.desc_extra[id].next;
1374
1375 /* Store token and indirect buffer state. */
1376 vq->packed.desc_state[id].num = 1;
1377 vq->packed.desc_state[id].data = data;
1378 vq->packed.desc_state[id].indir_desc = desc;
1379 vq->packed.desc_state[id].last = id;
1380
1381 vq->num_added += 1;
1382
1383 pr_debug("Added buffer head %i to %p\n", head, vq);
1384 END_USE(vq);
1385
1386 return 0;
1387
1388unmap_release:
1389 err_idx = i;
1390
1391 for (i = 0; i < err_idx; i++)
1392 vring_unmap_desc_packed(vq, &desc[i]);
1393
1394free_desc:
1395 kfree(desc);
1396
1397 END_USE(vq);
1398 return -ENOMEM;
1399}
1400
1401static inline int virtqueue_add_packed(struct virtqueue *_vq,
1402 struct scatterlist *sgs[],
1403 unsigned int total_sg,
1404 unsigned int out_sgs,
1405 unsigned int in_sgs,
1406 void *data,
1407 void *ctx,
1408 gfp_t gfp)
1409{
1410 struct vring_virtqueue *vq = to_vvq(_vq);
1411 struct vring_packed_desc *desc;
1412 struct scatterlist *sg;
1413 unsigned int i, n, c, descs_used, err_idx;
1414 __le16 head_flags, flags;
1415 u16 head, id, prev, curr, avail_used_flags;
1416 int err;
1417
1418 START_USE(vq);
1419
1420 BUG_ON(data == NULL);
1421 BUG_ON(ctx && vq->indirect);
1422
1423 if (unlikely(vq->broken)) {
1424 END_USE(vq);
1425 return -EIO;
1426 }
1427
1428 LAST_ADD_TIME_UPDATE(vq);
1429
1430 BUG_ON(total_sg == 0);
1431
1432 if (virtqueue_use_indirect(vq, total_sg)) {
1433 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
1434 in_sgs, data, gfp);
1435 if (err != -ENOMEM) {
1436 END_USE(vq);
1437 return err;
1438 }
1439
1440 /* fall back on direct */
1441 }
1442
1443 head = vq->packed.next_avail_idx;
1444 avail_used_flags = vq->packed.avail_used_flags;
1445
1446 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1447
1448 desc = vq->packed.vring.desc;
1449 i = head;
1450 descs_used = total_sg;
1451
1452 if (unlikely(vq->vq.num_free < descs_used)) {
1453 pr_debug("Can't add buf len %i - avail = %i\n",
1454 descs_used, vq->vq.num_free);
1455 END_USE(vq);
1456 return -ENOSPC;
1457 }
1458
1459 id = vq->free_head;
1460 BUG_ON(id == vq->packed.vring.num);
1461
1462 curr = id;
1463 c = 0;
1464 for (n = 0; n < out_sgs + in_sgs; n++) {
1465 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1466 dma_addr_t addr;
1467
1468 if (vring_map_one_sg(vq, sg, n < out_sgs ?
1469 DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr))
1470 goto unmap_release;
1471
1472 flags = cpu_to_le16(vq->packed.avail_used_flags |
1473 (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1474 (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1475 if (i == head)
1476 head_flags = flags;
1477 else
1478 desc[i].flags = flags;
1479
1480 desc[i].addr = cpu_to_le64(addr);
1481 desc[i].len = cpu_to_le32(sg->length);
1482 desc[i].id = cpu_to_le16(id);
1483
1484 if (unlikely(vq->do_unmap)) {
1485 vq->packed.desc_extra[curr].addr = addr;
1486 vq->packed.desc_extra[curr].len = sg->length;
1487 vq->packed.desc_extra[curr].flags =
1488 le16_to_cpu(flags);
1489 }
1490 prev = curr;
1491 curr = vq->packed.desc_extra[curr].next;
1492
1493 if ((unlikely(++i >= vq->packed.vring.num))) {
1494 i = 0;
1495 vq->packed.avail_used_flags ^=
1496 1 << VRING_PACKED_DESC_F_AVAIL |
1497 1 << VRING_PACKED_DESC_F_USED;
1498 }
1499 }
1500 }
1501
1502 if (i <= head)
1503 vq->packed.avail_wrap_counter ^= 1;
1504
1505 /* We're using some buffers from the free list. */
1506 vq->vq.num_free -= descs_used;
1507
1508 /* Update free pointer */
1509 vq->packed.next_avail_idx = i;
1510 vq->free_head = curr;
1511
1512 /* Store token. */
1513 vq->packed.desc_state[id].num = descs_used;
1514 vq->packed.desc_state[id].data = data;
1515 vq->packed.desc_state[id].indir_desc = ctx;
1516 vq->packed.desc_state[id].last = prev;
1517
1518 /*
1519 * A driver MUST NOT make the first descriptor in the list
1520 * available before all subsequent descriptors comprising
1521 * the list are made available.
1522 */
1523 virtio_wmb(vq->weak_barriers);
1524 vq->packed.vring.desc[head].flags = head_flags;
1525 vq->num_added += descs_used;
1526
1527 pr_debug("Added buffer head %i to %p\n", head, vq);
1528 END_USE(vq);
1529
1530 return 0;
1531
1532unmap_release:
1533 err_idx = i;
1534 i = head;
1535 curr = vq->free_head;
1536
1537 vq->packed.avail_used_flags = avail_used_flags;
1538
1539 for (n = 0; n < total_sg; n++) {
1540 if (i == err_idx)
1541 break;
1542 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1543 curr = vq->packed.desc_extra[curr].next;
1544 i++;
1545 if (i >= vq->packed.vring.num)
1546 i = 0;
1547 }
1548
1549 END_USE(vq);
1550 return -EIO;
1551}
1552
1553static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1554{
1555 struct vring_virtqueue *vq = to_vvq(_vq);
1556 u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1557 bool needs_kick;
1558 union {
1559 struct {
1560 __le16 off_wrap;
1561 __le16 flags;
1562 };
1563 u32 u32;
1564 } snapshot;
1565
1566 START_USE(vq);
1567
1568 /*
1569 * We need to expose the new flags value before checking notification
1570 * suppressions.
1571 */
1572 virtio_mb(vq->weak_barriers);
1573
1574 old = vq->packed.next_avail_idx - vq->num_added;
1575 new = vq->packed.next_avail_idx;
1576 vq->num_added = 0;
1577
1578 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1579 flags = le16_to_cpu(snapshot.flags);
1580
1581 LAST_ADD_TIME_CHECK(vq);
1582 LAST_ADD_TIME_INVALID(vq);
1583
1584 if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1585 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1586 goto out;
1587 }
1588
1589 off_wrap = le16_to_cpu(snapshot.off_wrap);
1590
1591 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1592 event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1593 if (wrap_counter != vq->packed.avail_wrap_counter)
1594 event_idx -= vq->packed.vring.num;
1595
1596 needs_kick = vring_need_event(event_idx, new, old);
1597out:
1598 END_USE(vq);
1599 return needs_kick;
1600}
1601
1602static void detach_buf_packed(struct vring_virtqueue *vq,
1603 unsigned int id, void **ctx)
1604{
1605 struct vring_desc_state_packed *state = NULL;
1606 struct vring_packed_desc *desc;
1607 unsigned int i, curr;
1608
1609 state = &vq->packed.desc_state[id];
1610
1611 /* Clear data ptr. */
1612 state->data = NULL;
1613
1614 vq->packed.desc_extra[state->last].next = vq->free_head;
1615 vq->free_head = id;
1616 vq->vq.num_free += state->num;
1617
1618 if (unlikely(vq->do_unmap)) {
1619 curr = id;
1620 for (i = 0; i < state->num; i++) {
1621 vring_unmap_extra_packed(vq,
1622 &vq->packed.desc_extra[curr]);
1623 curr = vq->packed.desc_extra[curr].next;
1624 }
1625 }
1626
1627 if (vq->indirect) {
1628 u32 len;
1629
1630 /* Free the indirect table, if any, now that it's unmapped. */
1631 desc = state->indir_desc;
1632 if (!desc)
1633 return;
1634
1635 if (vq->do_unmap) {
1636 len = vq->packed.desc_extra[id].len;
1637 for (i = 0; i < len / sizeof(struct vring_packed_desc);
1638 i++)
1639 vring_unmap_desc_packed(vq, &desc[i]);
1640 }
1641 kfree(desc);
1642 state->indir_desc = NULL;
1643 } else if (ctx) {
1644 *ctx = state->indir_desc;
1645 }
1646}
1647
1648static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1649 u16 idx, bool used_wrap_counter)
1650{
1651 bool avail, used;
1652 u16 flags;
1653
1654 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1655 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1656 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1657
1658 return avail == used && used == used_wrap_counter;
1659}
1660
1661static bool more_used_packed(const struct vring_virtqueue *vq)
1662{
1663 u16 last_used;
1664 u16 last_used_idx;
1665 bool used_wrap_counter;
1666
1667 last_used_idx = READ_ONCE(vq->last_used_idx);
1668 last_used = packed_last_used(last_used_idx);
1669 used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1670 return is_used_desc_packed(vq, last_used, used_wrap_counter);
1671}
1672
1673static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1674 unsigned int *len,
1675 void **ctx)
1676{
1677 struct vring_virtqueue *vq = to_vvq(_vq);
1678 u16 last_used, id, last_used_idx;
1679 bool used_wrap_counter;
1680 void *ret;
1681
1682 START_USE(vq);
1683
1684 if (unlikely(vq->broken)) {
1685 END_USE(vq);
1686 return NULL;
1687 }
1688
1689 if (!more_used_packed(vq)) {
1690 pr_debug("No more buffers in queue\n");
1691 END_USE(vq);
1692 return NULL;
1693 }
1694
1695 /* Only get used elements after they have been exposed by host. */
1696 virtio_rmb(vq->weak_barriers);
1697
1698 last_used_idx = READ_ONCE(vq->last_used_idx);
1699 used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1700 last_used = packed_last_used(last_used_idx);
1701 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1702 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1703
1704 if (unlikely(id >= vq->packed.vring.num)) {
1705 BAD_RING(vq, "id %u out of range\n", id);
1706 return NULL;
1707 }
1708 if (unlikely(!vq->packed.desc_state[id].data)) {
1709 BAD_RING(vq, "id %u is not a head!\n", id);
1710 return NULL;
1711 }
1712
1713 /* detach_buf_packed clears data, so grab it now. */
1714 ret = vq->packed.desc_state[id].data;
1715 detach_buf_packed(vq, id, ctx);
1716
1717 last_used += vq->packed.desc_state[id].num;
1718 if (unlikely(last_used >= vq->packed.vring.num)) {
1719 last_used -= vq->packed.vring.num;
1720 used_wrap_counter ^= 1;
1721 }
1722
1723 last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1724 WRITE_ONCE(vq->last_used_idx, last_used);
1725
1726 /*
1727 * If we expect an interrupt for the next entry, tell host
1728 * by writing event index and flush out the write before
1729 * the read in the next get_buf call.
1730 */
1731 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1732 virtio_store_mb(vq->weak_barriers,
1733 &vq->packed.vring.driver->off_wrap,
1734 cpu_to_le16(vq->last_used_idx));
1735
1736 LAST_ADD_TIME_INVALID(vq);
1737
1738 END_USE(vq);
1739 return ret;
1740}
1741
1742static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1743{
1744 struct vring_virtqueue *vq = to_vvq(_vq);
1745
1746 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1747 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1748
1749 /*
1750 * If device triggered an event already it won't trigger one again:
1751 * no need to disable.
1752 */
1753 if (vq->event_triggered)
1754 return;
1755
1756 vq->packed.vring.driver->flags =
1757 cpu_to_le16(vq->packed.event_flags_shadow);
1758 }
1759}
1760
1761static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1762{
1763 struct vring_virtqueue *vq = to_vvq(_vq);
1764
1765 START_USE(vq);
1766
1767 /*
1768 * We optimistically turn back on interrupts, then check if there was
1769 * more to do.
1770 */
1771
1772 if (vq->event) {
1773 vq->packed.vring.driver->off_wrap =
1774 cpu_to_le16(vq->last_used_idx);
1775 /*
1776 * We need to update event offset and event wrap
1777 * counter first before updating event flags.
1778 */
1779 virtio_wmb(vq->weak_barriers);
1780 }
1781
1782 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1783 vq->packed.event_flags_shadow = vq->event ?
1784 VRING_PACKED_EVENT_FLAG_DESC :
1785 VRING_PACKED_EVENT_FLAG_ENABLE;
1786 vq->packed.vring.driver->flags =
1787 cpu_to_le16(vq->packed.event_flags_shadow);
1788 }
1789
1790 END_USE(vq);
1791 return vq->last_used_idx;
1792}
1793
1794static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1795{
1796 struct vring_virtqueue *vq = to_vvq(_vq);
1797 bool wrap_counter;
1798 u16 used_idx;
1799
1800 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1801 used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1802
1803 return is_used_desc_packed(vq, used_idx, wrap_counter);
1804}
1805
1806static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1807{
1808 struct vring_virtqueue *vq = to_vvq(_vq);
1809 u16 used_idx, wrap_counter, last_used_idx;
1810 u16 bufs;
1811
1812 START_USE(vq);
1813
1814 /*
1815 * We optimistically turn back on interrupts, then check if there was
1816 * more to do.
1817 */
1818
1819 if (vq->event) {
1820 /* TODO: tune this threshold */
1821 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1822 last_used_idx = READ_ONCE(vq->last_used_idx);
1823 wrap_counter = packed_used_wrap_counter(last_used_idx);
1824
1825 used_idx = packed_last_used(last_used_idx) + bufs;
1826 if (used_idx >= vq->packed.vring.num) {
1827 used_idx -= vq->packed.vring.num;
1828 wrap_counter ^= 1;
1829 }
1830
1831 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1832 (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1833
1834 /*
1835 * We need to update event offset and event wrap
1836 * counter first before updating event flags.
1837 */
1838 virtio_wmb(vq->weak_barriers);
1839 }
1840
1841 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1842 vq->packed.event_flags_shadow = vq->event ?
1843 VRING_PACKED_EVENT_FLAG_DESC :
1844 VRING_PACKED_EVENT_FLAG_ENABLE;
1845 vq->packed.vring.driver->flags =
1846 cpu_to_le16(vq->packed.event_flags_shadow);
1847 }
1848
1849 /*
1850 * We need to update event suppression structure first
1851 * before re-checking for more used buffers.
1852 */
1853 virtio_mb(vq->weak_barriers);
1854
1855 last_used_idx = READ_ONCE(vq->last_used_idx);
1856 wrap_counter = packed_used_wrap_counter(last_used_idx);
1857 used_idx = packed_last_used(last_used_idx);
1858 if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
1859 END_USE(vq);
1860 return false;
1861 }
1862
1863 END_USE(vq);
1864 return true;
1865}
1866
1867static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1868{
1869 struct vring_virtqueue *vq = to_vvq(_vq);
1870 unsigned int i;
1871 void *buf;
1872
1873 START_USE(vq);
1874
1875 for (i = 0; i < vq->packed.vring.num; i++) {
1876 if (!vq->packed.desc_state[i].data)
1877 continue;
1878 /* detach_buf clears data, so grab it now. */
1879 buf = vq->packed.desc_state[i].data;
1880 detach_buf_packed(vq, i, NULL);
1881 END_USE(vq);
1882 return buf;
1883 }
1884 /* That should have freed everything. */
1885 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1886
1887 END_USE(vq);
1888 return NULL;
1889}
1890
1891static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
1892{
1893 struct vring_desc_extra *desc_extra;
1894 unsigned int i;
1895
1896 desc_extra = kmalloc_array(num, sizeof(struct vring_desc_extra),
1897 GFP_KERNEL);
1898 if (!desc_extra)
1899 return NULL;
1900
1901 memset(desc_extra, 0, num * sizeof(struct vring_desc_extra));
1902
1903 for (i = 0; i < num - 1; i++)
1904 desc_extra[i].next = i + 1;
1905
1906 return desc_extra;
1907}
1908
1909static void vring_free_packed(struct vring_virtqueue_packed *vring_packed,
1910 struct virtio_device *vdev,
1911 struct device *dma_dev)
1912{
1913 if (vring_packed->vring.desc)
1914 vring_free_queue(vdev, vring_packed->ring_size_in_bytes,
1915 vring_packed->vring.desc,
1916 vring_packed->ring_dma_addr,
1917 dma_dev);
1918
1919 if (vring_packed->vring.driver)
1920 vring_free_queue(vdev, vring_packed->event_size_in_bytes,
1921 vring_packed->vring.driver,
1922 vring_packed->driver_event_dma_addr,
1923 dma_dev);
1924
1925 if (vring_packed->vring.device)
1926 vring_free_queue(vdev, vring_packed->event_size_in_bytes,
1927 vring_packed->vring.device,
1928 vring_packed->device_event_dma_addr,
1929 dma_dev);
1930
1931 kfree(vring_packed->desc_state);
1932 kfree(vring_packed->desc_extra);
1933}
1934
1935static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed,
1936 struct virtio_device *vdev,
1937 u32 num, struct device *dma_dev)
1938{
1939 struct vring_packed_desc *ring;
1940 struct vring_packed_desc_event *driver, *device;
1941 dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1942 size_t ring_size_in_bytes, event_size_in_bytes;
1943
1944 ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1945
1946 ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1947 &ring_dma_addr,
1948 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1949 dma_dev);
1950 if (!ring)
1951 goto err;
1952
1953 vring_packed->vring.desc = ring;
1954 vring_packed->ring_dma_addr = ring_dma_addr;
1955 vring_packed->ring_size_in_bytes = ring_size_in_bytes;
1956
1957 event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1958
1959 driver = vring_alloc_queue(vdev, event_size_in_bytes,
1960 &driver_event_dma_addr,
1961 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1962 dma_dev);
1963 if (!driver)
1964 goto err;
1965
1966 vring_packed->vring.driver = driver;
1967 vring_packed->event_size_in_bytes = event_size_in_bytes;
1968 vring_packed->driver_event_dma_addr = driver_event_dma_addr;
1969
1970 device = vring_alloc_queue(vdev, event_size_in_bytes,
1971 &device_event_dma_addr,
1972 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1973 dma_dev);
1974 if (!device)
1975 goto err;
1976
1977 vring_packed->vring.device = device;
1978 vring_packed->device_event_dma_addr = device_event_dma_addr;
1979
1980 vring_packed->vring.num = num;
1981
1982 return 0;
1983
1984err:
1985 vring_free_packed(vring_packed, vdev, dma_dev);
1986 return -ENOMEM;
1987}
1988
1989static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed)
1990{
1991 struct vring_desc_state_packed *state;
1992 struct vring_desc_extra *extra;
1993 u32 num = vring_packed->vring.num;
1994
1995 state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL);
1996 if (!state)
1997 goto err_desc_state;
1998
1999 memset(state, 0, num * sizeof(struct vring_desc_state_packed));
2000
2001 extra = vring_alloc_desc_extra(num);
2002 if (!extra)
2003 goto err_desc_extra;
2004
2005 vring_packed->desc_state = state;
2006 vring_packed->desc_extra = extra;
2007
2008 return 0;
2009
2010err_desc_extra:
2011 kfree(state);
2012err_desc_state:
2013 return -ENOMEM;
2014}
2015
2016static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
2017 bool callback)
2018{
2019 vring_packed->next_avail_idx = 0;
2020 vring_packed->avail_wrap_counter = 1;
2021 vring_packed->event_flags_shadow = 0;
2022 vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
2023
2024 /* No callback? Tell other side not to bother us. */
2025 if (!callback) {
2026 vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
2027 vring_packed->vring.driver->flags =
2028 cpu_to_le16(vring_packed->event_flags_shadow);
2029 }
2030}
2031
2032static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
2033 struct vring_virtqueue_packed *vring_packed)
2034{
2035 vq->packed = *vring_packed;
2036
2037 /* Put everything in free lists. */
2038 vq->free_head = 0;
2039}
2040
2041static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
2042{
2043 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
2044 memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
2045
2046 /* we need to reset the desc.flags. For more, see is_used_desc_packed() */
2047 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
2048
2049 virtqueue_init(vq, vq->packed.vring.num);
2050 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
2051}
2052
2053static struct virtqueue *vring_create_virtqueue_packed(
2054 unsigned int index,
2055 unsigned int num,
2056 unsigned int vring_align,
2057 struct virtio_device *vdev,
2058 bool weak_barriers,
2059 bool may_reduce_num,
2060 bool context,
2061 bool (*notify)(struct virtqueue *),
2062 void (*callback)(struct virtqueue *),
2063 const char *name,
2064 struct device *dma_dev)
2065{
2066 struct vring_virtqueue_packed vring_packed = {};
2067 struct vring_virtqueue *vq;
2068 int err;
2069
2070 if (vring_alloc_queue_packed(&vring_packed, vdev, num, dma_dev))
2071 goto err_ring;
2072
2073 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2074 if (!vq)
2075 goto err_vq;
2076
2077 vq->vq.callback = callback;
2078 vq->vq.vdev = vdev;
2079 vq->vq.name = name;
2080 vq->vq.index = index;
2081 vq->vq.reset = false;
2082 vq->we_own_ring = true;
2083 vq->notify = notify;
2084 vq->weak_barriers = weak_barriers;
2085#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2086 vq->broken = true;
2087#else
2088 vq->broken = false;
2089#endif
2090 vq->packed_ring = true;
2091 vq->dma_dev = dma_dev;
2092 vq->use_dma_api = vring_use_dma_api(vdev);
2093 vq->premapped = false;
2094 vq->do_unmap = vq->use_dma_api;
2095
2096 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2097 !context;
2098 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2099
2100 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2101 vq->weak_barriers = false;
2102
2103 err = vring_alloc_state_extra_packed(&vring_packed);
2104 if (err)
2105 goto err_state_extra;
2106
2107 virtqueue_vring_init_packed(&vring_packed, !!callback);
2108
2109 virtqueue_init(vq, num);
2110 virtqueue_vring_attach_packed(vq, &vring_packed);
2111
2112 spin_lock(&vdev->vqs_list_lock);
2113 list_add_tail(&vq->vq.list, &vdev->vqs);
2114 spin_unlock(&vdev->vqs_list_lock);
2115 return &vq->vq;
2116
2117err_state_extra:
2118 kfree(vq);
2119err_vq:
2120 vring_free_packed(&vring_packed, vdev, dma_dev);
2121err_ring:
2122 return NULL;
2123}
2124
2125static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
2126{
2127 struct vring_virtqueue_packed vring_packed = {};
2128 struct vring_virtqueue *vq = to_vvq(_vq);
2129 struct virtio_device *vdev = _vq->vdev;
2130 int err;
2131
2132 if (vring_alloc_queue_packed(&vring_packed, vdev, num, vring_dma_dev(vq)))
2133 goto err_ring;
2134
2135 err = vring_alloc_state_extra_packed(&vring_packed);
2136 if (err)
2137 goto err_state_extra;
2138
2139 vring_free(&vq->vq);
2140
2141 virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback);
2142
2143 virtqueue_init(vq, vring_packed.vring.num);
2144 virtqueue_vring_attach_packed(vq, &vring_packed);
2145
2146 return 0;
2147
2148err_state_extra:
2149 vring_free_packed(&vring_packed, vdev, vring_dma_dev(vq));
2150err_ring:
2151 virtqueue_reinit_packed(vq);
2152 return -ENOMEM;
2153}
2154
2155static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
2156 void (*recycle)(struct virtqueue *vq, void *buf))
2157{
2158 struct vring_virtqueue *vq = to_vvq(_vq);
2159 struct virtio_device *vdev = vq->vq.vdev;
2160 void *buf;
2161 int err;
2162
2163 if (!vq->we_own_ring)
2164 return -EPERM;
2165
2166 if (!vdev->config->disable_vq_and_reset)
2167 return -ENOENT;
2168
2169 if (!vdev->config->enable_vq_after_reset)
2170 return -ENOENT;
2171
2172 err = vdev->config->disable_vq_and_reset(_vq);
2173 if (err)
2174 return err;
2175
2176 while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
2177 recycle(_vq, buf);
2178
2179 return 0;
2180}
2181
2182static int virtqueue_enable_after_reset(struct virtqueue *_vq)
2183{
2184 struct vring_virtqueue *vq = to_vvq(_vq);
2185 struct virtio_device *vdev = vq->vq.vdev;
2186
2187 if (vdev->config->enable_vq_after_reset(_vq))
2188 return -EBUSY;
2189
2190 return 0;
2191}
2192
2193/*
2194 * Generic functions and exported symbols.
2195 */
2196
2197static inline int virtqueue_add(struct virtqueue *_vq,
2198 struct scatterlist *sgs[],
2199 unsigned int total_sg,
2200 unsigned int out_sgs,
2201 unsigned int in_sgs,
2202 void *data,
2203 void *ctx,
2204 gfp_t gfp)
2205{
2206 struct vring_virtqueue *vq = to_vvq(_vq);
2207
2208 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
2209 out_sgs, in_sgs, data, ctx, gfp) :
2210 virtqueue_add_split(_vq, sgs, total_sg,
2211 out_sgs, in_sgs, data, ctx, gfp);
2212}
2213
2214/**
2215 * virtqueue_add_sgs - expose buffers to other end
2216 * @_vq: the struct virtqueue we're talking about.
2217 * @sgs: array of terminated scatterlists.
2218 * @out_sgs: the number of scatterlists readable by other side
2219 * @in_sgs: the number of scatterlists which are writable (after readable ones)
2220 * @data: the token identifying the buffer.
2221 * @gfp: how to do memory allocations (if necessary).
2222 *
2223 * Caller must ensure we don't call this with other virtqueue operations
2224 * at the same time (except where noted).
2225 *
2226 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2227 */
2228int virtqueue_add_sgs(struct virtqueue *_vq,
2229 struct scatterlist *sgs[],
2230 unsigned int out_sgs,
2231 unsigned int in_sgs,
2232 void *data,
2233 gfp_t gfp)
2234{
2235 unsigned int i, total_sg = 0;
2236
2237 /* Count them first. */
2238 for (i = 0; i < out_sgs + in_sgs; i++) {
2239 struct scatterlist *sg;
2240
2241 for (sg = sgs[i]; sg; sg = sg_next(sg))
2242 total_sg++;
2243 }
2244 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
2245 data, NULL, gfp);
2246}
2247EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
2248
2249/**
2250 * virtqueue_add_outbuf - expose output buffers to other end
2251 * @vq: the struct virtqueue we're talking about.
2252 * @sg: scatterlist (must be well-formed and terminated!)
2253 * @num: the number of entries in @sg readable by other side
2254 * @data: the token identifying the buffer.
2255 * @gfp: how to do memory allocations (if necessary).
2256 *
2257 * Caller must ensure we don't call this with other virtqueue operations
2258 * at the same time (except where noted).
2259 *
2260 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2261 */
2262int virtqueue_add_outbuf(struct virtqueue *vq,
2263 struct scatterlist *sg, unsigned int num,
2264 void *data,
2265 gfp_t gfp)
2266{
2267 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
2268}
2269EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
2270
2271/**
2272 * virtqueue_add_inbuf - expose input buffers to other end
2273 * @vq: the struct virtqueue we're talking about.
2274 * @sg: scatterlist (must be well-formed and terminated!)
2275 * @num: the number of entries in @sg writable by other side
2276 * @data: the token identifying the buffer.
2277 * @gfp: how to do memory allocations (if necessary).
2278 *
2279 * Caller must ensure we don't call this with other virtqueue operations
2280 * at the same time (except where noted).
2281 *
2282 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2283 */
2284int virtqueue_add_inbuf(struct virtqueue *vq,
2285 struct scatterlist *sg, unsigned int num,
2286 void *data,
2287 gfp_t gfp)
2288{
2289 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
2290}
2291EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
2292
2293/**
2294 * virtqueue_add_inbuf_ctx - expose input buffers to other end
2295 * @vq: the struct virtqueue we're talking about.
2296 * @sg: scatterlist (must be well-formed and terminated!)
2297 * @num: the number of entries in @sg writable by other side
2298 * @data: the token identifying the buffer.
2299 * @ctx: extra context for the token
2300 * @gfp: how to do memory allocations (if necessary).
2301 *
2302 * Caller must ensure we don't call this with other virtqueue operations
2303 * at the same time (except where noted).
2304 *
2305 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2306 */
2307int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
2308 struct scatterlist *sg, unsigned int num,
2309 void *data,
2310 void *ctx,
2311 gfp_t gfp)
2312{
2313 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
2314}
2315EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
2316
2317/**
2318 * virtqueue_dma_dev - get the dma dev
2319 * @_vq: the struct virtqueue we're talking about.
2320 *
2321 * Returns the dma dev. That can been used for dma api.
2322 */
2323struct device *virtqueue_dma_dev(struct virtqueue *_vq)
2324{
2325 struct vring_virtqueue *vq = to_vvq(_vq);
2326
2327 if (vq->use_dma_api)
2328 return vring_dma_dev(vq);
2329 else
2330 return NULL;
2331}
2332EXPORT_SYMBOL_GPL(virtqueue_dma_dev);
2333
2334/**
2335 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
2336 * @_vq: the struct virtqueue
2337 *
2338 * Instead of virtqueue_kick(), you can do:
2339 * if (virtqueue_kick_prepare(vq))
2340 * virtqueue_notify(vq);
2341 *
2342 * This is sometimes useful because the virtqueue_kick_prepare() needs
2343 * to be serialized, but the actual virtqueue_notify() call does not.
2344 */
2345bool virtqueue_kick_prepare(struct virtqueue *_vq)
2346{
2347 struct vring_virtqueue *vq = to_vvq(_vq);
2348
2349 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
2350 virtqueue_kick_prepare_split(_vq);
2351}
2352EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
2353
2354/**
2355 * virtqueue_notify - second half of split virtqueue_kick call.
2356 * @_vq: the struct virtqueue
2357 *
2358 * This does not need to be serialized.
2359 *
2360 * Returns false if host notify failed or queue is broken, otherwise true.
2361 */
2362bool virtqueue_notify(struct virtqueue *_vq)
2363{
2364 struct vring_virtqueue *vq = to_vvq(_vq);
2365
2366 if (unlikely(vq->broken))
2367 return false;
2368
2369 /* Prod other side to tell it about changes. */
2370 if (!vq->notify(_vq)) {
2371 vq->broken = true;
2372 return false;
2373 }
2374 return true;
2375}
2376EXPORT_SYMBOL_GPL(virtqueue_notify);
2377
2378/**
2379 * virtqueue_kick - update after add_buf
2380 * @vq: the struct virtqueue
2381 *
2382 * After one or more virtqueue_add_* calls, invoke this to kick
2383 * the other side.
2384 *
2385 * Caller must ensure we don't call this with other virtqueue
2386 * operations at the same time (except where noted).
2387 *
2388 * Returns false if kick failed, otherwise true.
2389 */
2390bool virtqueue_kick(struct virtqueue *vq)
2391{
2392 if (virtqueue_kick_prepare(vq))
2393 return virtqueue_notify(vq);
2394 return true;
2395}
2396EXPORT_SYMBOL_GPL(virtqueue_kick);
2397
2398/**
2399 * virtqueue_get_buf_ctx - get the next used buffer
2400 * @_vq: the struct virtqueue we're talking about.
2401 * @len: the length written into the buffer
2402 * @ctx: extra context for the token
2403 *
2404 * If the device wrote data into the buffer, @len will be set to the
2405 * amount written. This means you don't need to clear the buffer
2406 * beforehand to ensure there's no data leakage in the case of short
2407 * writes.
2408 *
2409 * Caller must ensure we don't call this with other virtqueue
2410 * operations at the same time (except where noted).
2411 *
2412 * Returns NULL if there are no used buffers, or the "data" token
2413 * handed to virtqueue_add_*().
2414 */
2415void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
2416 void **ctx)
2417{
2418 struct vring_virtqueue *vq = to_vvq(_vq);
2419
2420 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
2421 virtqueue_get_buf_ctx_split(_vq, len, ctx);
2422}
2423EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
2424
2425void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
2426{
2427 return virtqueue_get_buf_ctx(_vq, len, NULL);
2428}
2429EXPORT_SYMBOL_GPL(virtqueue_get_buf);
2430/**
2431 * virtqueue_disable_cb - disable callbacks
2432 * @_vq: the struct virtqueue we're talking about.
2433 *
2434 * Note that this is not necessarily synchronous, hence unreliable and only
2435 * useful as an optimization.
2436 *
2437 * Unlike other operations, this need not be serialized.
2438 */
2439void virtqueue_disable_cb(struct virtqueue *_vq)
2440{
2441 struct vring_virtqueue *vq = to_vvq(_vq);
2442
2443 if (vq->packed_ring)
2444 virtqueue_disable_cb_packed(_vq);
2445 else
2446 virtqueue_disable_cb_split(_vq);
2447}
2448EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
2449
2450/**
2451 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
2452 * @_vq: the struct virtqueue we're talking about.
2453 *
2454 * This re-enables callbacks; it returns current queue state
2455 * in an opaque unsigned value. This value should be later tested by
2456 * virtqueue_poll, to detect a possible race between the driver checking for
2457 * more work, and enabling callbacks.
2458 *
2459 * Caller must ensure we don't call this with other virtqueue
2460 * operations at the same time (except where noted).
2461 */
2462unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
2463{
2464 struct vring_virtqueue *vq = to_vvq(_vq);
2465
2466 if (vq->event_triggered)
2467 vq->event_triggered = false;
2468
2469 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
2470 virtqueue_enable_cb_prepare_split(_vq);
2471}
2472EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
2473
2474/**
2475 * virtqueue_poll - query pending used buffers
2476 * @_vq: the struct virtqueue we're talking about.
2477 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
2478 *
2479 * Returns "true" if there are pending used buffers in the queue.
2480 *
2481 * This does not need to be serialized.
2482 */
2483bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
2484{
2485 struct vring_virtqueue *vq = to_vvq(_vq);
2486
2487 if (unlikely(vq->broken))
2488 return false;
2489
2490 virtio_mb(vq->weak_barriers);
2491 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
2492 virtqueue_poll_split(_vq, last_used_idx);
2493}
2494EXPORT_SYMBOL_GPL(virtqueue_poll);
2495
2496/**
2497 * virtqueue_enable_cb - restart callbacks after disable_cb.
2498 * @_vq: the struct virtqueue we're talking about.
2499 *
2500 * This re-enables callbacks; it returns "false" if there are pending
2501 * buffers in the queue, to detect a possible race between the driver
2502 * checking for more work, and enabling callbacks.
2503 *
2504 * Caller must ensure we don't call this with other virtqueue
2505 * operations at the same time (except where noted).
2506 */
2507bool virtqueue_enable_cb(struct virtqueue *_vq)
2508{
2509 unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);
2510
2511 return !virtqueue_poll(_vq, last_used_idx);
2512}
2513EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
2514
2515/**
2516 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
2517 * @_vq: the struct virtqueue we're talking about.
2518 *
2519 * This re-enables callbacks but hints to the other side to delay
2520 * interrupts until most of the available buffers have been processed;
2521 * it returns "false" if there are many pending buffers in the queue,
2522 * to detect a possible race between the driver checking for more work,
2523 * and enabling callbacks.
2524 *
2525 * Caller must ensure we don't call this with other virtqueue
2526 * operations at the same time (except where noted).
2527 */
2528bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2529{
2530 struct vring_virtqueue *vq = to_vvq(_vq);
2531
2532 if (vq->event_triggered)
2533 vq->event_triggered = false;
2534
2535 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2536 virtqueue_enable_cb_delayed_split(_vq);
2537}
2538EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2539
2540/**
2541 * virtqueue_detach_unused_buf - detach first unused buffer
2542 * @_vq: the struct virtqueue we're talking about.
2543 *
2544 * Returns NULL or the "data" token handed to virtqueue_add_*().
2545 * This is not valid on an active queue; it is useful for device
2546 * shutdown or the reset queue.
2547 */
2548void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2549{
2550 struct vring_virtqueue *vq = to_vvq(_vq);
2551
2552 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2553 virtqueue_detach_unused_buf_split(_vq);
2554}
2555EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2556
2557static inline bool more_used(const struct vring_virtqueue *vq)
2558{
2559 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2560}
2561
2562/**
2563 * vring_interrupt - notify a virtqueue on an interrupt
2564 * @irq: the IRQ number (ignored)
2565 * @_vq: the struct virtqueue to notify
2566 *
2567 * Calls the callback function of @_vq to process the virtqueue
2568 * notification.
2569 */
2570irqreturn_t vring_interrupt(int irq, void *_vq)
2571{
2572 struct vring_virtqueue *vq = to_vvq(_vq);
2573
2574 if (!more_used(vq)) {
2575 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2576 return IRQ_NONE;
2577 }
2578
2579 if (unlikely(vq->broken)) {
2580#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2581 dev_warn_once(&vq->vq.vdev->dev,
2582 "virtio vring IRQ raised before DRIVER_OK");
2583 return IRQ_NONE;
2584#else
2585 return IRQ_HANDLED;
2586#endif
2587 }
2588
2589 /* Just a hint for performance: so it's ok that this can be racy! */
2590 if (vq->event)
2591 vq->event_triggered = true;
2592
2593 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2594 if (vq->vq.callback)
2595 vq->vq.callback(&vq->vq);
2596
2597 return IRQ_HANDLED;
2598}
2599EXPORT_SYMBOL_GPL(vring_interrupt);
2600
2601/* Only available for split ring */
2602static struct virtqueue *__vring_new_virtqueue(unsigned int index,
2603 struct vring_virtqueue_split *vring_split,
2604 struct virtio_device *vdev,
2605 bool weak_barriers,
2606 bool context,
2607 bool (*notify)(struct virtqueue *),
2608 void (*callback)(struct virtqueue *),
2609 const char *name,
2610 struct device *dma_dev)
2611{
2612 struct vring_virtqueue *vq;
2613 int err;
2614
2615 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2616 return NULL;
2617
2618 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2619 if (!vq)
2620 return NULL;
2621
2622 vq->packed_ring = false;
2623 vq->vq.callback = callback;
2624 vq->vq.vdev = vdev;
2625 vq->vq.name = name;
2626 vq->vq.index = index;
2627 vq->vq.reset = false;
2628 vq->we_own_ring = false;
2629 vq->notify = notify;
2630 vq->weak_barriers = weak_barriers;
2631#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2632 vq->broken = true;
2633#else
2634 vq->broken = false;
2635#endif
2636 vq->dma_dev = dma_dev;
2637 vq->use_dma_api = vring_use_dma_api(vdev);
2638 vq->premapped = false;
2639 vq->do_unmap = vq->use_dma_api;
2640
2641 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2642 !context;
2643 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2644
2645 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2646 vq->weak_barriers = false;
2647
2648 err = vring_alloc_state_extra_split(vring_split);
2649 if (err) {
2650 kfree(vq);
2651 return NULL;
2652 }
2653
2654 virtqueue_vring_init_split(vring_split, vq);
2655
2656 virtqueue_init(vq, vring_split->vring.num);
2657 virtqueue_vring_attach_split(vq, vring_split);
2658
2659 spin_lock(&vdev->vqs_list_lock);
2660 list_add_tail(&vq->vq.list, &vdev->vqs);
2661 spin_unlock(&vdev->vqs_list_lock);
2662 return &vq->vq;
2663}
2664
2665struct virtqueue *vring_create_virtqueue(
2666 unsigned int index,
2667 unsigned int num,
2668 unsigned int vring_align,
2669 struct virtio_device *vdev,
2670 bool weak_barriers,
2671 bool may_reduce_num,
2672 bool context,
2673 bool (*notify)(struct virtqueue *),
2674 void (*callback)(struct virtqueue *),
2675 const char *name)
2676{
2677
2678 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2679 return vring_create_virtqueue_packed(index, num, vring_align,
2680 vdev, weak_barriers, may_reduce_num,
2681 context, notify, callback, name, vdev->dev.parent);
2682
2683 return vring_create_virtqueue_split(index, num, vring_align,
2684 vdev, weak_barriers, may_reduce_num,
2685 context, notify, callback, name, vdev->dev.parent);
2686}
2687EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2688
2689struct virtqueue *vring_create_virtqueue_dma(
2690 unsigned int index,
2691 unsigned int num,
2692 unsigned int vring_align,
2693 struct virtio_device *vdev,
2694 bool weak_barriers,
2695 bool may_reduce_num,
2696 bool context,
2697 bool (*notify)(struct virtqueue *),
2698 void (*callback)(struct virtqueue *),
2699 const char *name,
2700 struct device *dma_dev)
2701{
2702
2703 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2704 return vring_create_virtqueue_packed(index, num, vring_align,
2705 vdev, weak_barriers, may_reduce_num,
2706 context, notify, callback, name, dma_dev);
2707
2708 return vring_create_virtqueue_split(index, num, vring_align,
2709 vdev, weak_barriers, may_reduce_num,
2710 context, notify, callback, name, dma_dev);
2711}
2712EXPORT_SYMBOL_GPL(vring_create_virtqueue_dma);
2713
2714/**
2715 * virtqueue_resize - resize the vring of vq
2716 * @_vq: the struct virtqueue we're talking about.
2717 * @num: new ring num
2718 * @recycle: callback to recycle unused buffers
2719 *
2720 * When it is really necessary to create a new vring, it will set the current vq
2721 * into the reset state. Then call the passed callback to recycle the buffer
2722 * that is no longer used. Only after the new vring is successfully created, the
2723 * old vring will be released.
2724 *
2725 * Caller must ensure we don't call this with other virtqueue operations
2726 * at the same time (except where noted).
2727 *
2728 * Returns zero or a negative error.
2729 * 0: success.
2730 * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size.
2731 * vq can still work normally
2732 * -EBUSY: Failed to sync with device, vq may not work properly
2733 * -ENOENT: Transport or device not supported
2734 * -E2BIG/-EINVAL: num error
2735 * -EPERM: Operation not permitted
2736 *
2737 */
2738int virtqueue_resize(struct virtqueue *_vq, u32 num,
2739 void (*recycle)(struct virtqueue *vq, void *buf))
2740{
2741 struct vring_virtqueue *vq = to_vvq(_vq);
2742 int err;
2743
2744 if (num > vq->vq.num_max)
2745 return -E2BIG;
2746
2747 if (!num)
2748 return -EINVAL;
2749
2750 if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
2751 return 0;
2752
2753 err = virtqueue_disable_and_recycle(_vq, recycle);
2754 if (err)
2755 return err;
2756
2757 if (vq->packed_ring)
2758 err = virtqueue_resize_packed(_vq, num);
2759 else
2760 err = virtqueue_resize_split(_vq, num);
2761
2762 return virtqueue_enable_after_reset(_vq);
2763}
2764EXPORT_SYMBOL_GPL(virtqueue_resize);
2765
2766/**
2767 * virtqueue_set_dma_premapped - set the vring premapped mode
2768 * @_vq: the struct virtqueue we're talking about.
2769 *
2770 * Enable the premapped mode of the vq.
2771 *
2772 * The vring in premapped mode does not do dma internally, so the driver must
2773 * do dma mapping in advance. The driver must pass the dma_address through
2774 * dma_address of scatterlist. When the driver got a used buffer from
2775 * the vring, it has to unmap the dma address.
2776 *
2777 * This function must be called immediately after creating the vq, or after vq
2778 * reset, and before adding any buffers to it.
2779 *
2780 * Caller must ensure we don't call this with other virtqueue operations
2781 * at the same time (except where noted).
2782 *
2783 * Returns zero or a negative error.
2784 * 0: success.
2785 * -EINVAL: vring does not use the dma api, so we can not enable premapped mode.
2786 */
2787int virtqueue_set_dma_premapped(struct virtqueue *_vq)
2788{
2789 struct vring_virtqueue *vq = to_vvq(_vq);
2790 u32 num;
2791
2792 START_USE(vq);
2793
2794 num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2795
2796 if (num != vq->vq.num_free) {
2797 END_USE(vq);
2798 return -EINVAL;
2799 }
2800
2801 if (!vq->use_dma_api) {
2802 END_USE(vq);
2803 return -EINVAL;
2804 }
2805
2806 vq->premapped = true;
2807 vq->do_unmap = false;
2808
2809 END_USE(vq);
2810
2811 return 0;
2812}
2813EXPORT_SYMBOL_GPL(virtqueue_set_dma_premapped);
2814
2815/**
2816 * virtqueue_reset - detach and recycle all unused buffers
2817 * @_vq: the struct virtqueue we're talking about.
2818 * @recycle: callback to recycle unused buffers
2819 *
2820 * Caller must ensure we don't call this with other virtqueue operations
2821 * at the same time (except where noted).
2822 *
2823 * Returns zero or a negative error.
2824 * 0: success.
2825 * -EBUSY: Failed to sync with device, vq may not work properly
2826 * -ENOENT: Transport or device not supported
2827 * -EPERM: Operation not permitted
2828 */
2829int virtqueue_reset(struct virtqueue *_vq,
2830 void (*recycle)(struct virtqueue *vq, void *buf))
2831{
2832 struct vring_virtqueue *vq = to_vvq(_vq);
2833 int err;
2834
2835 err = virtqueue_disable_and_recycle(_vq, recycle);
2836 if (err)
2837 return err;
2838
2839 if (vq->packed_ring)
2840 virtqueue_reinit_packed(vq);
2841 else
2842 virtqueue_reinit_split(vq);
2843
2844 return virtqueue_enable_after_reset(_vq);
2845}
2846EXPORT_SYMBOL_GPL(virtqueue_reset);
2847
2848/* Only available for split ring */
2849struct virtqueue *vring_new_virtqueue(unsigned int index,
2850 unsigned int num,
2851 unsigned int vring_align,
2852 struct virtio_device *vdev,
2853 bool weak_barriers,
2854 bool context,
2855 void *pages,
2856 bool (*notify)(struct virtqueue *vq),
2857 void (*callback)(struct virtqueue *vq),
2858 const char *name)
2859{
2860 struct vring_virtqueue_split vring_split = {};
2861
2862 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2863 return NULL;
2864
2865 vring_init(&vring_split.vring, num, pages, vring_align);
2866 return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
2867 context, notify, callback, name,
2868 vdev->dev.parent);
2869}
2870EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2871
2872static void vring_free(struct virtqueue *_vq)
2873{
2874 struct vring_virtqueue *vq = to_vvq(_vq);
2875
2876 if (vq->we_own_ring) {
2877 if (vq->packed_ring) {
2878 vring_free_queue(vq->vq.vdev,
2879 vq->packed.ring_size_in_bytes,
2880 vq->packed.vring.desc,
2881 vq->packed.ring_dma_addr,
2882 vring_dma_dev(vq));
2883
2884 vring_free_queue(vq->vq.vdev,
2885 vq->packed.event_size_in_bytes,
2886 vq->packed.vring.driver,
2887 vq->packed.driver_event_dma_addr,
2888 vring_dma_dev(vq));
2889
2890 vring_free_queue(vq->vq.vdev,
2891 vq->packed.event_size_in_bytes,
2892 vq->packed.vring.device,
2893 vq->packed.device_event_dma_addr,
2894 vring_dma_dev(vq));
2895
2896 kfree(vq->packed.desc_state);
2897 kfree(vq->packed.desc_extra);
2898 } else {
2899 vring_free_queue(vq->vq.vdev,
2900 vq->split.queue_size_in_bytes,
2901 vq->split.vring.desc,
2902 vq->split.queue_dma_addr,
2903 vring_dma_dev(vq));
2904 }
2905 }
2906 if (!vq->packed_ring) {
2907 kfree(vq->split.desc_state);
2908 kfree(vq->split.desc_extra);
2909 }
2910}
2911
2912void vring_del_virtqueue(struct virtqueue *_vq)
2913{
2914 struct vring_virtqueue *vq = to_vvq(_vq);
2915
2916 spin_lock(&vq->vq.vdev->vqs_list_lock);
2917 list_del(&_vq->list);
2918 spin_unlock(&vq->vq.vdev->vqs_list_lock);
2919
2920 vring_free(_vq);
2921
2922 kfree(vq);
2923}
2924EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2925
2926u32 vring_notification_data(struct virtqueue *_vq)
2927{
2928 struct vring_virtqueue *vq = to_vvq(_vq);
2929 u16 next;
2930
2931 if (vq->packed_ring)
2932 next = (vq->packed.next_avail_idx &
2933 ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
2934 vq->packed.avail_wrap_counter <<
2935 VRING_PACKED_EVENT_F_WRAP_CTR;
2936 else
2937 next = vq->split.avail_idx_shadow;
2938
2939 return next << 16 | _vq->index;
2940}
2941EXPORT_SYMBOL_GPL(vring_notification_data);
2942
2943/* Manipulates transport-specific feature bits. */
2944void vring_transport_features(struct virtio_device *vdev)
2945{
2946 unsigned int i;
2947
2948 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2949 switch (i) {
2950 case VIRTIO_RING_F_INDIRECT_DESC:
2951 break;
2952 case VIRTIO_RING_F_EVENT_IDX:
2953 break;
2954 case VIRTIO_F_VERSION_1:
2955 break;
2956 case VIRTIO_F_ACCESS_PLATFORM:
2957 break;
2958 case VIRTIO_F_RING_PACKED:
2959 break;
2960 case VIRTIO_F_ORDER_PLATFORM:
2961 break;
2962 case VIRTIO_F_NOTIFICATION_DATA:
2963 break;
2964 default:
2965 /* We don't understand this bit. */
2966 __virtio_clear_bit(vdev, i);
2967 }
2968 }
2969}
2970EXPORT_SYMBOL_GPL(vring_transport_features);
2971
2972/**
2973 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2974 * @_vq: the struct virtqueue containing the vring of interest.
2975 *
2976 * Returns the size of the vring. This is mainly used for boasting to
2977 * userspace. Unlike other operations, this need not be serialized.
2978 */
2979unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
2980{
2981
2982 const struct vring_virtqueue *vq = to_vvq(_vq);
2983
2984 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2985}
2986EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2987
2988/*
2989 * This function should only be called by the core, not directly by the driver.
2990 */
2991void __virtqueue_break(struct virtqueue *_vq)
2992{
2993 struct vring_virtqueue *vq = to_vvq(_vq);
2994
2995 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2996 WRITE_ONCE(vq->broken, true);
2997}
2998EXPORT_SYMBOL_GPL(__virtqueue_break);
2999
3000/*
3001 * This function should only be called by the core, not directly by the driver.
3002 */
3003void __virtqueue_unbreak(struct virtqueue *_vq)
3004{
3005 struct vring_virtqueue *vq = to_vvq(_vq);
3006
3007 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3008 WRITE_ONCE(vq->broken, false);
3009}
3010EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
3011
3012bool virtqueue_is_broken(const struct virtqueue *_vq)
3013{
3014 const struct vring_virtqueue *vq = to_vvq(_vq);
3015
3016 return READ_ONCE(vq->broken);
3017}
3018EXPORT_SYMBOL_GPL(virtqueue_is_broken);
3019
3020/*
3021 * This should prevent the device from being used, allowing drivers to
3022 * recover. You may need to grab appropriate locks to flush.
3023 */
3024void virtio_break_device(struct virtio_device *dev)
3025{
3026 struct virtqueue *_vq;
3027
3028 spin_lock(&dev->vqs_list_lock);
3029 list_for_each_entry(_vq, &dev->vqs, list) {
3030 struct vring_virtqueue *vq = to_vvq(_vq);
3031
3032 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3033 WRITE_ONCE(vq->broken, true);
3034 }
3035 spin_unlock(&dev->vqs_list_lock);
3036}
3037EXPORT_SYMBOL_GPL(virtio_break_device);
3038
3039/*
3040 * This should allow the device to be used by the driver. You may
3041 * need to grab appropriate locks to flush the write to
3042 * vq->broken. This should only be used in some specific case e.g
3043 * (probing and restoring). This function should only be called by the
3044 * core, not directly by the driver.
3045 */
3046void __virtio_unbreak_device(struct virtio_device *dev)
3047{
3048 struct virtqueue *_vq;
3049
3050 spin_lock(&dev->vqs_list_lock);
3051 list_for_each_entry(_vq, &dev->vqs, list) {
3052 struct vring_virtqueue *vq = to_vvq(_vq);
3053
3054 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3055 WRITE_ONCE(vq->broken, false);
3056 }
3057 spin_unlock(&dev->vqs_list_lock);
3058}
3059EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
3060
3061dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
3062{
3063 const struct vring_virtqueue *vq = to_vvq(_vq);
3064
3065 BUG_ON(!vq->we_own_ring);
3066
3067 if (vq->packed_ring)
3068 return vq->packed.ring_dma_addr;
3069
3070 return vq->split.queue_dma_addr;
3071}
3072EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
3073
3074dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
3075{
3076 const struct vring_virtqueue *vq = to_vvq(_vq);
3077
3078 BUG_ON(!vq->we_own_ring);
3079
3080 if (vq->packed_ring)
3081 return vq->packed.driver_event_dma_addr;
3082
3083 return vq->split.queue_dma_addr +
3084 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
3085}
3086EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
3087
3088dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
3089{
3090 const struct vring_virtqueue *vq = to_vvq(_vq);
3091
3092 BUG_ON(!vq->we_own_ring);
3093
3094 if (vq->packed_ring)
3095 return vq->packed.device_event_dma_addr;
3096
3097 return vq->split.queue_dma_addr +
3098 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
3099}
3100EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
3101
3102/* Only available for split ring */
3103const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
3104{
3105 return &to_vvq(vq)->split.vring;
3106}
3107EXPORT_SYMBOL_GPL(virtqueue_get_vring);
3108
3109/**
3110 * virtqueue_dma_map_single_attrs - map DMA for _vq
3111 * @_vq: the struct virtqueue we're talking about.
3112 * @ptr: the pointer of the buffer to do dma
3113 * @size: the size of the buffer to do dma
3114 * @dir: DMA direction
3115 * @attrs: DMA Attrs
3116 *
3117 * The caller calls this to do dma mapping in advance. The DMA address can be
3118 * passed to this _vq when it is in pre-mapped mode.
3119 *
3120 * return DMA address. Caller should check that by virtqueue_dma_mapping_error().
3121 */
3122dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr,
3123 size_t size,
3124 enum dma_data_direction dir,
3125 unsigned long attrs)
3126{
3127 struct vring_virtqueue *vq = to_vvq(_vq);
3128
3129 if (!vq->use_dma_api)
3130 return (dma_addr_t)virt_to_phys(ptr);
3131
3132 return dma_map_single_attrs(vring_dma_dev(vq), ptr, size, dir, attrs);
3133}
3134EXPORT_SYMBOL_GPL(virtqueue_dma_map_single_attrs);
3135
3136/**
3137 * virtqueue_dma_unmap_single_attrs - unmap DMA for _vq
3138 * @_vq: the struct virtqueue we're talking about.
3139 * @addr: the dma address to unmap
3140 * @size: the size of the buffer
3141 * @dir: DMA direction
3142 * @attrs: DMA Attrs
3143 *
3144 * Unmap the address that is mapped by the virtqueue_dma_map_* APIs.
3145 *
3146 */
3147void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr,
3148 size_t size, enum dma_data_direction dir,
3149 unsigned long attrs)
3150{
3151 struct vring_virtqueue *vq = to_vvq(_vq);
3152
3153 if (!vq->use_dma_api)
3154 return;
3155
3156 dma_unmap_single_attrs(vring_dma_dev(vq), addr, size, dir, attrs);
3157}
3158EXPORT_SYMBOL_GPL(virtqueue_dma_unmap_single_attrs);
3159
3160/**
3161 * virtqueue_dma_mapping_error - check dma address
3162 * @_vq: the struct virtqueue we're talking about.
3163 * @addr: DMA address
3164 *
3165 * Returns 0 means dma valid. Other means invalid dma address.
3166 */
3167int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr)
3168{
3169 struct vring_virtqueue *vq = to_vvq(_vq);
3170
3171 if (!vq->use_dma_api)
3172 return 0;
3173
3174 return dma_mapping_error(vring_dma_dev(vq), addr);
3175}
3176EXPORT_SYMBOL_GPL(virtqueue_dma_mapping_error);
3177
3178/**
3179 * virtqueue_dma_need_sync - check a dma address needs sync
3180 * @_vq: the struct virtqueue we're talking about.
3181 * @addr: DMA address
3182 *
3183 * Check if the dma address mapped by the virtqueue_dma_map_* APIs needs to be
3184 * synchronized
3185 *
3186 * return bool
3187 */
3188bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr)
3189{
3190 struct vring_virtqueue *vq = to_vvq(_vq);
3191
3192 if (!vq->use_dma_api)
3193 return false;
3194
3195 return dma_need_sync(vring_dma_dev(vq), addr);
3196}
3197EXPORT_SYMBOL_GPL(virtqueue_dma_need_sync);
3198
3199/**
3200 * virtqueue_dma_sync_single_range_for_cpu - dma sync for cpu
3201 * @_vq: the struct virtqueue we're talking about.
3202 * @addr: DMA address
3203 * @offset: DMA address offset
3204 * @size: buf size for sync
3205 * @dir: DMA direction
3206 *
3207 * Before calling this function, use virtqueue_dma_need_sync() to confirm that
3208 * the DMA address really needs to be synchronized
3209 *
3210 */
3211void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq,
3212 dma_addr_t addr,
3213 unsigned long offset, size_t size,
3214 enum dma_data_direction dir)
3215{
3216 struct vring_virtqueue *vq = to_vvq(_vq);
3217 struct device *dev = vring_dma_dev(vq);
3218
3219 if (!vq->use_dma_api)
3220 return;
3221
3222 dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
3223}
3224EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_cpu);
3225
3226/**
3227 * virtqueue_dma_sync_single_range_for_device - dma sync for device
3228 * @_vq: the struct virtqueue we're talking about.
3229 * @addr: DMA address
3230 * @offset: DMA address offset
3231 * @size: buf size for sync
3232 * @dir: DMA direction
3233 *
3234 * Before calling this function, use virtqueue_dma_need_sync() to confirm that
3235 * the DMA address really needs to be synchronized
3236 */
3237void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq,
3238 dma_addr_t addr,
3239 unsigned long offset, size_t size,
3240 enum dma_data_direction dir)
3241{
3242 struct vring_virtqueue *vq = to_vvq(_vq);
3243 struct device *dev = vring_dma_dev(vq);
3244
3245 if (!vq->use_dma_api)
3246 return;
3247
3248 dma_sync_single_range_for_device(dev, addr, offset, size, dir);
3249}
3250EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_device);
3251
3252MODULE_LICENSE("GPL");
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Virtio ring implementation.
3 *
4 * Copyright 2007 Rusty Russell IBM Corporation
5 */
6#include <linux/virtio.h>
7#include <linux/virtio_ring.h>
8#include <linux/virtio_config.h>
9#include <linux/device.h>
10#include <linux/slab.h>
11#include <linux/module.h>
12#include <linux/hrtimer.h>
13#include <linux/dma-mapping.h>
14#include <xen/xen.h>
15
16#ifdef DEBUG
17/* For development, we want to crash whenever the ring is screwed. */
18#define BAD_RING(_vq, fmt, args...) \
19 do { \
20 dev_err(&(_vq)->vq.vdev->dev, \
21 "%s:"fmt, (_vq)->vq.name, ##args); \
22 BUG(); \
23 } while (0)
24/* Caller is supposed to guarantee no reentry. */
25#define START_USE(_vq) \
26 do { \
27 if ((_vq)->in_use) \
28 panic("%s:in_use = %i\n", \
29 (_vq)->vq.name, (_vq)->in_use); \
30 (_vq)->in_use = __LINE__; \
31 } while (0)
32#define END_USE(_vq) \
33 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
34#define LAST_ADD_TIME_UPDATE(_vq) \
35 do { \
36 ktime_t now = ktime_get(); \
37 \
38 /* No kick or get, with .1 second between? Warn. */ \
39 if ((_vq)->last_add_time_valid) \
40 WARN_ON(ktime_to_ms(ktime_sub(now, \
41 (_vq)->last_add_time)) > 100); \
42 (_vq)->last_add_time = now; \
43 (_vq)->last_add_time_valid = true; \
44 } while (0)
45#define LAST_ADD_TIME_CHECK(_vq) \
46 do { \
47 if ((_vq)->last_add_time_valid) { \
48 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
49 (_vq)->last_add_time)) > 100); \
50 } \
51 } while (0)
52#define LAST_ADD_TIME_INVALID(_vq) \
53 ((_vq)->last_add_time_valid = false)
54#else
55#define BAD_RING(_vq, fmt, args...) \
56 do { \
57 dev_err(&_vq->vq.vdev->dev, \
58 "%s:"fmt, (_vq)->vq.name, ##args); \
59 (_vq)->broken = true; \
60 } while (0)
61#define START_USE(vq)
62#define END_USE(vq)
63#define LAST_ADD_TIME_UPDATE(vq)
64#define LAST_ADD_TIME_CHECK(vq)
65#define LAST_ADD_TIME_INVALID(vq)
66#endif
67
68struct vring_desc_state_split {
69 void *data; /* Data for callback. */
70 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
71};
72
73struct vring_desc_state_packed {
74 void *data; /* Data for callback. */
75 struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
76 u16 num; /* Descriptor list length. */
77 u16 next; /* The next desc state in a list. */
78 u16 last; /* The last desc state in a list. */
79};
80
81struct vring_desc_extra_packed {
82 dma_addr_t addr; /* Buffer DMA addr. */
83 u32 len; /* Buffer length. */
84 u16 flags; /* Descriptor flags. */
85};
86
87struct vring_virtqueue {
88 struct virtqueue vq;
89
90 /* Is this a packed ring? */
91 bool packed_ring;
92
93 /* Is DMA API used? */
94 bool use_dma_api;
95
96 /* Can we use weak barriers? */
97 bool weak_barriers;
98
99 /* Other side has made a mess, don't try any more. */
100 bool broken;
101
102 /* Host supports indirect buffers */
103 bool indirect;
104
105 /* Host publishes avail event idx */
106 bool event;
107
108 /* Head of free buffer list. */
109 unsigned int free_head;
110 /* Number we've added since last sync. */
111 unsigned int num_added;
112
113 /* Last used index we've seen. */
114 u16 last_used_idx;
115
116 union {
117 /* Available for split ring */
118 struct {
119 /* Actual memory layout for this queue. */
120 struct vring vring;
121
122 /* Last written value to avail->flags */
123 u16 avail_flags_shadow;
124
125 /*
126 * Last written value to avail->idx in
127 * guest byte order.
128 */
129 u16 avail_idx_shadow;
130
131 /* Per-descriptor state. */
132 struct vring_desc_state_split *desc_state;
133
134 /* DMA address and size information */
135 dma_addr_t queue_dma_addr;
136 size_t queue_size_in_bytes;
137 } split;
138
139 /* Available for packed ring */
140 struct {
141 /* Actual memory layout for this queue. */
142 struct {
143 unsigned int num;
144 struct vring_packed_desc *desc;
145 struct vring_packed_desc_event *driver;
146 struct vring_packed_desc_event *device;
147 } vring;
148
149 /* Driver ring wrap counter. */
150 bool avail_wrap_counter;
151
152 /* Device ring wrap counter. */
153 bool used_wrap_counter;
154
155 /* Avail used flags. */
156 u16 avail_used_flags;
157
158 /* Index of the next avail descriptor. */
159 u16 next_avail_idx;
160
161 /*
162 * Last written value to driver->flags in
163 * guest byte order.
164 */
165 u16 event_flags_shadow;
166
167 /* Per-descriptor state. */
168 struct vring_desc_state_packed *desc_state;
169 struct vring_desc_extra_packed *desc_extra;
170
171 /* DMA address and size information */
172 dma_addr_t ring_dma_addr;
173 dma_addr_t driver_event_dma_addr;
174 dma_addr_t device_event_dma_addr;
175 size_t ring_size_in_bytes;
176 size_t event_size_in_bytes;
177 } packed;
178 };
179
180 /* How to notify other side. FIXME: commonalize hcalls! */
181 bool (*notify)(struct virtqueue *vq);
182
183 /* DMA, allocation, and size information */
184 bool we_own_ring;
185
186#ifdef DEBUG
187 /* They're supposed to lock for us. */
188 unsigned int in_use;
189
190 /* Figure out if their kicks are too delayed. */
191 bool last_add_time_valid;
192 ktime_t last_add_time;
193#endif
194};
195
196
197/*
198 * Helpers.
199 */
200
201#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
202
203static inline bool virtqueue_use_indirect(struct virtqueue *_vq,
204 unsigned int total_sg)
205{
206 struct vring_virtqueue *vq = to_vvq(_vq);
207
208 /*
209 * If the host supports indirect descriptor tables, and we have multiple
210 * buffers, then go indirect. FIXME: tune this threshold
211 */
212 return (vq->indirect && total_sg > 1 && vq->vq.num_free);
213}
214
215/*
216 * Modern virtio devices have feature bits to specify whether they need a
217 * quirk and bypass the IOMMU. If not there, just use the DMA API.
218 *
219 * If there, the interaction between virtio and DMA API is messy.
220 *
221 * On most systems with virtio, physical addresses match bus addresses,
222 * and it doesn't particularly matter whether we use the DMA API.
223 *
224 * On some systems, including Xen and any system with a physical device
225 * that speaks virtio behind a physical IOMMU, we must use the DMA API
226 * for virtio DMA to work at all.
227 *
228 * On other systems, including SPARC and PPC64, virtio-pci devices are
229 * enumerated as though they are behind an IOMMU, but the virtio host
230 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
231 * there or somehow map everything as the identity.
232 *
233 * For the time being, we preserve historic behavior and bypass the DMA
234 * API.
235 *
236 * TODO: install a per-device DMA ops structure that does the right thing
237 * taking into account all the above quirks, and use the DMA API
238 * unconditionally on data path.
239 */
240
241static bool vring_use_dma_api(struct virtio_device *vdev)
242{
243 if (!virtio_has_iommu_quirk(vdev))
244 return true;
245
246 /* Otherwise, we are left to guess. */
247 /*
248 * In theory, it's possible to have a buggy QEMU-supposed
249 * emulated Q35 IOMMU and Xen enabled at the same time. On
250 * such a configuration, virtio has never worked and will
251 * not work without an even larger kludge. Instead, enable
252 * the DMA API if we're a Xen guest, which at least allows
253 * all of the sensible Xen configurations to work correctly.
254 */
255 if (xen_domain())
256 return true;
257
258 return false;
259}
260
261size_t virtio_max_dma_size(struct virtio_device *vdev)
262{
263 size_t max_segment_size = SIZE_MAX;
264
265 if (vring_use_dma_api(vdev))
266 max_segment_size = dma_max_mapping_size(&vdev->dev);
267
268 return max_segment_size;
269}
270EXPORT_SYMBOL_GPL(virtio_max_dma_size);
271
272static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
273 dma_addr_t *dma_handle, gfp_t flag)
274{
275 if (vring_use_dma_api(vdev)) {
276 return dma_alloc_coherent(vdev->dev.parent, size,
277 dma_handle, flag);
278 } else {
279 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
280
281 if (queue) {
282 phys_addr_t phys_addr = virt_to_phys(queue);
283 *dma_handle = (dma_addr_t)phys_addr;
284
285 /*
286 * Sanity check: make sure we dind't truncate
287 * the address. The only arches I can find that
288 * have 64-bit phys_addr_t but 32-bit dma_addr_t
289 * are certain non-highmem MIPS and x86
290 * configurations, but these configurations
291 * should never allocate physical pages above 32
292 * bits, so this is fine. Just in case, throw a
293 * warning and abort if we end up with an
294 * unrepresentable address.
295 */
296 if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
297 free_pages_exact(queue, PAGE_ALIGN(size));
298 return NULL;
299 }
300 }
301 return queue;
302 }
303}
304
305static void vring_free_queue(struct virtio_device *vdev, size_t size,
306 void *queue, dma_addr_t dma_handle)
307{
308 if (vring_use_dma_api(vdev))
309 dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
310 else
311 free_pages_exact(queue, PAGE_ALIGN(size));
312}
313
314/*
315 * The DMA ops on various arches are rather gnarly right now, and
316 * making all of the arch DMA ops work on the vring device itself
317 * is a mess. For now, we use the parent device for DMA ops.
318 */
319static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
320{
321 return vq->vq.vdev->dev.parent;
322}
323
324/* Map one sg entry. */
325static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
326 struct scatterlist *sg,
327 enum dma_data_direction direction)
328{
329 if (!vq->use_dma_api)
330 return (dma_addr_t)sg_phys(sg);
331
332 /*
333 * We can't use dma_map_sg, because we don't use scatterlists in
334 * the way it expects (we don't guarantee that the scatterlist
335 * will exist for the lifetime of the mapping).
336 */
337 return dma_map_page(vring_dma_dev(vq),
338 sg_page(sg), sg->offset, sg->length,
339 direction);
340}
341
342static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
343 void *cpu_addr, size_t size,
344 enum dma_data_direction direction)
345{
346 if (!vq->use_dma_api)
347 return (dma_addr_t)virt_to_phys(cpu_addr);
348
349 return dma_map_single(vring_dma_dev(vq),
350 cpu_addr, size, direction);
351}
352
353static int vring_mapping_error(const struct vring_virtqueue *vq,
354 dma_addr_t addr)
355{
356 if (!vq->use_dma_api)
357 return 0;
358
359 return dma_mapping_error(vring_dma_dev(vq), addr);
360}
361
362
363/*
364 * Split ring specific functions - *_split().
365 */
366
367static void vring_unmap_one_split(const struct vring_virtqueue *vq,
368 struct vring_desc *desc)
369{
370 u16 flags;
371
372 if (!vq->use_dma_api)
373 return;
374
375 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
376
377 if (flags & VRING_DESC_F_INDIRECT) {
378 dma_unmap_single(vring_dma_dev(vq),
379 virtio64_to_cpu(vq->vq.vdev, desc->addr),
380 virtio32_to_cpu(vq->vq.vdev, desc->len),
381 (flags & VRING_DESC_F_WRITE) ?
382 DMA_FROM_DEVICE : DMA_TO_DEVICE);
383 } else {
384 dma_unmap_page(vring_dma_dev(vq),
385 virtio64_to_cpu(vq->vq.vdev, desc->addr),
386 virtio32_to_cpu(vq->vq.vdev, desc->len),
387 (flags & VRING_DESC_F_WRITE) ?
388 DMA_FROM_DEVICE : DMA_TO_DEVICE);
389 }
390}
391
392static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
393 unsigned int total_sg,
394 gfp_t gfp)
395{
396 struct vring_desc *desc;
397 unsigned int i;
398
399 /*
400 * We require lowmem mappings for the descriptors because
401 * otherwise virt_to_phys will give us bogus addresses in the
402 * virtqueue.
403 */
404 gfp &= ~__GFP_HIGHMEM;
405
406 desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
407 if (!desc)
408 return NULL;
409
410 for (i = 0; i < total_sg; i++)
411 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
412 return desc;
413}
414
415static inline int virtqueue_add_split(struct virtqueue *_vq,
416 struct scatterlist *sgs[],
417 unsigned int total_sg,
418 unsigned int out_sgs,
419 unsigned int in_sgs,
420 void *data,
421 void *ctx,
422 gfp_t gfp)
423{
424 struct vring_virtqueue *vq = to_vvq(_vq);
425 struct scatterlist *sg;
426 struct vring_desc *desc;
427 unsigned int i, n, avail, descs_used, uninitialized_var(prev), err_idx;
428 int head;
429 bool indirect;
430
431 START_USE(vq);
432
433 BUG_ON(data == NULL);
434 BUG_ON(ctx && vq->indirect);
435
436 if (unlikely(vq->broken)) {
437 END_USE(vq);
438 return -EIO;
439 }
440
441 LAST_ADD_TIME_UPDATE(vq);
442
443 BUG_ON(total_sg == 0);
444
445 head = vq->free_head;
446
447 if (virtqueue_use_indirect(_vq, total_sg))
448 desc = alloc_indirect_split(_vq, total_sg, gfp);
449 else {
450 desc = NULL;
451 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
452 }
453
454 if (desc) {
455 /* Use a single buffer which doesn't continue */
456 indirect = true;
457 /* Set up rest to use this indirect table. */
458 i = 0;
459 descs_used = 1;
460 } else {
461 indirect = false;
462 desc = vq->split.vring.desc;
463 i = head;
464 descs_used = total_sg;
465 }
466
467 if (vq->vq.num_free < descs_used) {
468 pr_debug("Can't add buf len %i - avail = %i\n",
469 descs_used, vq->vq.num_free);
470 /* FIXME: for historical reasons, we force a notify here if
471 * there are outgoing parts to the buffer. Presumably the
472 * host should service the ring ASAP. */
473 if (out_sgs)
474 vq->notify(&vq->vq);
475 if (indirect)
476 kfree(desc);
477 END_USE(vq);
478 return -ENOSPC;
479 }
480
481 for (n = 0; n < out_sgs; n++) {
482 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
483 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
484 if (vring_mapping_error(vq, addr))
485 goto unmap_release;
486
487 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);
488 desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
489 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
490 prev = i;
491 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
492 }
493 }
494 for (; n < (out_sgs + in_sgs); n++) {
495 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
496 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
497 if (vring_mapping_error(vq, addr))
498 goto unmap_release;
499
500 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);
501 desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
502 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
503 prev = i;
504 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
505 }
506 }
507 /* Last one doesn't continue. */
508 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
509
510 if (indirect) {
511 /* Now that the indirect table is filled in, map it. */
512 dma_addr_t addr = vring_map_single(
513 vq, desc, total_sg * sizeof(struct vring_desc),
514 DMA_TO_DEVICE);
515 if (vring_mapping_error(vq, addr))
516 goto unmap_release;
517
518 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev,
519 VRING_DESC_F_INDIRECT);
520 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev,
521 addr);
522
523 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev,
524 total_sg * sizeof(struct vring_desc));
525 }
526
527 /* We're using some buffers from the free list. */
528 vq->vq.num_free -= descs_used;
529
530 /* Update free pointer */
531 if (indirect)
532 vq->free_head = virtio16_to_cpu(_vq->vdev,
533 vq->split.vring.desc[head].next);
534 else
535 vq->free_head = i;
536
537 /* Store token and indirect buffer state. */
538 vq->split.desc_state[head].data = data;
539 if (indirect)
540 vq->split.desc_state[head].indir_desc = desc;
541 else
542 vq->split.desc_state[head].indir_desc = ctx;
543
544 /* Put entry in available array (but don't update avail->idx until they
545 * do sync). */
546 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
547 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
548
549 /* Descriptors and available array need to be set before we expose the
550 * new available array entries. */
551 virtio_wmb(vq->weak_barriers);
552 vq->split.avail_idx_shadow++;
553 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
554 vq->split.avail_idx_shadow);
555 vq->num_added++;
556
557 pr_debug("Added buffer head %i to %p\n", head, vq);
558 END_USE(vq);
559
560 /* This is very unlikely, but theoretically possible. Kick
561 * just in case. */
562 if (unlikely(vq->num_added == (1 << 16) - 1))
563 virtqueue_kick(_vq);
564
565 return 0;
566
567unmap_release:
568 err_idx = i;
569
570 if (indirect)
571 i = 0;
572 else
573 i = head;
574
575 for (n = 0; n < total_sg; n++) {
576 if (i == err_idx)
577 break;
578 vring_unmap_one_split(vq, &desc[i]);
579 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
580 }
581
582 if (indirect)
583 kfree(desc);
584
585 END_USE(vq);
586 return -ENOMEM;
587}
588
589static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
590{
591 struct vring_virtqueue *vq = to_vvq(_vq);
592 u16 new, old;
593 bool needs_kick;
594
595 START_USE(vq);
596 /* We need to expose available array entries before checking avail
597 * event. */
598 virtio_mb(vq->weak_barriers);
599
600 old = vq->split.avail_idx_shadow - vq->num_added;
601 new = vq->split.avail_idx_shadow;
602 vq->num_added = 0;
603
604 LAST_ADD_TIME_CHECK(vq);
605 LAST_ADD_TIME_INVALID(vq);
606
607 if (vq->event) {
608 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
609 vring_avail_event(&vq->split.vring)),
610 new, old);
611 } else {
612 needs_kick = !(vq->split.vring.used->flags &
613 cpu_to_virtio16(_vq->vdev,
614 VRING_USED_F_NO_NOTIFY));
615 }
616 END_USE(vq);
617 return needs_kick;
618}
619
620static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
621 void **ctx)
622{
623 unsigned int i, j;
624 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
625
626 /* Clear data ptr. */
627 vq->split.desc_state[head].data = NULL;
628
629 /* Put back on free list: unmap first-level descriptors and find end */
630 i = head;
631
632 while (vq->split.vring.desc[i].flags & nextflag) {
633 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
634 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
635 vq->vq.num_free++;
636 }
637
638 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
639 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
640 vq->free_head);
641 vq->free_head = head;
642
643 /* Plus final descriptor */
644 vq->vq.num_free++;
645
646 if (vq->indirect) {
647 struct vring_desc *indir_desc =
648 vq->split.desc_state[head].indir_desc;
649 u32 len;
650
651 /* Free the indirect table, if any, now that it's unmapped. */
652 if (!indir_desc)
653 return;
654
655 len = virtio32_to_cpu(vq->vq.vdev,
656 vq->split.vring.desc[head].len);
657
658 BUG_ON(!(vq->split.vring.desc[head].flags &
659 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
660 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
661
662 for (j = 0; j < len / sizeof(struct vring_desc); j++)
663 vring_unmap_one_split(vq, &indir_desc[j]);
664
665 kfree(indir_desc);
666 vq->split.desc_state[head].indir_desc = NULL;
667 } else if (ctx) {
668 *ctx = vq->split.desc_state[head].indir_desc;
669 }
670}
671
672static inline bool more_used_split(const struct vring_virtqueue *vq)
673{
674 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
675 vq->split.vring.used->idx);
676}
677
678static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
679 unsigned int *len,
680 void **ctx)
681{
682 struct vring_virtqueue *vq = to_vvq(_vq);
683 void *ret;
684 unsigned int i;
685 u16 last_used;
686
687 START_USE(vq);
688
689 if (unlikely(vq->broken)) {
690 END_USE(vq);
691 return NULL;
692 }
693
694 if (!more_used_split(vq)) {
695 pr_debug("No more buffers in queue\n");
696 END_USE(vq);
697 return NULL;
698 }
699
700 /* Only get used array entries after they have been exposed by host. */
701 virtio_rmb(vq->weak_barriers);
702
703 last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
704 i = virtio32_to_cpu(_vq->vdev,
705 vq->split.vring.used->ring[last_used].id);
706 *len = virtio32_to_cpu(_vq->vdev,
707 vq->split.vring.used->ring[last_used].len);
708
709 if (unlikely(i >= vq->split.vring.num)) {
710 BAD_RING(vq, "id %u out of range\n", i);
711 return NULL;
712 }
713 if (unlikely(!vq->split.desc_state[i].data)) {
714 BAD_RING(vq, "id %u is not a head!\n", i);
715 return NULL;
716 }
717
718 /* detach_buf_split clears data, so grab it now. */
719 ret = vq->split.desc_state[i].data;
720 detach_buf_split(vq, i, ctx);
721 vq->last_used_idx++;
722 /* If we expect an interrupt for the next entry, tell host
723 * by writing event index and flush out the write before
724 * the read in the next get_buf call. */
725 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
726 virtio_store_mb(vq->weak_barriers,
727 &vring_used_event(&vq->split.vring),
728 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
729
730 LAST_ADD_TIME_INVALID(vq);
731
732 END_USE(vq);
733 return ret;
734}
735
736static void virtqueue_disable_cb_split(struct virtqueue *_vq)
737{
738 struct vring_virtqueue *vq = to_vvq(_vq);
739
740 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
741 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
742 if (!vq->event)
743 vq->split.vring.avail->flags =
744 cpu_to_virtio16(_vq->vdev,
745 vq->split.avail_flags_shadow);
746 }
747}
748
749static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
750{
751 struct vring_virtqueue *vq = to_vvq(_vq);
752 u16 last_used_idx;
753
754 START_USE(vq);
755
756 /* We optimistically turn back on interrupts, then check if there was
757 * more to do. */
758 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
759 * either clear the flags bit or point the event index at the next
760 * entry. Always do both to keep code simple. */
761 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
762 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
763 if (!vq->event)
764 vq->split.vring.avail->flags =
765 cpu_to_virtio16(_vq->vdev,
766 vq->split.avail_flags_shadow);
767 }
768 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
769 last_used_idx = vq->last_used_idx);
770 END_USE(vq);
771 return last_used_idx;
772}
773
774static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx)
775{
776 struct vring_virtqueue *vq = to_vvq(_vq);
777
778 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
779 vq->split.vring.used->idx);
780}
781
782static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
783{
784 struct vring_virtqueue *vq = to_vvq(_vq);
785 u16 bufs;
786
787 START_USE(vq);
788
789 /* We optimistically turn back on interrupts, then check if there was
790 * more to do. */
791 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
792 * either clear the flags bit or point the event index at the next
793 * entry. Always update the event index to keep code simple. */
794 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
795 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
796 if (!vq->event)
797 vq->split.vring.avail->flags =
798 cpu_to_virtio16(_vq->vdev,
799 vq->split.avail_flags_shadow);
800 }
801 /* TODO: tune this threshold */
802 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
803
804 virtio_store_mb(vq->weak_barriers,
805 &vring_used_event(&vq->split.vring),
806 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
807
808 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
809 - vq->last_used_idx) > bufs)) {
810 END_USE(vq);
811 return false;
812 }
813
814 END_USE(vq);
815 return true;
816}
817
818static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
819{
820 struct vring_virtqueue *vq = to_vvq(_vq);
821 unsigned int i;
822 void *buf;
823
824 START_USE(vq);
825
826 for (i = 0; i < vq->split.vring.num; i++) {
827 if (!vq->split.desc_state[i].data)
828 continue;
829 /* detach_buf_split clears data, so grab it now. */
830 buf = vq->split.desc_state[i].data;
831 detach_buf_split(vq, i, NULL);
832 vq->split.avail_idx_shadow--;
833 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
834 vq->split.avail_idx_shadow);
835 END_USE(vq);
836 return buf;
837 }
838 /* That should have freed everything. */
839 BUG_ON(vq->vq.num_free != vq->split.vring.num);
840
841 END_USE(vq);
842 return NULL;
843}
844
845static struct virtqueue *vring_create_virtqueue_split(
846 unsigned int index,
847 unsigned int num,
848 unsigned int vring_align,
849 struct virtio_device *vdev,
850 bool weak_barriers,
851 bool may_reduce_num,
852 bool context,
853 bool (*notify)(struct virtqueue *),
854 void (*callback)(struct virtqueue *),
855 const char *name)
856{
857 struct virtqueue *vq;
858 void *queue = NULL;
859 dma_addr_t dma_addr;
860 size_t queue_size_in_bytes;
861 struct vring vring;
862
863 /* We assume num is a power of 2. */
864 if (num & (num - 1)) {
865 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
866 return NULL;
867 }
868
869 /* TODO: allocate each queue chunk individually */
870 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
871 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
872 &dma_addr,
873 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
874 if (queue)
875 break;
876 if (!may_reduce_num)
877 return NULL;
878 }
879
880 if (!num)
881 return NULL;
882
883 if (!queue) {
884 /* Try to get a single page. You are my only hope! */
885 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
886 &dma_addr, GFP_KERNEL|__GFP_ZERO);
887 }
888 if (!queue)
889 return NULL;
890
891 queue_size_in_bytes = vring_size(num, vring_align);
892 vring_init(&vring, num, queue, vring_align);
893
894 vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
895 notify, callback, name);
896 if (!vq) {
897 vring_free_queue(vdev, queue_size_in_bytes, queue,
898 dma_addr);
899 return NULL;
900 }
901
902 to_vvq(vq)->split.queue_dma_addr = dma_addr;
903 to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
904 to_vvq(vq)->we_own_ring = true;
905
906 return vq;
907}
908
909
910/*
911 * Packed ring specific functions - *_packed().
912 */
913
914static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
915 struct vring_desc_extra_packed *state)
916{
917 u16 flags;
918
919 if (!vq->use_dma_api)
920 return;
921
922 flags = state->flags;
923
924 if (flags & VRING_DESC_F_INDIRECT) {
925 dma_unmap_single(vring_dma_dev(vq),
926 state->addr, state->len,
927 (flags & VRING_DESC_F_WRITE) ?
928 DMA_FROM_DEVICE : DMA_TO_DEVICE);
929 } else {
930 dma_unmap_page(vring_dma_dev(vq),
931 state->addr, state->len,
932 (flags & VRING_DESC_F_WRITE) ?
933 DMA_FROM_DEVICE : DMA_TO_DEVICE);
934 }
935}
936
937static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
938 struct vring_packed_desc *desc)
939{
940 u16 flags;
941
942 if (!vq->use_dma_api)
943 return;
944
945 flags = le16_to_cpu(desc->flags);
946
947 if (flags & VRING_DESC_F_INDIRECT) {
948 dma_unmap_single(vring_dma_dev(vq),
949 le64_to_cpu(desc->addr),
950 le32_to_cpu(desc->len),
951 (flags & VRING_DESC_F_WRITE) ?
952 DMA_FROM_DEVICE : DMA_TO_DEVICE);
953 } else {
954 dma_unmap_page(vring_dma_dev(vq),
955 le64_to_cpu(desc->addr),
956 le32_to_cpu(desc->len),
957 (flags & VRING_DESC_F_WRITE) ?
958 DMA_FROM_DEVICE : DMA_TO_DEVICE);
959 }
960}
961
962static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
963 gfp_t gfp)
964{
965 struct vring_packed_desc *desc;
966
967 /*
968 * We require lowmem mappings for the descriptors because
969 * otherwise virt_to_phys will give us bogus addresses in the
970 * virtqueue.
971 */
972 gfp &= ~__GFP_HIGHMEM;
973
974 desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
975
976 return desc;
977}
978
979static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
980 struct scatterlist *sgs[],
981 unsigned int total_sg,
982 unsigned int out_sgs,
983 unsigned int in_sgs,
984 void *data,
985 gfp_t gfp)
986{
987 struct vring_packed_desc *desc;
988 struct scatterlist *sg;
989 unsigned int i, n, err_idx;
990 u16 head, id;
991 dma_addr_t addr;
992
993 head = vq->packed.next_avail_idx;
994 desc = alloc_indirect_packed(total_sg, gfp);
995
996 if (unlikely(vq->vq.num_free < 1)) {
997 pr_debug("Can't add buf len 1 - avail = 0\n");
998 kfree(desc);
999 END_USE(vq);
1000 return -ENOSPC;
1001 }
1002
1003 i = 0;
1004 id = vq->free_head;
1005 BUG_ON(id == vq->packed.vring.num);
1006
1007 for (n = 0; n < out_sgs + in_sgs; n++) {
1008 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1009 addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1010 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1011 if (vring_mapping_error(vq, addr))
1012 goto unmap_release;
1013
1014 desc[i].flags = cpu_to_le16(n < out_sgs ?
1015 0 : VRING_DESC_F_WRITE);
1016 desc[i].addr = cpu_to_le64(addr);
1017 desc[i].len = cpu_to_le32(sg->length);
1018 i++;
1019 }
1020 }
1021
1022 /* Now that the indirect table is filled in, map it. */
1023 addr = vring_map_single(vq, desc,
1024 total_sg * sizeof(struct vring_packed_desc),
1025 DMA_TO_DEVICE);
1026 if (vring_mapping_error(vq, addr))
1027 goto unmap_release;
1028
1029 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1030 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1031 sizeof(struct vring_packed_desc));
1032 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1033
1034 if (vq->use_dma_api) {
1035 vq->packed.desc_extra[id].addr = addr;
1036 vq->packed.desc_extra[id].len = total_sg *
1037 sizeof(struct vring_packed_desc);
1038 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1039 vq->packed.avail_used_flags;
1040 }
1041
1042 /*
1043 * A driver MUST NOT make the first descriptor in the list
1044 * available before all subsequent descriptors comprising
1045 * the list are made available.
1046 */
1047 virtio_wmb(vq->weak_barriers);
1048 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1049 vq->packed.avail_used_flags);
1050
1051 /* We're using some buffers from the free list. */
1052 vq->vq.num_free -= 1;
1053
1054 /* Update free pointer */
1055 n = head + 1;
1056 if (n >= vq->packed.vring.num) {
1057 n = 0;
1058 vq->packed.avail_wrap_counter ^= 1;
1059 vq->packed.avail_used_flags ^=
1060 1 << VRING_PACKED_DESC_F_AVAIL |
1061 1 << VRING_PACKED_DESC_F_USED;
1062 }
1063 vq->packed.next_avail_idx = n;
1064 vq->free_head = vq->packed.desc_state[id].next;
1065
1066 /* Store token and indirect buffer state. */
1067 vq->packed.desc_state[id].num = 1;
1068 vq->packed.desc_state[id].data = data;
1069 vq->packed.desc_state[id].indir_desc = desc;
1070 vq->packed.desc_state[id].last = id;
1071
1072 vq->num_added += 1;
1073
1074 pr_debug("Added buffer head %i to %p\n", head, vq);
1075 END_USE(vq);
1076
1077 return 0;
1078
1079unmap_release:
1080 err_idx = i;
1081
1082 for (i = 0; i < err_idx; i++)
1083 vring_unmap_desc_packed(vq, &desc[i]);
1084
1085 kfree(desc);
1086
1087 END_USE(vq);
1088 return -ENOMEM;
1089}
1090
1091static inline int virtqueue_add_packed(struct virtqueue *_vq,
1092 struct scatterlist *sgs[],
1093 unsigned int total_sg,
1094 unsigned int out_sgs,
1095 unsigned int in_sgs,
1096 void *data,
1097 void *ctx,
1098 gfp_t gfp)
1099{
1100 struct vring_virtqueue *vq = to_vvq(_vq);
1101 struct vring_packed_desc *desc;
1102 struct scatterlist *sg;
1103 unsigned int i, n, c, descs_used, err_idx;
1104 __le16 uninitialized_var(head_flags), flags;
1105 u16 head, id, uninitialized_var(prev), curr, avail_used_flags;
1106
1107 START_USE(vq);
1108
1109 BUG_ON(data == NULL);
1110 BUG_ON(ctx && vq->indirect);
1111
1112 if (unlikely(vq->broken)) {
1113 END_USE(vq);
1114 return -EIO;
1115 }
1116
1117 LAST_ADD_TIME_UPDATE(vq);
1118
1119 BUG_ON(total_sg == 0);
1120
1121 if (virtqueue_use_indirect(_vq, total_sg))
1122 return virtqueue_add_indirect_packed(vq, sgs, total_sg,
1123 out_sgs, in_sgs, data, gfp);
1124
1125 head = vq->packed.next_avail_idx;
1126 avail_used_flags = vq->packed.avail_used_flags;
1127
1128 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1129
1130 desc = vq->packed.vring.desc;
1131 i = head;
1132 descs_used = total_sg;
1133
1134 if (unlikely(vq->vq.num_free < descs_used)) {
1135 pr_debug("Can't add buf len %i - avail = %i\n",
1136 descs_used, vq->vq.num_free);
1137 END_USE(vq);
1138 return -ENOSPC;
1139 }
1140
1141 id = vq->free_head;
1142 BUG_ON(id == vq->packed.vring.num);
1143
1144 curr = id;
1145 c = 0;
1146 for (n = 0; n < out_sgs + in_sgs; n++) {
1147 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1148 dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1149 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1150 if (vring_mapping_error(vq, addr))
1151 goto unmap_release;
1152
1153 flags = cpu_to_le16(vq->packed.avail_used_flags |
1154 (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1155 (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1156 if (i == head)
1157 head_flags = flags;
1158 else
1159 desc[i].flags = flags;
1160
1161 desc[i].addr = cpu_to_le64(addr);
1162 desc[i].len = cpu_to_le32(sg->length);
1163 desc[i].id = cpu_to_le16(id);
1164
1165 if (unlikely(vq->use_dma_api)) {
1166 vq->packed.desc_extra[curr].addr = addr;
1167 vq->packed.desc_extra[curr].len = sg->length;
1168 vq->packed.desc_extra[curr].flags =
1169 le16_to_cpu(flags);
1170 }
1171 prev = curr;
1172 curr = vq->packed.desc_state[curr].next;
1173
1174 if ((unlikely(++i >= vq->packed.vring.num))) {
1175 i = 0;
1176 vq->packed.avail_used_flags ^=
1177 1 << VRING_PACKED_DESC_F_AVAIL |
1178 1 << VRING_PACKED_DESC_F_USED;
1179 }
1180 }
1181 }
1182
1183 if (i < head)
1184 vq->packed.avail_wrap_counter ^= 1;
1185
1186 /* We're using some buffers from the free list. */
1187 vq->vq.num_free -= descs_used;
1188
1189 /* Update free pointer */
1190 vq->packed.next_avail_idx = i;
1191 vq->free_head = curr;
1192
1193 /* Store token. */
1194 vq->packed.desc_state[id].num = descs_used;
1195 vq->packed.desc_state[id].data = data;
1196 vq->packed.desc_state[id].indir_desc = ctx;
1197 vq->packed.desc_state[id].last = prev;
1198
1199 /*
1200 * A driver MUST NOT make the first descriptor in the list
1201 * available before all subsequent descriptors comprising
1202 * the list are made available.
1203 */
1204 virtio_wmb(vq->weak_barriers);
1205 vq->packed.vring.desc[head].flags = head_flags;
1206 vq->num_added += descs_used;
1207
1208 pr_debug("Added buffer head %i to %p\n", head, vq);
1209 END_USE(vq);
1210
1211 return 0;
1212
1213unmap_release:
1214 err_idx = i;
1215 i = head;
1216
1217 vq->packed.avail_used_flags = avail_used_flags;
1218
1219 for (n = 0; n < total_sg; n++) {
1220 if (i == err_idx)
1221 break;
1222 vring_unmap_desc_packed(vq, &desc[i]);
1223 i++;
1224 if (i >= vq->packed.vring.num)
1225 i = 0;
1226 }
1227
1228 END_USE(vq);
1229 return -EIO;
1230}
1231
1232static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1233{
1234 struct vring_virtqueue *vq = to_vvq(_vq);
1235 u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1236 bool needs_kick;
1237 union {
1238 struct {
1239 __le16 off_wrap;
1240 __le16 flags;
1241 };
1242 u32 u32;
1243 } snapshot;
1244
1245 START_USE(vq);
1246
1247 /*
1248 * We need to expose the new flags value before checking notification
1249 * suppressions.
1250 */
1251 virtio_mb(vq->weak_barriers);
1252
1253 old = vq->packed.next_avail_idx - vq->num_added;
1254 new = vq->packed.next_avail_idx;
1255 vq->num_added = 0;
1256
1257 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1258 flags = le16_to_cpu(snapshot.flags);
1259
1260 LAST_ADD_TIME_CHECK(vq);
1261 LAST_ADD_TIME_INVALID(vq);
1262
1263 if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1264 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1265 goto out;
1266 }
1267
1268 off_wrap = le16_to_cpu(snapshot.off_wrap);
1269
1270 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1271 event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1272 if (wrap_counter != vq->packed.avail_wrap_counter)
1273 event_idx -= vq->packed.vring.num;
1274
1275 needs_kick = vring_need_event(event_idx, new, old);
1276out:
1277 END_USE(vq);
1278 return needs_kick;
1279}
1280
1281static void detach_buf_packed(struct vring_virtqueue *vq,
1282 unsigned int id, void **ctx)
1283{
1284 struct vring_desc_state_packed *state = NULL;
1285 struct vring_packed_desc *desc;
1286 unsigned int i, curr;
1287
1288 state = &vq->packed.desc_state[id];
1289
1290 /* Clear data ptr. */
1291 state->data = NULL;
1292
1293 vq->packed.desc_state[state->last].next = vq->free_head;
1294 vq->free_head = id;
1295 vq->vq.num_free += state->num;
1296
1297 if (unlikely(vq->use_dma_api)) {
1298 curr = id;
1299 for (i = 0; i < state->num; i++) {
1300 vring_unmap_state_packed(vq,
1301 &vq->packed.desc_extra[curr]);
1302 curr = vq->packed.desc_state[curr].next;
1303 }
1304 }
1305
1306 if (vq->indirect) {
1307 u32 len;
1308
1309 /* Free the indirect table, if any, now that it's unmapped. */
1310 desc = state->indir_desc;
1311 if (!desc)
1312 return;
1313
1314 if (vq->use_dma_api) {
1315 len = vq->packed.desc_extra[id].len;
1316 for (i = 0; i < len / sizeof(struct vring_packed_desc);
1317 i++)
1318 vring_unmap_desc_packed(vq, &desc[i]);
1319 }
1320 kfree(desc);
1321 state->indir_desc = NULL;
1322 } else if (ctx) {
1323 *ctx = state->indir_desc;
1324 }
1325}
1326
1327static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1328 u16 idx, bool used_wrap_counter)
1329{
1330 bool avail, used;
1331 u16 flags;
1332
1333 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1334 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1335 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1336
1337 return avail == used && used == used_wrap_counter;
1338}
1339
1340static inline bool more_used_packed(const struct vring_virtqueue *vq)
1341{
1342 return is_used_desc_packed(vq, vq->last_used_idx,
1343 vq->packed.used_wrap_counter);
1344}
1345
1346static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1347 unsigned int *len,
1348 void **ctx)
1349{
1350 struct vring_virtqueue *vq = to_vvq(_vq);
1351 u16 last_used, id;
1352 void *ret;
1353
1354 START_USE(vq);
1355
1356 if (unlikely(vq->broken)) {
1357 END_USE(vq);
1358 return NULL;
1359 }
1360
1361 if (!more_used_packed(vq)) {
1362 pr_debug("No more buffers in queue\n");
1363 END_USE(vq);
1364 return NULL;
1365 }
1366
1367 /* Only get used elements after they have been exposed by host. */
1368 virtio_rmb(vq->weak_barriers);
1369
1370 last_used = vq->last_used_idx;
1371 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1372 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1373
1374 if (unlikely(id >= vq->packed.vring.num)) {
1375 BAD_RING(vq, "id %u out of range\n", id);
1376 return NULL;
1377 }
1378 if (unlikely(!vq->packed.desc_state[id].data)) {
1379 BAD_RING(vq, "id %u is not a head!\n", id);
1380 return NULL;
1381 }
1382
1383 /* detach_buf_packed clears data, so grab it now. */
1384 ret = vq->packed.desc_state[id].data;
1385 detach_buf_packed(vq, id, ctx);
1386
1387 vq->last_used_idx += vq->packed.desc_state[id].num;
1388 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {
1389 vq->last_used_idx -= vq->packed.vring.num;
1390 vq->packed.used_wrap_counter ^= 1;
1391 }
1392
1393 /*
1394 * If we expect an interrupt for the next entry, tell host
1395 * by writing event index and flush out the write before
1396 * the read in the next get_buf call.
1397 */
1398 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1399 virtio_store_mb(vq->weak_barriers,
1400 &vq->packed.vring.driver->off_wrap,
1401 cpu_to_le16(vq->last_used_idx |
1402 (vq->packed.used_wrap_counter <<
1403 VRING_PACKED_EVENT_F_WRAP_CTR)));
1404
1405 LAST_ADD_TIME_INVALID(vq);
1406
1407 END_USE(vq);
1408 return ret;
1409}
1410
1411static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1412{
1413 struct vring_virtqueue *vq = to_vvq(_vq);
1414
1415 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1416 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1417 vq->packed.vring.driver->flags =
1418 cpu_to_le16(vq->packed.event_flags_shadow);
1419 }
1420}
1421
1422static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1423{
1424 struct vring_virtqueue *vq = to_vvq(_vq);
1425
1426 START_USE(vq);
1427
1428 /*
1429 * We optimistically turn back on interrupts, then check if there was
1430 * more to do.
1431 */
1432
1433 if (vq->event) {
1434 vq->packed.vring.driver->off_wrap =
1435 cpu_to_le16(vq->last_used_idx |
1436 (vq->packed.used_wrap_counter <<
1437 VRING_PACKED_EVENT_F_WRAP_CTR));
1438 /*
1439 * We need to update event offset and event wrap
1440 * counter first before updating event flags.
1441 */
1442 virtio_wmb(vq->weak_barriers);
1443 }
1444
1445 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1446 vq->packed.event_flags_shadow = vq->event ?
1447 VRING_PACKED_EVENT_FLAG_DESC :
1448 VRING_PACKED_EVENT_FLAG_ENABLE;
1449 vq->packed.vring.driver->flags =
1450 cpu_to_le16(vq->packed.event_flags_shadow);
1451 }
1452
1453 END_USE(vq);
1454 return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<
1455 VRING_PACKED_EVENT_F_WRAP_CTR);
1456}
1457
1458static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1459{
1460 struct vring_virtqueue *vq = to_vvq(_vq);
1461 bool wrap_counter;
1462 u16 used_idx;
1463
1464 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1465 used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1466
1467 return is_used_desc_packed(vq, used_idx, wrap_counter);
1468}
1469
1470static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1471{
1472 struct vring_virtqueue *vq = to_vvq(_vq);
1473 u16 used_idx, wrap_counter;
1474 u16 bufs;
1475
1476 START_USE(vq);
1477
1478 /*
1479 * We optimistically turn back on interrupts, then check if there was
1480 * more to do.
1481 */
1482
1483 if (vq->event) {
1484 /* TODO: tune this threshold */
1485 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1486 wrap_counter = vq->packed.used_wrap_counter;
1487
1488 used_idx = vq->last_used_idx + bufs;
1489 if (used_idx >= vq->packed.vring.num) {
1490 used_idx -= vq->packed.vring.num;
1491 wrap_counter ^= 1;
1492 }
1493
1494 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1495 (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1496
1497 /*
1498 * We need to update event offset and event wrap
1499 * counter first before updating event flags.
1500 */
1501 virtio_wmb(vq->weak_barriers);
1502 }
1503
1504 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1505 vq->packed.event_flags_shadow = vq->event ?
1506 VRING_PACKED_EVENT_FLAG_DESC :
1507 VRING_PACKED_EVENT_FLAG_ENABLE;
1508 vq->packed.vring.driver->flags =
1509 cpu_to_le16(vq->packed.event_flags_shadow);
1510 }
1511
1512 /*
1513 * We need to update event suppression structure first
1514 * before re-checking for more used buffers.
1515 */
1516 virtio_mb(vq->weak_barriers);
1517
1518 if (is_used_desc_packed(vq,
1519 vq->last_used_idx,
1520 vq->packed.used_wrap_counter)) {
1521 END_USE(vq);
1522 return false;
1523 }
1524
1525 END_USE(vq);
1526 return true;
1527}
1528
1529static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1530{
1531 struct vring_virtqueue *vq = to_vvq(_vq);
1532 unsigned int i;
1533 void *buf;
1534
1535 START_USE(vq);
1536
1537 for (i = 0; i < vq->packed.vring.num; i++) {
1538 if (!vq->packed.desc_state[i].data)
1539 continue;
1540 /* detach_buf clears data, so grab it now. */
1541 buf = vq->packed.desc_state[i].data;
1542 detach_buf_packed(vq, i, NULL);
1543 END_USE(vq);
1544 return buf;
1545 }
1546 /* That should have freed everything. */
1547 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1548
1549 END_USE(vq);
1550 return NULL;
1551}
1552
1553static struct virtqueue *vring_create_virtqueue_packed(
1554 unsigned int index,
1555 unsigned int num,
1556 unsigned int vring_align,
1557 struct virtio_device *vdev,
1558 bool weak_barriers,
1559 bool may_reduce_num,
1560 bool context,
1561 bool (*notify)(struct virtqueue *),
1562 void (*callback)(struct virtqueue *),
1563 const char *name)
1564{
1565 struct vring_virtqueue *vq;
1566 struct vring_packed_desc *ring;
1567 struct vring_packed_desc_event *driver, *device;
1568 dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1569 size_t ring_size_in_bytes, event_size_in_bytes;
1570 unsigned int i;
1571
1572 ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1573
1574 ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1575 &ring_dma_addr,
1576 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1577 if (!ring)
1578 goto err_ring;
1579
1580 event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1581
1582 driver = vring_alloc_queue(vdev, event_size_in_bytes,
1583 &driver_event_dma_addr,
1584 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1585 if (!driver)
1586 goto err_driver;
1587
1588 device = vring_alloc_queue(vdev, event_size_in_bytes,
1589 &device_event_dma_addr,
1590 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1591 if (!device)
1592 goto err_device;
1593
1594 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1595 if (!vq)
1596 goto err_vq;
1597
1598 vq->vq.callback = callback;
1599 vq->vq.vdev = vdev;
1600 vq->vq.name = name;
1601 vq->vq.num_free = num;
1602 vq->vq.index = index;
1603 vq->we_own_ring = true;
1604 vq->notify = notify;
1605 vq->weak_barriers = weak_barriers;
1606 vq->broken = false;
1607 vq->last_used_idx = 0;
1608 vq->num_added = 0;
1609 vq->packed_ring = true;
1610 vq->use_dma_api = vring_use_dma_api(vdev);
1611 list_add_tail(&vq->vq.list, &vdev->vqs);
1612#ifdef DEBUG
1613 vq->in_use = false;
1614 vq->last_add_time_valid = false;
1615#endif
1616
1617 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1618 !context;
1619 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1620
1621 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1622 vq->weak_barriers = false;
1623
1624 vq->packed.ring_dma_addr = ring_dma_addr;
1625 vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1626 vq->packed.device_event_dma_addr = device_event_dma_addr;
1627
1628 vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1629 vq->packed.event_size_in_bytes = event_size_in_bytes;
1630
1631 vq->packed.vring.num = num;
1632 vq->packed.vring.desc = ring;
1633 vq->packed.vring.driver = driver;
1634 vq->packed.vring.device = device;
1635
1636 vq->packed.next_avail_idx = 0;
1637 vq->packed.avail_wrap_counter = 1;
1638 vq->packed.used_wrap_counter = 1;
1639 vq->packed.event_flags_shadow = 0;
1640 vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1641
1642 vq->packed.desc_state = kmalloc_array(num,
1643 sizeof(struct vring_desc_state_packed),
1644 GFP_KERNEL);
1645 if (!vq->packed.desc_state)
1646 goto err_desc_state;
1647
1648 memset(vq->packed.desc_state, 0,
1649 num * sizeof(struct vring_desc_state_packed));
1650
1651 /* Put everything in free lists. */
1652 vq->free_head = 0;
1653 for (i = 0; i < num-1; i++)
1654 vq->packed.desc_state[i].next = i + 1;
1655
1656 vq->packed.desc_extra = kmalloc_array(num,
1657 sizeof(struct vring_desc_extra_packed),
1658 GFP_KERNEL);
1659 if (!vq->packed.desc_extra)
1660 goto err_desc_extra;
1661
1662 memset(vq->packed.desc_extra, 0,
1663 num * sizeof(struct vring_desc_extra_packed));
1664
1665 /* No callback? Tell other side not to bother us. */
1666 if (!callback) {
1667 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1668 vq->packed.vring.driver->flags =
1669 cpu_to_le16(vq->packed.event_flags_shadow);
1670 }
1671
1672 return &vq->vq;
1673
1674err_desc_extra:
1675 kfree(vq->packed.desc_state);
1676err_desc_state:
1677 kfree(vq);
1678err_vq:
1679 vring_free_queue(vdev, event_size_in_bytes, device, ring_dma_addr);
1680err_device:
1681 vring_free_queue(vdev, event_size_in_bytes, driver, ring_dma_addr);
1682err_driver:
1683 vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
1684err_ring:
1685 return NULL;
1686}
1687
1688
1689/*
1690 * Generic functions and exported symbols.
1691 */
1692
1693static inline int virtqueue_add(struct virtqueue *_vq,
1694 struct scatterlist *sgs[],
1695 unsigned int total_sg,
1696 unsigned int out_sgs,
1697 unsigned int in_sgs,
1698 void *data,
1699 void *ctx,
1700 gfp_t gfp)
1701{
1702 struct vring_virtqueue *vq = to_vvq(_vq);
1703
1704 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
1705 out_sgs, in_sgs, data, ctx, gfp) :
1706 virtqueue_add_split(_vq, sgs, total_sg,
1707 out_sgs, in_sgs, data, ctx, gfp);
1708}
1709
1710/**
1711 * virtqueue_add_sgs - expose buffers to other end
1712 * @_vq: the struct virtqueue we're talking about.
1713 * @sgs: array of terminated scatterlists.
1714 * @out_sgs: the number of scatterlists readable by other side
1715 * @in_sgs: the number of scatterlists which are writable (after readable ones)
1716 * @data: the token identifying the buffer.
1717 * @gfp: how to do memory allocations (if necessary).
1718 *
1719 * Caller must ensure we don't call this with other virtqueue operations
1720 * at the same time (except where noted).
1721 *
1722 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1723 */
1724int virtqueue_add_sgs(struct virtqueue *_vq,
1725 struct scatterlist *sgs[],
1726 unsigned int out_sgs,
1727 unsigned int in_sgs,
1728 void *data,
1729 gfp_t gfp)
1730{
1731 unsigned int i, total_sg = 0;
1732
1733 /* Count them first. */
1734 for (i = 0; i < out_sgs + in_sgs; i++) {
1735 struct scatterlist *sg;
1736
1737 for (sg = sgs[i]; sg; sg = sg_next(sg))
1738 total_sg++;
1739 }
1740 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
1741 data, NULL, gfp);
1742}
1743EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
1744
1745/**
1746 * virtqueue_add_outbuf - expose output buffers to other end
1747 * @vq: the struct virtqueue we're talking about.
1748 * @sg: scatterlist (must be well-formed and terminated!)
1749 * @num: the number of entries in @sg readable by other side
1750 * @data: the token identifying the buffer.
1751 * @gfp: how to do memory allocations (if necessary).
1752 *
1753 * Caller must ensure we don't call this with other virtqueue operations
1754 * at the same time (except where noted).
1755 *
1756 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1757 */
1758int virtqueue_add_outbuf(struct virtqueue *vq,
1759 struct scatterlist *sg, unsigned int num,
1760 void *data,
1761 gfp_t gfp)
1762{
1763 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
1764}
1765EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
1766
1767/**
1768 * virtqueue_add_inbuf - expose input buffers to other end
1769 * @vq: the struct virtqueue we're talking about.
1770 * @sg: scatterlist (must be well-formed and terminated!)
1771 * @num: the number of entries in @sg writable by other side
1772 * @data: the token identifying the buffer.
1773 * @gfp: how to do memory allocations (if necessary).
1774 *
1775 * Caller must ensure we don't call this with other virtqueue operations
1776 * at the same time (except where noted).
1777 *
1778 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1779 */
1780int virtqueue_add_inbuf(struct virtqueue *vq,
1781 struct scatterlist *sg, unsigned int num,
1782 void *data,
1783 gfp_t gfp)
1784{
1785 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
1786}
1787EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
1788
1789/**
1790 * virtqueue_add_inbuf_ctx - expose input buffers to other end
1791 * @vq: the struct virtqueue we're talking about.
1792 * @sg: scatterlist (must be well-formed and terminated!)
1793 * @num: the number of entries in @sg writable by other side
1794 * @data: the token identifying the buffer.
1795 * @ctx: extra context for the token
1796 * @gfp: how to do memory allocations (if necessary).
1797 *
1798 * Caller must ensure we don't call this with other virtqueue operations
1799 * at the same time (except where noted).
1800 *
1801 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1802 */
1803int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
1804 struct scatterlist *sg, unsigned int num,
1805 void *data,
1806 void *ctx,
1807 gfp_t gfp)
1808{
1809 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
1810}
1811EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
1812
1813/**
1814 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1815 * @_vq: the struct virtqueue
1816 *
1817 * Instead of virtqueue_kick(), you can do:
1818 * if (virtqueue_kick_prepare(vq))
1819 * virtqueue_notify(vq);
1820 *
1821 * This is sometimes useful because the virtqueue_kick_prepare() needs
1822 * to be serialized, but the actual virtqueue_notify() call does not.
1823 */
1824bool virtqueue_kick_prepare(struct virtqueue *_vq)
1825{
1826 struct vring_virtqueue *vq = to_vvq(_vq);
1827
1828 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
1829 virtqueue_kick_prepare_split(_vq);
1830}
1831EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
1832
1833/**
1834 * virtqueue_notify - second half of split virtqueue_kick call.
1835 * @_vq: the struct virtqueue
1836 *
1837 * This does not need to be serialized.
1838 *
1839 * Returns false if host notify failed or queue is broken, otherwise true.
1840 */
1841bool virtqueue_notify(struct virtqueue *_vq)
1842{
1843 struct vring_virtqueue *vq = to_vvq(_vq);
1844
1845 if (unlikely(vq->broken))
1846 return false;
1847
1848 /* Prod other side to tell it about changes. */
1849 if (!vq->notify(_vq)) {
1850 vq->broken = true;
1851 return false;
1852 }
1853 return true;
1854}
1855EXPORT_SYMBOL_GPL(virtqueue_notify);
1856
1857/**
1858 * virtqueue_kick - update after add_buf
1859 * @vq: the struct virtqueue
1860 *
1861 * After one or more virtqueue_add_* calls, invoke this to kick
1862 * the other side.
1863 *
1864 * Caller must ensure we don't call this with other virtqueue
1865 * operations at the same time (except where noted).
1866 *
1867 * Returns false if kick failed, otherwise true.
1868 */
1869bool virtqueue_kick(struct virtqueue *vq)
1870{
1871 if (virtqueue_kick_prepare(vq))
1872 return virtqueue_notify(vq);
1873 return true;
1874}
1875EXPORT_SYMBOL_GPL(virtqueue_kick);
1876
1877/**
1878 * virtqueue_get_buf - get the next used buffer
1879 * @_vq: the struct virtqueue we're talking about.
1880 * @len: the length written into the buffer
1881 * @ctx: extra context for the token
1882 *
1883 * If the device wrote data into the buffer, @len will be set to the
1884 * amount written. This means you don't need to clear the buffer
1885 * beforehand to ensure there's no data leakage in the case of short
1886 * writes.
1887 *
1888 * Caller must ensure we don't call this with other virtqueue
1889 * operations at the same time (except where noted).
1890 *
1891 * Returns NULL if there are no used buffers, or the "data" token
1892 * handed to virtqueue_add_*().
1893 */
1894void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
1895 void **ctx)
1896{
1897 struct vring_virtqueue *vq = to_vvq(_vq);
1898
1899 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
1900 virtqueue_get_buf_ctx_split(_vq, len, ctx);
1901}
1902EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
1903
1904void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
1905{
1906 return virtqueue_get_buf_ctx(_vq, len, NULL);
1907}
1908EXPORT_SYMBOL_GPL(virtqueue_get_buf);
1909/**
1910 * virtqueue_disable_cb - disable callbacks
1911 * @_vq: the struct virtqueue we're talking about.
1912 *
1913 * Note that this is not necessarily synchronous, hence unreliable and only
1914 * useful as an optimization.
1915 *
1916 * Unlike other operations, this need not be serialized.
1917 */
1918void virtqueue_disable_cb(struct virtqueue *_vq)
1919{
1920 struct vring_virtqueue *vq = to_vvq(_vq);
1921
1922 if (vq->packed_ring)
1923 virtqueue_disable_cb_packed(_vq);
1924 else
1925 virtqueue_disable_cb_split(_vq);
1926}
1927EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
1928
1929/**
1930 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
1931 * @_vq: the struct virtqueue we're talking about.
1932 *
1933 * This re-enables callbacks; it returns current queue state
1934 * in an opaque unsigned value. This value should be later tested by
1935 * virtqueue_poll, to detect a possible race between the driver checking for
1936 * more work, and enabling callbacks.
1937 *
1938 * Caller must ensure we don't call this with other virtqueue
1939 * operations at the same time (except where noted).
1940 */
1941unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
1942{
1943 struct vring_virtqueue *vq = to_vvq(_vq);
1944
1945 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
1946 virtqueue_enable_cb_prepare_split(_vq);
1947}
1948EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
1949
1950/**
1951 * virtqueue_poll - query pending used buffers
1952 * @_vq: the struct virtqueue we're talking about.
1953 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
1954 *
1955 * Returns "true" if there are pending used buffers in the queue.
1956 *
1957 * This does not need to be serialized.
1958 */
1959bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
1960{
1961 struct vring_virtqueue *vq = to_vvq(_vq);
1962
1963 virtio_mb(vq->weak_barriers);
1964 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
1965 virtqueue_poll_split(_vq, last_used_idx);
1966}
1967EXPORT_SYMBOL_GPL(virtqueue_poll);
1968
1969/**
1970 * virtqueue_enable_cb - restart callbacks after disable_cb.
1971 * @_vq: the struct virtqueue we're talking about.
1972 *
1973 * This re-enables callbacks; it returns "false" if there are pending
1974 * buffers in the queue, to detect a possible race between the driver
1975 * checking for more work, and enabling callbacks.
1976 *
1977 * Caller must ensure we don't call this with other virtqueue
1978 * operations at the same time (except where noted).
1979 */
1980bool virtqueue_enable_cb(struct virtqueue *_vq)
1981{
1982 unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);
1983
1984 return !virtqueue_poll(_vq, last_used_idx);
1985}
1986EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
1987
1988/**
1989 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
1990 * @_vq: the struct virtqueue we're talking about.
1991 *
1992 * This re-enables callbacks but hints to the other side to delay
1993 * interrupts until most of the available buffers have been processed;
1994 * it returns "false" if there are many pending buffers in the queue,
1995 * to detect a possible race between the driver checking for more work,
1996 * and enabling callbacks.
1997 *
1998 * Caller must ensure we don't call this with other virtqueue
1999 * operations at the same time (except where noted).
2000 */
2001bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2002{
2003 struct vring_virtqueue *vq = to_vvq(_vq);
2004
2005 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2006 virtqueue_enable_cb_delayed_split(_vq);
2007}
2008EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2009
2010/**
2011 * virtqueue_detach_unused_buf - detach first unused buffer
2012 * @_vq: the struct virtqueue we're talking about.
2013 *
2014 * Returns NULL or the "data" token handed to virtqueue_add_*().
2015 * This is not valid on an active queue; it is useful only for device
2016 * shutdown.
2017 */
2018void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2019{
2020 struct vring_virtqueue *vq = to_vvq(_vq);
2021
2022 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2023 virtqueue_detach_unused_buf_split(_vq);
2024}
2025EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2026
2027static inline bool more_used(const struct vring_virtqueue *vq)
2028{
2029 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2030}
2031
2032irqreturn_t vring_interrupt(int irq, void *_vq)
2033{
2034 struct vring_virtqueue *vq = to_vvq(_vq);
2035
2036 if (!more_used(vq)) {
2037 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2038 return IRQ_NONE;
2039 }
2040
2041 if (unlikely(vq->broken))
2042 return IRQ_HANDLED;
2043
2044 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2045 if (vq->vq.callback)
2046 vq->vq.callback(&vq->vq);
2047
2048 return IRQ_HANDLED;
2049}
2050EXPORT_SYMBOL_GPL(vring_interrupt);
2051
2052/* Only available for split ring */
2053struct virtqueue *__vring_new_virtqueue(unsigned int index,
2054 struct vring vring,
2055 struct virtio_device *vdev,
2056 bool weak_barriers,
2057 bool context,
2058 bool (*notify)(struct virtqueue *),
2059 void (*callback)(struct virtqueue *),
2060 const char *name)
2061{
2062 unsigned int i;
2063 struct vring_virtqueue *vq;
2064
2065 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2066 return NULL;
2067
2068 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2069 if (!vq)
2070 return NULL;
2071
2072 vq->packed_ring = false;
2073 vq->vq.callback = callback;
2074 vq->vq.vdev = vdev;
2075 vq->vq.name = name;
2076 vq->vq.num_free = vring.num;
2077 vq->vq.index = index;
2078 vq->we_own_ring = false;
2079 vq->notify = notify;
2080 vq->weak_barriers = weak_barriers;
2081 vq->broken = false;
2082 vq->last_used_idx = 0;
2083 vq->num_added = 0;
2084 vq->use_dma_api = vring_use_dma_api(vdev);
2085 list_add_tail(&vq->vq.list, &vdev->vqs);
2086#ifdef DEBUG
2087 vq->in_use = false;
2088 vq->last_add_time_valid = false;
2089#endif
2090
2091 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2092 !context;
2093 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2094
2095 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2096 vq->weak_barriers = false;
2097
2098 vq->split.queue_dma_addr = 0;
2099 vq->split.queue_size_in_bytes = 0;
2100
2101 vq->split.vring = vring;
2102 vq->split.avail_flags_shadow = 0;
2103 vq->split.avail_idx_shadow = 0;
2104
2105 /* No callback? Tell other side not to bother us. */
2106 if (!callback) {
2107 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
2108 if (!vq->event)
2109 vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2110 vq->split.avail_flags_shadow);
2111 }
2112
2113 vq->split.desc_state = kmalloc_array(vring.num,
2114 sizeof(struct vring_desc_state_split), GFP_KERNEL);
2115 if (!vq->split.desc_state) {
2116 kfree(vq);
2117 return NULL;
2118 }
2119
2120 /* Put everything in free lists. */
2121 vq->free_head = 0;
2122 for (i = 0; i < vring.num-1; i++)
2123 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
2124 memset(vq->split.desc_state, 0, vring.num *
2125 sizeof(struct vring_desc_state_split));
2126
2127 return &vq->vq;
2128}
2129EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2130
2131struct virtqueue *vring_create_virtqueue(
2132 unsigned int index,
2133 unsigned int num,
2134 unsigned int vring_align,
2135 struct virtio_device *vdev,
2136 bool weak_barriers,
2137 bool may_reduce_num,
2138 bool context,
2139 bool (*notify)(struct virtqueue *),
2140 void (*callback)(struct virtqueue *),
2141 const char *name)
2142{
2143
2144 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2145 return vring_create_virtqueue_packed(index, num, vring_align,
2146 vdev, weak_barriers, may_reduce_num,
2147 context, notify, callback, name);
2148
2149 return vring_create_virtqueue_split(index, num, vring_align,
2150 vdev, weak_barriers, may_reduce_num,
2151 context, notify, callback, name);
2152}
2153EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2154
2155/* Only available for split ring */
2156struct virtqueue *vring_new_virtqueue(unsigned int index,
2157 unsigned int num,
2158 unsigned int vring_align,
2159 struct virtio_device *vdev,
2160 bool weak_barriers,
2161 bool context,
2162 void *pages,
2163 bool (*notify)(struct virtqueue *vq),
2164 void (*callback)(struct virtqueue *vq),
2165 const char *name)
2166{
2167 struct vring vring;
2168
2169 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2170 return NULL;
2171
2172 vring_init(&vring, num, pages, vring_align);
2173 return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
2174 notify, callback, name);
2175}
2176EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2177
2178void vring_del_virtqueue(struct virtqueue *_vq)
2179{
2180 struct vring_virtqueue *vq = to_vvq(_vq);
2181
2182 if (vq->we_own_ring) {
2183 if (vq->packed_ring) {
2184 vring_free_queue(vq->vq.vdev,
2185 vq->packed.ring_size_in_bytes,
2186 vq->packed.vring.desc,
2187 vq->packed.ring_dma_addr);
2188
2189 vring_free_queue(vq->vq.vdev,
2190 vq->packed.event_size_in_bytes,
2191 vq->packed.vring.driver,
2192 vq->packed.driver_event_dma_addr);
2193
2194 vring_free_queue(vq->vq.vdev,
2195 vq->packed.event_size_in_bytes,
2196 vq->packed.vring.device,
2197 vq->packed.device_event_dma_addr);
2198
2199 kfree(vq->packed.desc_state);
2200 kfree(vq->packed.desc_extra);
2201 } else {
2202 vring_free_queue(vq->vq.vdev,
2203 vq->split.queue_size_in_bytes,
2204 vq->split.vring.desc,
2205 vq->split.queue_dma_addr);
2206
2207 kfree(vq->split.desc_state);
2208 }
2209 }
2210 list_del(&_vq->list);
2211 kfree(vq);
2212}
2213EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2214
2215/* Manipulates transport-specific feature bits. */
2216void vring_transport_features(struct virtio_device *vdev)
2217{
2218 unsigned int i;
2219
2220 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2221 switch (i) {
2222 case VIRTIO_RING_F_INDIRECT_DESC:
2223 break;
2224 case VIRTIO_RING_F_EVENT_IDX:
2225 break;
2226 case VIRTIO_F_VERSION_1:
2227 break;
2228 case VIRTIO_F_IOMMU_PLATFORM:
2229 break;
2230 case VIRTIO_F_RING_PACKED:
2231 break;
2232 case VIRTIO_F_ORDER_PLATFORM:
2233 break;
2234 default:
2235 /* We don't understand this bit. */
2236 __virtio_clear_bit(vdev, i);
2237 }
2238 }
2239}
2240EXPORT_SYMBOL_GPL(vring_transport_features);
2241
2242/**
2243 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2244 * @_vq: the struct virtqueue containing the vring of interest.
2245 *
2246 * Returns the size of the vring. This is mainly used for boasting to
2247 * userspace. Unlike other operations, this need not be serialized.
2248 */
2249unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
2250{
2251
2252 struct vring_virtqueue *vq = to_vvq(_vq);
2253
2254 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2255}
2256EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2257
2258bool virtqueue_is_broken(struct virtqueue *_vq)
2259{
2260 struct vring_virtqueue *vq = to_vvq(_vq);
2261
2262 return vq->broken;
2263}
2264EXPORT_SYMBOL_GPL(virtqueue_is_broken);
2265
2266/*
2267 * This should prevent the device from being used, allowing drivers to
2268 * recover. You may need to grab appropriate locks to flush.
2269 */
2270void virtio_break_device(struct virtio_device *dev)
2271{
2272 struct virtqueue *_vq;
2273
2274 list_for_each_entry(_vq, &dev->vqs, list) {
2275 struct vring_virtqueue *vq = to_vvq(_vq);
2276 vq->broken = true;
2277 }
2278}
2279EXPORT_SYMBOL_GPL(virtio_break_device);
2280
2281dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
2282{
2283 struct vring_virtqueue *vq = to_vvq(_vq);
2284
2285 BUG_ON(!vq->we_own_ring);
2286
2287 if (vq->packed_ring)
2288 return vq->packed.ring_dma_addr;
2289
2290 return vq->split.queue_dma_addr;
2291}
2292EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
2293
2294dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
2295{
2296 struct vring_virtqueue *vq = to_vvq(_vq);
2297
2298 BUG_ON(!vq->we_own_ring);
2299
2300 if (vq->packed_ring)
2301 return vq->packed.driver_event_dma_addr;
2302
2303 return vq->split.queue_dma_addr +
2304 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2305}
2306EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
2307
2308dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
2309{
2310 struct vring_virtqueue *vq = to_vvq(_vq);
2311
2312 BUG_ON(!vq->we_own_ring);
2313
2314 if (vq->packed_ring)
2315 return vq->packed.device_event_dma_addr;
2316
2317 return vq->split.queue_dma_addr +
2318 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2319}
2320EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
2321
2322/* Only available for split ring */
2323const struct vring *virtqueue_get_vring(struct virtqueue *vq)
2324{
2325 return &to_vvq(vq)->split.vring;
2326}
2327EXPORT_SYMBOL_GPL(virtqueue_get_vring);
2328
2329MODULE_LICENSE("GPL");