Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Virtio ring implementation.
3 *
4 * Copyright 2007 Rusty Russell IBM Corporation
5 */
6#include <linux/virtio.h>
7#include <linux/virtio_ring.h>
8#include <linux/virtio_config.h>
9#include <linux/device.h>
10#include <linux/slab.h>
11#include <linux/module.h>
12#include <linux/hrtimer.h>
13#include <linux/dma-mapping.h>
14#include <linux/kmsan.h>
15#include <linux/spinlock.h>
16#include <xen/xen.h>
17
18#ifdef DEBUG
19/* For development, we want to crash whenever the ring is screwed. */
20#define BAD_RING(_vq, fmt, args...) \
21 do { \
22 dev_err(&(_vq)->vq.vdev->dev, \
23 "%s:"fmt, (_vq)->vq.name, ##args); \
24 BUG(); \
25 } while (0)
26/* Caller is supposed to guarantee no reentry. */
27#define START_USE(_vq) \
28 do { \
29 if ((_vq)->in_use) \
30 panic("%s:in_use = %i\n", \
31 (_vq)->vq.name, (_vq)->in_use); \
32 (_vq)->in_use = __LINE__; \
33 } while (0)
34#define END_USE(_vq) \
35 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36#define LAST_ADD_TIME_UPDATE(_vq) \
37 do { \
38 ktime_t now = ktime_get(); \
39 \
40 /* No kick or get, with .1 second between? Warn. */ \
41 if ((_vq)->last_add_time_valid) \
42 WARN_ON(ktime_to_ms(ktime_sub(now, \
43 (_vq)->last_add_time)) > 100); \
44 (_vq)->last_add_time = now; \
45 (_vq)->last_add_time_valid = true; \
46 } while (0)
47#define LAST_ADD_TIME_CHECK(_vq) \
48 do { \
49 if ((_vq)->last_add_time_valid) { \
50 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
51 (_vq)->last_add_time)) > 100); \
52 } \
53 } while (0)
54#define LAST_ADD_TIME_INVALID(_vq) \
55 ((_vq)->last_add_time_valid = false)
56#else
57#define BAD_RING(_vq, fmt, args...) \
58 do { \
59 dev_err(&_vq->vq.vdev->dev, \
60 "%s:"fmt, (_vq)->vq.name, ##args); \
61 (_vq)->broken = true; \
62 } while (0)
63#define START_USE(vq)
64#define END_USE(vq)
65#define LAST_ADD_TIME_UPDATE(vq)
66#define LAST_ADD_TIME_CHECK(vq)
67#define LAST_ADD_TIME_INVALID(vq)
68#endif
69
70struct vring_desc_state_split {
71 void *data; /* Data for callback. */
72 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
73};
74
75struct vring_desc_state_packed {
76 void *data; /* Data for callback. */
77 struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
78 u16 num; /* Descriptor list length. */
79 u16 last; /* The last desc state in a list. */
80};
81
82struct vring_desc_extra {
83 dma_addr_t addr; /* Descriptor DMA addr. */
84 u32 len; /* Descriptor length. */
85 u16 flags; /* Descriptor flags. */
86 u16 next; /* The next desc state in a list. */
87};
88
89struct vring_virtqueue_split {
90 /* Actual memory layout for this queue. */
91 struct vring vring;
92
93 /* Last written value to avail->flags */
94 u16 avail_flags_shadow;
95
96 /*
97 * Last written value to avail->idx in
98 * guest byte order.
99 */
100 u16 avail_idx_shadow;
101
102 /* Per-descriptor state. */
103 struct vring_desc_state_split *desc_state;
104 struct vring_desc_extra *desc_extra;
105
106 /* DMA address and size information */
107 dma_addr_t queue_dma_addr;
108 size_t queue_size_in_bytes;
109
110 /*
111 * The parameters for creating vrings are reserved for creating new
112 * vring.
113 */
114 u32 vring_align;
115 bool may_reduce_num;
116};
117
118struct vring_virtqueue_packed {
119 /* Actual memory layout for this queue. */
120 struct {
121 unsigned int num;
122 struct vring_packed_desc *desc;
123 struct vring_packed_desc_event *driver;
124 struct vring_packed_desc_event *device;
125 } vring;
126
127 /* Driver ring wrap counter. */
128 bool avail_wrap_counter;
129
130 /* Avail used flags. */
131 u16 avail_used_flags;
132
133 /* Index of the next avail descriptor. */
134 u16 next_avail_idx;
135
136 /*
137 * Last written value to driver->flags in
138 * guest byte order.
139 */
140 u16 event_flags_shadow;
141
142 /* Per-descriptor state. */
143 struct vring_desc_state_packed *desc_state;
144 struct vring_desc_extra *desc_extra;
145
146 /* DMA address and size information */
147 dma_addr_t ring_dma_addr;
148 dma_addr_t driver_event_dma_addr;
149 dma_addr_t device_event_dma_addr;
150 size_t ring_size_in_bytes;
151 size_t event_size_in_bytes;
152};
153
154struct vring_virtqueue {
155 struct virtqueue vq;
156
157 /* Is this a packed ring? */
158 bool packed_ring;
159
160 /* Is DMA API used? */
161 bool use_dma_api;
162
163 /* Can we use weak barriers? */
164 bool weak_barriers;
165
166 /* Other side has made a mess, don't try any more. */
167 bool broken;
168
169 /* Host supports indirect buffers */
170 bool indirect;
171
172 /* Host publishes avail event idx */
173 bool event;
174
175 /* Do DMA mapping by driver */
176 bool premapped;
177
178 /* Do unmap or not for desc. Just when premapped is False and
179 * use_dma_api is true, this is true.
180 */
181 bool do_unmap;
182
183 /* Head of free buffer list. */
184 unsigned int free_head;
185 /* Number we've added since last sync. */
186 unsigned int num_added;
187
188 /* Last used index we've seen.
189 * for split ring, it just contains last used index
190 * for packed ring:
191 * bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index.
192 * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter.
193 */
194 u16 last_used_idx;
195
196 /* Hint for event idx: already triggered no need to disable. */
197 bool event_triggered;
198
199 union {
200 /* Available for split ring */
201 struct vring_virtqueue_split split;
202
203 /* Available for packed ring */
204 struct vring_virtqueue_packed packed;
205 };
206
207 /* How to notify other side. FIXME: commonalize hcalls! */
208 bool (*notify)(struct virtqueue *vq);
209
210 /* DMA, allocation, and size information */
211 bool we_own_ring;
212
213 /* Device used for doing DMA */
214 struct device *dma_dev;
215
216#ifdef DEBUG
217 /* They're supposed to lock for us. */
218 unsigned int in_use;
219
220 /* Figure out if their kicks are too delayed. */
221 bool last_add_time_valid;
222 ktime_t last_add_time;
223#endif
224};
225
226static struct virtqueue *__vring_new_virtqueue(unsigned int index,
227 struct vring_virtqueue_split *vring_split,
228 struct virtio_device *vdev,
229 bool weak_barriers,
230 bool context,
231 bool (*notify)(struct virtqueue *),
232 void (*callback)(struct virtqueue *),
233 const char *name,
234 struct device *dma_dev);
235static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
236static void vring_free(struct virtqueue *_vq);
237
238/*
239 * Helpers.
240 */
241
242#define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
243
244static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
245 unsigned int total_sg)
246{
247 /*
248 * If the host supports indirect descriptor tables, and we have multiple
249 * buffers, then go indirect. FIXME: tune this threshold
250 */
251 return (vq->indirect && total_sg > 1 && vq->vq.num_free);
252}
253
254/*
255 * Modern virtio devices have feature bits to specify whether they need a
256 * quirk and bypass the IOMMU. If not there, just use the DMA API.
257 *
258 * If there, the interaction between virtio and DMA API is messy.
259 *
260 * On most systems with virtio, physical addresses match bus addresses,
261 * and it doesn't particularly matter whether we use the DMA API.
262 *
263 * On some systems, including Xen and any system with a physical device
264 * that speaks virtio behind a physical IOMMU, we must use the DMA API
265 * for virtio DMA to work at all.
266 *
267 * On other systems, including SPARC and PPC64, virtio-pci devices are
268 * enumerated as though they are behind an IOMMU, but the virtio host
269 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
270 * there or somehow map everything as the identity.
271 *
272 * For the time being, we preserve historic behavior and bypass the DMA
273 * API.
274 *
275 * TODO: install a per-device DMA ops structure that does the right thing
276 * taking into account all the above quirks, and use the DMA API
277 * unconditionally on data path.
278 */
279
280static bool vring_use_dma_api(const struct virtio_device *vdev)
281{
282 if (!virtio_has_dma_quirk(vdev))
283 return true;
284
285 /* Otherwise, we are left to guess. */
286 /*
287 * In theory, it's possible to have a buggy QEMU-supposed
288 * emulated Q35 IOMMU and Xen enabled at the same time. On
289 * such a configuration, virtio has never worked and will
290 * not work without an even larger kludge. Instead, enable
291 * the DMA API if we're a Xen guest, which at least allows
292 * all of the sensible Xen configurations to work correctly.
293 */
294 if (xen_domain())
295 return true;
296
297 return false;
298}
299
300size_t virtio_max_dma_size(const struct virtio_device *vdev)
301{
302 size_t max_segment_size = SIZE_MAX;
303
304 if (vring_use_dma_api(vdev))
305 max_segment_size = dma_max_mapping_size(vdev->dev.parent);
306
307 return max_segment_size;
308}
309EXPORT_SYMBOL_GPL(virtio_max_dma_size);
310
311static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
312 dma_addr_t *dma_handle, gfp_t flag,
313 struct device *dma_dev)
314{
315 if (vring_use_dma_api(vdev)) {
316 return dma_alloc_coherent(dma_dev, size,
317 dma_handle, flag);
318 } else {
319 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
320
321 if (queue) {
322 phys_addr_t phys_addr = virt_to_phys(queue);
323 *dma_handle = (dma_addr_t)phys_addr;
324
325 /*
326 * Sanity check: make sure we dind't truncate
327 * the address. The only arches I can find that
328 * have 64-bit phys_addr_t but 32-bit dma_addr_t
329 * are certain non-highmem MIPS and x86
330 * configurations, but these configurations
331 * should never allocate physical pages above 32
332 * bits, so this is fine. Just in case, throw a
333 * warning and abort if we end up with an
334 * unrepresentable address.
335 */
336 if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
337 free_pages_exact(queue, PAGE_ALIGN(size));
338 return NULL;
339 }
340 }
341 return queue;
342 }
343}
344
345static void vring_free_queue(struct virtio_device *vdev, size_t size,
346 void *queue, dma_addr_t dma_handle,
347 struct device *dma_dev)
348{
349 if (vring_use_dma_api(vdev))
350 dma_free_coherent(dma_dev, size, queue, dma_handle);
351 else
352 free_pages_exact(queue, PAGE_ALIGN(size));
353}
354
355/*
356 * The DMA ops on various arches are rather gnarly right now, and
357 * making all of the arch DMA ops work on the vring device itself
358 * is a mess.
359 */
360static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
361{
362 return vq->dma_dev;
363}
364
365/* Map one sg entry. */
366static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg,
367 enum dma_data_direction direction, dma_addr_t *addr)
368{
369 if (vq->premapped) {
370 *addr = sg_dma_address(sg);
371 return 0;
372 }
373
374 if (!vq->use_dma_api) {
375 /*
376 * If DMA is not used, KMSAN doesn't know that the scatterlist
377 * is initialized by the hardware. Explicitly check/unpoison it
378 * depending on the direction.
379 */
380 kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction);
381 *addr = (dma_addr_t)sg_phys(sg);
382 return 0;
383 }
384
385 /*
386 * We can't use dma_map_sg, because we don't use scatterlists in
387 * the way it expects (we don't guarantee that the scatterlist
388 * will exist for the lifetime of the mapping).
389 */
390 *addr = dma_map_page(vring_dma_dev(vq),
391 sg_page(sg), sg->offset, sg->length,
392 direction);
393
394 if (dma_mapping_error(vring_dma_dev(vq), *addr))
395 return -ENOMEM;
396
397 return 0;
398}
399
400static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
401 void *cpu_addr, size_t size,
402 enum dma_data_direction direction)
403{
404 if (!vq->use_dma_api)
405 return (dma_addr_t)virt_to_phys(cpu_addr);
406
407 return dma_map_single(vring_dma_dev(vq),
408 cpu_addr, size, direction);
409}
410
411static int vring_mapping_error(const struct vring_virtqueue *vq,
412 dma_addr_t addr)
413{
414 if (!vq->use_dma_api)
415 return 0;
416
417 return dma_mapping_error(vring_dma_dev(vq), addr);
418}
419
420static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
421{
422 vq->vq.num_free = num;
423
424 if (vq->packed_ring)
425 vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
426 else
427 vq->last_used_idx = 0;
428
429 vq->event_triggered = false;
430 vq->num_added = 0;
431
432#ifdef DEBUG
433 vq->in_use = false;
434 vq->last_add_time_valid = false;
435#endif
436}
437
438
439/*
440 * Split ring specific functions - *_split().
441 */
442
443static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
444 const struct vring_desc *desc)
445{
446 u16 flags;
447
448 if (!vq->do_unmap)
449 return;
450
451 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
452
453 dma_unmap_page(vring_dma_dev(vq),
454 virtio64_to_cpu(vq->vq.vdev, desc->addr),
455 virtio32_to_cpu(vq->vq.vdev, desc->len),
456 (flags & VRING_DESC_F_WRITE) ?
457 DMA_FROM_DEVICE : DMA_TO_DEVICE);
458}
459
460static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
461 unsigned int i)
462{
463 struct vring_desc_extra *extra = vq->split.desc_extra;
464 u16 flags;
465
466 flags = extra[i].flags;
467
468 if (flags & VRING_DESC_F_INDIRECT) {
469 if (!vq->use_dma_api)
470 goto out;
471
472 dma_unmap_single(vring_dma_dev(vq),
473 extra[i].addr,
474 extra[i].len,
475 (flags & VRING_DESC_F_WRITE) ?
476 DMA_FROM_DEVICE : DMA_TO_DEVICE);
477 } else {
478 if (!vq->do_unmap)
479 goto out;
480
481 dma_unmap_page(vring_dma_dev(vq),
482 extra[i].addr,
483 extra[i].len,
484 (flags & VRING_DESC_F_WRITE) ?
485 DMA_FROM_DEVICE : DMA_TO_DEVICE);
486 }
487
488out:
489 return extra[i].next;
490}
491
492static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
493 unsigned int total_sg,
494 gfp_t gfp)
495{
496 struct vring_desc *desc;
497 unsigned int i;
498
499 /*
500 * We require lowmem mappings for the descriptors because
501 * otherwise virt_to_phys will give us bogus addresses in the
502 * virtqueue.
503 */
504 gfp &= ~__GFP_HIGHMEM;
505
506 desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
507 if (!desc)
508 return NULL;
509
510 for (i = 0; i < total_sg; i++)
511 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
512 return desc;
513}
514
515static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
516 struct vring_desc *desc,
517 unsigned int i,
518 dma_addr_t addr,
519 unsigned int len,
520 u16 flags,
521 bool indirect)
522{
523 struct vring_virtqueue *vring = to_vvq(vq);
524 struct vring_desc_extra *extra = vring->split.desc_extra;
525 u16 next;
526
527 desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
528 desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
529 desc[i].len = cpu_to_virtio32(vq->vdev, len);
530
531 if (!indirect) {
532 next = extra[i].next;
533 desc[i].next = cpu_to_virtio16(vq->vdev, next);
534
535 extra[i].addr = addr;
536 extra[i].len = len;
537 extra[i].flags = flags;
538 } else
539 next = virtio16_to_cpu(vq->vdev, desc[i].next);
540
541 return next;
542}
543
544static inline int virtqueue_add_split(struct virtqueue *_vq,
545 struct scatterlist *sgs[],
546 unsigned int total_sg,
547 unsigned int out_sgs,
548 unsigned int in_sgs,
549 void *data,
550 void *ctx,
551 gfp_t gfp)
552{
553 struct vring_virtqueue *vq = to_vvq(_vq);
554 struct scatterlist *sg;
555 struct vring_desc *desc;
556 unsigned int i, n, avail, descs_used, prev, err_idx;
557 int head;
558 bool indirect;
559
560 START_USE(vq);
561
562 BUG_ON(data == NULL);
563 BUG_ON(ctx && vq->indirect);
564
565 if (unlikely(vq->broken)) {
566 END_USE(vq);
567 return -EIO;
568 }
569
570 LAST_ADD_TIME_UPDATE(vq);
571
572 BUG_ON(total_sg == 0);
573
574 head = vq->free_head;
575
576 if (virtqueue_use_indirect(vq, total_sg))
577 desc = alloc_indirect_split(_vq, total_sg, gfp);
578 else {
579 desc = NULL;
580 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
581 }
582
583 if (desc) {
584 /* Use a single buffer which doesn't continue */
585 indirect = true;
586 /* Set up rest to use this indirect table. */
587 i = 0;
588 descs_used = 1;
589 } else {
590 indirect = false;
591 desc = vq->split.vring.desc;
592 i = head;
593 descs_used = total_sg;
594 }
595
596 if (unlikely(vq->vq.num_free < descs_used)) {
597 pr_debug("Can't add buf len %i - avail = %i\n",
598 descs_used, vq->vq.num_free);
599 /* FIXME: for historical reasons, we force a notify here if
600 * there are outgoing parts to the buffer. Presumably the
601 * host should service the ring ASAP. */
602 if (out_sgs)
603 vq->notify(&vq->vq);
604 if (indirect)
605 kfree(desc);
606 END_USE(vq);
607 return -ENOSPC;
608 }
609
610 for (n = 0; n < out_sgs; n++) {
611 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
612 dma_addr_t addr;
613
614 if (vring_map_one_sg(vq, sg, DMA_TO_DEVICE, &addr))
615 goto unmap_release;
616
617 prev = i;
618 /* Note that we trust indirect descriptor
619 * table since it use stream DMA mapping.
620 */
621 i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,
622 VRING_DESC_F_NEXT,
623 indirect);
624 }
625 }
626 for (; n < (out_sgs + in_sgs); n++) {
627 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
628 dma_addr_t addr;
629
630 if (vring_map_one_sg(vq, sg, DMA_FROM_DEVICE, &addr))
631 goto unmap_release;
632
633 prev = i;
634 /* Note that we trust indirect descriptor
635 * table since it use stream DMA mapping.
636 */
637 i = virtqueue_add_desc_split(_vq, desc, i, addr,
638 sg->length,
639 VRING_DESC_F_NEXT |
640 VRING_DESC_F_WRITE,
641 indirect);
642 }
643 }
644 /* Last one doesn't continue. */
645 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
646 if (!indirect && vq->do_unmap)
647 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
648 ~VRING_DESC_F_NEXT;
649
650 if (indirect) {
651 /* Now that the indirect table is filled in, map it. */
652 dma_addr_t addr = vring_map_single(
653 vq, desc, total_sg * sizeof(struct vring_desc),
654 DMA_TO_DEVICE);
655 if (vring_mapping_error(vq, addr)) {
656 if (vq->premapped)
657 goto free_indirect;
658
659 goto unmap_release;
660 }
661
662 virtqueue_add_desc_split(_vq, vq->split.vring.desc,
663 head, addr,
664 total_sg * sizeof(struct vring_desc),
665 VRING_DESC_F_INDIRECT,
666 false);
667 }
668
669 /* We're using some buffers from the free list. */
670 vq->vq.num_free -= descs_used;
671
672 /* Update free pointer */
673 if (indirect)
674 vq->free_head = vq->split.desc_extra[head].next;
675 else
676 vq->free_head = i;
677
678 /* Store token and indirect buffer state. */
679 vq->split.desc_state[head].data = data;
680 if (indirect)
681 vq->split.desc_state[head].indir_desc = desc;
682 else
683 vq->split.desc_state[head].indir_desc = ctx;
684
685 /* Put entry in available array (but don't update avail->idx until they
686 * do sync). */
687 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
688 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
689
690 /* Descriptors and available array need to be set before we expose the
691 * new available array entries. */
692 virtio_wmb(vq->weak_barriers);
693 vq->split.avail_idx_shadow++;
694 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
695 vq->split.avail_idx_shadow);
696 vq->num_added++;
697
698 pr_debug("Added buffer head %i to %p\n", head, vq);
699 END_USE(vq);
700
701 /* This is very unlikely, but theoretically possible. Kick
702 * just in case. */
703 if (unlikely(vq->num_added == (1 << 16) - 1))
704 virtqueue_kick(_vq);
705
706 return 0;
707
708unmap_release:
709 err_idx = i;
710
711 if (indirect)
712 i = 0;
713 else
714 i = head;
715
716 for (n = 0; n < total_sg; n++) {
717 if (i == err_idx)
718 break;
719 if (indirect) {
720 vring_unmap_one_split_indirect(vq, &desc[i]);
721 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
722 } else
723 i = vring_unmap_one_split(vq, i);
724 }
725
726free_indirect:
727 if (indirect)
728 kfree(desc);
729
730 END_USE(vq);
731 return -ENOMEM;
732}
733
734static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
735{
736 struct vring_virtqueue *vq = to_vvq(_vq);
737 u16 new, old;
738 bool needs_kick;
739
740 START_USE(vq);
741 /* We need to expose available array entries before checking avail
742 * event. */
743 virtio_mb(vq->weak_barriers);
744
745 old = vq->split.avail_idx_shadow - vq->num_added;
746 new = vq->split.avail_idx_shadow;
747 vq->num_added = 0;
748
749 LAST_ADD_TIME_CHECK(vq);
750 LAST_ADD_TIME_INVALID(vq);
751
752 if (vq->event) {
753 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
754 vring_avail_event(&vq->split.vring)),
755 new, old);
756 } else {
757 needs_kick = !(vq->split.vring.used->flags &
758 cpu_to_virtio16(_vq->vdev,
759 VRING_USED_F_NO_NOTIFY));
760 }
761 END_USE(vq);
762 return needs_kick;
763}
764
765static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
766 void **ctx)
767{
768 unsigned int i, j;
769 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
770
771 /* Clear data ptr. */
772 vq->split.desc_state[head].data = NULL;
773
774 /* Put back on free list: unmap first-level descriptors and find end */
775 i = head;
776
777 while (vq->split.vring.desc[i].flags & nextflag) {
778 vring_unmap_one_split(vq, i);
779 i = vq->split.desc_extra[i].next;
780 vq->vq.num_free++;
781 }
782
783 vring_unmap_one_split(vq, i);
784 vq->split.desc_extra[i].next = vq->free_head;
785 vq->free_head = head;
786
787 /* Plus final descriptor */
788 vq->vq.num_free++;
789
790 if (vq->indirect) {
791 struct vring_desc *indir_desc =
792 vq->split.desc_state[head].indir_desc;
793 u32 len;
794
795 /* Free the indirect table, if any, now that it's unmapped. */
796 if (!indir_desc)
797 return;
798
799 len = vq->split.desc_extra[head].len;
800
801 BUG_ON(!(vq->split.desc_extra[head].flags &
802 VRING_DESC_F_INDIRECT));
803 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
804
805 if (vq->do_unmap) {
806 for (j = 0; j < len / sizeof(struct vring_desc); j++)
807 vring_unmap_one_split_indirect(vq, &indir_desc[j]);
808 }
809
810 kfree(indir_desc);
811 vq->split.desc_state[head].indir_desc = NULL;
812 } else if (ctx) {
813 *ctx = vq->split.desc_state[head].indir_desc;
814 }
815}
816
817static bool more_used_split(const struct vring_virtqueue *vq)
818{
819 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
820 vq->split.vring.used->idx);
821}
822
823static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
824 unsigned int *len,
825 void **ctx)
826{
827 struct vring_virtqueue *vq = to_vvq(_vq);
828 void *ret;
829 unsigned int i;
830 u16 last_used;
831
832 START_USE(vq);
833
834 if (unlikely(vq->broken)) {
835 END_USE(vq);
836 return NULL;
837 }
838
839 if (!more_used_split(vq)) {
840 pr_debug("No more buffers in queue\n");
841 END_USE(vq);
842 return NULL;
843 }
844
845 /* Only get used array entries after they have been exposed by host. */
846 virtio_rmb(vq->weak_barriers);
847
848 last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
849 i = virtio32_to_cpu(_vq->vdev,
850 vq->split.vring.used->ring[last_used].id);
851 *len = virtio32_to_cpu(_vq->vdev,
852 vq->split.vring.used->ring[last_used].len);
853
854 if (unlikely(i >= vq->split.vring.num)) {
855 BAD_RING(vq, "id %u out of range\n", i);
856 return NULL;
857 }
858 if (unlikely(!vq->split.desc_state[i].data)) {
859 BAD_RING(vq, "id %u is not a head!\n", i);
860 return NULL;
861 }
862
863 /* detach_buf_split clears data, so grab it now. */
864 ret = vq->split.desc_state[i].data;
865 detach_buf_split(vq, i, ctx);
866 vq->last_used_idx++;
867 /* If we expect an interrupt for the next entry, tell host
868 * by writing event index and flush out the write before
869 * the read in the next get_buf call. */
870 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
871 virtio_store_mb(vq->weak_barriers,
872 &vring_used_event(&vq->split.vring),
873 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
874
875 LAST_ADD_TIME_INVALID(vq);
876
877 END_USE(vq);
878 return ret;
879}
880
881static void virtqueue_disable_cb_split(struct virtqueue *_vq)
882{
883 struct vring_virtqueue *vq = to_vvq(_vq);
884
885 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
886 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
887
888 /*
889 * If device triggered an event already it won't trigger one again:
890 * no need to disable.
891 */
892 if (vq->event_triggered)
893 return;
894
895 if (vq->event)
896 /* TODO: this is a hack. Figure out a cleaner value to write. */
897 vring_used_event(&vq->split.vring) = 0x0;
898 else
899 vq->split.vring.avail->flags =
900 cpu_to_virtio16(_vq->vdev,
901 vq->split.avail_flags_shadow);
902 }
903}
904
905static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
906{
907 struct vring_virtqueue *vq = to_vvq(_vq);
908 u16 last_used_idx;
909
910 START_USE(vq);
911
912 /* We optimistically turn back on interrupts, then check if there was
913 * more to do. */
914 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
915 * either clear the flags bit or point the event index at the next
916 * entry. Always do both to keep code simple. */
917 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
918 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
919 if (!vq->event)
920 vq->split.vring.avail->flags =
921 cpu_to_virtio16(_vq->vdev,
922 vq->split.avail_flags_shadow);
923 }
924 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
925 last_used_idx = vq->last_used_idx);
926 END_USE(vq);
927 return last_used_idx;
928}
929
930static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
931{
932 struct vring_virtqueue *vq = to_vvq(_vq);
933
934 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
935 vq->split.vring.used->idx);
936}
937
938static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
939{
940 struct vring_virtqueue *vq = to_vvq(_vq);
941 u16 bufs;
942
943 START_USE(vq);
944
945 /* We optimistically turn back on interrupts, then check if there was
946 * more to do. */
947 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
948 * either clear the flags bit or point the event index at the next
949 * entry. Always update the event index to keep code simple. */
950 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
951 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
952 if (!vq->event)
953 vq->split.vring.avail->flags =
954 cpu_to_virtio16(_vq->vdev,
955 vq->split.avail_flags_shadow);
956 }
957 /* TODO: tune this threshold */
958 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
959
960 virtio_store_mb(vq->weak_barriers,
961 &vring_used_event(&vq->split.vring),
962 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
963
964 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
965 - vq->last_used_idx) > bufs)) {
966 END_USE(vq);
967 return false;
968 }
969
970 END_USE(vq);
971 return true;
972}
973
974static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
975{
976 struct vring_virtqueue *vq = to_vvq(_vq);
977 unsigned int i;
978 void *buf;
979
980 START_USE(vq);
981
982 for (i = 0; i < vq->split.vring.num; i++) {
983 if (!vq->split.desc_state[i].data)
984 continue;
985 /* detach_buf_split clears data, so grab it now. */
986 buf = vq->split.desc_state[i].data;
987 detach_buf_split(vq, i, NULL);
988 vq->split.avail_idx_shadow--;
989 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
990 vq->split.avail_idx_shadow);
991 END_USE(vq);
992 return buf;
993 }
994 /* That should have freed everything. */
995 BUG_ON(vq->vq.num_free != vq->split.vring.num);
996
997 END_USE(vq);
998 return NULL;
999}
1000
1001static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split,
1002 struct vring_virtqueue *vq)
1003{
1004 struct virtio_device *vdev;
1005
1006 vdev = vq->vq.vdev;
1007
1008 vring_split->avail_flags_shadow = 0;
1009 vring_split->avail_idx_shadow = 0;
1010
1011 /* No callback? Tell other side not to bother us. */
1012 if (!vq->vq.callback) {
1013 vring_split->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
1014 if (!vq->event)
1015 vring_split->vring.avail->flags = cpu_to_virtio16(vdev,
1016 vring_split->avail_flags_shadow);
1017 }
1018}
1019
1020static void virtqueue_reinit_split(struct vring_virtqueue *vq)
1021{
1022 int num;
1023
1024 num = vq->split.vring.num;
1025
1026 vq->split.vring.avail->flags = 0;
1027 vq->split.vring.avail->idx = 0;
1028
1029 /* reset avail event */
1030 vq->split.vring.avail->ring[num] = 0;
1031
1032 vq->split.vring.used->flags = 0;
1033 vq->split.vring.used->idx = 0;
1034
1035 /* reset used event */
1036 *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;
1037
1038 virtqueue_init(vq, num);
1039
1040 virtqueue_vring_init_split(&vq->split, vq);
1041}
1042
1043static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
1044 struct vring_virtqueue_split *vring_split)
1045{
1046 vq->split = *vring_split;
1047
1048 /* Put everything in free lists. */
1049 vq->free_head = 0;
1050}
1051
1052static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
1053{
1054 struct vring_desc_state_split *state;
1055 struct vring_desc_extra *extra;
1056 u32 num = vring_split->vring.num;
1057
1058 state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL);
1059 if (!state)
1060 goto err_state;
1061
1062 extra = vring_alloc_desc_extra(num);
1063 if (!extra)
1064 goto err_extra;
1065
1066 memset(state, 0, num * sizeof(struct vring_desc_state_split));
1067
1068 vring_split->desc_state = state;
1069 vring_split->desc_extra = extra;
1070 return 0;
1071
1072err_extra:
1073 kfree(state);
1074err_state:
1075 return -ENOMEM;
1076}
1077
1078static void vring_free_split(struct vring_virtqueue_split *vring_split,
1079 struct virtio_device *vdev, struct device *dma_dev)
1080{
1081 vring_free_queue(vdev, vring_split->queue_size_in_bytes,
1082 vring_split->vring.desc,
1083 vring_split->queue_dma_addr,
1084 dma_dev);
1085
1086 kfree(vring_split->desc_state);
1087 kfree(vring_split->desc_extra);
1088}
1089
1090static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
1091 struct virtio_device *vdev,
1092 u32 num,
1093 unsigned int vring_align,
1094 bool may_reduce_num,
1095 struct device *dma_dev)
1096{
1097 void *queue = NULL;
1098 dma_addr_t dma_addr;
1099
1100 /* We assume num is a power of 2. */
1101 if (!is_power_of_2(num)) {
1102 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
1103 return -EINVAL;
1104 }
1105
1106 /* TODO: allocate each queue chunk individually */
1107 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
1108 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1109 &dma_addr,
1110 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1111 dma_dev);
1112 if (queue)
1113 break;
1114 if (!may_reduce_num)
1115 return -ENOMEM;
1116 }
1117
1118 if (!num)
1119 return -ENOMEM;
1120
1121 if (!queue) {
1122 /* Try to get a single page. You are my only hope! */
1123 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1124 &dma_addr, GFP_KERNEL | __GFP_ZERO,
1125 dma_dev);
1126 }
1127 if (!queue)
1128 return -ENOMEM;
1129
1130 vring_init(&vring_split->vring, num, queue, vring_align);
1131
1132 vring_split->queue_dma_addr = dma_addr;
1133 vring_split->queue_size_in_bytes = vring_size(num, vring_align);
1134
1135 vring_split->vring_align = vring_align;
1136 vring_split->may_reduce_num = may_reduce_num;
1137
1138 return 0;
1139}
1140
1141static struct virtqueue *vring_create_virtqueue_split(
1142 unsigned int index,
1143 unsigned int num,
1144 unsigned int vring_align,
1145 struct virtio_device *vdev,
1146 bool weak_barriers,
1147 bool may_reduce_num,
1148 bool context,
1149 bool (*notify)(struct virtqueue *),
1150 void (*callback)(struct virtqueue *),
1151 const char *name,
1152 struct device *dma_dev)
1153{
1154 struct vring_virtqueue_split vring_split = {};
1155 struct virtqueue *vq;
1156 int err;
1157
1158 err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align,
1159 may_reduce_num, dma_dev);
1160 if (err)
1161 return NULL;
1162
1163 vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
1164 context, notify, callback, name, dma_dev);
1165 if (!vq) {
1166 vring_free_split(&vring_split, vdev, dma_dev);
1167 return NULL;
1168 }
1169
1170 to_vvq(vq)->we_own_ring = true;
1171
1172 return vq;
1173}
1174
1175static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
1176{
1177 struct vring_virtqueue_split vring_split = {};
1178 struct vring_virtqueue *vq = to_vvq(_vq);
1179 struct virtio_device *vdev = _vq->vdev;
1180 int err;
1181
1182 err = vring_alloc_queue_split(&vring_split, vdev, num,
1183 vq->split.vring_align,
1184 vq->split.may_reduce_num,
1185 vring_dma_dev(vq));
1186 if (err)
1187 goto err;
1188
1189 err = vring_alloc_state_extra_split(&vring_split);
1190 if (err)
1191 goto err_state_extra;
1192
1193 vring_free(&vq->vq);
1194
1195 virtqueue_vring_init_split(&vring_split, vq);
1196
1197 virtqueue_init(vq, vring_split.vring.num);
1198 virtqueue_vring_attach_split(vq, &vring_split);
1199
1200 return 0;
1201
1202err_state_extra:
1203 vring_free_split(&vring_split, vdev, vring_dma_dev(vq));
1204err:
1205 virtqueue_reinit_split(vq);
1206 return -ENOMEM;
1207}
1208
1209
1210/*
1211 * Packed ring specific functions - *_packed().
1212 */
1213static bool packed_used_wrap_counter(u16 last_used_idx)
1214{
1215 return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1216}
1217
1218static u16 packed_last_used(u16 last_used_idx)
1219{
1220 return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1221}
1222
1223static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
1224 const struct vring_desc_extra *extra)
1225{
1226 u16 flags;
1227
1228 flags = extra->flags;
1229
1230 if (flags & VRING_DESC_F_INDIRECT) {
1231 if (!vq->use_dma_api)
1232 return;
1233
1234 dma_unmap_single(vring_dma_dev(vq),
1235 extra->addr, extra->len,
1236 (flags & VRING_DESC_F_WRITE) ?
1237 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1238 } else {
1239 if (!vq->do_unmap)
1240 return;
1241
1242 dma_unmap_page(vring_dma_dev(vq),
1243 extra->addr, extra->len,
1244 (flags & VRING_DESC_F_WRITE) ?
1245 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1246 }
1247}
1248
1249static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
1250 const struct vring_packed_desc *desc)
1251{
1252 u16 flags;
1253
1254 if (!vq->do_unmap)
1255 return;
1256
1257 flags = le16_to_cpu(desc->flags);
1258
1259 dma_unmap_page(vring_dma_dev(vq),
1260 le64_to_cpu(desc->addr),
1261 le32_to_cpu(desc->len),
1262 (flags & VRING_DESC_F_WRITE) ?
1263 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1264}
1265
1266static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
1267 gfp_t gfp)
1268{
1269 struct vring_packed_desc *desc;
1270
1271 /*
1272 * We require lowmem mappings for the descriptors because
1273 * otherwise virt_to_phys will give us bogus addresses in the
1274 * virtqueue.
1275 */
1276 gfp &= ~__GFP_HIGHMEM;
1277
1278 desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
1279
1280 return desc;
1281}
1282
1283static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
1284 struct scatterlist *sgs[],
1285 unsigned int total_sg,
1286 unsigned int out_sgs,
1287 unsigned int in_sgs,
1288 void *data,
1289 gfp_t gfp)
1290{
1291 struct vring_packed_desc *desc;
1292 struct scatterlist *sg;
1293 unsigned int i, n, err_idx;
1294 u16 head, id;
1295 dma_addr_t addr;
1296
1297 head = vq->packed.next_avail_idx;
1298 desc = alloc_indirect_packed(total_sg, gfp);
1299 if (!desc)
1300 return -ENOMEM;
1301
1302 if (unlikely(vq->vq.num_free < 1)) {
1303 pr_debug("Can't add buf len 1 - avail = 0\n");
1304 kfree(desc);
1305 END_USE(vq);
1306 return -ENOSPC;
1307 }
1308
1309 i = 0;
1310 id = vq->free_head;
1311 BUG_ON(id == vq->packed.vring.num);
1312
1313 for (n = 0; n < out_sgs + in_sgs; n++) {
1314 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1315 if (vring_map_one_sg(vq, sg, n < out_sgs ?
1316 DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr))
1317 goto unmap_release;
1318
1319 desc[i].flags = cpu_to_le16(n < out_sgs ?
1320 0 : VRING_DESC_F_WRITE);
1321 desc[i].addr = cpu_to_le64(addr);
1322 desc[i].len = cpu_to_le32(sg->length);
1323 i++;
1324 }
1325 }
1326
1327 /* Now that the indirect table is filled in, map it. */
1328 addr = vring_map_single(vq, desc,
1329 total_sg * sizeof(struct vring_packed_desc),
1330 DMA_TO_DEVICE);
1331 if (vring_mapping_error(vq, addr)) {
1332 if (vq->premapped)
1333 goto free_desc;
1334
1335 goto unmap_release;
1336 }
1337
1338 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1339 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1340 sizeof(struct vring_packed_desc));
1341 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1342
1343 if (vq->do_unmap) {
1344 vq->packed.desc_extra[id].addr = addr;
1345 vq->packed.desc_extra[id].len = total_sg *
1346 sizeof(struct vring_packed_desc);
1347 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1348 vq->packed.avail_used_flags;
1349 }
1350
1351 /*
1352 * A driver MUST NOT make the first descriptor in the list
1353 * available before all subsequent descriptors comprising
1354 * the list are made available.
1355 */
1356 virtio_wmb(vq->weak_barriers);
1357 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1358 vq->packed.avail_used_flags);
1359
1360 /* We're using some buffers from the free list. */
1361 vq->vq.num_free -= 1;
1362
1363 /* Update free pointer */
1364 n = head + 1;
1365 if (n >= vq->packed.vring.num) {
1366 n = 0;
1367 vq->packed.avail_wrap_counter ^= 1;
1368 vq->packed.avail_used_flags ^=
1369 1 << VRING_PACKED_DESC_F_AVAIL |
1370 1 << VRING_PACKED_DESC_F_USED;
1371 }
1372 vq->packed.next_avail_idx = n;
1373 vq->free_head = vq->packed.desc_extra[id].next;
1374
1375 /* Store token and indirect buffer state. */
1376 vq->packed.desc_state[id].num = 1;
1377 vq->packed.desc_state[id].data = data;
1378 vq->packed.desc_state[id].indir_desc = desc;
1379 vq->packed.desc_state[id].last = id;
1380
1381 vq->num_added += 1;
1382
1383 pr_debug("Added buffer head %i to %p\n", head, vq);
1384 END_USE(vq);
1385
1386 return 0;
1387
1388unmap_release:
1389 err_idx = i;
1390
1391 for (i = 0; i < err_idx; i++)
1392 vring_unmap_desc_packed(vq, &desc[i]);
1393
1394free_desc:
1395 kfree(desc);
1396
1397 END_USE(vq);
1398 return -ENOMEM;
1399}
1400
1401static inline int virtqueue_add_packed(struct virtqueue *_vq,
1402 struct scatterlist *sgs[],
1403 unsigned int total_sg,
1404 unsigned int out_sgs,
1405 unsigned int in_sgs,
1406 void *data,
1407 void *ctx,
1408 gfp_t gfp)
1409{
1410 struct vring_virtqueue *vq = to_vvq(_vq);
1411 struct vring_packed_desc *desc;
1412 struct scatterlist *sg;
1413 unsigned int i, n, c, descs_used, err_idx;
1414 __le16 head_flags, flags;
1415 u16 head, id, prev, curr, avail_used_flags;
1416 int err;
1417
1418 START_USE(vq);
1419
1420 BUG_ON(data == NULL);
1421 BUG_ON(ctx && vq->indirect);
1422
1423 if (unlikely(vq->broken)) {
1424 END_USE(vq);
1425 return -EIO;
1426 }
1427
1428 LAST_ADD_TIME_UPDATE(vq);
1429
1430 BUG_ON(total_sg == 0);
1431
1432 if (virtqueue_use_indirect(vq, total_sg)) {
1433 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
1434 in_sgs, data, gfp);
1435 if (err != -ENOMEM) {
1436 END_USE(vq);
1437 return err;
1438 }
1439
1440 /* fall back on direct */
1441 }
1442
1443 head = vq->packed.next_avail_idx;
1444 avail_used_flags = vq->packed.avail_used_flags;
1445
1446 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1447
1448 desc = vq->packed.vring.desc;
1449 i = head;
1450 descs_used = total_sg;
1451
1452 if (unlikely(vq->vq.num_free < descs_used)) {
1453 pr_debug("Can't add buf len %i - avail = %i\n",
1454 descs_used, vq->vq.num_free);
1455 END_USE(vq);
1456 return -ENOSPC;
1457 }
1458
1459 id = vq->free_head;
1460 BUG_ON(id == vq->packed.vring.num);
1461
1462 curr = id;
1463 c = 0;
1464 for (n = 0; n < out_sgs + in_sgs; n++) {
1465 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1466 dma_addr_t addr;
1467
1468 if (vring_map_one_sg(vq, sg, n < out_sgs ?
1469 DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr))
1470 goto unmap_release;
1471
1472 flags = cpu_to_le16(vq->packed.avail_used_flags |
1473 (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1474 (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1475 if (i == head)
1476 head_flags = flags;
1477 else
1478 desc[i].flags = flags;
1479
1480 desc[i].addr = cpu_to_le64(addr);
1481 desc[i].len = cpu_to_le32(sg->length);
1482 desc[i].id = cpu_to_le16(id);
1483
1484 if (unlikely(vq->do_unmap)) {
1485 vq->packed.desc_extra[curr].addr = addr;
1486 vq->packed.desc_extra[curr].len = sg->length;
1487 vq->packed.desc_extra[curr].flags =
1488 le16_to_cpu(flags);
1489 }
1490 prev = curr;
1491 curr = vq->packed.desc_extra[curr].next;
1492
1493 if ((unlikely(++i >= vq->packed.vring.num))) {
1494 i = 0;
1495 vq->packed.avail_used_flags ^=
1496 1 << VRING_PACKED_DESC_F_AVAIL |
1497 1 << VRING_PACKED_DESC_F_USED;
1498 }
1499 }
1500 }
1501
1502 if (i <= head)
1503 vq->packed.avail_wrap_counter ^= 1;
1504
1505 /* We're using some buffers from the free list. */
1506 vq->vq.num_free -= descs_used;
1507
1508 /* Update free pointer */
1509 vq->packed.next_avail_idx = i;
1510 vq->free_head = curr;
1511
1512 /* Store token. */
1513 vq->packed.desc_state[id].num = descs_used;
1514 vq->packed.desc_state[id].data = data;
1515 vq->packed.desc_state[id].indir_desc = ctx;
1516 vq->packed.desc_state[id].last = prev;
1517
1518 /*
1519 * A driver MUST NOT make the first descriptor in the list
1520 * available before all subsequent descriptors comprising
1521 * the list are made available.
1522 */
1523 virtio_wmb(vq->weak_barriers);
1524 vq->packed.vring.desc[head].flags = head_flags;
1525 vq->num_added += descs_used;
1526
1527 pr_debug("Added buffer head %i to %p\n", head, vq);
1528 END_USE(vq);
1529
1530 return 0;
1531
1532unmap_release:
1533 err_idx = i;
1534 i = head;
1535 curr = vq->free_head;
1536
1537 vq->packed.avail_used_flags = avail_used_flags;
1538
1539 for (n = 0; n < total_sg; n++) {
1540 if (i == err_idx)
1541 break;
1542 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1543 curr = vq->packed.desc_extra[curr].next;
1544 i++;
1545 if (i >= vq->packed.vring.num)
1546 i = 0;
1547 }
1548
1549 END_USE(vq);
1550 return -EIO;
1551}
1552
1553static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1554{
1555 struct vring_virtqueue *vq = to_vvq(_vq);
1556 u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1557 bool needs_kick;
1558 union {
1559 struct {
1560 __le16 off_wrap;
1561 __le16 flags;
1562 };
1563 u32 u32;
1564 } snapshot;
1565
1566 START_USE(vq);
1567
1568 /*
1569 * We need to expose the new flags value before checking notification
1570 * suppressions.
1571 */
1572 virtio_mb(vq->weak_barriers);
1573
1574 old = vq->packed.next_avail_idx - vq->num_added;
1575 new = vq->packed.next_avail_idx;
1576 vq->num_added = 0;
1577
1578 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1579 flags = le16_to_cpu(snapshot.flags);
1580
1581 LAST_ADD_TIME_CHECK(vq);
1582 LAST_ADD_TIME_INVALID(vq);
1583
1584 if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1585 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1586 goto out;
1587 }
1588
1589 off_wrap = le16_to_cpu(snapshot.off_wrap);
1590
1591 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1592 event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1593 if (wrap_counter != vq->packed.avail_wrap_counter)
1594 event_idx -= vq->packed.vring.num;
1595
1596 needs_kick = vring_need_event(event_idx, new, old);
1597out:
1598 END_USE(vq);
1599 return needs_kick;
1600}
1601
1602static void detach_buf_packed(struct vring_virtqueue *vq,
1603 unsigned int id, void **ctx)
1604{
1605 struct vring_desc_state_packed *state = NULL;
1606 struct vring_packed_desc *desc;
1607 unsigned int i, curr;
1608
1609 state = &vq->packed.desc_state[id];
1610
1611 /* Clear data ptr. */
1612 state->data = NULL;
1613
1614 vq->packed.desc_extra[state->last].next = vq->free_head;
1615 vq->free_head = id;
1616 vq->vq.num_free += state->num;
1617
1618 if (unlikely(vq->do_unmap)) {
1619 curr = id;
1620 for (i = 0; i < state->num; i++) {
1621 vring_unmap_extra_packed(vq,
1622 &vq->packed.desc_extra[curr]);
1623 curr = vq->packed.desc_extra[curr].next;
1624 }
1625 }
1626
1627 if (vq->indirect) {
1628 u32 len;
1629
1630 /* Free the indirect table, if any, now that it's unmapped. */
1631 desc = state->indir_desc;
1632 if (!desc)
1633 return;
1634
1635 if (vq->do_unmap) {
1636 len = vq->packed.desc_extra[id].len;
1637 for (i = 0; i < len / sizeof(struct vring_packed_desc);
1638 i++)
1639 vring_unmap_desc_packed(vq, &desc[i]);
1640 }
1641 kfree(desc);
1642 state->indir_desc = NULL;
1643 } else if (ctx) {
1644 *ctx = state->indir_desc;
1645 }
1646}
1647
1648static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1649 u16 idx, bool used_wrap_counter)
1650{
1651 bool avail, used;
1652 u16 flags;
1653
1654 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1655 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1656 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1657
1658 return avail == used && used == used_wrap_counter;
1659}
1660
1661static bool more_used_packed(const struct vring_virtqueue *vq)
1662{
1663 u16 last_used;
1664 u16 last_used_idx;
1665 bool used_wrap_counter;
1666
1667 last_used_idx = READ_ONCE(vq->last_used_idx);
1668 last_used = packed_last_used(last_used_idx);
1669 used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1670 return is_used_desc_packed(vq, last_used, used_wrap_counter);
1671}
1672
1673static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1674 unsigned int *len,
1675 void **ctx)
1676{
1677 struct vring_virtqueue *vq = to_vvq(_vq);
1678 u16 last_used, id, last_used_idx;
1679 bool used_wrap_counter;
1680 void *ret;
1681
1682 START_USE(vq);
1683
1684 if (unlikely(vq->broken)) {
1685 END_USE(vq);
1686 return NULL;
1687 }
1688
1689 if (!more_used_packed(vq)) {
1690 pr_debug("No more buffers in queue\n");
1691 END_USE(vq);
1692 return NULL;
1693 }
1694
1695 /* Only get used elements after they have been exposed by host. */
1696 virtio_rmb(vq->weak_barriers);
1697
1698 last_used_idx = READ_ONCE(vq->last_used_idx);
1699 used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1700 last_used = packed_last_used(last_used_idx);
1701 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1702 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1703
1704 if (unlikely(id >= vq->packed.vring.num)) {
1705 BAD_RING(vq, "id %u out of range\n", id);
1706 return NULL;
1707 }
1708 if (unlikely(!vq->packed.desc_state[id].data)) {
1709 BAD_RING(vq, "id %u is not a head!\n", id);
1710 return NULL;
1711 }
1712
1713 /* detach_buf_packed clears data, so grab it now. */
1714 ret = vq->packed.desc_state[id].data;
1715 detach_buf_packed(vq, id, ctx);
1716
1717 last_used += vq->packed.desc_state[id].num;
1718 if (unlikely(last_used >= vq->packed.vring.num)) {
1719 last_used -= vq->packed.vring.num;
1720 used_wrap_counter ^= 1;
1721 }
1722
1723 last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1724 WRITE_ONCE(vq->last_used_idx, last_used);
1725
1726 /*
1727 * If we expect an interrupt for the next entry, tell host
1728 * by writing event index and flush out the write before
1729 * the read in the next get_buf call.
1730 */
1731 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1732 virtio_store_mb(vq->weak_barriers,
1733 &vq->packed.vring.driver->off_wrap,
1734 cpu_to_le16(vq->last_used_idx));
1735
1736 LAST_ADD_TIME_INVALID(vq);
1737
1738 END_USE(vq);
1739 return ret;
1740}
1741
1742static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1743{
1744 struct vring_virtqueue *vq = to_vvq(_vq);
1745
1746 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1747 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1748
1749 /*
1750 * If device triggered an event already it won't trigger one again:
1751 * no need to disable.
1752 */
1753 if (vq->event_triggered)
1754 return;
1755
1756 vq->packed.vring.driver->flags =
1757 cpu_to_le16(vq->packed.event_flags_shadow);
1758 }
1759}
1760
1761static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1762{
1763 struct vring_virtqueue *vq = to_vvq(_vq);
1764
1765 START_USE(vq);
1766
1767 /*
1768 * We optimistically turn back on interrupts, then check if there was
1769 * more to do.
1770 */
1771
1772 if (vq->event) {
1773 vq->packed.vring.driver->off_wrap =
1774 cpu_to_le16(vq->last_used_idx);
1775 /*
1776 * We need to update event offset and event wrap
1777 * counter first before updating event flags.
1778 */
1779 virtio_wmb(vq->weak_barriers);
1780 }
1781
1782 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1783 vq->packed.event_flags_shadow = vq->event ?
1784 VRING_PACKED_EVENT_FLAG_DESC :
1785 VRING_PACKED_EVENT_FLAG_ENABLE;
1786 vq->packed.vring.driver->flags =
1787 cpu_to_le16(vq->packed.event_flags_shadow);
1788 }
1789
1790 END_USE(vq);
1791 return vq->last_used_idx;
1792}
1793
1794static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1795{
1796 struct vring_virtqueue *vq = to_vvq(_vq);
1797 bool wrap_counter;
1798 u16 used_idx;
1799
1800 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1801 used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1802
1803 return is_used_desc_packed(vq, used_idx, wrap_counter);
1804}
1805
1806static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1807{
1808 struct vring_virtqueue *vq = to_vvq(_vq);
1809 u16 used_idx, wrap_counter, last_used_idx;
1810 u16 bufs;
1811
1812 START_USE(vq);
1813
1814 /*
1815 * We optimistically turn back on interrupts, then check if there was
1816 * more to do.
1817 */
1818
1819 if (vq->event) {
1820 /* TODO: tune this threshold */
1821 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1822 last_used_idx = READ_ONCE(vq->last_used_idx);
1823 wrap_counter = packed_used_wrap_counter(last_used_idx);
1824
1825 used_idx = packed_last_used(last_used_idx) + bufs;
1826 if (used_idx >= vq->packed.vring.num) {
1827 used_idx -= vq->packed.vring.num;
1828 wrap_counter ^= 1;
1829 }
1830
1831 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1832 (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1833
1834 /*
1835 * We need to update event offset and event wrap
1836 * counter first before updating event flags.
1837 */
1838 virtio_wmb(vq->weak_barriers);
1839 }
1840
1841 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1842 vq->packed.event_flags_shadow = vq->event ?
1843 VRING_PACKED_EVENT_FLAG_DESC :
1844 VRING_PACKED_EVENT_FLAG_ENABLE;
1845 vq->packed.vring.driver->flags =
1846 cpu_to_le16(vq->packed.event_flags_shadow);
1847 }
1848
1849 /*
1850 * We need to update event suppression structure first
1851 * before re-checking for more used buffers.
1852 */
1853 virtio_mb(vq->weak_barriers);
1854
1855 last_used_idx = READ_ONCE(vq->last_used_idx);
1856 wrap_counter = packed_used_wrap_counter(last_used_idx);
1857 used_idx = packed_last_used(last_used_idx);
1858 if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
1859 END_USE(vq);
1860 return false;
1861 }
1862
1863 END_USE(vq);
1864 return true;
1865}
1866
1867static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1868{
1869 struct vring_virtqueue *vq = to_vvq(_vq);
1870 unsigned int i;
1871 void *buf;
1872
1873 START_USE(vq);
1874
1875 for (i = 0; i < vq->packed.vring.num; i++) {
1876 if (!vq->packed.desc_state[i].data)
1877 continue;
1878 /* detach_buf clears data, so grab it now. */
1879 buf = vq->packed.desc_state[i].data;
1880 detach_buf_packed(vq, i, NULL);
1881 END_USE(vq);
1882 return buf;
1883 }
1884 /* That should have freed everything. */
1885 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1886
1887 END_USE(vq);
1888 return NULL;
1889}
1890
1891static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
1892{
1893 struct vring_desc_extra *desc_extra;
1894 unsigned int i;
1895
1896 desc_extra = kmalloc_array(num, sizeof(struct vring_desc_extra),
1897 GFP_KERNEL);
1898 if (!desc_extra)
1899 return NULL;
1900
1901 memset(desc_extra, 0, num * sizeof(struct vring_desc_extra));
1902
1903 for (i = 0; i < num - 1; i++)
1904 desc_extra[i].next = i + 1;
1905
1906 return desc_extra;
1907}
1908
1909static void vring_free_packed(struct vring_virtqueue_packed *vring_packed,
1910 struct virtio_device *vdev,
1911 struct device *dma_dev)
1912{
1913 if (vring_packed->vring.desc)
1914 vring_free_queue(vdev, vring_packed->ring_size_in_bytes,
1915 vring_packed->vring.desc,
1916 vring_packed->ring_dma_addr,
1917 dma_dev);
1918
1919 if (vring_packed->vring.driver)
1920 vring_free_queue(vdev, vring_packed->event_size_in_bytes,
1921 vring_packed->vring.driver,
1922 vring_packed->driver_event_dma_addr,
1923 dma_dev);
1924
1925 if (vring_packed->vring.device)
1926 vring_free_queue(vdev, vring_packed->event_size_in_bytes,
1927 vring_packed->vring.device,
1928 vring_packed->device_event_dma_addr,
1929 dma_dev);
1930
1931 kfree(vring_packed->desc_state);
1932 kfree(vring_packed->desc_extra);
1933}
1934
1935static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed,
1936 struct virtio_device *vdev,
1937 u32 num, struct device *dma_dev)
1938{
1939 struct vring_packed_desc *ring;
1940 struct vring_packed_desc_event *driver, *device;
1941 dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1942 size_t ring_size_in_bytes, event_size_in_bytes;
1943
1944 ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1945
1946 ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1947 &ring_dma_addr,
1948 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1949 dma_dev);
1950 if (!ring)
1951 goto err;
1952
1953 vring_packed->vring.desc = ring;
1954 vring_packed->ring_dma_addr = ring_dma_addr;
1955 vring_packed->ring_size_in_bytes = ring_size_in_bytes;
1956
1957 event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1958
1959 driver = vring_alloc_queue(vdev, event_size_in_bytes,
1960 &driver_event_dma_addr,
1961 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1962 dma_dev);
1963 if (!driver)
1964 goto err;
1965
1966 vring_packed->vring.driver = driver;
1967 vring_packed->event_size_in_bytes = event_size_in_bytes;
1968 vring_packed->driver_event_dma_addr = driver_event_dma_addr;
1969
1970 device = vring_alloc_queue(vdev, event_size_in_bytes,
1971 &device_event_dma_addr,
1972 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1973 dma_dev);
1974 if (!device)
1975 goto err;
1976
1977 vring_packed->vring.device = device;
1978 vring_packed->device_event_dma_addr = device_event_dma_addr;
1979
1980 vring_packed->vring.num = num;
1981
1982 return 0;
1983
1984err:
1985 vring_free_packed(vring_packed, vdev, dma_dev);
1986 return -ENOMEM;
1987}
1988
1989static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed)
1990{
1991 struct vring_desc_state_packed *state;
1992 struct vring_desc_extra *extra;
1993 u32 num = vring_packed->vring.num;
1994
1995 state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL);
1996 if (!state)
1997 goto err_desc_state;
1998
1999 memset(state, 0, num * sizeof(struct vring_desc_state_packed));
2000
2001 extra = vring_alloc_desc_extra(num);
2002 if (!extra)
2003 goto err_desc_extra;
2004
2005 vring_packed->desc_state = state;
2006 vring_packed->desc_extra = extra;
2007
2008 return 0;
2009
2010err_desc_extra:
2011 kfree(state);
2012err_desc_state:
2013 return -ENOMEM;
2014}
2015
2016static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
2017 bool callback)
2018{
2019 vring_packed->next_avail_idx = 0;
2020 vring_packed->avail_wrap_counter = 1;
2021 vring_packed->event_flags_shadow = 0;
2022 vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
2023
2024 /* No callback? Tell other side not to bother us. */
2025 if (!callback) {
2026 vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
2027 vring_packed->vring.driver->flags =
2028 cpu_to_le16(vring_packed->event_flags_shadow);
2029 }
2030}
2031
2032static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
2033 struct vring_virtqueue_packed *vring_packed)
2034{
2035 vq->packed = *vring_packed;
2036
2037 /* Put everything in free lists. */
2038 vq->free_head = 0;
2039}
2040
2041static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
2042{
2043 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
2044 memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
2045
2046 /* we need to reset the desc.flags. For more, see is_used_desc_packed() */
2047 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
2048
2049 virtqueue_init(vq, vq->packed.vring.num);
2050 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
2051}
2052
2053static struct virtqueue *vring_create_virtqueue_packed(
2054 unsigned int index,
2055 unsigned int num,
2056 unsigned int vring_align,
2057 struct virtio_device *vdev,
2058 bool weak_barriers,
2059 bool may_reduce_num,
2060 bool context,
2061 bool (*notify)(struct virtqueue *),
2062 void (*callback)(struct virtqueue *),
2063 const char *name,
2064 struct device *dma_dev)
2065{
2066 struct vring_virtqueue_packed vring_packed = {};
2067 struct vring_virtqueue *vq;
2068 int err;
2069
2070 if (vring_alloc_queue_packed(&vring_packed, vdev, num, dma_dev))
2071 goto err_ring;
2072
2073 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2074 if (!vq)
2075 goto err_vq;
2076
2077 vq->vq.callback = callback;
2078 vq->vq.vdev = vdev;
2079 vq->vq.name = name;
2080 vq->vq.index = index;
2081 vq->vq.reset = false;
2082 vq->we_own_ring = true;
2083 vq->notify = notify;
2084 vq->weak_barriers = weak_barriers;
2085#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2086 vq->broken = true;
2087#else
2088 vq->broken = false;
2089#endif
2090 vq->packed_ring = true;
2091 vq->dma_dev = dma_dev;
2092 vq->use_dma_api = vring_use_dma_api(vdev);
2093 vq->premapped = false;
2094 vq->do_unmap = vq->use_dma_api;
2095
2096 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2097 !context;
2098 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2099
2100 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2101 vq->weak_barriers = false;
2102
2103 err = vring_alloc_state_extra_packed(&vring_packed);
2104 if (err)
2105 goto err_state_extra;
2106
2107 virtqueue_vring_init_packed(&vring_packed, !!callback);
2108
2109 virtqueue_init(vq, num);
2110 virtqueue_vring_attach_packed(vq, &vring_packed);
2111
2112 spin_lock(&vdev->vqs_list_lock);
2113 list_add_tail(&vq->vq.list, &vdev->vqs);
2114 spin_unlock(&vdev->vqs_list_lock);
2115 return &vq->vq;
2116
2117err_state_extra:
2118 kfree(vq);
2119err_vq:
2120 vring_free_packed(&vring_packed, vdev, dma_dev);
2121err_ring:
2122 return NULL;
2123}
2124
2125static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
2126{
2127 struct vring_virtqueue_packed vring_packed = {};
2128 struct vring_virtqueue *vq = to_vvq(_vq);
2129 struct virtio_device *vdev = _vq->vdev;
2130 int err;
2131
2132 if (vring_alloc_queue_packed(&vring_packed, vdev, num, vring_dma_dev(vq)))
2133 goto err_ring;
2134
2135 err = vring_alloc_state_extra_packed(&vring_packed);
2136 if (err)
2137 goto err_state_extra;
2138
2139 vring_free(&vq->vq);
2140
2141 virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback);
2142
2143 virtqueue_init(vq, vring_packed.vring.num);
2144 virtqueue_vring_attach_packed(vq, &vring_packed);
2145
2146 return 0;
2147
2148err_state_extra:
2149 vring_free_packed(&vring_packed, vdev, vring_dma_dev(vq));
2150err_ring:
2151 virtqueue_reinit_packed(vq);
2152 return -ENOMEM;
2153}
2154
2155static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
2156 void (*recycle)(struct virtqueue *vq, void *buf))
2157{
2158 struct vring_virtqueue *vq = to_vvq(_vq);
2159 struct virtio_device *vdev = vq->vq.vdev;
2160 void *buf;
2161 int err;
2162
2163 if (!vq->we_own_ring)
2164 return -EPERM;
2165
2166 if (!vdev->config->disable_vq_and_reset)
2167 return -ENOENT;
2168
2169 if (!vdev->config->enable_vq_after_reset)
2170 return -ENOENT;
2171
2172 err = vdev->config->disable_vq_and_reset(_vq);
2173 if (err)
2174 return err;
2175
2176 while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
2177 recycle(_vq, buf);
2178
2179 return 0;
2180}
2181
2182static int virtqueue_enable_after_reset(struct virtqueue *_vq)
2183{
2184 struct vring_virtqueue *vq = to_vvq(_vq);
2185 struct virtio_device *vdev = vq->vq.vdev;
2186
2187 if (vdev->config->enable_vq_after_reset(_vq))
2188 return -EBUSY;
2189
2190 return 0;
2191}
2192
2193/*
2194 * Generic functions and exported symbols.
2195 */
2196
2197static inline int virtqueue_add(struct virtqueue *_vq,
2198 struct scatterlist *sgs[],
2199 unsigned int total_sg,
2200 unsigned int out_sgs,
2201 unsigned int in_sgs,
2202 void *data,
2203 void *ctx,
2204 gfp_t gfp)
2205{
2206 struct vring_virtqueue *vq = to_vvq(_vq);
2207
2208 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
2209 out_sgs, in_sgs, data, ctx, gfp) :
2210 virtqueue_add_split(_vq, sgs, total_sg,
2211 out_sgs, in_sgs, data, ctx, gfp);
2212}
2213
2214/**
2215 * virtqueue_add_sgs - expose buffers to other end
2216 * @_vq: the struct virtqueue we're talking about.
2217 * @sgs: array of terminated scatterlists.
2218 * @out_sgs: the number of scatterlists readable by other side
2219 * @in_sgs: the number of scatterlists which are writable (after readable ones)
2220 * @data: the token identifying the buffer.
2221 * @gfp: how to do memory allocations (if necessary).
2222 *
2223 * Caller must ensure we don't call this with other virtqueue operations
2224 * at the same time (except where noted).
2225 *
2226 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2227 */
2228int virtqueue_add_sgs(struct virtqueue *_vq,
2229 struct scatterlist *sgs[],
2230 unsigned int out_sgs,
2231 unsigned int in_sgs,
2232 void *data,
2233 gfp_t gfp)
2234{
2235 unsigned int i, total_sg = 0;
2236
2237 /* Count them first. */
2238 for (i = 0; i < out_sgs + in_sgs; i++) {
2239 struct scatterlist *sg;
2240
2241 for (sg = sgs[i]; sg; sg = sg_next(sg))
2242 total_sg++;
2243 }
2244 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
2245 data, NULL, gfp);
2246}
2247EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
2248
2249/**
2250 * virtqueue_add_outbuf - expose output buffers to other end
2251 * @vq: the struct virtqueue we're talking about.
2252 * @sg: scatterlist (must be well-formed and terminated!)
2253 * @num: the number of entries in @sg readable by other side
2254 * @data: the token identifying the buffer.
2255 * @gfp: how to do memory allocations (if necessary).
2256 *
2257 * Caller must ensure we don't call this with other virtqueue operations
2258 * at the same time (except where noted).
2259 *
2260 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2261 */
2262int virtqueue_add_outbuf(struct virtqueue *vq,
2263 struct scatterlist *sg, unsigned int num,
2264 void *data,
2265 gfp_t gfp)
2266{
2267 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
2268}
2269EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
2270
2271/**
2272 * virtqueue_add_inbuf - expose input buffers to other end
2273 * @vq: the struct virtqueue we're talking about.
2274 * @sg: scatterlist (must be well-formed and terminated!)
2275 * @num: the number of entries in @sg writable by other side
2276 * @data: the token identifying the buffer.
2277 * @gfp: how to do memory allocations (if necessary).
2278 *
2279 * Caller must ensure we don't call this with other virtqueue operations
2280 * at the same time (except where noted).
2281 *
2282 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2283 */
2284int virtqueue_add_inbuf(struct virtqueue *vq,
2285 struct scatterlist *sg, unsigned int num,
2286 void *data,
2287 gfp_t gfp)
2288{
2289 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
2290}
2291EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
2292
2293/**
2294 * virtqueue_add_inbuf_ctx - expose input buffers to other end
2295 * @vq: the struct virtqueue we're talking about.
2296 * @sg: scatterlist (must be well-formed and terminated!)
2297 * @num: the number of entries in @sg writable by other side
2298 * @data: the token identifying the buffer.
2299 * @ctx: extra context for the token
2300 * @gfp: how to do memory allocations (if necessary).
2301 *
2302 * Caller must ensure we don't call this with other virtqueue operations
2303 * at the same time (except where noted).
2304 *
2305 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2306 */
2307int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
2308 struct scatterlist *sg, unsigned int num,
2309 void *data,
2310 void *ctx,
2311 gfp_t gfp)
2312{
2313 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
2314}
2315EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
2316
2317/**
2318 * virtqueue_dma_dev - get the dma dev
2319 * @_vq: the struct virtqueue we're talking about.
2320 *
2321 * Returns the dma dev. That can been used for dma api.
2322 */
2323struct device *virtqueue_dma_dev(struct virtqueue *_vq)
2324{
2325 struct vring_virtqueue *vq = to_vvq(_vq);
2326
2327 if (vq->use_dma_api)
2328 return vring_dma_dev(vq);
2329 else
2330 return NULL;
2331}
2332EXPORT_SYMBOL_GPL(virtqueue_dma_dev);
2333
2334/**
2335 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
2336 * @_vq: the struct virtqueue
2337 *
2338 * Instead of virtqueue_kick(), you can do:
2339 * if (virtqueue_kick_prepare(vq))
2340 * virtqueue_notify(vq);
2341 *
2342 * This is sometimes useful because the virtqueue_kick_prepare() needs
2343 * to be serialized, but the actual virtqueue_notify() call does not.
2344 */
2345bool virtqueue_kick_prepare(struct virtqueue *_vq)
2346{
2347 struct vring_virtqueue *vq = to_vvq(_vq);
2348
2349 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
2350 virtqueue_kick_prepare_split(_vq);
2351}
2352EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
2353
2354/**
2355 * virtqueue_notify - second half of split virtqueue_kick call.
2356 * @_vq: the struct virtqueue
2357 *
2358 * This does not need to be serialized.
2359 *
2360 * Returns false if host notify failed or queue is broken, otherwise true.
2361 */
2362bool virtqueue_notify(struct virtqueue *_vq)
2363{
2364 struct vring_virtqueue *vq = to_vvq(_vq);
2365
2366 if (unlikely(vq->broken))
2367 return false;
2368
2369 /* Prod other side to tell it about changes. */
2370 if (!vq->notify(_vq)) {
2371 vq->broken = true;
2372 return false;
2373 }
2374 return true;
2375}
2376EXPORT_SYMBOL_GPL(virtqueue_notify);
2377
2378/**
2379 * virtqueue_kick - update after add_buf
2380 * @vq: the struct virtqueue
2381 *
2382 * After one or more virtqueue_add_* calls, invoke this to kick
2383 * the other side.
2384 *
2385 * Caller must ensure we don't call this with other virtqueue
2386 * operations at the same time (except where noted).
2387 *
2388 * Returns false if kick failed, otherwise true.
2389 */
2390bool virtqueue_kick(struct virtqueue *vq)
2391{
2392 if (virtqueue_kick_prepare(vq))
2393 return virtqueue_notify(vq);
2394 return true;
2395}
2396EXPORT_SYMBOL_GPL(virtqueue_kick);
2397
2398/**
2399 * virtqueue_get_buf_ctx - get the next used buffer
2400 * @_vq: the struct virtqueue we're talking about.
2401 * @len: the length written into the buffer
2402 * @ctx: extra context for the token
2403 *
2404 * If the device wrote data into the buffer, @len will be set to the
2405 * amount written. This means you don't need to clear the buffer
2406 * beforehand to ensure there's no data leakage in the case of short
2407 * writes.
2408 *
2409 * Caller must ensure we don't call this with other virtqueue
2410 * operations at the same time (except where noted).
2411 *
2412 * Returns NULL if there are no used buffers, or the "data" token
2413 * handed to virtqueue_add_*().
2414 */
2415void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
2416 void **ctx)
2417{
2418 struct vring_virtqueue *vq = to_vvq(_vq);
2419
2420 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
2421 virtqueue_get_buf_ctx_split(_vq, len, ctx);
2422}
2423EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
2424
2425void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
2426{
2427 return virtqueue_get_buf_ctx(_vq, len, NULL);
2428}
2429EXPORT_SYMBOL_GPL(virtqueue_get_buf);
2430/**
2431 * virtqueue_disable_cb - disable callbacks
2432 * @_vq: the struct virtqueue we're talking about.
2433 *
2434 * Note that this is not necessarily synchronous, hence unreliable and only
2435 * useful as an optimization.
2436 *
2437 * Unlike other operations, this need not be serialized.
2438 */
2439void virtqueue_disable_cb(struct virtqueue *_vq)
2440{
2441 struct vring_virtqueue *vq = to_vvq(_vq);
2442
2443 if (vq->packed_ring)
2444 virtqueue_disable_cb_packed(_vq);
2445 else
2446 virtqueue_disable_cb_split(_vq);
2447}
2448EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
2449
2450/**
2451 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
2452 * @_vq: the struct virtqueue we're talking about.
2453 *
2454 * This re-enables callbacks; it returns current queue state
2455 * in an opaque unsigned value. This value should be later tested by
2456 * virtqueue_poll, to detect a possible race between the driver checking for
2457 * more work, and enabling callbacks.
2458 *
2459 * Caller must ensure we don't call this with other virtqueue
2460 * operations at the same time (except where noted).
2461 */
2462unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
2463{
2464 struct vring_virtqueue *vq = to_vvq(_vq);
2465
2466 if (vq->event_triggered)
2467 vq->event_triggered = false;
2468
2469 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
2470 virtqueue_enable_cb_prepare_split(_vq);
2471}
2472EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
2473
2474/**
2475 * virtqueue_poll - query pending used buffers
2476 * @_vq: the struct virtqueue we're talking about.
2477 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
2478 *
2479 * Returns "true" if there are pending used buffers in the queue.
2480 *
2481 * This does not need to be serialized.
2482 */
2483bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
2484{
2485 struct vring_virtqueue *vq = to_vvq(_vq);
2486
2487 if (unlikely(vq->broken))
2488 return false;
2489
2490 virtio_mb(vq->weak_barriers);
2491 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
2492 virtqueue_poll_split(_vq, last_used_idx);
2493}
2494EXPORT_SYMBOL_GPL(virtqueue_poll);
2495
2496/**
2497 * virtqueue_enable_cb - restart callbacks after disable_cb.
2498 * @_vq: the struct virtqueue we're talking about.
2499 *
2500 * This re-enables callbacks; it returns "false" if there are pending
2501 * buffers in the queue, to detect a possible race between the driver
2502 * checking for more work, and enabling callbacks.
2503 *
2504 * Caller must ensure we don't call this with other virtqueue
2505 * operations at the same time (except where noted).
2506 */
2507bool virtqueue_enable_cb(struct virtqueue *_vq)
2508{
2509 unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);
2510
2511 return !virtqueue_poll(_vq, last_used_idx);
2512}
2513EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
2514
2515/**
2516 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
2517 * @_vq: the struct virtqueue we're talking about.
2518 *
2519 * This re-enables callbacks but hints to the other side to delay
2520 * interrupts until most of the available buffers have been processed;
2521 * it returns "false" if there are many pending buffers in the queue,
2522 * to detect a possible race between the driver checking for more work,
2523 * and enabling callbacks.
2524 *
2525 * Caller must ensure we don't call this with other virtqueue
2526 * operations at the same time (except where noted).
2527 */
2528bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2529{
2530 struct vring_virtqueue *vq = to_vvq(_vq);
2531
2532 if (vq->event_triggered)
2533 vq->event_triggered = false;
2534
2535 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2536 virtqueue_enable_cb_delayed_split(_vq);
2537}
2538EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2539
2540/**
2541 * virtqueue_detach_unused_buf - detach first unused buffer
2542 * @_vq: the struct virtqueue we're talking about.
2543 *
2544 * Returns NULL or the "data" token handed to virtqueue_add_*().
2545 * This is not valid on an active queue; it is useful for device
2546 * shutdown or the reset queue.
2547 */
2548void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2549{
2550 struct vring_virtqueue *vq = to_vvq(_vq);
2551
2552 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2553 virtqueue_detach_unused_buf_split(_vq);
2554}
2555EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2556
2557static inline bool more_used(const struct vring_virtqueue *vq)
2558{
2559 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2560}
2561
2562/**
2563 * vring_interrupt - notify a virtqueue on an interrupt
2564 * @irq: the IRQ number (ignored)
2565 * @_vq: the struct virtqueue to notify
2566 *
2567 * Calls the callback function of @_vq to process the virtqueue
2568 * notification.
2569 */
2570irqreturn_t vring_interrupt(int irq, void *_vq)
2571{
2572 struct vring_virtqueue *vq = to_vvq(_vq);
2573
2574 if (!more_used(vq)) {
2575 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2576 return IRQ_NONE;
2577 }
2578
2579 if (unlikely(vq->broken)) {
2580#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2581 dev_warn_once(&vq->vq.vdev->dev,
2582 "virtio vring IRQ raised before DRIVER_OK");
2583 return IRQ_NONE;
2584#else
2585 return IRQ_HANDLED;
2586#endif
2587 }
2588
2589 /* Just a hint for performance: so it's ok that this can be racy! */
2590 if (vq->event)
2591 vq->event_triggered = true;
2592
2593 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2594 if (vq->vq.callback)
2595 vq->vq.callback(&vq->vq);
2596
2597 return IRQ_HANDLED;
2598}
2599EXPORT_SYMBOL_GPL(vring_interrupt);
2600
2601/* Only available for split ring */
2602static struct virtqueue *__vring_new_virtqueue(unsigned int index,
2603 struct vring_virtqueue_split *vring_split,
2604 struct virtio_device *vdev,
2605 bool weak_barriers,
2606 bool context,
2607 bool (*notify)(struct virtqueue *),
2608 void (*callback)(struct virtqueue *),
2609 const char *name,
2610 struct device *dma_dev)
2611{
2612 struct vring_virtqueue *vq;
2613 int err;
2614
2615 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2616 return NULL;
2617
2618 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2619 if (!vq)
2620 return NULL;
2621
2622 vq->packed_ring = false;
2623 vq->vq.callback = callback;
2624 vq->vq.vdev = vdev;
2625 vq->vq.name = name;
2626 vq->vq.index = index;
2627 vq->vq.reset = false;
2628 vq->we_own_ring = false;
2629 vq->notify = notify;
2630 vq->weak_barriers = weak_barriers;
2631#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2632 vq->broken = true;
2633#else
2634 vq->broken = false;
2635#endif
2636 vq->dma_dev = dma_dev;
2637 vq->use_dma_api = vring_use_dma_api(vdev);
2638 vq->premapped = false;
2639 vq->do_unmap = vq->use_dma_api;
2640
2641 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2642 !context;
2643 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2644
2645 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2646 vq->weak_barriers = false;
2647
2648 err = vring_alloc_state_extra_split(vring_split);
2649 if (err) {
2650 kfree(vq);
2651 return NULL;
2652 }
2653
2654 virtqueue_vring_init_split(vring_split, vq);
2655
2656 virtqueue_init(vq, vring_split->vring.num);
2657 virtqueue_vring_attach_split(vq, vring_split);
2658
2659 spin_lock(&vdev->vqs_list_lock);
2660 list_add_tail(&vq->vq.list, &vdev->vqs);
2661 spin_unlock(&vdev->vqs_list_lock);
2662 return &vq->vq;
2663}
2664
2665struct virtqueue *vring_create_virtqueue(
2666 unsigned int index,
2667 unsigned int num,
2668 unsigned int vring_align,
2669 struct virtio_device *vdev,
2670 bool weak_barriers,
2671 bool may_reduce_num,
2672 bool context,
2673 bool (*notify)(struct virtqueue *),
2674 void (*callback)(struct virtqueue *),
2675 const char *name)
2676{
2677
2678 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2679 return vring_create_virtqueue_packed(index, num, vring_align,
2680 vdev, weak_barriers, may_reduce_num,
2681 context, notify, callback, name, vdev->dev.parent);
2682
2683 return vring_create_virtqueue_split(index, num, vring_align,
2684 vdev, weak_barriers, may_reduce_num,
2685 context, notify, callback, name, vdev->dev.parent);
2686}
2687EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2688
2689struct virtqueue *vring_create_virtqueue_dma(
2690 unsigned int index,
2691 unsigned int num,
2692 unsigned int vring_align,
2693 struct virtio_device *vdev,
2694 bool weak_barriers,
2695 bool may_reduce_num,
2696 bool context,
2697 bool (*notify)(struct virtqueue *),
2698 void (*callback)(struct virtqueue *),
2699 const char *name,
2700 struct device *dma_dev)
2701{
2702
2703 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2704 return vring_create_virtqueue_packed(index, num, vring_align,
2705 vdev, weak_barriers, may_reduce_num,
2706 context, notify, callback, name, dma_dev);
2707
2708 return vring_create_virtqueue_split(index, num, vring_align,
2709 vdev, weak_barriers, may_reduce_num,
2710 context, notify, callback, name, dma_dev);
2711}
2712EXPORT_SYMBOL_GPL(vring_create_virtqueue_dma);
2713
2714/**
2715 * virtqueue_resize - resize the vring of vq
2716 * @_vq: the struct virtqueue we're talking about.
2717 * @num: new ring num
2718 * @recycle: callback to recycle unused buffers
2719 *
2720 * When it is really necessary to create a new vring, it will set the current vq
2721 * into the reset state. Then call the passed callback to recycle the buffer
2722 * that is no longer used. Only after the new vring is successfully created, the
2723 * old vring will be released.
2724 *
2725 * Caller must ensure we don't call this with other virtqueue operations
2726 * at the same time (except where noted).
2727 *
2728 * Returns zero or a negative error.
2729 * 0: success.
2730 * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size.
2731 * vq can still work normally
2732 * -EBUSY: Failed to sync with device, vq may not work properly
2733 * -ENOENT: Transport or device not supported
2734 * -E2BIG/-EINVAL: num error
2735 * -EPERM: Operation not permitted
2736 *
2737 */
2738int virtqueue_resize(struct virtqueue *_vq, u32 num,
2739 void (*recycle)(struct virtqueue *vq, void *buf))
2740{
2741 struct vring_virtqueue *vq = to_vvq(_vq);
2742 int err;
2743
2744 if (num > vq->vq.num_max)
2745 return -E2BIG;
2746
2747 if (!num)
2748 return -EINVAL;
2749
2750 if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
2751 return 0;
2752
2753 err = virtqueue_disable_and_recycle(_vq, recycle);
2754 if (err)
2755 return err;
2756
2757 if (vq->packed_ring)
2758 err = virtqueue_resize_packed(_vq, num);
2759 else
2760 err = virtqueue_resize_split(_vq, num);
2761
2762 return virtqueue_enable_after_reset(_vq);
2763}
2764EXPORT_SYMBOL_GPL(virtqueue_resize);
2765
2766/**
2767 * virtqueue_set_dma_premapped - set the vring premapped mode
2768 * @_vq: the struct virtqueue we're talking about.
2769 *
2770 * Enable the premapped mode of the vq.
2771 *
2772 * The vring in premapped mode does not do dma internally, so the driver must
2773 * do dma mapping in advance. The driver must pass the dma_address through
2774 * dma_address of scatterlist. When the driver got a used buffer from
2775 * the vring, it has to unmap the dma address.
2776 *
2777 * This function must be called immediately after creating the vq, or after vq
2778 * reset, and before adding any buffers to it.
2779 *
2780 * Caller must ensure we don't call this with other virtqueue operations
2781 * at the same time (except where noted).
2782 *
2783 * Returns zero or a negative error.
2784 * 0: success.
2785 * -EINVAL: vring does not use the dma api, so we can not enable premapped mode.
2786 */
2787int virtqueue_set_dma_premapped(struct virtqueue *_vq)
2788{
2789 struct vring_virtqueue *vq = to_vvq(_vq);
2790 u32 num;
2791
2792 START_USE(vq);
2793
2794 num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2795
2796 if (num != vq->vq.num_free) {
2797 END_USE(vq);
2798 return -EINVAL;
2799 }
2800
2801 if (!vq->use_dma_api) {
2802 END_USE(vq);
2803 return -EINVAL;
2804 }
2805
2806 vq->premapped = true;
2807 vq->do_unmap = false;
2808
2809 END_USE(vq);
2810
2811 return 0;
2812}
2813EXPORT_SYMBOL_GPL(virtqueue_set_dma_premapped);
2814
2815/**
2816 * virtqueue_reset - detach and recycle all unused buffers
2817 * @_vq: the struct virtqueue we're talking about.
2818 * @recycle: callback to recycle unused buffers
2819 *
2820 * Caller must ensure we don't call this with other virtqueue operations
2821 * at the same time (except where noted).
2822 *
2823 * Returns zero or a negative error.
2824 * 0: success.
2825 * -EBUSY: Failed to sync with device, vq may not work properly
2826 * -ENOENT: Transport or device not supported
2827 * -EPERM: Operation not permitted
2828 */
2829int virtqueue_reset(struct virtqueue *_vq,
2830 void (*recycle)(struct virtqueue *vq, void *buf))
2831{
2832 struct vring_virtqueue *vq = to_vvq(_vq);
2833 int err;
2834
2835 err = virtqueue_disable_and_recycle(_vq, recycle);
2836 if (err)
2837 return err;
2838
2839 if (vq->packed_ring)
2840 virtqueue_reinit_packed(vq);
2841 else
2842 virtqueue_reinit_split(vq);
2843
2844 return virtqueue_enable_after_reset(_vq);
2845}
2846EXPORT_SYMBOL_GPL(virtqueue_reset);
2847
2848/* Only available for split ring */
2849struct virtqueue *vring_new_virtqueue(unsigned int index,
2850 unsigned int num,
2851 unsigned int vring_align,
2852 struct virtio_device *vdev,
2853 bool weak_barriers,
2854 bool context,
2855 void *pages,
2856 bool (*notify)(struct virtqueue *vq),
2857 void (*callback)(struct virtqueue *vq),
2858 const char *name)
2859{
2860 struct vring_virtqueue_split vring_split = {};
2861
2862 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2863 return NULL;
2864
2865 vring_init(&vring_split.vring, num, pages, vring_align);
2866 return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
2867 context, notify, callback, name,
2868 vdev->dev.parent);
2869}
2870EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2871
2872static void vring_free(struct virtqueue *_vq)
2873{
2874 struct vring_virtqueue *vq = to_vvq(_vq);
2875
2876 if (vq->we_own_ring) {
2877 if (vq->packed_ring) {
2878 vring_free_queue(vq->vq.vdev,
2879 vq->packed.ring_size_in_bytes,
2880 vq->packed.vring.desc,
2881 vq->packed.ring_dma_addr,
2882 vring_dma_dev(vq));
2883
2884 vring_free_queue(vq->vq.vdev,
2885 vq->packed.event_size_in_bytes,
2886 vq->packed.vring.driver,
2887 vq->packed.driver_event_dma_addr,
2888 vring_dma_dev(vq));
2889
2890 vring_free_queue(vq->vq.vdev,
2891 vq->packed.event_size_in_bytes,
2892 vq->packed.vring.device,
2893 vq->packed.device_event_dma_addr,
2894 vring_dma_dev(vq));
2895
2896 kfree(vq->packed.desc_state);
2897 kfree(vq->packed.desc_extra);
2898 } else {
2899 vring_free_queue(vq->vq.vdev,
2900 vq->split.queue_size_in_bytes,
2901 vq->split.vring.desc,
2902 vq->split.queue_dma_addr,
2903 vring_dma_dev(vq));
2904 }
2905 }
2906 if (!vq->packed_ring) {
2907 kfree(vq->split.desc_state);
2908 kfree(vq->split.desc_extra);
2909 }
2910}
2911
2912void vring_del_virtqueue(struct virtqueue *_vq)
2913{
2914 struct vring_virtqueue *vq = to_vvq(_vq);
2915
2916 spin_lock(&vq->vq.vdev->vqs_list_lock);
2917 list_del(&_vq->list);
2918 spin_unlock(&vq->vq.vdev->vqs_list_lock);
2919
2920 vring_free(_vq);
2921
2922 kfree(vq);
2923}
2924EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2925
2926u32 vring_notification_data(struct virtqueue *_vq)
2927{
2928 struct vring_virtqueue *vq = to_vvq(_vq);
2929 u16 next;
2930
2931 if (vq->packed_ring)
2932 next = (vq->packed.next_avail_idx &
2933 ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
2934 vq->packed.avail_wrap_counter <<
2935 VRING_PACKED_EVENT_F_WRAP_CTR;
2936 else
2937 next = vq->split.avail_idx_shadow;
2938
2939 return next << 16 | _vq->index;
2940}
2941EXPORT_SYMBOL_GPL(vring_notification_data);
2942
2943/* Manipulates transport-specific feature bits. */
2944void vring_transport_features(struct virtio_device *vdev)
2945{
2946 unsigned int i;
2947
2948 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2949 switch (i) {
2950 case VIRTIO_RING_F_INDIRECT_DESC:
2951 break;
2952 case VIRTIO_RING_F_EVENT_IDX:
2953 break;
2954 case VIRTIO_F_VERSION_1:
2955 break;
2956 case VIRTIO_F_ACCESS_PLATFORM:
2957 break;
2958 case VIRTIO_F_RING_PACKED:
2959 break;
2960 case VIRTIO_F_ORDER_PLATFORM:
2961 break;
2962 case VIRTIO_F_NOTIFICATION_DATA:
2963 break;
2964 default:
2965 /* We don't understand this bit. */
2966 __virtio_clear_bit(vdev, i);
2967 }
2968 }
2969}
2970EXPORT_SYMBOL_GPL(vring_transport_features);
2971
2972/**
2973 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2974 * @_vq: the struct virtqueue containing the vring of interest.
2975 *
2976 * Returns the size of the vring. This is mainly used for boasting to
2977 * userspace. Unlike other operations, this need not be serialized.
2978 */
2979unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
2980{
2981
2982 const struct vring_virtqueue *vq = to_vvq(_vq);
2983
2984 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2985}
2986EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2987
2988/*
2989 * This function should only be called by the core, not directly by the driver.
2990 */
2991void __virtqueue_break(struct virtqueue *_vq)
2992{
2993 struct vring_virtqueue *vq = to_vvq(_vq);
2994
2995 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2996 WRITE_ONCE(vq->broken, true);
2997}
2998EXPORT_SYMBOL_GPL(__virtqueue_break);
2999
3000/*
3001 * This function should only be called by the core, not directly by the driver.
3002 */
3003void __virtqueue_unbreak(struct virtqueue *_vq)
3004{
3005 struct vring_virtqueue *vq = to_vvq(_vq);
3006
3007 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3008 WRITE_ONCE(vq->broken, false);
3009}
3010EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
3011
3012bool virtqueue_is_broken(const struct virtqueue *_vq)
3013{
3014 const struct vring_virtqueue *vq = to_vvq(_vq);
3015
3016 return READ_ONCE(vq->broken);
3017}
3018EXPORT_SYMBOL_GPL(virtqueue_is_broken);
3019
3020/*
3021 * This should prevent the device from being used, allowing drivers to
3022 * recover. You may need to grab appropriate locks to flush.
3023 */
3024void virtio_break_device(struct virtio_device *dev)
3025{
3026 struct virtqueue *_vq;
3027
3028 spin_lock(&dev->vqs_list_lock);
3029 list_for_each_entry(_vq, &dev->vqs, list) {
3030 struct vring_virtqueue *vq = to_vvq(_vq);
3031
3032 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3033 WRITE_ONCE(vq->broken, true);
3034 }
3035 spin_unlock(&dev->vqs_list_lock);
3036}
3037EXPORT_SYMBOL_GPL(virtio_break_device);
3038
3039/*
3040 * This should allow the device to be used by the driver. You may
3041 * need to grab appropriate locks to flush the write to
3042 * vq->broken. This should only be used in some specific case e.g
3043 * (probing and restoring). This function should only be called by the
3044 * core, not directly by the driver.
3045 */
3046void __virtio_unbreak_device(struct virtio_device *dev)
3047{
3048 struct virtqueue *_vq;
3049
3050 spin_lock(&dev->vqs_list_lock);
3051 list_for_each_entry(_vq, &dev->vqs, list) {
3052 struct vring_virtqueue *vq = to_vvq(_vq);
3053
3054 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3055 WRITE_ONCE(vq->broken, false);
3056 }
3057 spin_unlock(&dev->vqs_list_lock);
3058}
3059EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
3060
3061dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
3062{
3063 const struct vring_virtqueue *vq = to_vvq(_vq);
3064
3065 BUG_ON(!vq->we_own_ring);
3066
3067 if (vq->packed_ring)
3068 return vq->packed.ring_dma_addr;
3069
3070 return vq->split.queue_dma_addr;
3071}
3072EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
3073
3074dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
3075{
3076 const struct vring_virtqueue *vq = to_vvq(_vq);
3077
3078 BUG_ON(!vq->we_own_ring);
3079
3080 if (vq->packed_ring)
3081 return vq->packed.driver_event_dma_addr;
3082
3083 return vq->split.queue_dma_addr +
3084 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
3085}
3086EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
3087
3088dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
3089{
3090 const struct vring_virtqueue *vq = to_vvq(_vq);
3091
3092 BUG_ON(!vq->we_own_ring);
3093
3094 if (vq->packed_ring)
3095 return vq->packed.device_event_dma_addr;
3096
3097 return vq->split.queue_dma_addr +
3098 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
3099}
3100EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
3101
3102/* Only available for split ring */
3103const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
3104{
3105 return &to_vvq(vq)->split.vring;
3106}
3107EXPORT_SYMBOL_GPL(virtqueue_get_vring);
3108
3109/**
3110 * virtqueue_dma_map_single_attrs - map DMA for _vq
3111 * @_vq: the struct virtqueue we're talking about.
3112 * @ptr: the pointer of the buffer to do dma
3113 * @size: the size of the buffer to do dma
3114 * @dir: DMA direction
3115 * @attrs: DMA Attrs
3116 *
3117 * The caller calls this to do dma mapping in advance. The DMA address can be
3118 * passed to this _vq when it is in pre-mapped mode.
3119 *
3120 * return DMA address. Caller should check that by virtqueue_dma_mapping_error().
3121 */
3122dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr,
3123 size_t size,
3124 enum dma_data_direction dir,
3125 unsigned long attrs)
3126{
3127 struct vring_virtqueue *vq = to_vvq(_vq);
3128
3129 if (!vq->use_dma_api)
3130 return (dma_addr_t)virt_to_phys(ptr);
3131
3132 return dma_map_single_attrs(vring_dma_dev(vq), ptr, size, dir, attrs);
3133}
3134EXPORT_SYMBOL_GPL(virtqueue_dma_map_single_attrs);
3135
3136/**
3137 * virtqueue_dma_unmap_single_attrs - unmap DMA for _vq
3138 * @_vq: the struct virtqueue we're talking about.
3139 * @addr: the dma address to unmap
3140 * @size: the size of the buffer
3141 * @dir: DMA direction
3142 * @attrs: DMA Attrs
3143 *
3144 * Unmap the address that is mapped by the virtqueue_dma_map_* APIs.
3145 *
3146 */
3147void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr,
3148 size_t size, enum dma_data_direction dir,
3149 unsigned long attrs)
3150{
3151 struct vring_virtqueue *vq = to_vvq(_vq);
3152
3153 if (!vq->use_dma_api)
3154 return;
3155
3156 dma_unmap_single_attrs(vring_dma_dev(vq), addr, size, dir, attrs);
3157}
3158EXPORT_SYMBOL_GPL(virtqueue_dma_unmap_single_attrs);
3159
3160/**
3161 * virtqueue_dma_mapping_error - check dma address
3162 * @_vq: the struct virtqueue we're talking about.
3163 * @addr: DMA address
3164 *
3165 * Returns 0 means dma valid. Other means invalid dma address.
3166 */
3167int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr)
3168{
3169 struct vring_virtqueue *vq = to_vvq(_vq);
3170
3171 if (!vq->use_dma_api)
3172 return 0;
3173
3174 return dma_mapping_error(vring_dma_dev(vq), addr);
3175}
3176EXPORT_SYMBOL_GPL(virtqueue_dma_mapping_error);
3177
3178/**
3179 * virtqueue_dma_need_sync - check a dma address needs sync
3180 * @_vq: the struct virtqueue we're talking about.
3181 * @addr: DMA address
3182 *
3183 * Check if the dma address mapped by the virtqueue_dma_map_* APIs needs to be
3184 * synchronized
3185 *
3186 * return bool
3187 */
3188bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr)
3189{
3190 struct vring_virtqueue *vq = to_vvq(_vq);
3191
3192 if (!vq->use_dma_api)
3193 return false;
3194
3195 return dma_need_sync(vring_dma_dev(vq), addr);
3196}
3197EXPORT_SYMBOL_GPL(virtqueue_dma_need_sync);
3198
3199/**
3200 * virtqueue_dma_sync_single_range_for_cpu - dma sync for cpu
3201 * @_vq: the struct virtqueue we're talking about.
3202 * @addr: DMA address
3203 * @offset: DMA address offset
3204 * @size: buf size for sync
3205 * @dir: DMA direction
3206 *
3207 * Before calling this function, use virtqueue_dma_need_sync() to confirm that
3208 * the DMA address really needs to be synchronized
3209 *
3210 */
3211void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq,
3212 dma_addr_t addr,
3213 unsigned long offset, size_t size,
3214 enum dma_data_direction dir)
3215{
3216 struct vring_virtqueue *vq = to_vvq(_vq);
3217 struct device *dev = vring_dma_dev(vq);
3218
3219 if (!vq->use_dma_api)
3220 return;
3221
3222 dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
3223}
3224EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_cpu);
3225
3226/**
3227 * virtqueue_dma_sync_single_range_for_device - dma sync for device
3228 * @_vq: the struct virtqueue we're talking about.
3229 * @addr: DMA address
3230 * @offset: DMA address offset
3231 * @size: buf size for sync
3232 * @dir: DMA direction
3233 *
3234 * Before calling this function, use virtqueue_dma_need_sync() to confirm that
3235 * the DMA address really needs to be synchronized
3236 */
3237void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq,
3238 dma_addr_t addr,
3239 unsigned long offset, size_t size,
3240 enum dma_data_direction dir)
3241{
3242 struct vring_virtqueue *vq = to_vvq(_vq);
3243 struct device *dev = vring_dma_dev(vq);
3244
3245 if (!vq->use_dma_api)
3246 return;
3247
3248 dma_sync_single_range_for_device(dev, addr, offset, size, dir);
3249}
3250EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_device);
3251
3252MODULE_LICENSE("GPL");
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Virtio ring implementation.
3 *
4 * Copyright 2007 Rusty Russell IBM Corporation
5 */
6#include <linux/virtio.h>
7#include <linux/virtio_ring.h>
8#include <linux/virtio_config.h>
9#include <linux/device.h>
10#include <linux/slab.h>
11#include <linux/module.h>
12#include <linux/hrtimer.h>
13#include <linux/dma-mapping.h>
14#include <linux/spinlock.h>
15#include <xen/xen.h>
16
17#ifdef DEBUG
18/* For development, we want to crash whenever the ring is screwed. */
19#define BAD_RING(_vq, fmt, args...) \
20 do { \
21 dev_err(&(_vq)->vq.vdev->dev, \
22 "%s:"fmt, (_vq)->vq.name, ##args); \
23 BUG(); \
24 } while (0)
25/* Caller is supposed to guarantee no reentry. */
26#define START_USE(_vq) \
27 do { \
28 if ((_vq)->in_use) \
29 panic("%s:in_use = %i\n", \
30 (_vq)->vq.name, (_vq)->in_use); \
31 (_vq)->in_use = __LINE__; \
32 } while (0)
33#define END_USE(_vq) \
34 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
35#define LAST_ADD_TIME_UPDATE(_vq) \
36 do { \
37 ktime_t now = ktime_get(); \
38 \
39 /* No kick or get, with .1 second between? Warn. */ \
40 if ((_vq)->last_add_time_valid) \
41 WARN_ON(ktime_to_ms(ktime_sub(now, \
42 (_vq)->last_add_time)) > 100); \
43 (_vq)->last_add_time = now; \
44 (_vq)->last_add_time_valid = true; \
45 } while (0)
46#define LAST_ADD_TIME_CHECK(_vq) \
47 do { \
48 if ((_vq)->last_add_time_valid) { \
49 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
50 (_vq)->last_add_time)) > 100); \
51 } \
52 } while (0)
53#define LAST_ADD_TIME_INVALID(_vq) \
54 ((_vq)->last_add_time_valid = false)
55#else
56#define BAD_RING(_vq, fmt, args...) \
57 do { \
58 dev_err(&_vq->vq.vdev->dev, \
59 "%s:"fmt, (_vq)->vq.name, ##args); \
60 (_vq)->broken = true; \
61 } while (0)
62#define START_USE(vq)
63#define END_USE(vq)
64#define LAST_ADD_TIME_UPDATE(vq)
65#define LAST_ADD_TIME_CHECK(vq)
66#define LAST_ADD_TIME_INVALID(vq)
67#endif
68
69struct vring_desc_state_split {
70 void *data; /* Data for callback. */
71 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
72};
73
74struct vring_desc_state_packed {
75 void *data; /* Data for callback. */
76 struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
77 u16 num; /* Descriptor list length. */
78 u16 last; /* The last desc state in a list. */
79};
80
81struct vring_desc_extra {
82 dma_addr_t addr; /* Buffer DMA addr. */
83 u32 len; /* Buffer length. */
84 u16 flags; /* Descriptor flags. */
85 u16 next; /* The next desc state in a list. */
86};
87
88struct vring_virtqueue {
89 struct virtqueue vq;
90
91 /* Is this a packed ring? */
92 bool packed_ring;
93
94 /* Is DMA API used? */
95 bool use_dma_api;
96
97 /* Can we use weak barriers? */
98 bool weak_barriers;
99
100 /* Other side has made a mess, don't try any more. */
101 bool broken;
102
103 /* Host supports indirect buffers */
104 bool indirect;
105
106 /* Host publishes avail event idx */
107 bool event;
108
109 /* Head of free buffer list. */
110 unsigned int free_head;
111 /* Number we've added since last sync. */
112 unsigned int num_added;
113
114 /* Last used index we've seen. */
115 u16 last_used_idx;
116
117 /* Hint for event idx: already triggered no need to disable. */
118 bool event_triggered;
119
120 union {
121 /* Available for split ring */
122 struct {
123 /* Actual memory layout for this queue. */
124 struct vring vring;
125
126 /* Last written value to avail->flags */
127 u16 avail_flags_shadow;
128
129 /*
130 * Last written value to avail->idx in
131 * guest byte order.
132 */
133 u16 avail_idx_shadow;
134
135 /* Per-descriptor state. */
136 struct vring_desc_state_split *desc_state;
137 struct vring_desc_extra *desc_extra;
138
139 /* DMA address and size information */
140 dma_addr_t queue_dma_addr;
141 size_t queue_size_in_bytes;
142 } split;
143
144 /* Available for packed ring */
145 struct {
146 /* Actual memory layout for this queue. */
147 struct {
148 unsigned int num;
149 struct vring_packed_desc *desc;
150 struct vring_packed_desc_event *driver;
151 struct vring_packed_desc_event *device;
152 } vring;
153
154 /* Driver ring wrap counter. */
155 bool avail_wrap_counter;
156
157 /* Device ring wrap counter. */
158 bool used_wrap_counter;
159
160 /* Avail used flags. */
161 u16 avail_used_flags;
162
163 /* Index of the next avail descriptor. */
164 u16 next_avail_idx;
165
166 /*
167 * Last written value to driver->flags in
168 * guest byte order.
169 */
170 u16 event_flags_shadow;
171
172 /* Per-descriptor state. */
173 struct vring_desc_state_packed *desc_state;
174 struct vring_desc_extra *desc_extra;
175
176 /* DMA address and size information */
177 dma_addr_t ring_dma_addr;
178 dma_addr_t driver_event_dma_addr;
179 dma_addr_t device_event_dma_addr;
180 size_t ring_size_in_bytes;
181 size_t event_size_in_bytes;
182 } packed;
183 };
184
185 /* How to notify other side. FIXME: commonalize hcalls! */
186 bool (*notify)(struct virtqueue *vq);
187
188 /* DMA, allocation, and size information */
189 bool we_own_ring;
190
191#ifdef DEBUG
192 /* They're supposed to lock for us. */
193 unsigned int in_use;
194
195 /* Figure out if their kicks are too delayed. */
196 bool last_add_time_valid;
197 ktime_t last_add_time;
198#endif
199};
200
201
202/*
203 * Helpers.
204 */
205
206#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
207
208static inline bool virtqueue_use_indirect(struct virtqueue *_vq,
209 unsigned int total_sg)
210{
211 struct vring_virtqueue *vq = to_vvq(_vq);
212
213 /*
214 * If the host supports indirect descriptor tables, and we have multiple
215 * buffers, then go indirect. FIXME: tune this threshold
216 */
217 return (vq->indirect && total_sg > 1 && vq->vq.num_free);
218}
219
220/*
221 * Modern virtio devices have feature bits to specify whether they need a
222 * quirk and bypass the IOMMU. If not there, just use the DMA API.
223 *
224 * If there, the interaction between virtio and DMA API is messy.
225 *
226 * On most systems with virtio, physical addresses match bus addresses,
227 * and it doesn't particularly matter whether we use the DMA API.
228 *
229 * On some systems, including Xen and any system with a physical device
230 * that speaks virtio behind a physical IOMMU, we must use the DMA API
231 * for virtio DMA to work at all.
232 *
233 * On other systems, including SPARC and PPC64, virtio-pci devices are
234 * enumerated as though they are behind an IOMMU, but the virtio host
235 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
236 * there or somehow map everything as the identity.
237 *
238 * For the time being, we preserve historic behavior and bypass the DMA
239 * API.
240 *
241 * TODO: install a per-device DMA ops structure that does the right thing
242 * taking into account all the above quirks, and use the DMA API
243 * unconditionally on data path.
244 */
245
246static bool vring_use_dma_api(struct virtio_device *vdev)
247{
248 if (!virtio_has_dma_quirk(vdev))
249 return true;
250
251 /* Otherwise, we are left to guess. */
252 /*
253 * In theory, it's possible to have a buggy QEMU-supposed
254 * emulated Q35 IOMMU and Xen enabled at the same time. On
255 * such a configuration, virtio has never worked and will
256 * not work without an even larger kludge. Instead, enable
257 * the DMA API if we're a Xen guest, which at least allows
258 * all of the sensible Xen configurations to work correctly.
259 */
260 if (xen_domain())
261 return true;
262
263 return false;
264}
265
266size_t virtio_max_dma_size(struct virtio_device *vdev)
267{
268 size_t max_segment_size = SIZE_MAX;
269
270 if (vring_use_dma_api(vdev))
271 max_segment_size = dma_max_mapping_size(&vdev->dev);
272
273 return max_segment_size;
274}
275EXPORT_SYMBOL_GPL(virtio_max_dma_size);
276
277static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
278 dma_addr_t *dma_handle, gfp_t flag)
279{
280 if (vring_use_dma_api(vdev)) {
281 return dma_alloc_coherent(vdev->dev.parent, size,
282 dma_handle, flag);
283 } else {
284 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
285
286 if (queue) {
287 phys_addr_t phys_addr = virt_to_phys(queue);
288 *dma_handle = (dma_addr_t)phys_addr;
289
290 /*
291 * Sanity check: make sure we dind't truncate
292 * the address. The only arches I can find that
293 * have 64-bit phys_addr_t but 32-bit dma_addr_t
294 * are certain non-highmem MIPS and x86
295 * configurations, but these configurations
296 * should never allocate physical pages above 32
297 * bits, so this is fine. Just in case, throw a
298 * warning and abort if we end up with an
299 * unrepresentable address.
300 */
301 if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
302 free_pages_exact(queue, PAGE_ALIGN(size));
303 return NULL;
304 }
305 }
306 return queue;
307 }
308}
309
310static void vring_free_queue(struct virtio_device *vdev, size_t size,
311 void *queue, dma_addr_t dma_handle)
312{
313 if (vring_use_dma_api(vdev))
314 dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
315 else
316 free_pages_exact(queue, PAGE_ALIGN(size));
317}
318
319/*
320 * The DMA ops on various arches are rather gnarly right now, and
321 * making all of the arch DMA ops work on the vring device itself
322 * is a mess. For now, we use the parent device for DMA ops.
323 */
324static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
325{
326 return vq->vq.vdev->dev.parent;
327}
328
329/* Map one sg entry. */
330static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
331 struct scatterlist *sg,
332 enum dma_data_direction direction)
333{
334 if (!vq->use_dma_api)
335 return (dma_addr_t)sg_phys(sg);
336
337 /*
338 * We can't use dma_map_sg, because we don't use scatterlists in
339 * the way it expects (we don't guarantee that the scatterlist
340 * will exist for the lifetime of the mapping).
341 */
342 return dma_map_page(vring_dma_dev(vq),
343 sg_page(sg), sg->offset, sg->length,
344 direction);
345}
346
347static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
348 void *cpu_addr, size_t size,
349 enum dma_data_direction direction)
350{
351 if (!vq->use_dma_api)
352 return (dma_addr_t)virt_to_phys(cpu_addr);
353
354 return dma_map_single(vring_dma_dev(vq),
355 cpu_addr, size, direction);
356}
357
358static int vring_mapping_error(const struct vring_virtqueue *vq,
359 dma_addr_t addr)
360{
361 if (!vq->use_dma_api)
362 return 0;
363
364 return dma_mapping_error(vring_dma_dev(vq), addr);
365}
366
367
368/*
369 * Split ring specific functions - *_split().
370 */
371
372static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
373 struct vring_desc *desc)
374{
375 u16 flags;
376
377 if (!vq->use_dma_api)
378 return;
379
380 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
381
382 if (flags & VRING_DESC_F_INDIRECT) {
383 dma_unmap_single(vring_dma_dev(vq),
384 virtio64_to_cpu(vq->vq.vdev, desc->addr),
385 virtio32_to_cpu(vq->vq.vdev, desc->len),
386 (flags & VRING_DESC_F_WRITE) ?
387 DMA_FROM_DEVICE : DMA_TO_DEVICE);
388 } else {
389 dma_unmap_page(vring_dma_dev(vq),
390 virtio64_to_cpu(vq->vq.vdev, desc->addr),
391 virtio32_to_cpu(vq->vq.vdev, desc->len),
392 (flags & VRING_DESC_F_WRITE) ?
393 DMA_FROM_DEVICE : DMA_TO_DEVICE);
394 }
395}
396
397static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
398 unsigned int i)
399{
400 struct vring_desc_extra *extra = vq->split.desc_extra;
401 u16 flags;
402
403 if (!vq->use_dma_api)
404 goto out;
405
406 flags = extra[i].flags;
407
408 if (flags & VRING_DESC_F_INDIRECT) {
409 dma_unmap_single(vring_dma_dev(vq),
410 extra[i].addr,
411 extra[i].len,
412 (flags & VRING_DESC_F_WRITE) ?
413 DMA_FROM_DEVICE : DMA_TO_DEVICE);
414 } else {
415 dma_unmap_page(vring_dma_dev(vq),
416 extra[i].addr,
417 extra[i].len,
418 (flags & VRING_DESC_F_WRITE) ?
419 DMA_FROM_DEVICE : DMA_TO_DEVICE);
420 }
421
422out:
423 return extra[i].next;
424}
425
426static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
427 unsigned int total_sg,
428 gfp_t gfp)
429{
430 struct vring_desc *desc;
431 unsigned int i;
432
433 /*
434 * We require lowmem mappings for the descriptors because
435 * otherwise virt_to_phys will give us bogus addresses in the
436 * virtqueue.
437 */
438 gfp &= ~__GFP_HIGHMEM;
439
440 desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
441 if (!desc)
442 return NULL;
443
444 for (i = 0; i < total_sg; i++)
445 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
446 return desc;
447}
448
449static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
450 struct vring_desc *desc,
451 unsigned int i,
452 dma_addr_t addr,
453 unsigned int len,
454 u16 flags,
455 bool indirect)
456{
457 struct vring_virtqueue *vring = to_vvq(vq);
458 struct vring_desc_extra *extra = vring->split.desc_extra;
459 u16 next;
460
461 desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
462 desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
463 desc[i].len = cpu_to_virtio32(vq->vdev, len);
464
465 if (!indirect) {
466 next = extra[i].next;
467 desc[i].next = cpu_to_virtio16(vq->vdev, next);
468
469 extra[i].addr = addr;
470 extra[i].len = len;
471 extra[i].flags = flags;
472 } else
473 next = virtio16_to_cpu(vq->vdev, desc[i].next);
474
475 return next;
476}
477
478static inline int virtqueue_add_split(struct virtqueue *_vq,
479 struct scatterlist *sgs[],
480 unsigned int total_sg,
481 unsigned int out_sgs,
482 unsigned int in_sgs,
483 void *data,
484 void *ctx,
485 gfp_t gfp)
486{
487 struct vring_virtqueue *vq = to_vvq(_vq);
488 struct scatterlist *sg;
489 struct vring_desc *desc;
490 unsigned int i, n, avail, descs_used, prev, err_idx;
491 int head;
492 bool indirect;
493
494 START_USE(vq);
495
496 BUG_ON(data == NULL);
497 BUG_ON(ctx && vq->indirect);
498
499 if (unlikely(vq->broken)) {
500 END_USE(vq);
501 return -EIO;
502 }
503
504 LAST_ADD_TIME_UPDATE(vq);
505
506 BUG_ON(total_sg == 0);
507
508 head = vq->free_head;
509
510 if (virtqueue_use_indirect(_vq, total_sg))
511 desc = alloc_indirect_split(_vq, total_sg, gfp);
512 else {
513 desc = NULL;
514 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
515 }
516
517 if (desc) {
518 /* Use a single buffer which doesn't continue */
519 indirect = true;
520 /* Set up rest to use this indirect table. */
521 i = 0;
522 descs_used = 1;
523 } else {
524 indirect = false;
525 desc = vq->split.vring.desc;
526 i = head;
527 descs_used = total_sg;
528 }
529
530 if (vq->vq.num_free < descs_used) {
531 pr_debug("Can't add buf len %i - avail = %i\n",
532 descs_used, vq->vq.num_free);
533 /* FIXME: for historical reasons, we force a notify here if
534 * there are outgoing parts to the buffer. Presumably the
535 * host should service the ring ASAP. */
536 if (out_sgs)
537 vq->notify(&vq->vq);
538 if (indirect)
539 kfree(desc);
540 END_USE(vq);
541 return -ENOSPC;
542 }
543
544 for (n = 0; n < out_sgs; n++) {
545 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
546 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
547 if (vring_mapping_error(vq, addr))
548 goto unmap_release;
549
550 prev = i;
551 /* Note that we trust indirect descriptor
552 * table since it use stream DMA mapping.
553 */
554 i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,
555 VRING_DESC_F_NEXT,
556 indirect);
557 }
558 }
559 for (; n < (out_sgs + in_sgs); n++) {
560 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
561 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
562 if (vring_mapping_error(vq, addr))
563 goto unmap_release;
564
565 prev = i;
566 /* Note that we trust indirect descriptor
567 * table since it use stream DMA mapping.
568 */
569 i = virtqueue_add_desc_split(_vq, desc, i, addr,
570 sg->length,
571 VRING_DESC_F_NEXT |
572 VRING_DESC_F_WRITE,
573 indirect);
574 }
575 }
576 /* Last one doesn't continue. */
577 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
578 if (!indirect && vq->use_dma_api)
579 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags =
580 ~VRING_DESC_F_NEXT;
581
582 if (indirect) {
583 /* Now that the indirect table is filled in, map it. */
584 dma_addr_t addr = vring_map_single(
585 vq, desc, total_sg * sizeof(struct vring_desc),
586 DMA_TO_DEVICE);
587 if (vring_mapping_error(vq, addr))
588 goto unmap_release;
589
590 virtqueue_add_desc_split(_vq, vq->split.vring.desc,
591 head, addr,
592 total_sg * sizeof(struct vring_desc),
593 VRING_DESC_F_INDIRECT,
594 false);
595 }
596
597 /* We're using some buffers from the free list. */
598 vq->vq.num_free -= descs_used;
599
600 /* Update free pointer */
601 if (indirect)
602 vq->free_head = vq->split.desc_extra[head].next;
603 else
604 vq->free_head = i;
605
606 /* Store token and indirect buffer state. */
607 vq->split.desc_state[head].data = data;
608 if (indirect)
609 vq->split.desc_state[head].indir_desc = desc;
610 else
611 vq->split.desc_state[head].indir_desc = ctx;
612
613 /* Put entry in available array (but don't update avail->idx until they
614 * do sync). */
615 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
616 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
617
618 /* Descriptors and available array need to be set before we expose the
619 * new available array entries. */
620 virtio_wmb(vq->weak_barriers);
621 vq->split.avail_idx_shadow++;
622 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
623 vq->split.avail_idx_shadow);
624 vq->num_added++;
625
626 pr_debug("Added buffer head %i to %p\n", head, vq);
627 END_USE(vq);
628
629 /* This is very unlikely, but theoretically possible. Kick
630 * just in case. */
631 if (unlikely(vq->num_added == (1 << 16) - 1))
632 virtqueue_kick(_vq);
633
634 return 0;
635
636unmap_release:
637 err_idx = i;
638
639 if (indirect)
640 i = 0;
641 else
642 i = head;
643
644 for (n = 0; n < total_sg; n++) {
645 if (i == err_idx)
646 break;
647 if (indirect) {
648 vring_unmap_one_split_indirect(vq, &desc[i]);
649 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
650 } else
651 i = vring_unmap_one_split(vq, i);
652 }
653
654 if (indirect)
655 kfree(desc);
656
657 END_USE(vq);
658 return -ENOMEM;
659}
660
661static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
662{
663 struct vring_virtqueue *vq = to_vvq(_vq);
664 u16 new, old;
665 bool needs_kick;
666
667 START_USE(vq);
668 /* We need to expose available array entries before checking avail
669 * event. */
670 virtio_mb(vq->weak_barriers);
671
672 old = vq->split.avail_idx_shadow - vq->num_added;
673 new = vq->split.avail_idx_shadow;
674 vq->num_added = 0;
675
676 LAST_ADD_TIME_CHECK(vq);
677 LAST_ADD_TIME_INVALID(vq);
678
679 if (vq->event) {
680 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
681 vring_avail_event(&vq->split.vring)),
682 new, old);
683 } else {
684 needs_kick = !(vq->split.vring.used->flags &
685 cpu_to_virtio16(_vq->vdev,
686 VRING_USED_F_NO_NOTIFY));
687 }
688 END_USE(vq);
689 return needs_kick;
690}
691
692static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
693 void **ctx)
694{
695 unsigned int i, j;
696 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
697
698 /* Clear data ptr. */
699 vq->split.desc_state[head].data = NULL;
700
701 /* Put back on free list: unmap first-level descriptors and find end */
702 i = head;
703
704 while (vq->split.vring.desc[i].flags & nextflag) {
705 vring_unmap_one_split(vq, i);
706 i = vq->split.desc_extra[i].next;
707 vq->vq.num_free++;
708 }
709
710 vring_unmap_one_split(vq, i);
711 vq->split.desc_extra[i].next = vq->free_head;
712 vq->free_head = head;
713
714 /* Plus final descriptor */
715 vq->vq.num_free++;
716
717 if (vq->indirect) {
718 struct vring_desc *indir_desc =
719 vq->split.desc_state[head].indir_desc;
720 u32 len;
721
722 /* Free the indirect table, if any, now that it's unmapped. */
723 if (!indir_desc)
724 return;
725
726 len = vq->split.desc_extra[head].len;
727
728 BUG_ON(!(vq->split.desc_extra[head].flags &
729 VRING_DESC_F_INDIRECT));
730 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
731
732 for (j = 0; j < len / sizeof(struct vring_desc); j++)
733 vring_unmap_one_split_indirect(vq, &indir_desc[j]);
734
735 kfree(indir_desc);
736 vq->split.desc_state[head].indir_desc = NULL;
737 } else if (ctx) {
738 *ctx = vq->split.desc_state[head].indir_desc;
739 }
740}
741
742static inline bool more_used_split(const struct vring_virtqueue *vq)
743{
744 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
745 vq->split.vring.used->idx);
746}
747
748static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
749 unsigned int *len,
750 void **ctx)
751{
752 struct vring_virtqueue *vq = to_vvq(_vq);
753 void *ret;
754 unsigned int i;
755 u16 last_used;
756
757 START_USE(vq);
758
759 if (unlikely(vq->broken)) {
760 END_USE(vq);
761 return NULL;
762 }
763
764 if (!more_used_split(vq)) {
765 pr_debug("No more buffers in queue\n");
766 END_USE(vq);
767 return NULL;
768 }
769
770 /* Only get used array entries after they have been exposed by host. */
771 virtio_rmb(vq->weak_barriers);
772
773 last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
774 i = virtio32_to_cpu(_vq->vdev,
775 vq->split.vring.used->ring[last_used].id);
776 *len = virtio32_to_cpu(_vq->vdev,
777 vq->split.vring.used->ring[last_used].len);
778
779 if (unlikely(i >= vq->split.vring.num)) {
780 BAD_RING(vq, "id %u out of range\n", i);
781 return NULL;
782 }
783 if (unlikely(!vq->split.desc_state[i].data)) {
784 BAD_RING(vq, "id %u is not a head!\n", i);
785 return NULL;
786 }
787
788 /* detach_buf_split clears data, so grab it now. */
789 ret = vq->split.desc_state[i].data;
790 detach_buf_split(vq, i, ctx);
791 vq->last_used_idx++;
792 /* If we expect an interrupt for the next entry, tell host
793 * by writing event index and flush out the write before
794 * the read in the next get_buf call. */
795 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
796 virtio_store_mb(vq->weak_barriers,
797 &vring_used_event(&vq->split.vring),
798 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
799
800 LAST_ADD_TIME_INVALID(vq);
801
802 END_USE(vq);
803 return ret;
804}
805
806static void virtqueue_disable_cb_split(struct virtqueue *_vq)
807{
808 struct vring_virtqueue *vq = to_vvq(_vq);
809
810 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
811 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
812 if (vq->event)
813 /* TODO: this is a hack. Figure out a cleaner value to write. */
814 vring_used_event(&vq->split.vring) = 0x0;
815 else
816 vq->split.vring.avail->flags =
817 cpu_to_virtio16(_vq->vdev,
818 vq->split.avail_flags_shadow);
819 }
820}
821
822static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
823{
824 struct vring_virtqueue *vq = to_vvq(_vq);
825 u16 last_used_idx;
826
827 START_USE(vq);
828
829 /* We optimistically turn back on interrupts, then check if there was
830 * more to do. */
831 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
832 * either clear the flags bit or point the event index at the next
833 * entry. Always do both to keep code simple. */
834 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
835 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
836 if (!vq->event)
837 vq->split.vring.avail->flags =
838 cpu_to_virtio16(_vq->vdev,
839 vq->split.avail_flags_shadow);
840 }
841 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
842 last_used_idx = vq->last_used_idx);
843 END_USE(vq);
844 return last_used_idx;
845}
846
847static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx)
848{
849 struct vring_virtqueue *vq = to_vvq(_vq);
850
851 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
852 vq->split.vring.used->idx);
853}
854
855static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
856{
857 struct vring_virtqueue *vq = to_vvq(_vq);
858 u16 bufs;
859
860 START_USE(vq);
861
862 /* We optimistically turn back on interrupts, then check if there was
863 * more to do. */
864 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
865 * either clear the flags bit or point the event index at the next
866 * entry. Always update the event index to keep code simple. */
867 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
868 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
869 if (!vq->event)
870 vq->split.vring.avail->flags =
871 cpu_to_virtio16(_vq->vdev,
872 vq->split.avail_flags_shadow);
873 }
874 /* TODO: tune this threshold */
875 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
876
877 virtio_store_mb(vq->weak_barriers,
878 &vring_used_event(&vq->split.vring),
879 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
880
881 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
882 - vq->last_used_idx) > bufs)) {
883 END_USE(vq);
884 return false;
885 }
886
887 END_USE(vq);
888 return true;
889}
890
891static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
892{
893 struct vring_virtqueue *vq = to_vvq(_vq);
894 unsigned int i;
895 void *buf;
896
897 START_USE(vq);
898
899 for (i = 0; i < vq->split.vring.num; i++) {
900 if (!vq->split.desc_state[i].data)
901 continue;
902 /* detach_buf_split clears data, so grab it now. */
903 buf = vq->split.desc_state[i].data;
904 detach_buf_split(vq, i, NULL);
905 vq->split.avail_idx_shadow--;
906 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
907 vq->split.avail_idx_shadow);
908 END_USE(vq);
909 return buf;
910 }
911 /* That should have freed everything. */
912 BUG_ON(vq->vq.num_free != vq->split.vring.num);
913
914 END_USE(vq);
915 return NULL;
916}
917
918static struct virtqueue *vring_create_virtqueue_split(
919 unsigned int index,
920 unsigned int num,
921 unsigned int vring_align,
922 struct virtio_device *vdev,
923 bool weak_barriers,
924 bool may_reduce_num,
925 bool context,
926 bool (*notify)(struct virtqueue *),
927 void (*callback)(struct virtqueue *),
928 const char *name)
929{
930 struct virtqueue *vq;
931 void *queue = NULL;
932 dma_addr_t dma_addr;
933 size_t queue_size_in_bytes;
934 struct vring vring;
935
936 /* We assume num is a power of 2. */
937 if (num & (num - 1)) {
938 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
939 return NULL;
940 }
941
942 /* TODO: allocate each queue chunk individually */
943 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
944 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
945 &dma_addr,
946 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
947 if (queue)
948 break;
949 if (!may_reduce_num)
950 return NULL;
951 }
952
953 if (!num)
954 return NULL;
955
956 if (!queue) {
957 /* Try to get a single page. You are my only hope! */
958 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
959 &dma_addr, GFP_KERNEL|__GFP_ZERO);
960 }
961 if (!queue)
962 return NULL;
963
964 queue_size_in_bytes = vring_size(num, vring_align);
965 vring_init(&vring, num, queue, vring_align);
966
967 vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
968 notify, callback, name);
969 if (!vq) {
970 vring_free_queue(vdev, queue_size_in_bytes, queue,
971 dma_addr);
972 return NULL;
973 }
974
975 to_vvq(vq)->split.queue_dma_addr = dma_addr;
976 to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
977 to_vvq(vq)->we_own_ring = true;
978
979 return vq;
980}
981
982
983/*
984 * Packed ring specific functions - *_packed().
985 */
986
987static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
988 struct vring_desc_extra *state)
989{
990 u16 flags;
991
992 if (!vq->use_dma_api)
993 return;
994
995 flags = state->flags;
996
997 if (flags & VRING_DESC_F_INDIRECT) {
998 dma_unmap_single(vring_dma_dev(vq),
999 state->addr, state->len,
1000 (flags & VRING_DESC_F_WRITE) ?
1001 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1002 } else {
1003 dma_unmap_page(vring_dma_dev(vq),
1004 state->addr, state->len,
1005 (flags & VRING_DESC_F_WRITE) ?
1006 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1007 }
1008}
1009
1010static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
1011 struct vring_packed_desc *desc)
1012{
1013 u16 flags;
1014
1015 if (!vq->use_dma_api)
1016 return;
1017
1018 flags = le16_to_cpu(desc->flags);
1019
1020 if (flags & VRING_DESC_F_INDIRECT) {
1021 dma_unmap_single(vring_dma_dev(vq),
1022 le64_to_cpu(desc->addr),
1023 le32_to_cpu(desc->len),
1024 (flags & VRING_DESC_F_WRITE) ?
1025 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1026 } else {
1027 dma_unmap_page(vring_dma_dev(vq),
1028 le64_to_cpu(desc->addr),
1029 le32_to_cpu(desc->len),
1030 (flags & VRING_DESC_F_WRITE) ?
1031 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1032 }
1033}
1034
1035static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
1036 gfp_t gfp)
1037{
1038 struct vring_packed_desc *desc;
1039
1040 /*
1041 * We require lowmem mappings for the descriptors because
1042 * otherwise virt_to_phys will give us bogus addresses in the
1043 * virtqueue.
1044 */
1045 gfp &= ~__GFP_HIGHMEM;
1046
1047 desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
1048
1049 return desc;
1050}
1051
1052static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
1053 struct scatterlist *sgs[],
1054 unsigned int total_sg,
1055 unsigned int out_sgs,
1056 unsigned int in_sgs,
1057 void *data,
1058 gfp_t gfp)
1059{
1060 struct vring_packed_desc *desc;
1061 struct scatterlist *sg;
1062 unsigned int i, n, err_idx;
1063 u16 head, id;
1064 dma_addr_t addr;
1065
1066 head = vq->packed.next_avail_idx;
1067 desc = alloc_indirect_packed(total_sg, gfp);
1068
1069 if (unlikely(vq->vq.num_free < 1)) {
1070 pr_debug("Can't add buf len 1 - avail = 0\n");
1071 kfree(desc);
1072 END_USE(vq);
1073 return -ENOSPC;
1074 }
1075
1076 i = 0;
1077 id = vq->free_head;
1078 BUG_ON(id == vq->packed.vring.num);
1079
1080 for (n = 0; n < out_sgs + in_sgs; n++) {
1081 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1082 addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1083 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1084 if (vring_mapping_error(vq, addr))
1085 goto unmap_release;
1086
1087 desc[i].flags = cpu_to_le16(n < out_sgs ?
1088 0 : VRING_DESC_F_WRITE);
1089 desc[i].addr = cpu_to_le64(addr);
1090 desc[i].len = cpu_to_le32(sg->length);
1091 i++;
1092 }
1093 }
1094
1095 /* Now that the indirect table is filled in, map it. */
1096 addr = vring_map_single(vq, desc,
1097 total_sg * sizeof(struct vring_packed_desc),
1098 DMA_TO_DEVICE);
1099 if (vring_mapping_error(vq, addr))
1100 goto unmap_release;
1101
1102 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1103 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1104 sizeof(struct vring_packed_desc));
1105 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1106
1107 if (vq->use_dma_api) {
1108 vq->packed.desc_extra[id].addr = addr;
1109 vq->packed.desc_extra[id].len = total_sg *
1110 sizeof(struct vring_packed_desc);
1111 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1112 vq->packed.avail_used_flags;
1113 }
1114
1115 /*
1116 * A driver MUST NOT make the first descriptor in the list
1117 * available before all subsequent descriptors comprising
1118 * the list are made available.
1119 */
1120 virtio_wmb(vq->weak_barriers);
1121 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1122 vq->packed.avail_used_flags);
1123
1124 /* We're using some buffers from the free list. */
1125 vq->vq.num_free -= 1;
1126
1127 /* Update free pointer */
1128 n = head + 1;
1129 if (n >= vq->packed.vring.num) {
1130 n = 0;
1131 vq->packed.avail_wrap_counter ^= 1;
1132 vq->packed.avail_used_flags ^=
1133 1 << VRING_PACKED_DESC_F_AVAIL |
1134 1 << VRING_PACKED_DESC_F_USED;
1135 }
1136 vq->packed.next_avail_idx = n;
1137 vq->free_head = vq->packed.desc_extra[id].next;
1138
1139 /* Store token and indirect buffer state. */
1140 vq->packed.desc_state[id].num = 1;
1141 vq->packed.desc_state[id].data = data;
1142 vq->packed.desc_state[id].indir_desc = desc;
1143 vq->packed.desc_state[id].last = id;
1144
1145 vq->num_added += 1;
1146
1147 pr_debug("Added buffer head %i to %p\n", head, vq);
1148 END_USE(vq);
1149
1150 return 0;
1151
1152unmap_release:
1153 err_idx = i;
1154
1155 for (i = 0; i < err_idx; i++)
1156 vring_unmap_desc_packed(vq, &desc[i]);
1157
1158 kfree(desc);
1159
1160 END_USE(vq);
1161 return -ENOMEM;
1162}
1163
1164static inline int virtqueue_add_packed(struct virtqueue *_vq,
1165 struct scatterlist *sgs[],
1166 unsigned int total_sg,
1167 unsigned int out_sgs,
1168 unsigned int in_sgs,
1169 void *data,
1170 void *ctx,
1171 gfp_t gfp)
1172{
1173 struct vring_virtqueue *vq = to_vvq(_vq);
1174 struct vring_packed_desc *desc;
1175 struct scatterlist *sg;
1176 unsigned int i, n, c, descs_used, err_idx;
1177 __le16 head_flags, flags;
1178 u16 head, id, prev, curr, avail_used_flags;
1179
1180 START_USE(vq);
1181
1182 BUG_ON(data == NULL);
1183 BUG_ON(ctx && vq->indirect);
1184
1185 if (unlikely(vq->broken)) {
1186 END_USE(vq);
1187 return -EIO;
1188 }
1189
1190 LAST_ADD_TIME_UPDATE(vq);
1191
1192 BUG_ON(total_sg == 0);
1193
1194 if (virtqueue_use_indirect(_vq, total_sg))
1195 return virtqueue_add_indirect_packed(vq, sgs, total_sg,
1196 out_sgs, in_sgs, data, gfp);
1197
1198 head = vq->packed.next_avail_idx;
1199 avail_used_flags = vq->packed.avail_used_flags;
1200
1201 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1202
1203 desc = vq->packed.vring.desc;
1204 i = head;
1205 descs_used = total_sg;
1206
1207 if (unlikely(vq->vq.num_free < descs_used)) {
1208 pr_debug("Can't add buf len %i - avail = %i\n",
1209 descs_used, vq->vq.num_free);
1210 END_USE(vq);
1211 return -ENOSPC;
1212 }
1213
1214 id = vq->free_head;
1215 BUG_ON(id == vq->packed.vring.num);
1216
1217 curr = id;
1218 c = 0;
1219 for (n = 0; n < out_sgs + in_sgs; n++) {
1220 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1221 dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1222 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1223 if (vring_mapping_error(vq, addr))
1224 goto unmap_release;
1225
1226 flags = cpu_to_le16(vq->packed.avail_used_flags |
1227 (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1228 (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1229 if (i == head)
1230 head_flags = flags;
1231 else
1232 desc[i].flags = flags;
1233
1234 desc[i].addr = cpu_to_le64(addr);
1235 desc[i].len = cpu_to_le32(sg->length);
1236 desc[i].id = cpu_to_le16(id);
1237
1238 if (unlikely(vq->use_dma_api)) {
1239 vq->packed.desc_extra[curr].addr = addr;
1240 vq->packed.desc_extra[curr].len = sg->length;
1241 vq->packed.desc_extra[curr].flags =
1242 le16_to_cpu(flags);
1243 }
1244 prev = curr;
1245 curr = vq->packed.desc_extra[curr].next;
1246
1247 if ((unlikely(++i >= vq->packed.vring.num))) {
1248 i = 0;
1249 vq->packed.avail_used_flags ^=
1250 1 << VRING_PACKED_DESC_F_AVAIL |
1251 1 << VRING_PACKED_DESC_F_USED;
1252 }
1253 }
1254 }
1255
1256 if (i < head)
1257 vq->packed.avail_wrap_counter ^= 1;
1258
1259 /* We're using some buffers from the free list. */
1260 vq->vq.num_free -= descs_used;
1261
1262 /* Update free pointer */
1263 vq->packed.next_avail_idx = i;
1264 vq->free_head = curr;
1265
1266 /* Store token. */
1267 vq->packed.desc_state[id].num = descs_used;
1268 vq->packed.desc_state[id].data = data;
1269 vq->packed.desc_state[id].indir_desc = ctx;
1270 vq->packed.desc_state[id].last = prev;
1271
1272 /*
1273 * A driver MUST NOT make the first descriptor in the list
1274 * available before all subsequent descriptors comprising
1275 * the list are made available.
1276 */
1277 virtio_wmb(vq->weak_barriers);
1278 vq->packed.vring.desc[head].flags = head_flags;
1279 vq->num_added += descs_used;
1280
1281 pr_debug("Added buffer head %i to %p\n", head, vq);
1282 END_USE(vq);
1283
1284 return 0;
1285
1286unmap_release:
1287 err_idx = i;
1288 i = head;
1289 curr = vq->free_head;
1290
1291 vq->packed.avail_used_flags = avail_used_flags;
1292
1293 for (n = 0; n < total_sg; n++) {
1294 if (i == err_idx)
1295 break;
1296 vring_unmap_state_packed(vq,
1297 &vq->packed.desc_extra[curr]);
1298 curr = vq->packed.desc_extra[curr].next;
1299 i++;
1300 if (i >= vq->packed.vring.num)
1301 i = 0;
1302 }
1303
1304 END_USE(vq);
1305 return -EIO;
1306}
1307
1308static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1309{
1310 struct vring_virtqueue *vq = to_vvq(_vq);
1311 u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1312 bool needs_kick;
1313 union {
1314 struct {
1315 __le16 off_wrap;
1316 __le16 flags;
1317 };
1318 u32 u32;
1319 } snapshot;
1320
1321 START_USE(vq);
1322
1323 /*
1324 * We need to expose the new flags value before checking notification
1325 * suppressions.
1326 */
1327 virtio_mb(vq->weak_barriers);
1328
1329 old = vq->packed.next_avail_idx - vq->num_added;
1330 new = vq->packed.next_avail_idx;
1331 vq->num_added = 0;
1332
1333 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1334 flags = le16_to_cpu(snapshot.flags);
1335
1336 LAST_ADD_TIME_CHECK(vq);
1337 LAST_ADD_TIME_INVALID(vq);
1338
1339 if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1340 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1341 goto out;
1342 }
1343
1344 off_wrap = le16_to_cpu(snapshot.off_wrap);
1345
1346 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1347 event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1348 if (wrap_counter != vq->packed.avail_wrap_counter)
1349 event_idx -= vq->packed.vring.num;
1350
1351 needs_kick = vring_need_event(event_idx, new, old);
1352out:
1353 END_USE(vq);
1354 return needs_kick;
1355}
1356
1357static void detach_buf_packed(struct vring_virtqueue *vq,
1358 unsigned int id, void **ctx)
1359{
1360 struct vring_desc_state_packed *state = NULL;
1361 struct vring_packed_desc *desc;
1362 unsigned int i, curr;
1363
1364 state = &vq->packed.desc_state[id];
1365
1366 /* Clear data ptr. */
1367 state->data = NULL;
1368
1369 vq->packed.desc_extra[state->last].next = vq->free_head;
1370 vq->free_head = id;
1371 vq->vq.num_free += state->num;
1372
1373 if (unlikely(vq->use_dma_api)) {
1374 curr = id;
1375 for (i = 0; i < state->num; i++) {
1376 vring_unmap_state_packed(vq,
1377 &vq->packed.desc_extra[curr]);
1378 curr = vq->packed.desc_extra[curr].next;
1379 }
1380 }
1381
1382 if (vq->indirect) {
1383 u32 len;
1384
1385 /* Free the indirect table, if any, now that it's unmapped. */
1386 desc = state->indir_desc;
1387 if (!desc)
1388 return;
1389
1390 if (vq->use_dma_api) {
1391 len = vq->packed.desc_extra[id].len;
1392 for (i = 0; i < len / sizeof(struct vring_packed_desc);
1393 i++)
1394 vring_unmap_desc_packed(vq, &desc[i]);
1395 }
1396 kfree(desc);
1397 state->indir_desc = NULL;
1398 } else if (ctx) {
1399 *ctx = state->indir_desc;
1400 }
1401}
1402
1403static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1404 u16 idx, bool used_wrap_counter)
1405{
1406 bool avail, used;
1407 u16 flags;
1408
1409 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1410 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1411 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1412
1413 return avail == used && used == used_wrap_counter;
1414}
1415
1416static inline bool more_used_packed(const struct vring_virtqueue *vq)
1417{
1418 return is_used_desc_packed(vq, vq->last_used_idx,
1419 vq->packed.used_wrap_counter);
1420}
1421
1422static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1423 unsigned int *len,
1424 void **ctx)
1425{
1426 struct vring_virtqueue *vq = to_vvq(_vq);
1427 u16 last_used, id;
1428 void *ret;
1429
1430 START_USE(vq);
1431
1432 if (unlikely(vq->broken)) {
1433 END_USE(vq);
1434 return NULL;
1435 }
1436
1437 if (!more_used_packed(vq)) {
1438 pr_debug("No more buffers in queue\n");
1439 END_USE(vq);
1440 return NULL;
1441 }
1442
1443 /* Only get used elements after they have been exposed by host. */
1444 virtio_rmb(vq->weak_barriers);
1445
1446 last_used = vq->last_used_idx;
1447 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1448 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1449
1450 if (unlikely(id >= vq->packed.vring.num)) {
1451 BAD_RING(vq, "id %u out of range\n", id);
1452 return NULL;
1453 }
1454 if (unlikely(!vq->packed.desc_state[id].data)) {
1455 BAD_RING(vq, "id %u is not a head!\n", id);
1456 return NULL;
1457 }
1458
1459 /* detach_buf_packed clears data, so grab it now. */
1460 ret = vq->packed.desc_state[id].data;
1461 detach_buf_packed(vq, id, ctx);
1462
1463 vq->last_used_idx += vq->packed.desc_state[id].num;
1464 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {
1465 vq->last_used_idx -= vq->packed.vring.num;
1466 vq->packed.used_wrap_counter ^= 1;
1467 }
1468
1469 /*
1470 * If we expect an interrupt for the next entry, tell host
1471 * by writing event index and flush out the write before
1472 * the read in the next get_buf call.
1473 */
1474 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1475 virtio_store_mb(vq->weak_barriers,
1476 &vq->packed.vring.driver->off_wrap,
1477 cpu_to_le16(vq->last_used_idx |
1478 (vq->packed.used_wrap_counter <<
1479 VRING_PACKED_EVENT_F_WRAP_CTR)));
1480
1481 LAST_ADD_TIME_INVALID(vq);
1482
1483 END_USE(vq);
1484 return ret;
1485}
1486
1487static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1488{
1489 struct vring_virtqueue *vq = to_vvq(_vq);
1490
1491 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1492 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1493 vq->packed.vring.driver->flags =
1494 cpu_to_le16(vq->packed.event_flags_shadow);
1495 }
1496}
1497
1498static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1499{
1500 struct vring_virtqueue *vq = to_vvq(_vq);
1501
1502 START_USE(vq);
1503
1504 /*
1505 * We optimistically turn back on interrupts, then check if there was
1506 * more to do.
1507 */
1508
1509 if (vq->event) {
1510 vq->packed.vring.driver->off_wrap =
1511 cpu_to_le16(vq->last_used_idx |
1512 (vq->packed.used_wrap_counter <<
1513 VRING_PACKED_EVENT_F_WRAP_CTR));
1514 /*
1515 * We need to update event offset and event wrap
1516 * counter first before updating event flags.
1517 */
1518 virtio_wmb(vq->weak_barriers);
1519 }
1520
1521 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1522 vq->packed.event_flags_shadow = vq->event ?
1523 VRING_PACKED_EVENT_FLAG_DESC :
1524 VRING_PACKED_EVENT_FLAG_ENABLE;
1525 vq->packed.vring.driver->flags =
1526 cpu_to_le16(vq->packed.event_flags_shadow);
1527 }
1528
1529 END_USE(vq);
1530 return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<
1531 VRING_PACKED_EVENT_F_WRAP_CTR);
1532}
1533
1534static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1535{
1536 struct vring_virtqueue *vq = to_vvq(_vq);
1537 bool wrap_counter;
1538 u16 used_idx;
1539
1540 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1541 used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1542
1543 return is_used_desc_packed(vq, used_idx, wrap_counter);
1544}
1545
1546static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1547{
1548 struct vring_virtqueue *vq = to_vvq(_vq);
1549 u16 used_idx, wrap_counter;
1550 u16 bufs;
1551
1552 START_USE(vq);
1553
1554 /*
1555 * We optimistically turn back on interrupts, then check if there was
1556 * more to do.
1557 */
1558
1559 if (vq->event) {
1560 /* TODO: tune this threshold */
1561 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1562 wrap_counter = vq->packed.used_wrap_counter;
1563
1564 used_idx = vq->last_used_idx + bufs;
1565 if (used_idx >= vq->packed.vring.num) {
1566 used_idx -= vq->packed.vring.num;
1567 wrap_counter ^= 1;
1568 }
1569
1570 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1571 (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1572
1573 /*
1574 * We need to update event offset and event wrap
1575 * counter first before updating event flags.
1576 */
1577 virtio_wmb(vq->weak_barriers);
1578 }
1579
1580 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1581 vq->packed.event_flags_shadow = vq->event ?
1582 VRING_PACKED_EVENT_FLAG_DESC :
1583 VRING_PACKED_EVENT_FLAG_ENABLE;
1584 vq->packed.vring.driver->flags =
1585 cpu_to_le16(vq->packed.event_flags_shadow);
1586 }
1587
1588 /*
1589 * We need to update event suppression structure first
1590 * before re-checking for more used buffers.
1591 */
1592 virtio_mb(vq->weak_barriers);
1593
1594 if (is_used_desc_packed(vq,
1595 vq->last_used_idx,
1596 vq->packed.used_wrap_counter)) {
1597 END_USE(vq);
1598 return false;
1599 }
1600
1601 END_USE(vq);
1602 return true;
1603}
1604
1605static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1606{
1607 struct vring_virtqueue *vq = to_vvq(_vq);
1608 unsigned int i;
1609 void *buf;
1610
1611 START_USE(vq);
1612
1613 for (i = 0; i < vq->packed.vring.num; i++) {
1614 if (!vq->packed.desc_state[i].data)
1615 continue;
1616 /* detach_buf clears data, so grab it now. */
1617 buf = vq->packed.desc_state[i].data;
1618 detach_buf_packed(vq, i, NULL);
1619 END_USE(vq);
1620 return buf;
1621 }
1622 /* That should have freed everything. */
1623 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1624
1625 END_USE(vq);
1626 return NULL;
1627}
1628
1629static struct vring_desc_extra *vring_alloc_desc_extra(struct vring_virtqueue *vq,
1630 unsigned int num)
1631{
1632 struct vring_desc_extra *desc_extra;
1633 unsigned int i;
1634
1635 desc_extra = kmalloc_array(num, sizeof(struct vring_desc_extra),
1636 GFP_KERNEL);
1637 if (!desc_extra)
1638 return NULL;
1639
1640 memset(desc_extra, 0, num * sizeof(struct vring_desc_extra));
1641
1642 for (i = 0; i < num - 1; i++)
1643 desc_extra[i].next = i + 1;
1644
1645 return desc_extra;
1646}
1647
1648static struct virtqueue *vring_create_virtqueue_packed(
1649 unsigned int index,
1650 unsigned int num,
1651 unsigned int vring_align,
1652 struct virtio_device *vdev,
1653 bool weak_barriers,
1654 bool may_reduce_num,
1655 bool context,
1656 bool (*notify)(struct virtqueue *),
1657 void (*callback)(struct virtqueue *),
1658 const char *name)
1659{
1660 struct vring_virtqueue *vq;
1661 struct vring_packed_desc *ring;
1662 struct vring_packed_desc_event *driver, *device;
1663 dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1664 size_t ring_size_in_bytes, event_size_in_bytes;
1665
1666 ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1667
1668 ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1669 &ring_dma_addr,
1670 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1671 if (!ring)
1672 goto err_ring;
1673
1674 event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1675
1676 driver = vring_alloc_queue(vdev, event_size_in_bytes,
1677 &driver_event_dma_addr,
1678 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1679 if (!driver)
1680 goto err_driver;
1681
1682 device = vring_alloc_queue(vdev, event_size_in_bytes,
1683 &device_event_dma_addr,
1684 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1685 if (!device)
1686 goto err_device;
1687
1688 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1689 if (!vq)
1690 goto err_vq;
1691
1692 vq->vq.callback = callback;
1693 vq->vq.vdev = vdev;
1694 vq->vq.name = name;
1695 vq->vq.num_free = num;
1696 vq->vq.index = index;
1697 vq->we_own_ring = true;
1698 vq->notify = notify;
1699 vq->weak_barriers = weak_barriers;
1700 vq->broken = false;
1701 vq->last_used_idx = 0;
1702 vq->event_triggered = false;
1703 vq->num_added = 0;
1704 vq->packed_ring = true;
1705 vq->use_dma_api = vring_use_dma_api(vdev);
1706#ifdef DEBUG
1707 vq->in_use = false;
1708 vq->last_add_time_valid = false;
1709#endif
1710
1711 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1712 !context;
1713 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1714
1715 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1716 vq->weak_barriers = false;
1717
1718 vq->packed.ring_dma_addr = ring_dma_addr;
1719 vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1720 vq->packed.device_event_dma_addr = device_event_dma_addr;
1721
1722 vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1723 vq->packed.event_size_in_bytes = event_size_in_bytes;
1724
1725 vq->packed.vring.num = num;
1726 vq->packed.vring.desc = ring;
1727 vq->packed.vring.driver = driver;
1728 vq->packed.vring.device = device;
1729
1730 vq->packed.next_avail_idx = 0;
1731 vq->packed.avail_wrap_counter = 1;
1732 vq->packed.used_wrap_counter = 1;
1733 vq->packed.event_flags_shadow = 0;
1734 vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1735
1736 vq->packed.desc_state = kmalloc_array(num,
1737 sizeof(struct vring_desc_state_packed),
1738 GFP_KERNEL);
1739 if (!vq->packed.desc_state)
1740 goto err_desc_state;
1741
1742 memset(vq->packed.desc_state, 0,
1743 num * sizeof(struct vring_desc_state_packed));
1744
1745 /* Put everything in free lists. */
1746 vq->free_head = 0;
1747
1748 vq->packed.desc_extra = vring_alloc_desc_extra(vq, num);
1749 if (!vq->packed.desc_extra)
1750 goto err_desc_extra;
1751
1752 /* No callback? Tell other side not to bother us. */
1753 if (!callback) {
1754 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1755 vq->packed.vring.driver->flags =
1756 cpu_to_le16(vq->packed.event_flags_shadow);
1757 }
1758
1759 spin_lock(&vdev->vqs_list_lock);
1760 list_add_tail(&vq->vq.list, &vdev->vqs);
1761 spin_unlock(&vdev->vqs_list_lock);
1762 return &vq->vq;
1763
1764err_desc_extra:
1765 kfree(vq->packed.desc_state);
1766err_desc_state:
1767 kfree(vq);
1768err_vq:
1769 vring_free_queue(vdev, event_size_in_bytes, device, device_event_dma_addr);
1770err_device:
1771 vring_free_queue(vdev, event_size_in_bytes, driver, driver_event_dma_addr);
1772err_driver:
1773 vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
1774err_ring:
1775 return NULL;
1776}
1777
1778
1779/*
1780 * Generic functions and exported symbols.
1781 */
1782
1783static inline int virtqueue_add(struct virtqueue *_vq,
1784 struct scatterlist *sgs[],
1785 unsigned int total_sg,
1786 unsigned int out_sgs,
1787 unsigned int in_sgs,
1788 void *data,
1789 void *ctx,
1790 gfp_t gfp)
1791{
1792 struct vring_virtqueue *vq = to_vvq(_vq);
1793
1794 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
1795 out_sgs, in_sgs, data, ctx, gfp) :
1796 virtqueue_add_split(_vq, sgs, total_sg,
1797 out_sgs, in_sgs, data, ctx, gfp);
1798}
1799
1800/**
1801 * virtqueue_add_sgs - expose buffers to other end
1802 * @_vq: the struct virtqueue we're talking about.
1803 * @sgs: array of terminated scatterlists.
1804 * @out_sgs: the number of scatterlists readable by other side
1805 * @in_sgs: the number of scatterlists which are writable (after readable ones)
1806 * @data: the token identifying the buffer.
1807 * @gfp: how to do memory allocations (if necessary).
1808 *
1809 * Caller must ensure we don't call this with other virtqueue operations
1810 * at the same time (except where noted).
1811 *
1812 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1813 */
1814int virtqueue_add_sgs(struct virtqueue *_vq,
1815 struct scatterlist *sgs[],
1816 unsigned int out_sgs,
1817 unsigned int in_sgs,
1818 void *data,
1819 gfp_t gfp)
1820{
1821 unsigned int i, total_sg = 0;
1822
1823 /* Count them first. */
1824 for (i = 0; i < out_sgs + in_sgs; i++) {
1825 struct scatterlist *sg;
1826
1827 for (sg = sgs[i]; sg; sg = sg_next(sg))
1828 total_sg++;
1829 }
1830 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
1831 data, NULL, gfp);
1832}
1833EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
1834
1835/**
1836 * virtqueue_add_outbuf - expose output buffers to other end
1837 * @vq: the struct virtqueue we're talking about.
1838 * @sg: scatterlist (must be well-formed and terminated!)
1839 * @num: the number of entries in @sg readable by other side
1840 * @data: the token identifying the buffer.
1841 * @gfp: how to do memory allocations (if necessary).
1842 *
1843 * Caller must ensure we don't call this with other virtqueue operations
1844 * at the same time (except where noted).
1845 *
1846 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1847 */
1848int virtqueue_add_outbuf(struct virtqueue *vq,
1849 struct scatterlist *sg, unsigned int num,
1850 void *data,
1851 gfp_t gfp)
1852{
1853 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
1854}
1855EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
1856
1857/**
1858 * virtqueue_add_inbuf - expose input buffers to other end
1859 * @vq: the struct virtqueue we're talking about.
1860 * @sg: scatterlist (must be well-formed and terminated!)
1861 * @num: the number of entries in @sg writable by other side
1862 * @data: the token identifying the buffer.
1863 * @gfp: how to do memory allocations (if necessary).
1864 *
1865 * Caller must ensure we don't call this with other virtqueue operations
1866 * at the same time (except where noted).
1867 *
1868 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1869 */
1870int virtqueue_add_inbuf(struct virtqueue *vq,
1871 struct scatterlist *sg, unsigned int num,
1872 void *data,
1873 gfp_t gfp)
1874{
1875 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
1876}
1877EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
1878
1879/**
1880 * virtqueue_add_inbuf_ctx - expose input buffers to other end
1881 * @vq: the struct virtqueue we're talking about.
1882 * @sg: scatterlist (must be well-formed and terminated!)
1883 * @num: the number of entries in @sg writable by other side
1884 * @data: the token identifying the buffer.
1885 * @ctx: extra context for the token
1886 * @gfp: how to do memory allocations (if necessary).
1887 *
1888 * Caller must ensure we don't call this with other virtqueue operations
1889 * at the same time (except where noted).
1890 *
1891 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1892 */
1893int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
1894 struct scatterlist *sg, unsigned int num,
1895 void *data,
1896 void *ctx,
1897 gfp_t gfp)
1898{
1899 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
1900}
1901EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
1902
1903/**
1904 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1905 * @_vq: the struct virtqueue
1906 *
1907 * Instead of virtqueue_kick(), you can do:
1908 * if (virtqueue_kick_prepare(vq))
1909 * virtqueue_notify(vq);
1910 *
1911 * This is sometimes useful because the virtqueue_kick_prepare() needs
1912 * to be serialized, but the actual virtqueue_notify() call does not.
1913 */
1914bool virtqueue_kick_prepare(struct virtqueue *_vq)
1915{
1916 struct vring_virtqueue *vq = to_vvq(_vq);
1917
1918 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
1919 virtqueue_kick_prepare_split(_vq);
1920}
1921EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
1922
1923/**
1924 * virtqueue_notify - second half of split virtqueue_kick call.
1925 * @_vq: the struct virtqueue
1926 *
1927 * This does not need to be serialized.
1928 *
1929 * Returns false if host notify failed or queue is broken, otherwise true.
1930 */
1931bool virtqueue_notify(struct virtqueue *_vq)
1932{
1933 struct vring_virtqueue *vq = to_vvq(_vq);
1934
1935 if (unlikely(vq->broken))
1936 return false;
1937
1938 /* Prod other side to tell it about changes. */
1939 if (!vq->notify(_vq)) {
1940 vq->broken = true;
1941 return false;
1942 }
1943 return true;
1944}
1945EXPORT_SYMBOL_GPL(virtqueue_notify);
1946
1947/**
1948 * virtqueue_kick - update after add_buf
1949 * @vq: the struct virtqueue
1950 *
1951 * After one or more virtqueue_add_* calls, invoke this to kick
1952 * the other side.
1953 *
1954 * Caller must ensure we don't call this with other virtqueue
1955 * operations at the same time (except where noted).
1956 *
1957 * Returns false if kick failed, otherwise true.
1958 */
1959bool virtqueue_kick(struct virtqueue *vq)
1960{
1961 if (virtqueue_kick_prepare(vq))
1962 return virtqueue_notify(vq);
1963 return true;
1964}
1965EXPORT_SYMBOL_GPL(virtqueue_kick);
1966
1967/**
1968 * virtqueue_get_buf_ctx - get the next used buffer
1969 * @_vq: the struct virtqueue we're talking about.
1970 * @len: the length written into the buffer
1971 * @ctx: extra context for the token
1972 *
1973 * If the device wrote data into the buffer, @len will be set to the
1974 * amount written. This means you don't need to clear the buffer
1975 * beforehand to ensure there's no data leakage in the case of short
1976 * writes.
1977 *
1978 * Caller must ensure we don't call this with other virtqueue
1979 * operations at the same time (except where noted).
1980 *
1981 * Returns NULL if there are no used buffers, or the "data" token
1982 * handed to virtqueue_add_*().
1983 */
1984void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
1985 void **ctx)
1986{
1987 struct vring_virtqueue *vq = to_vvq(_vq);
1988
1989 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
1990 virtqueue_get_buf_ctx_split(_vq, len, ctx);
1991}
1992EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
1993
1994void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
1995{
1996 return virtqueue_get_buf_ctx(_vq, len, NULL);
1997}
1998EXPORT_SYMBOL_GPL(virtqueue_get_buf);
1999/**
2000 * virtqueue_disable_cb - disable callbacks
2001 * @_vq: the struct virtqueue we're talking about.
2002 *
2003 * Note that this is not necessarily synchronous, hence unreliable and only
2004 * useful as an optimization.
2005 *
2006 * Unlike other operations, this need not be serialized.
2007 */
2008void virtqueue_disable_cb(struct virtqueue *_vq)
2009{
2010 struct vring_virtqueue *vq = to_vvq(_vq);
2011
2012 /* If device triggered an event already it won't trigger one again:
2013 * no need to disable.
2014 */
2015 if (vq->event_triggered)
2016 return;
2017
2018 if (vq->packed_ring)
2019 virtqueue_disable_cb_packed(_vq);
2020 else
2021 virtqueue_disable_cb_split(_vq);
2022}
2023EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
2024
2025/**
2026 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
2027 * @_vq: the struct virtqueue we're talking about.
2028 *
2029 * This re-enables callbacks; it returns current queue state
2030 * in an opaque unsigned value. This value should be later tested by
2031 * virtqueue_poll, to detect a possible race between the driver checking for
2032 * more work, and enabling callbacks.
2033 *
2034 * Caller must ensure we don't call this with other virtqueue
2035 * operations at the same time (except where noted).
2036 */
2037unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
2038{
2039 struct vring_virtqueue *vq = to_vvq(_vq);
2040
2041 if (vq->event_triggered)
2042 vq->event_triggered = false;
2043
2044 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
2045 virtqueue_enable_cb_prepare_split(_vq);
2046}
2047EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
2048
2049/**
2050 * virtqueue_poll - query pending used buffers
2051 * @_vq: the struct virtqueue we're talking about.
2052 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
2053 *
2054 * Returns "true" if there are pending used buffers in the queue.
2055 *
2056 * This does not need to be serialized.
2057 */
2058bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
2059{
2060 struct vring_virtqueue *vq = to_vvq(_vq);
2061
2062 if (unlikely(vq->broken))
2063 return false;
2064
2065 virtio_mb(vq->weak_barriers);
2066 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
2067 virtqueue_poll_split(_vq, last_used_idx);
2068}
2069EXPORT_SYMBOL_GPL(virtqueue_poll);
2070
2071/**
2072 * virtqueue_enable_cb - restart callbacks after disable_cb.
2073 * @_vq: the struct virtqueue we're talking about.
2074 *
2075 * This re-enables callbacks; it returns "false" if there are pending
2076 * buffers in the queue, to detect a possible race between the driver
2077 * checking for more work, and enabling callbacks.
2078 *
2079 * Caller must ensure we don't call this with other virtqueue
2080 * operations at the same time (except where noted).
2081 */
2082bool virtqueue_enable_cb(struct virtqueue *_vq)
2083{
2084 unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);
2085
2086 return !virtqueue_poll(_vq, last_used_idx);
2087}
2088EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
2089
2090/**
2091 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
2092 * @_vq: the struct virtqueue we're talking about.
2093 *
2094 * This re-enables callbacks but hints to the other side to delay
2095 * interrupts until most of the available buffers have been processed;
2096 * it returns "false" if there are many pending buffers in the queue,
2097 * to detect a possible race between the driver checking for more work,
2098 * and enabling callbacks.
2099 *
2100 * Caller must ensure we don't call this with other virtqueue
2101 * operations at the same time (except where noted).
2102 */
2103bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2104{
2105 struct vring_virtqueue *vq = to_vvq(_vq);
2106
2107 if (vq->event_triggered)
2108 vq->event_triggered = false;
2109
2110 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2111 virtqueue_enable_cb_delayed_split(_vq);
2112}
2113EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2114
2115/**
2116 * virtqueue_detach_unused_buf - detach first unused buffer
2117 * @_vq: the struct virtqueue we're talking about.
2118 *
2119 * Returns NULL or the "data" token handed to virtqueue_add_*().
2120 * This is not valid on an active queue; it is useful only for device
2121 * shutdown.
2122 */
2123void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2124{
2125 struct vring_virtqueue *vq = to_vvq(_vq);
2126
2127 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2128 virtqueue_detach_unused_buf_split(_vq);
2129}
2130EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2131
2132static inline bool more_used(const struct vring_virtqueue *vq)
2133{
2134 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2135}
2136
2137irqreturn_t vring_interrupt(int irq, void *_vq)
2138{
2139 struct vring_virtqueue *vq = to_vvq(_vq);
2140
2141 if (!more_used(vq)) {
2142 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2143 return IRQ_NONE;
2144 }
2145
2146 if (unlikely(vq->broken))
2147 return IRQ_HANDLED;
2148
2149 /* Just a hint for performance: so it's ok that this can be racy! */
2150 if (vq->event)
2151 vq->event_triggered = true;
2152
2153 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2154 if (vq->vq.callback)
2155 vq->vq.callback(&vq->vq);
2156
2157 return IRQ_HANDLED;
2158}
2159EXPORT_SYMBOL_GPL(vring_interrupt);
2160
2161/* Only available for split ring */
2162struct virtqueue *__vring_new_virtqueue(unsigned int index,
2163 struct vring vring,
2164 struct virtio_device *vdev,
2165 bool weak_barriers,
2166 bool context,
2167 bool (*notify)(struct virtqueue *),
2168 void (*callback)(struct virtqueue *),
2169 const char *name)
2170{
2171 struct vring_virtqueue *vq;
2172
2173 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2174 return NULL;
2175
2176 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2177 if (!vq)
2178 return NULL;
2179
2180 vq->packed_ring = false;
2181 vq->vq.callback = callback;
2182 vq->vq.vdev = vdev;
2183 vq->vq.name = name;
2184 vq->vq.num_free = vring.num;
2185 vq->vq.index = index;
2186 vq->we_own_ring = false;
2187 vq->notify = notify;
2188 vq->weak_barriers = weak_barriers;
2189 vq->broken = false;
2190 vq->last_used_idx = 0;
2191 vq->event_triggered = false;
2192 vq->num_added = 0;
2193 vq->use_dma_api = vring_use_dma_api(vdev);
2194#ifdef DEBUG
2195 vq->in_use = false;
2196 vq->last_add_time_valid = false;
2197#endif
2198
2199 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2200 !context;
2201 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2202
2203 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2204 vq->weak_barriers = false;
2205
2206 vq->split.queue_dma_addr = 0;
2207 vq->split.queue_size_in_bytes = 0;
2208
2209 vq->split.vring = vring;
2210 vq->split.avail_flags_shadow = 0;
2211 vq->split.avail_idx_shadow = 0;
2212
2213 /* No callback? Tell other side not to bother us. */
2214 if (!callback) {
2215 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
2216 if (!vq->event)
2217 vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2218 vq->split.avail_flags_shadow);
2219 }
2220
2221 vq->split.desc_state = kmalloc_array(vring.num,
2222 sizeof(struct vring_desc_state_split), GFP_KERNEL);
2223 if (!vq->split.desc_state)
2224 goto err_state;
2225
2226 vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num);
2227 if (!vq->split.desc_extra)
2228 goto err_extra;
2229
2230 /* Put everything in free lists. */
2231 vq->free_head = 0;
2232 memset(vq->split.desc_state, 0, vring.num *
2233 sizeof(struct vring_desc_state_split));
2234
2235 spin_lock(&vdev->vqs_list_lock);
2236 list_add_tail(&vq->vq.list, &vdev->vqs);
2237 spin_unlock(&vdev->vqs_list_lock);
2238 return &vq->vq;
2239
2240err_extra:
2241 kfree(vq->split.desc_state);
2242err_state:
2243 kfree(vq);
2244 return NULL;
2245}
2246EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2247
2248struct virtqueue *vring_create_virtqueue(
2249 unsigned int index,
2250 unsigned int num,
2251 unsigned int vring_align,
2252 struct virtio_device *vdev,
2253 bool weak_barriers,
2254 bool may_reduce_num,
2255 bool context,
2256 bool (*notify)(struct virtqueue *),
2257 void (*callback)(struct virtqueue *),
2258 const char *name)
2259{
2260
2261 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2262 return vring_create_virtqueue_packed(index, num, vring_align,
2263 vdev, weak_barriers, may_reduce_num,
2264 context, notify, callback, name);
2265
2266 return vring_create_virtqueue_split(index, num, vring_align,
2267 vdev, weak_barriers, may_reduce_num,
2268 context, notify, callback, name);
2269}
2270EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2271
2272/* Only available for split ring */
2273struct virtqueue *vring_new_virtqueue(unsigned int index,
2274 unsigned int num,
2275 unsigned int vring_align,
2276 struct virtio_device *vdev,
2277 bool weak_barriers,
2278 bool context,
2279 void *pages,
2280 bool (*notify)(struct virtqueue *vq),
2281 void (*callback)(struct virtqueue *vq),
2282 const char *name)
2283{
2284 struct vring vring;
2285
2286 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2287 return NULL;
2288
2289 vring_init(&vring, num, pages, vring_align);
2290 return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
2291 notify, callback, name);
2292}
2293EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2294
2295void vring_del_virtqueue(struct virtqueue *_vq)
2296{
2297 struct vring_virtqueue *vq = to_vvq(_vq);
2298
2299 spin_lock(&vq->vq.vdev->vqs_list_lock);
2300 list_del(&_vq->list);
2301 spin_unlock(&vq->vq.vdev->vqs_list_lock);
2302
2303 if (vq->we_own_ring) {
2304 if (vq->packed_ring) {
2305 vring_free_queue(vq->vq.vdev,
2306 vq->packed.ring_size_in_bytes,
2307 vq->packed.vring.desc,
2308 vq->packed.ring_dma_addr);
2309
2310 vring_free_queue(vq->vq.vdev,
2311 vq->packed.event_size_in_bytes,
2312 vq->packed.vring.driver,
2313 vq->packed.driver_event_dma_addr);
2314
2315 vring_free_queue(vq->vq.vdev,
2316 vq->packed.event_size_in_bytes,
2317 vq->packed.vring.device,
2318 vq->packed.device_event_dma_addr);
2319
2320 kfree(vq->packed.desc_state);
2321 kfree(vq->packed.desc_extra);
2322 } else {
2323 vring_free_queue(vq->vq.vdev,
2324 vq->split.queue_size_in_bytes,
2325 vq->split.vring.desc,
2326 vq->split.queue_dma_addr);
2327 }
2328 }
2329 if (!vq->packed_ring) {
2330 kfree(vq->split.desc_state);
2331 kfree(vq->split.desc_extra);
2332 }
2333 kfree(vq);
2334}
2335EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2336
2337/* Manipulates transport-specific feature bits. */
2338void vring_transport_features(struct virtio_device *vdev)
2339{
2340 unsigned int i;
2341
2342 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2343 switch (i) {
2344 case VIRTIO_RING_F_INDIRECT_DESC:
2345 break;
2346 case VIRTIO_RING_F_EVENT_IDX:
2347 break;
2348 case VIRTIO_F_VERSION_1:
2349 break;
2350 case VIRTIO_F_ACCESS_PLATFORM:
2351 break;
2352 case VIRTIO_F_RING_PACKED:
2353 break;
2354 case VIRTIO_F_ORDER_PLATFORM:
2355 break;
2356 default:
2357 /* We don't understand this bit. */
2358 __virtio_clear_bit(vdev, i);
2359 }
2360 }
2361}
2362EXPORT_SYMBOL_GPL(vring_transport_features);
2363
2364/**
2365 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2366 * @_vq: the struct virtqueue containing the vring of interest.
2367 *
2368 * Returns the size of the vring. This is mainly used for boasting to
2369 * userspace. Unlike other operations, this need not be serialized.
2370 */
2371unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
2372{
2373
2374 struct vring_virtqueue *vq = to_vvq(_vq);
2375
2376 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2377}
2378EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2379
2380bool virtqueue_is_broken(struct virtqueue *_vq)
2381{
2382 struct vring_virtqueue *vq = to_vvq(_vq);
2383
2384 return READ_ONCE(vq->broken);
2385}
2386EXPORT_SYMBOL_GPL(virtqueue_is_broken);
2387
2388/*
2389 * This should prevent the device from being used, allowing drivers to
2390 * recover. You may need to grab appropriate locks to flush.
2391 */
2392void virtio_break_device(struct virtio_device *dev)
2393{
2394 struct virtqueue *_vq;
2395
2396 spin_lock(&dev->vqs_list_lock);
2397 list_for_each_entry(_vq, &dev->vqs, list) {
2398 struct vring_virtqueue *vq = to_vvq(_vq);
2399
2400 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2401 WRITE_ONCE(vq->broken, true);
2402 }
2403 spin_unlock(&dev->vqs_list_lock);
2404}
2405EXPORT_SYMBOL_GPL(virtio_break_device);
2406
2407dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
2408{
2409 struct vring_virtqueue *vq = to_vvq(_vq);
2410
2411 BUG_ON(!vq->we_own_ring);
2412
2413 if (vq->packed_ring)
2414 return vq->packed.ring_dma_addr;
2415
2416 return vq->split.queue_dma_addr;
2417}
2418EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
2419
2420dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
2421{
2422 struct vring_virtqueue *vq = to_vvq(_vq);
2423
2424 BUG_ON(!vq->we_own_ring);
2425
2426 if (vq->packed_ring)
2427 return vq->packed.driver_event_dma_addr;
2428
2429 return vq->split.queue_dma_addr +
2430 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2431}
2432EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
2433
2434dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
2435{
2436 struct vring_virtqueue *vq = to_vvq(_vq);
2437
2438 BUG_ON(!vq->we_own_ring);
2439
2440 if (vq->packed_ring)
2441 return vq->packed.device_event_dma_addr;
2442
2443 return vq->split.queue_dma_addr +
2444 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2445}
2446EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
2447
2448/* Only available for split ring */
2449const struct vring *virtqueue_get_vring(struct virtqueue *vq)
2450{
2451 return &to_vvq(vq)->split.vring;
2452}
2453EXPORT_SYMBOL_GPL(virtqueue_get_vring);
2454
2455MODULE_LICENSE("GPL");