Loading...
1/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19#include <linux/virtio.h>
20#include <linux/virtio_ring.h>
21#include <linux/virtio_config.h>
22#include <linux/device.h>
23#include <linux/slab.h>
24
25/* virtio guest is communicating with a virtual "device" that actually runs on
26 * a host processor. Memory barriers are used to control SMP effects. */
27#ifdef CONFIG_SMP
28/* Where possible, use SMP barriers which are more lightweight than mandatory
29 * barriers, because mandatory barriers control MMIO effects on accesses
30 * through relaxed memory I/O windows (which virtio does not use). */
31#define virtio_mb() smp_mb()
32#define virtio_rmb() smp_rmb()
33#define virtio_wmb() smp_wmb()
34#else
35/* We must force memory ordering even if guest is UP since host could be
36 * running on another CPU, but SMP barriers are defined to barrier() in that
37 * configuration. So fall back to mandatory barriers instead. */
38#define virtio_mb() mb()
39#define virtio_rmb() rmb()
40#define virtio_wmb() wmb()
41#endif
42
43#ifdef DEBUG
44/* For development, we want to crash whenever the ring is screwed. */
45#define BAD_RING(_vq, fmt, args...) \
46 do { \
47 dev_err(&(_vq)->vq.vdev->dev, \
48 "%s:"fmt, (_vq)->vq.name, ##args); \
49 BUG(); \
50 } while (0)
51/* Caller is supposed to guarantee no reentry. */
52#define START_USE(_vq) \
53 do { \
54 if ((_vq)->in_use) \
55 panic("%s:in_use = %i\n", \
56 (_vq)->vq.name, (_vq)->in_use); \
57 (_vq)->in_use = __LINE__; \
58 } while (0)
59#define END_USE(_vq) \
60 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
61#else
62#define BAD_RING(_vq, fmt, args...) \
63 do { \
64 dev_err(&_vq->vq.vdev->dev, \
65 "%s:"fmt, (_vq)->vq.name, ##args); \
66 (_vq)->broken = true; \
67 } while (0)
68#define START_USE(vq)
69#define END_USE(vq)
70#endif
71
72struct vring_virtqueue
73{
74 struct virtqueue vq;
75
76 /* Actual memory layout for this queue */
77 struct vring vring;
78
79 /* Other side has made a mess, don't try any more. */
80 bool broken;
81
82 /* Host supports indirect buffers */
83 bool indirect;
84
85 /* Host publishes avail event idx */
86 bool event;
87
88 /* Number of free buffers */
89 unsigned int num_free;
90 /* Head of free buffer list. */
91 unsigned int free_head;
92 /* Number we've added since last sync. */
93 unsigned int num_added;
94
95 /* Last used index we've seen. */
96 u16 last_used_idx;
97
98 /* How to notify other side. FIXME: commonalize hcalls! */
99 void (*notify)(struct virtqueue *vq);
100
101#ifdef DEBUG
102 /* They're supposed to lock for us. */
103 unsigned int in_use;
104#endif
105
106 /* Tokens for callbacks. */
107 void *data[];
108};
109
110#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
111
112/* Set up an indirect table of descriptors and add it to the queue. */
113static int vring_add_indirect(struct vring_virtqueue *vq,
114 struct scatterlist sg[],
115 unsigned int out,
116 unsigned int in,
117 gfp_t gfp)
118{
119 struct vring_desc *desc;
120 unsigned head;
121 int i;
122
123 desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
124 if (!desc)
125 return -ENOMEM;
126
127 /* Transfer entries from the sg list into the indirect page */
128 for (i = 0; i < out; i++) {
129 desc[i].flags = VRING_DESC_F_NEXT;
130 desc[i].addr = sg_phys(sg);
131 desc[i].len = sg->length;
132 desc[i].next = i+1;
133 sg++;
134 }
135 for (; i < (out + in); i++) {
136 desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
137 desc[i].addr = sg_phys(sg);
138 desc[i].len = sg->length;
139 desc[i].next = i+1;
140 sg++;
141 }
142
143 /* Last one doesn't continue. */
144 desc[i-1].flags &= ~VRING_DESC_F_NEXT;
145 desc[i-1].next = 0;
146
147 /* We're about to use a buffer */
148 vq->num_free--;
149
150 /* Use a single buffer which doesn't continue */
151 head = vq->free_head;
152 vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT;
153 vq->vring.desc[head].addr = virt_to_phys(desc);
154 vq->vring.desc[head].len = i * sizeof(struct vring_desc);
155
156 /* Update free pointer */
157 vq->free_head = vq->vring.desc[head].next;
158
159 return head;
160}
161
162int virtqueue_add_buf_gfp(struct virtqueue *_vq,
163 struct scatterlist sg[],
164 unsigned int out,
165 unsigned int in,
166 void *data,
167 gfp_t gfp)
168{
169 struct vring_virtqueue *vq = to_vvq(_vq);
170 unsigned int i, avail, uninitialized_var(prev);
171 int head;
172
173 START_USE(vq);
174
175 BUG_ON(data == NULL);
176
177 /* If the host supports indirect descriptor tables, and we have multiple
178 * buffers, then go indirect. FIXME: tune this threshold */
179 if (vq->indirect && (out + in) > 1 && vq->num_free) {
180 head = vring_add_indirect(vq, sg, out, in, gfp);
181 if (likely(head >= 0))
182 goto add_head;
183 }
184
185 BUG_ON(out + in > vq->vring.num);
186 BUG_ON(out + in == 0);
187
188 if (vq->num_free < out + in) {
189 pr_debug("Can't add buf len %i - avail = %i\n",
190 out + in, vq->num_free);
191 /* FIXME: for historical reasons, we force a notify here if
192 * there are outgoing parts to the buffer. Presumably the
193 * host should service the ring ASAP. */
194 if (out)
195 vq->notify(&vq->vq);
196 END_USE(vq);
197 return -ENOSPC;
198 }
199
200 /* We're about to use some buffers from the free list. */
201 vq->num_free -= out + in;
202
203 head = vq->free_head;
204 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) {
205 vq->vring.desc[i].flags = VRING_DESC_F_NEXT;
206 vq->vring.desc[i].addr = sg_phys(sg);
207 vq->vring.desc[i].len = sg->length;
208 prev = i;
209 sg++;
210 }
211 for (; in; i = vq->vring.desc[i].next, in--) {
212 vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
213 vq->vring.desc[i].addr = sg_phys(sg);
214 vq->vring.desc[i].len = sg->length;
215 prev = i;
216 sg++;
217 }
218 /* Last one doesn't continue. */
219 vq->vring.desc[prev].flags &= ~VRING_DESC_F_NEXT;
220
221 /* Update free pointer */
222 vq->free_head = i;
223
224add_head:
225 /* Set token. */
226 vq->data[head] = data;
227
228 /* Put entry in available array (but don't update avail->idx until they
229 * do sync). FIXME: avoid modulus here? */
230 avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num;
231 vq->vring.avail->ring[avail] = head;
232
233 pr_debug("Added buffer head %i to %p\n", head, vq);
234 END_USE(vq);
235
236 return vq->num_free;
237}
238EXPORT_SYMBOL_GPL(virtqueue_add_buf_gfp);
239
240void virtqueue_kick(struct virtqueue *_vq)
241{
242 struct vring_virtqueue *vq = to_vvq(_vq);
243 u16 new, old;
244 START_USE(vq);
245 /* Descriptors and available array need to be set before we expose the
246 * new available array entries. */
247 virtio_wmb();
248
249 old = vq->vring.avail->idx;
250 new = vq->vring.avail->idx = old + vq->num_added;
251 vq->num_added = 0;
252
253 /* Need to update avail index before checking if we should notify */
254 virtio_mb();
255
256 if (vq->event ?
257 vring_need_event(vring_avail_event(&vq->vring), new, old) :
258 !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
259 /* Prod other side to tell it about changes. */
260 vq->notify(&vq->vq);
261
262 END_USE(vq);
263}
264EXPORT_SYMBOL_GPL(virtqueue_kick);
265
266static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
267{
268 unsigned int i;
269
270 /* Clear data ptr. */
271 vq->data[head] = NULL;
272
273 /* Put back on free list: find end */
274 i = head;
275
276 /* Free the indirect table */
277 if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
278 kfree(phys_to_virt(vq->vring.desc[i].addr));
279
280 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
281 i = vq->vring.desc[i].next;
282 vq->num_free++;
283 }
284
285 vq->vring.desc[i].next = vq->free_head;
286 vq->free_head = head;
287 /* Plus final descriptor */
288 vq->num_free++;
289}
290
291static inline bool more_used(const struct vring_virtqueue *vq)
292{
293 return vq->last_used_idx != vq->vring.used->idx;
294}
295
296void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
297{
298 struct vring_virtqueue *vq = to_vvq(_vq);
299 void *ret;
300 unsigned int i;
301
302 START_USE(vq);
303
304 if (unlikely(vq->broken)) {
305 END_USE(vq);
306 return NULL;
307 }
308
309 if (!more_used(vq)) {
310 pr_debug("No more buffers in queue\n");
311 END_USE(vq);
312 return NULL;
313 }
314
315 /* Only get used array entries after they have been exposed by host. */
316 virtio_rmb();
317
318 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
319 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
320
321 if (unlikely(i >= vq->vring.num)) {
322 BAD_RING(vq, "id %u out of range\n", i);
323 return NULL;
324 }
325 if (unlikely(!vq->data[i])) {
326 BAD_RING(vq, "id %u is not a head!\n", i);
327 return NULL;
328 }
329
330 /* detach_buf clears data, so grab it now. */
331 ret = vq->data[i];
332 detach_buf(vq, i);
333 vq->last_used_idx++;
334 /* If we expect an interrupt for the next entry, tell host
335 * by writing event index and flush out the write before
336 * the read in the next get_buf call. */
337 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
338 vring_used_event(&vq->vring) = vq->last_used_idx;
339 virtio_mb();
340 }
341
342 END_USE(vq);
343 return ret;
344}
345EXPORT_SYMBOL_GPL(virtqueue_get_buf);
346
347void virtqueue_disable_cb(struct virtqueue *_vq)
348{
349 struct vring_virtqueue *vq = to_vvq(_vq);
350
351 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
352}
353EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
354
355bool virtqueue_enable_cb(struct virtqueue *_vq)
356{
357 struct vring_virtqueue *vq = to_vvq(_vq);
358
359 START_USE(vq);
360
361 /* We optimistically turn back on interrupts, then check if there was
362 * more to do. */
363 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
364 * either clear the flags bit or point the event index at the next
365 * entry. Always do both to keep code simple. */
366 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
367 vring_used_event(&vq->vring) = vq->last_used_idx;
368 virtio_mb();
369 if (unlikely(more_used(vq))) {
370 END_USE(vq);
371 return false;
372 }
373
374 END_USE(vq);
375 return true;
376}
377EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
378
379bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
380{
381 struct vring_virtqueue *vq = to_vvq(_vq);
382 u16 bufs;
383
384 START_USE(vq);
385
386 /* We optimistically turn back on interrupts, then check if there was
387 * more to do. */
388 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
389 * either clear the flags bit or point the event index at the next
390 * entry. Always do both to keep code simple. */
391 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
392 /* TODO: tune this threshold */
393 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
394 vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
395 virtio_mb();
396 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
397 END_USE(vq);
398 return false;
399 }
400
401 END_USE(vq);
402 return true;
403}
404EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
405
406void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
407{
408 struct vring_virtqueue *vq = to_vvq(_vq);
409 unsigned int i;
410 void *buf;
411
412 START_USE(vq);
413
414 for (i = 0; i < vq->vring.num; i++) {
415 if (!vq->data[i])
416 continue;
417 /* detach_buf clears data, so grab it now. */
418 buf = vq->data[i];
419 detach_buf(vq, i);
420 vq->vring.avail->idx--;
421 END_USE(vq);
422 return buf;
423 }
424 /* That should have freed everything. */
425 BUG_ON(vq->num_free != vq->vring.num);
426
427 END_USE(vq);
428 return NULL;
429}
430EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
431
432irqreturn_t vring_interrupt(int irq, void *_vq)
433{
434 struct vring_virtqueue *vq = to_vvq(_vq);
435
436 if (!more_used(vq)) {
437 pr_debug("virtqueue interrupt with no work for %p\n", vq);
438 return IRQ_NONE;
439 }
440
441 if (unlikely(vq->broken))
442 return IRQ_HANDLED;
443
444 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
445 if (vq->vq.callback)
446 vq->vq.callback(&vq->vq);
447
448 return IRQ_HANDLED;
449}
450EXPORT_SYMBOL_GPL(vring_interrupt);
451
452struct virtqueue *vring_new_virtqueue(unsigned int num,
453 unsigned int vring_align,
454 struct virtio_device *vdev,
455 void *pages,
456 void (*notify)(struct virtqueue *),
457 void (*callback)(struct virtqueue *),
458 const char *name)
459{
460 struct vring_virtqueue *vq;
461 unsigned int i;
462
463 /* We assume num is a power of 2. */
464 if (num & (num - 1)) {
465 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
466 return NULL;
467 }
468
469 vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);
470 if (!vq)
471 return NULL;
472
473 vring_init(&vq->vring, num, pages, vring_align);
474 vq->vq.callback = callback;
475 vq->vq.vdev = vdev;
476 vq->vq.name = name;
477 vq->notify = notify;
478 vq->broken = false;
479 vq->last_used_idx = 0;
480 vq->num_added = 0;
481 list_add_tail(&vq->vq.list, &vdev->vqs);
482#ifdef DEBUG
483 vq->in_use = false;
484#endif
485
486 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
487 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
488
489 /* No callback? Tell other side not to bother us. */
490 if (!callback)
491 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
492
493 /* Put everything in free lists. */
494 vq->num_free = num;
495 vq->free_head = 0;
496 for (i = 0; i < num-1; i++) {
497 vq->vring.desc[i].next = i+1;
498 vq->data[i] = NULL;
499 }
500 vq->data[i] = NULL;
501
502 return &vq->vq;
503}
504EXPORT_SYMBOL_GPL(vring_new_virtqueue);
505
506void vring_del_virtqueue(struct virtqueue *vq)
507{
508 list_del(&vq->list);
509 kfree(to_vvq(vq));
510}
511EXPORT_SYMBOL_GPL(vring_del_virtqueue);
512
513/* Manipulates transport-specific feature bits. */
514void vring_transport_features(struct virtio_device *vdev)
515{
516 unsigned int i;
517
518 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
519 switch (i) {
520 case VIRTIO_RING_F_INDIRECT_DESC:
521 break;
522 case VIRTIO_RING_F_EVENT_IDX:
523 break;
524 default:
525 /* We don't understand this bit. */
526 clear_bit(i, vdev->features);
527 }
528 }
529}
530EXPORT_SYMBOL_GPL(vring_transport_features);
531
532MODULE_LICENSE("GPL");
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Virtio ring implementation.
3 *
4 * Copyright 2007 Rusty Russell IBM Corporation
5 */
6#include <linux/virtio.h>
7#include <linux/virtio_ring.h>
8#include <linux/virtio_config.h>
9#include <linux/device.h>
10#include <linux/slab.h>
11#include <linux/module.h>
12#include <linux/hrtimer.h>
13#include <linux/dma-mapping.h>
14#include <linux/kmsan.h>
15#include <linux/spinlock.h>
16#include <xen/xen.h>
17
18#ifdef DEBUG
19/* For development, we want to crash whenever the ring is screwed. */
20#define BAD_RING(_vq, fmt, args...) \
21 do { \
22 dev_err(&(_vq)->vq.vdev->dev, \
23 "%s:"fmt, (_vq)->vq.name, ##args); \
24 BUG(); \
25 } while (0)
26/* Caller is supposed to guarantee no reentry. */
27#define START_USE(_vq) \
28 do { \
29 if ((_vq)->in_use) \
30 panic("%s:in_use = %i\n", \
31 (_vq)->vq.name, (_vq)->in_use); \
32 (_vq)->in_use = __LINE__; \
33 } while (0)
34#define END_USE(_vq) \
35 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36#define LAST_ADD_TIME_UPDATE(_vq) \
37 do { \
38 ktime_t now = ktime_get(); \
39 \
40 /* No kick or get, with .1 second between? Warn. */ \
41 if ((_vq)->last_add_time_valid) \
42 WARN_ON(ktime_to_ms(ktime_sub(now, \
43 (_vq)->last_add_time)) > 100); \
44 (_vq)->last_add_time = now; \
45 (_vq)->last_add_time_valid = true; \
46 } while (0)
47#define LAST_ADD_TIME_CHECK(_vq) \
48 do { \
49 if ((_vq)->last_add_time_valid) { \
50 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
51 (_vq)->last_add_time)) > 100); \
52 } \
53 } while (0)
54#define LAST_ADD_TIME_INVALID(_vq) \
55 ((_vq)->last_add_time_valid = false)
56#else
57#define BAD_RING(_vq, fmt, args...) \
58 do { \
59 dev_err(&_vq->vq.vdev->dev, \
60 "%s:"fmt, (_vq)->vq.name, ##args); \
61 (_vq)->broken = true; \
62 } while (0)
63#define START_USE(vq)
64#define END_USE(vq)
65#define LAST_ADD_TIME_UPDATE(vq)
66#define LAST_ADD_TIME_CHECK(vq)
67#define LAST_ADD_TIME_INVALID(vq)
68#endif
69
70struct vring_desc_state_split {
71 void *data; /* Data for callback. */
72 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
73};
74
75struct vring_desc_state_packed {
76 void *data; /* Data for callback. */
77 struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
78 u16 num; /* Descriptor list length. */
79 u16 last; /* The last desc state in a list. */
80};
81
82struct vring_desc_extra {
83 dma_addr_t addr; /* Descriptor DMA addr. */
84 u32 len; /* Descriptor length. */
85 u16 flags; /* Descriptor flags. */
86 u16 next; /* The next desc state in a list. */
87};
88
89struct vring_virtqueue_split {
90 /* Actual memory layout for this queue. */
91 struct vring vring;
92
93 /* Last written value to avail->flags */
94 u16 avail_flags_shadow;
95
96 /*
97 * Last written value to avail->idx in
98 * guest byte order.
99 */
100 u16 avail_idx_shadow;
101
102 /* Per-descriptor state. */
103 struct vring_desc_state_split *desc_state;
104 struct vring_desc_extra *desc_extra;
105
106 /* DMA address and size information */
107 dma_addr_t queue_dma_addr;
108 size_t queue_size_in_bytes;
109
110 /*
111 * The parameters for creating vrings are reserved for creating new
112 * vring.
113 */
114 u32 vring_align;
115 bool may_reduce_num;
116};
117
118struct vring_virtqueue_packed {
119 /* Actual memory layout for this queue. */
120 struct {
121 unsigned int num;
122 struct vring_packed_desc *desc;
123 struct vring_packed_desc_event *driver;
124 struct vring_packed_desc_event *device;
125 } vring;
126
127 /* Driver ring wrap counter. */
128 bool avail_wrap_counter;
129
130 /* Avail used flags. */
131 u16 avail_used_flags;
132
133 /* Index of the next avail descriptor. */
134 u16 next_avail_idx;
135
136 /*
137 * Last written value to driver->flags in
138 * guest byte order.
139 */
140 u16 event_flags_shadow;
141
142 /* Per-descriptor state. */
143 struct vring_desc_state_packed *desc_state;
144 struct vring_desc_extra *desc_extra;
145
146 /* DMA address and size information */
147 dma_addr_t ring_dma_addr;
148 dma_addr_t driver_event_dma_addr;
149 dma_addr_t device_event_dma_addr;
150 size_t ring_size_in_bytes;
151 size_t event_size_in_bytes;
152};
153
154struct vring_virtqueue {
155 struct virtqueue vq;
156
157 /* Is this a packed ring? */
158 bool packed_ring;
159
160 /* Is DMA API used? */
161 bool use_dma_api;
162
163 /* Can we use weak barriers? */
164 bool weak_barriers;
165
166 /* Other side has made a mess, don't try any more. */
167 bool broken;
168
169 /* Host supports indirect buffers */
170 bool indirect;
171
172 /* Host publishes avail event idx */
173 bool event;
174
175 /* Head of free buffer list. */
176 unsigned int free_head;
177 /* Number we've added since last sync. */
178 unsigned int num_added;
179
180 /* Last used index we've seen.
181 * for split ring, it just contains last used index
182 * for packed ring:
183 * bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index.
184 * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter.
185 */
186 u16 last_used_idx;
187
188 /* Hint for event idx: already triggered no need to disable. */
189 bool event_triggered;
190
191 union {
192 /* Available for split ring */
193 struct vring_virtqueue_split split;
194
195 /* Available for packed ring */
196 struct vring_virtqueue_packed packed;
197 };
198
199 /* How to notify other side. FIXME: commonalize hcalls! */
200 bool (*notify)(struct virtqueue *vq);
201
202 /* DMA, allocation, and size information */
203 bool we_own_ring;
204
205#ifdef DEBUG
206 /* They're supposed to lock for us. */
207 unsigned int in_use;
208
209 /* Figure out if their kicks are too delayed. */
210 bool last_add_time_valid;
211 ktime_t last_add_time;
212#endif
213};
214
215static struct virtqueue *__vring_new_virtqueue(unsigned int index,
216 struct vring_virtqueue_split *vring_split,
217 struct virtio_device *vdev,
218 bool weak_barriers,
219 bool context,
220 bool (*notify)(struct virtqueue *),
221 void (*callback)(struct virtqueue *),
222 const char *name);
223static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
224static void vring_free(struct virtqueue *_vq);
225
226/*
227 * Helpers.
228 */
229
230#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
231
232static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq,
233 unsigned int total_sg)
234{
235 /*
236 * If the host supports indirect descriptor tables, and we have multiple
237 * buffers, then go indirect. FIXME: tune this threshold
238 */
239 return (vq->indirect && total_sg > 1 && vq->vq.num_free);
240}
241
242/*
243 * Modern virtio devices have feature bits to specify whether they need a
244 * quirk and bypass the IOMMU. If not there, just use the DMA API.
245 *
246 * If there, the interaction between virtio and DMA API is messy.
247 *
248 * On most systems with virtio, physical addresses match bus addresses,
249 * and it doesn't particularly matter whether we use the DMA API.
250 *
251 * On some systems, including Xen and any system with a physical device
252 * that speaks virtio behind a physical IOMMU, we must use the DMA API
253 * for virtio DMA to work at all.
254 *
255 * On other systems, including SPARC and PPC64, virtio-pci devices are
256 * enumerated as though they are behind an IOMMU, but the virtio host
257 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
258 * there or somehow map everything as the identity.
259 *
260 * For the time being, we preserve historic behavior and bypass the DMA
261 * API.
262 *
263 * TODO: install a per-device DMA ops structure that does the right thing
264 * taking into account all the above quirks, and use the DMA API
265 * unconditionally on data path.
266 */
267
268static bool vring_use_dma_api(struct virtio_device *vdev)
269{
270 if (!virtio_has_dma_quirk(vdev))
271 return true;
272
273 /* Otherwise, we are left to guess. */
274 /*
275 * In theory, it's possible to have a buggy QEMU-supposed
276 * emulated Q35 IOMMU and Xen enabled at the same time. On
277 * such a configuration, virtio has never worked and will
278 * not work without an even larger kludge. Instead, enable
279 * the DMA API if we're a Xen guest, which at least allows
280 * all of the sensible Xen configurations to work correctly.
281 */
282 if (xen_domain())
283 return true;
284
285 return false;
286}
287
288size_t virtio_max_dma_size(struct virtio_device *vdev)
289{
290 size_t max_segment_size = SIZE_MAX;
291
292 if (vring_use_dma_api(vdev))
293 max_segment_size = dma_max_mapping_size(vdev->dev.parent);
294
295 return max_segment_size;
296}
297EXPORT_SYMBOL_GPL(virtio_max_dma_size);
298
299static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
300 dma_addr_t *dma_handle, gfp_t flag)
301{
302 if (vring_use_dma_api(vdev)) {
303 return dma_alloc_coherent(vdev->dev.parent, size,
304 dma_handle, flag);
305 } else {
306 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
307
308 if (queue) {
309 phys_addr_t phys_addr = virt_to_phys(queue);
310 *dma_handle = (dma_addr_t)phys_addr;
311
312 /*
313 * Sanity check: make sure we dind't truncate
314 * the address. The only arches I can find that
315 * have 64-bit phys_addr_t but 32-bit dma_addr_t
316 * are certain non-highmem MIPS and x86
317 * configurations, but these configurations
318 * should never allocate physical pages above 32
319 * bits, so this is fine. Just in case, throw a
320 * warning and abort if we end up with an
321 * unrepresentable address.
322 */
323 if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
324 free_pages_exact(queue, PAGE_ALIGN(size));
325 return NULL;
326 }
327 }
328 return queue;
329 }
330}
331
332static void vring_free_queue(struct virtio_device *vdev, size_t size,
333 void *queue, dma_addr_t dma_handle)
334{
335 if (vring_use_dma_api(vdev))
336 dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
337 else
338 free_pages_exact(queue, PAGE_ALIGN(size));
339}
340
341/*
342 * The DMA ops on various arches are rather gnarly right now, and
343 * making all of the arch DMA ops work on the vring device itself
344 * is a mess. For now, we use the parent device for DMA ops.
345 */
346static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
347{
348 return vq->vq.vdev->dev.parent;
349}
350
351/* Map one sg entry. */
352static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
353 struct scatterlist *sg,
354 enum dma_data_direction direction)
355{
356 if (!vq->use_dma_api) {
357 /*
358 * If DMA is not used, KMSAN doesn't know that the scatterlist
359 * is initialized by the hardware. Explicitly check/unpoison it
360 * depending on the direction.
361 */
362 kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction);
363 return (dma_addr_t)sg_phys(sg);
364 }
365
366 /*
367 * We can't use dma_map_sg, because we don't use scatterlists in
368 * the way it expects (we don't guarantee that the scatterlist
369 * will exist for the lifetime of the mapping).
370 */
371 return dma_map_page(vring_dma_dev(vq),
372 sg_page(sg), sg->offset, sg->length,
373 direction);
374}
375
376static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
377 void *cpu_addr, size_t size,
378 enum dma_data_direction direction)
379{
380 if (!vq->use_dma_api)
381 return (dma_addr_t)virt_to_phys(cpu_addr);
382
383 return dma_map_single(vring_dma_dev(vq),
384 cpu_addr, size, direction);
385}
386
387static int vring_mapping_error(const struct vring_virtqueue *vq,
388 dma_addr_t addr)
389{
390 if (!vq->use_dma_api)
391 return 0;
392
393 return dma_mapping_error(vring_dma_dev(vq), addr);
394}
395
396static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
397{
398 vq->vq.num_free = num;
399
400 if (vq->packed_ring)
401 vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
402 else
403 vq->last_used_idx = 0;
404
405 vq->event_triggered = false;
406 vq->num_added = 0;
407
408#ifdef DEBUG
409 vq->in_use = false;
410 vq->last_add_time_valid = false;
411#endif
412}
413
414
415/*
416 * Split ring specific functions - *_split().
417 */
418
419static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
420 struct vring_desc *desc)
421{
422 u16 flags;
423
424 if (!vq->use_dma_api)
425 return;
426
427 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
428
429 dma_unmap_page(vring_dma_dev(vq),
430 virtio64_to_cpu(vq->vq.vdev, desc->addr),
431 virtio32_to_cpu(vq->vq.vdev, desc->len),
432 (flags & VRING_DESC_F_WRITE) ?
433 DMA_FROM_DEVICE : DMA_TO_DEVICE);
434}
435
436static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
437 unsigned int i)
438{
439 struct vring_desc_extra *extra = vq->split.desc_extra;
440 u16 flags;
441
442 if (!vq->use_dma_api)
443 goto out;
444
445 flags = extra[i].flags;
446
447 if (flags & VRING_DESC_F_INDIRECT) {
448 dma_unmap_single(vring_dma_dev(vq),
449 extra[i].addr,
450 extra[i].len,
451 (flags & VRING_DESC_F_WRITE) ?
452 DMA_FROM_DEVICE : DMA_TO_DEVICE);
453 } else {
454 dma_unmap_page(vring_dma_dev(vq),
455 extra[i].addr,
456 extra[i].len,
457 (flags & VRING_DESC_F_WRITE) ?
458 DMA_FROM_DEVICE : DMA_TO_DEVICE);
459 }
460
461out:
462 return extra[i].next;
463}
464
465static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
466 unsigned int total_sg,
467 gfp_t gfp)
468{
469 struct vring_desc *desc;
470 unsigned int i;
471
472 /*
473 * We require lowmem mappings for the descriptors because
474 * otherwise virt_to_phys will give us bogus addresses in the
475 * virtqueue.
476 */
477 gfp &= ~__GFP_HIGHMEM;
478
479 desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
480 if (!desc)
481 return NULL;
482
483 for (i = 0; i < total_sg; i++)
484 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
485 return desc;
486}
487
488static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
489 struct vring_desc *desc,
490 unsigned int i,
491 dma_addr_t addr,
492 unsigned int len,
493 u16 flags,
494 bool indirect)
495{
496 struct vring_virtqueue *vring = to_vvq(vq);
497 struct vring_desc_extra *extra = vring->split.desc_extra;
498 u16 next;
499
500 desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
501 desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
502 desc[i].len = cpu_to_virtio32(vq->vdev, len);
503
504 if (!indirect) {
505 next = extra[i].next;
506 desc[i].next = cpu_to_virtio16(vq->vdev, next);
507
508 extra[i].addr = addr;
509 extra[i].len = len;
510 extra[i].flags = flags;
511 } else
512 next = virtio16_to_cpu(vq->vdev, desc[i].next);
513
514 return next;
515}
516
517static inline int virtqueue_add_split(struct virtqueue *_vq,
518 struct scatterlist *sgs[],
519 unsigned int total_sg,
520 unsigned int out_sgs,
521 unsigned int in_sgs,
522 void *data,
523 void *ctx,
524 gfp_t gfp)
525{
526 struct vring_virtqueue *vq = to_vvq(_vq);
527 struct scatterlist *sg;
528 struct vring_desc *desc;
529 unsigned int i, n, avail, descs_used, prev, err_idx;
530 int head;
531 bool indirect;
532
533 START_USE(vq);
534
535 BUG_ON(data == NULL);
536 BUG_ON(ctx && vq->indirect);
537
538 if (unlikely(vq->broken)) {
539 END_USE(vq);
540 return -EIO;
541 }
542
543 LAST_ADD_TIME_UPDATE(vq);
544
545 BUG_ON(total_sg == 0);
546
547 head = vq->free_head;
548
549 if (virtqueue_use_indirect(vq, total_sg))
550 desc = alloc_indirect_split(_vq, total_sg, gfp);
551 else {
552 desc = NULL;
553 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
554 }
555
556 if (desc) {
557 /* Use a single buffer which doesn't continue */
558 indirect = true;
559 /* Set up rest to use this indirect table. */
560 i = 0;
561 descs_used = 1;
562 } else {
563 indirect = false;
564 desc = vq->split.vring.desc;
565 i = head;
566 descs_used = total_sg;
567 }
568
569 if (unlikely(vq->vq.num_free < descs_used)) {
570 pr_debug("Can't add buf len %i - avail = %i\n",
571 descs_used, vq->vq.num_free);
572 /* FIXME: for historical reasons, we force a notify here if
573 * there are outgoing parts to the buffer. Presumably the
574 * host should service the ring ASAP. */
575 if (out_sgs)
576 vq->notify(&vq->vq);
577 if (indirect)
578 kfree(desc);
579 END_USE(vq);
580 return -ENOSPC;
581 }
582
583 for (n = 0; n < out_sgs; n++) {
584 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
585 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
586 if (vring_mapping_error(vq, addr))
587 goto unmap_release;
588
589 prev = i;
590 /* Note that we trust indirect descriptor
591 * table since it use stream DMA mapping.
592 */
593 i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,
594 VRING_DESC_F_NEXT,
595 indirect);
596 }
597 }
598 for (; n < (out_sgs + in_sgs); n++) {
599 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
600 dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
601 if (vring_mapping_error(vq, addr))
602 goto unmap_release;
603
604 prev = i;
605 /* Note that we trust indirect descriptor
606 * table since it use stream DMA mapping.
607 */
608 i = virtqueue_add_desc_split(_vq, desc, i, addr,
609 sg->length,
610 VRING_DESC_F_NEXT |
611 VRING_DESC_F_WRITE,
612 indirect);
613 }
614 }
615 /* Last one doesn't continue. */
616 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
617 if (!indirect && vq->use_dma_api)
618 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
619 ~VRING_DESC_F_NEXT;
620
621 if (indirect) {
622 /* Now that the indirect table is filled in, map it. */
623 dma_addr_t addr = vring_map_single(
624 vq, desc, total_sg * sizeof(struct vring_desc),
625 DMA_TO_DEVICE);
626 if (vring_mapping_error(vq, addr))
627 goto unmap_release;
628
629 virtqueue_add_desc_split(_vq, vq->split.vring.desc,
630 head, addr,
631 total_sg * sizeof(struct vring_desc),
632 VRING_DESC_F_INDIRECT,
633 false);
634 }
635
636 /* We're using some buffers from the free list. */
637 vq->vq.num_free -= descs_used;
638
639 /* Update free pointer */
640 if (indirect)
641 vq->free_head = vq->split.desc_extra[head].next;
642 else
643 vq->free_head = i;
644
645 /* Store token and indirect buffer state. */
646 vq->split.desc_state[head].data = data;
647 if (indirect)
648 vq->split.desc_state[head].indir_desc = desc;
649 else
650 vq->split.desc_state[head].indir_desc = ctx;
651
652 /* Put entry in available array (but don't update avail->idx until they
653 * do sync). */
654 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
655 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
656
657 /* Descriptors and available array need to be set before we expose the
658 * new available array entries. */
659 virtio_wmb(vq->weak_barriers);
660 vq->split.avail_idx_shadow++;
661 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
662 vq->split.avail_idx_shadow);
663 vq->num_added++;
664
665 pr_debug("Added buffer head %i to %p\n", head, vq);
666 END_USE(vq);
667
668 /* This is very unlikely, but theoretically possible. Kick
669 * just in case. */
670 if (unlikely(vq->num_added == (1 << 16) - 1))
671 virtqueue_kick(_vq);
672
673 return 0;
674
675unmap_release:
676 err_idx = i;
677
678 if (indirect)
679 i = 0;
680 else
681 i = head;
682
683 for (n = 0; n < total_sg; n++) {
684 if (i == err_idx)
685 break;
686 if (indirect) {
687 vring_unmap_one_split_indirect(vq, &desc[i]);
688 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
689 } else
690 i = vring_unmap_one_split(vq, i);
691 }
692
693 if (indirect)
694 kfree(desc);
695
696 END_USE(vq);
697 return -ENOMEM;
698}
699
700static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
701{
702 struct vring_virtqueue *vq = to_vvq(_vq);
703 u16 new, old;
704 bool needs_kick;
705
706 START_USE(vq);
707 /* We need to expose available array entries before checking avail
708 * event. */
709 virtio_mb(vq->weak_barriers);
710
711 old = vq->split.avail_idx_shadow - vq->num_added;
712 new = vq->split.avail_idx_shadow;
713 vq->num_added = 0;
714
715 LAST_ADD_TIME_CHECK(vq);
716 LAST_ADD_TIME_INVALID(vq);
717
718 if (vq->event) {
719 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
720 vring_avail_event(&vq->split.vring)),
721 new, old);
722 } else {
723 needs_kick = !(vq->split.vring.used->flags &
724 cpu_to_virtio16(_vq->vdev,
725 VRING_USED_F_NO_NOTIFY));
726 }
727 END_USE(vq);
728 return needs_kick;
729}
730
731static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
732 void **ctx)
733{
734 unsigned int i, j;
735 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
736
737 /* Clear data ptr. */
738 vq->split.desc_state[head].data = NULL;
739
740 /* Put back on free list: unmap first-level descriptors and find end */
741 i = head;
742
743 while (vq->split.vring.desc[i].flags & nextflag) {
744 vring_unmap_one_split(vq, i);
745 i = vq->split.desc_extra[i].next;
746 vq->vq.num_free++;
747 }
748
749 vring_unmap_one_split(vq, i);
750 vq->split.desc_extra[i].next = vq->free_head;
751 vq->free_head = head;
752
753 /* Plus final descriptor */
754 vq->vq.num_free++;
755
756 if (vq->indirect) {
757 struct vring_desc *indir_desc =
758 vq->split.desc_state[head].indir_desc;
759 u32 len;
760
761 /* Free the indirect table, if any, now that it's unmapped. */
762 if (!indir_desc)
763 return;
764
765 len = vq->split.desc_extra[head].len;
766
767 BUG_ON(!(vq->split.desc_extra[head].flags &
768 VRING_DESC_F_INDIRECT));
769 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
770
771 for (j = 0; j < len / sizeof(struct vring_desc); j++)
772 vring_unmap_one_split_indirect(vq, &indir_desc[j]);
773
774 kfree(indir_desc);
775 vq->split.desc_state[head].indir_desc = NULL;
776 } else if (ctx) {
777 *ctx = vq->split.desc_state[head].indir_desc;
778 }
779}
780
781static inline bool more_used_split(const struct vring_virtqueue *vq)
782{
783 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
784 vq->split.vring.used->idx);
785}
786
787static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
788 unsigned int *len,
789 void **ctx)
790{
791 struct vring_virtqueue *vq = to_vvq(_vq);
792 void *ret;
793 unsigned int i;
794 u16 last_used;
795
796 START_USE(vq);
797
798 if (unlikely(vq->broken)) {
799 END_USE(vq);
800 return NULL;
801 }
802
803 if (!more_used_split(vq)) {
804 pr_debug("No more buffers in queue\n");
805 END_USE(vq);
806 return NULL;
807 }
808
809 /* Only get used array entries after they have been exposed by host. */
810 virtio_rmb(vq->weak_barriers);
811
812 last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
813 i = virtio32_to_cpu(_vq->vdev,
814 vq->split.vring.used->ring[last_used].id);
815 *len = virtio32_to_cpu(_vq->vdev,
816 vq->split.vring.used->ring[last_used].len);
817
818 if (unlikely(i >= vq->split.vring.num)) {
819 BAD_RING(vq, "id %u out of range\n", i);
820 return NULL;
821 }
822 if (unlikely(!vq->split.desc_state[i].data)) {
823 BAD_RING(vq, "id %u is not a head!\n", i);
824 return NULL;
825 }
826
827 /* detach_buf_split clears data, so grab it now. */
828 ret = vq->split.desc_state[i].data;
829 detach_buf_split(vq, i, ctx);
830 vq->last_used_idx++;
831 /* If we expect an interrupt for the next entry, tell host
832 * by writing event index and flush out the write before
833 * the read in the next get_buf call. */
834 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
835 virtio_store_mb(vq->weak_barriers,
836 &vring_used_event(&vq->split.vring),
837 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
838
839 LAST_ADD_TIME_INVALID(vq);
840
841 END_USE(vq);
842 return ret;
843}
844
845static void virtqueue_disable_cb_split(struct virtqueue *_vq)
846{
847 struct vring_virtqueue *vq = to_vvq(_vq);
848
849 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
850 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
851 if (vq->event)
852 /* TODO: this is a hack. Figure out a cleaner value to write. */
853 vring_used_event(&vq->split.vring) = 0x0;
854 else
855 vq->split.vring.avail->flags =
856 cpu_to_virtio16(_vq->vdev,
857 vq->split.avail_flags_shadow);
858 }
859}
860
861static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
862{
863 struct vring_virtqueue *vq = to_vvq(_vq);
864 u16 last_used_idx;
865
866 START_USE(vq);
867
868 /* We optimistically turn back on interrupts, then check if there was
869 * more to do. */
870 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
871 * either clear the flags bit or point the event index at the next
872 * entry. Always do both to keep code simple. */
873 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
874 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
875 if (!vq->event)
876 vq->split.vring.avail->flags =
877 cpu_to_virtio16(_vq->vdev,
878 vq->split.avail_flags_shadow);
879 }
880 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
881 last_used_idx = vq->last_used_idx);
882 END_USE(vq);
883 return last_used_idx;
884}
885
886static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
887{
888 struct vring_virtqueue *vq = to_vvq(_vq);
889
890 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
891 vq->split.vring.used->idx);
892}
893
894static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
895{
896 struct vring_virtqueue *vq = to_vvq(_vq);
897 u16 bufs;
898
899 START_USE(vq);
900
901 /* We optimistically turn back on interrupts, then check if there was
902 * more to do. */
903 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
904 * either clear the flags bit or point the event index at the next
905 * entry. Always update the event index to keep code simple. */
906 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
907 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
908 if (!vq->event)
909 vq->split.vring.avail->flags =
910 cpu_to_virtio16(_vq->vdev,
911 vq->split.avail_flags_shadow);
912 }
913 /* TODO: tune this threshold */
914 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
915
916 virtio_store_mb(vq->weak_barriers,
917 &vring_used_event(&vq->split.vring),
918 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
919
920 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
921 - vq->last_used_idx) > bufs)) {
922 END_USE(vq);
923 return false;
924 }
925
926 END_USE(vq);
927 return true;
928}
929
930static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
931{
932 struct vring_virtqueue *vq = to_vvq(_vq);
933 unsigned int i;
934 void *buf;
935
936 START_USE(vq);
937
938 for (i = 0; i < vq->split.vring.num; i++) {
939 if (!vq->split.desc_state[i].data)
940 continue;
941 /* detach_buf_split clears data, so grab it now. */
942 buf = vq->split.desc_state[i].data;
943 detach_buf_split(vq, i, NULL);
944 vq->split.avail_idx_shadow--;
945 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
946 vq->split.avail_idx_shadow);
947 END_USE(vq);
948 return buf;
949 }
950 /* That should have freed everything. */
951 BUG_ON(vq->vq.num_free != vq->split.vring.num);
952
953 END_USE(vq);
954 return NULL;
955}
956
957static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split,
958 struct vring_virtqueue *vq)
959{
960 struct virtio_device *vdev;
961
962 vdev = vq->vq.vdev;
963
964 vring_split->avail_flags_shadow = 0;
965 vring_split->avail_idx_shadow = 0;
966
967 /* No callback? Tell other side not to bother us. */
968 if (!vq->vq.callback) {
969 vring_split->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
970 if (!vq->event)
971 vring_split->vring.avail->flags = cpu_to_virtio16(vdev,
972 vring_split->avail_flags_shadow);
973 }
974}
975
976static void virtqueue_reinit_split(struct vring_virtqueue *vq)
977{
978 int num;
979
980 num = vq->split.vring.num;
981
982 vq->split.vring.avail->flags = 0;
983 vq->split.vring.avail->idx = 0;
984
985 /* reset avail event */
986 vq->split.vring.avail->ring[num] = 0;
987
988 vq->split.vring.used->flags = 0;
989 vq->split.vring.used->idx = 0;
990
991 /* reset used event */
992 *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;
993
994 virtqueue_init(vq, num);
995
996 virtqueue_vring_init_split(&vq->split, vq);
997}
998
999static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
1000 struct vring_virtqueue_split *vring_split)
1001{
1002 vq->split = *vring_split;
1003
1004 /* Put everything in free lists. */
1005 vq->free_head = 0;
1006}
1007
1008static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
1009{
1010 struct vring_desc_state_split *state;
1011 struct vring_desc_extra *extra;
1012 u32 num = vring_split->vring.num;
1013
1014 state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL);
1015 if (!state)
1016 goto err_state;
1017
1018 extra = vring_alloc_desc_extra(num);
1019 if (!extra)
1020 goto err_extra;
1021
1022 memset(state, 0, num * sizeof(struct vring_desc_state_split));
1023
1024 vring_split->desc_state = state;
1025 vring_split->desc_extra = extra;
1026 return 0;
1027
1028err_extra:
1029 kfree(state);
1030err_state:
1031 return -ENOMEM;
1032}
1033
1034static void vring_free_split(struct vring_virtqueue_split *vring_split,
1035 struct virtio_device *vdev)
1036{
1037 vring_free_queue(vdev, vring_split->queue_size_in_bytes,
1038 vring_split->vring.desc,
1039 vring_split->queue_dma_addr);
1040
1041 kfree(vring_split->desc_state);
1042 kfree(vring_split->desc_extra);
1043}
1044
1045static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
1046 struct virtio_device *vdev,
1047 u32 num,
1048 unsigned int vring_align,
1049 bool may_reduce_num)
1050{
1051 void *queue = NULL;
1052 dma_addr_t dma_addr;
1053
1054 /* We assume num is a power of 2. */
1055 if (!is_power_of_2(num)) {
1056 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
1057 return -EINVAL;
1058 }
1059
1060 /* TODO: allocate each queue chunk individually */
1061 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
1062 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1063 &dma_addr,
1064 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
1065 if (queue)
1066 break;
1067 if (!may_reduce_num)
1068 return -ENOMEM;
1069 }
1070
1071 if (!num)
1072 return -ENOMEM;
1073
1074 if (!queue) {
1075 /* Try to get a single page. You are my only hope! */
1076 queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1077 &dma_addr, GFP_KERNEL | __GFP_ZERO);
1078 }
1079 if (!queue)
1080 return -ENOMEM;
1081
1082 vring_init(&vring_split->vring, num, queue, vring_align);
1083
1084 vring_split->queue_dma_addr = dma_addr;
1085 vring_split->queue_size_in_bytes = vring_size(num, vring_align);
1086
1087 vring_split->vring_align = vring_align;
1088 vring_split->may_reduce_num = may_reduce_num;
1089
1090 return 0;
1091}
1092
1093static struct virtqueue *vring_create_virtqueue_split(
1094 unsigned int index,
1095 unsigned int num,
1096 unsigned int vring_align,
1097 struct virtio_device *vdev,
1098 bool weak_barriers,
1099 bool may_reduce_num,
1100 bool context,
1101 bool (*notify)(struct virtqueue *),
1102 void (*callback)(struct virtqueue *),
1103 const char *name)
1104{
1105 struct vring_virtqueue_split vring_split = {};
1106 struct virtqueue *vq;
1107 int err;
1108
1109 err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align,
1110 may_reduce_num);
1111 if (err)
1112 return NULL;
1113
1114 vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
1115 context, notify, callback, name);
1116 if (!vq) {
1117 vring_free_split(&vring_split, vdev);
1118 return NULL;
1119 }
1120
1121 to_vvq(vq)->we_own_ring = true;
1122
1123 return vq;
1124}
1125
1126static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
1127{
1128 struct vring_virtqueue_split vring_split = {};
1129 struct vring_virtqueue *vq = to_vvq(_vq);
1130 struct virtio_device *vdev = _vq->vdev;
1131 int err;
1132
1133 err = vring_alloc_queue_split(&vring_split, vdev, num,
1134 vq->split.vring_align,
1135 vq->split.may_reduce_num);
1136 if (err)
1137 goto err;
1138
1139 err = vring_alloc_state_extra_split(&vring_split);
1140 if (err)
1141 goto err_state_extra;
1142
1143 vring_free(&vq->vq);
1144
1145 virtqueue_vring_init_split(&vring_split, vq);
1146
1147 virtqueue_init(vq, vring_split.vring.num);
1148 virtqueue_vring_attach_split(vq, &vring_split);
1149
1150 return 0;
1151
1152err_state_extra:
1153 vring_free_split(&vring_split, vdev);
1154err:
1155 virtqueue_reinit_split(vq);
1156 return -ENOMEM;
1157}
1158
1159
1160/*
1161 * Packed ring specific functions - *_packed().
1162 */
1163static inline bool packed_used_wrap_counter(u16 last_used_idx)
1164{
1165 return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1166}
1167
1168static inline u16 packed_last_used(u16 last_used_idx)
1169{
1170 return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1171}
1172
1173static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
1174 struct vring_desc_extra *extra)
1175{
1176 u16 flags;
1177
1178 if (!vq->use_dma_api)
1179 return;
1180
1181 flags = extra->flags;
1182
1183 if (flags & VRING_DESC_F_INDIRECT) {
1184 dma_unmap_single(vring_dma_dev(vq),
1185 extra->addr, extra->len,
1186 (flags & VRING_DESC_F_WRITE) ?
1187 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1188 } else {
1189 dma_unmap_page(vring_dma_dev(vq),
1190 extra->addr, extra->len,
1191 (flags & VRING_DESC_F_WRITE) ?
1192 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1193 }
1194}
1195
1196static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
1197 struct vring_packed_desc *desc)
1198{
1199 u16 flags;
1200
1201 if (!vq->use_dma_api)
1202 return;
1203
1204 flags = le16_to_cpu(desc->flags);
1205
1206 dma_unmap_page(vring_dma_dev(vq),
1207 le64_to_cpu(desc->addr),
1208 le32_to_cpu(desc->len),
1209 (flags & VRING_DESC_F_WRITE) ?
1210 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1211}
1212
1213static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
1214 gfp_t gfp)
1215{
1216 struct vring_packed_desc *desc;
1217
1218 /*
1219 * We require lowmem mappings for the descriptors because
1220 * otherwise virt_to_phys will give us bogus addresses in the
1221 * virtqueue.
1222 */
1223 gfp &= ~__GFP_HIGHMEM;
1224
1225 desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
1226
1227 return desc;
1228}
1229
1230static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
1231 struct scatterlist *sgs[],
1232 unsigned int total_sg,
1233 unsigned int out_sgs,
1234 unsigned int in_sgs,
1235 void *data,
1236 gfp_t gfp)
1237{
1238 struct vring_packed_desc *desc;
1239 struct scatterlist *sg;
1240 unsigned int i, n, err_idx;
1241 u16 head, id;
1242 dma_addr_t addr;
1243
1244 head = vq->packed.next_avail_idx;
1245 desc = alloc_indirect_packed(total_sg, gfp);
1246 if (!desc)
1247 return -ENOMEM;
1248
1249 if (unlikely(vq->vq.num_free < 1)) {
1250 pr_debug("Can't add buf len 1 - avail = 0\n");
1251 kfree(desc);
1252 END_USE(vq);
1253 return -ENOSPC;
1254 }
1255
1256 i = 0;
1257 id = vq->free_head;
1258 BUG_ON(id == vq->packed.vring.num);
1259
1260 for (n = 0; n < out_sgs + in_sgs; n++) {
1261 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1262 addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1263 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1264 if (vring_mapping_error(vq, addr))
1265 goto unmap_release;
1266
1267 desc[i].flags = cpu_to_le16(n < out_sgs ?
1268 0 : VRING_DESC_F_WRITE);
1269 desc[i].addr = cpu_to_le64(addr);
1270 desc[i].len = cpu_to_le32(sg->length);
1271 i++;
1272 }
1273 }
1274
1275 /* Now that the indirect table is filled in, map it. */
1276 addr = vring_map_single(vq, desc,
1277 total_sg * sizeof(struct vring_packed_desc),
1278 DMA_TO_DEVICE);
1279 if (vring_mapping_error(vq, addr))
1280 goto unmap_release;
1281
1282 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1283 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1284 sizeof(struct vring_packed_desc));
1285 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1286
1287 if (vq->use_dma_api) {
1288 vq->packed.desc_extra[id].addr = addr;
1289 vq->packed.desc_extra[id].len = total_sg *
1290 sizeof(struct vring_packed_desc);
1291 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1292 vq->packed.avail_used_flags;
1293 }
1294
1295 /*
1296 * A driver MUST NOT make the first descriptor in the list
1297 * available before all subsequent descriptors comprising
1298 * the list are made available.
1299 */
1300 virtio_wmb(vq->weak_barriers);
1301 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1302 vq->packed.avail_used_flags);
1303
1304 /* We're using some buffers from the free list. */
1305 vq->vq.num_free -= 1;
1306
1307 /* Update free pointer */
1308 n = head + 1;
1309 if (n >= vq->packed.vring.num) {
1310 n = 0;
1311 vq->packed.avail_wrap_counter ^= 1;
1312 vq->packed.avail_used_flags ^=
1313 1 << VRING_PACKED_DESC_F_AVAIL |
1314 1 << VRING_PACKED_DESC_F_USED;
1315 }
1316 vq->packed.next_avail_idx = n;
1317 vq->free_head = vq->packed.desc_extra[id].next;
1318
1319 /* Store token and indirect buffer state. */
1320 vq->packed.desc_state[id].num = 1;
1321 vq->packed.desc_state[id].data = data;
1322 vq->packed.desc_state[id].indir_desc = desc;
1323 vq->packed.desc_state[id].last = id;
1324
1325 vq->num_added += 1;
1326
1327 pr_debug("Added buffer head %i to %p\n", head, vq);
1328 END_USE(vq);
1329
1330 return 0;
1331
1332unmap_release:
1333 err_idx = i;
1334
1335 for (i = 0; i < err_idx; i++)
1336 vring_unmap_desc_packed(vq, &desc[i]);
1337
1338 kfree(desc);
1339
1340 END_USE(vq);
1341 return -ENOMEM;
1342}
1343
1344static inline int virtqueue_add_packed(struct virtqueue *_vq,
1345 struct scatterlist *sgs[],
1346 unsigned int total_sg,
1347 unsigned int out_sgs,
1348 unsigned int in_sgs,
1349 void *data,
1350 void *ctx,
1351 gfp_t gfp)
1352{
1353 struct vring_virtqueue *vq = to_vvq(_vq);
1354 struct vring_packed_desc *desc;
1355 struct scatterlist *sg;
1356 unsigned int i, n, c, descs_used, err_idx;
1357 __le16 head_flags, flags;
1358 u16 head, id, prev, curr, avail_used_flags;
1359 int err;
1360
1361 START_USE(vq);
1362
1363 BUG_ON(data == NULL);
1364 BUG_ON(ctx && vq->indirect);
1365
1366 if (unlikely(vq->broken)) {
1367 END_USE(vq);
1368 return -EIO;
1369 }
1370
1371 LAST_ADD_TIME_UPDATE(vq);
1372
1373 BUG_ON(total_sg == 0);
1374
1375 if (virtqueue_use_indirect(vq, total_sg)) {
1376 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
1377 in_sgs, data, gfp);
1378 if (err != -ENOMEM) {
1379 END_USE(vq);
1380 return err;
1381 }
1382
1383 /* fall back on direct */
1384 }
1385
1386 head = vq->packed.next_avail_idx;
1387 avail_used_flags = vq->packed.avail_used_flags;
1388
1389 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1390
1391 desc = vq->packed.vring.desc;
1392 i = head;
1393 descs_used = total_sg;
1394
1395 if (unlikely(vq->vq.num_free < descs_used)) {
1396 pr_debug("Can't add buf len %i - avail = %i\n",
1397 descs_used, vq->vq.num_free);
1398 END_USE(vq);
1399 return -ENOSPC;
1400 }
1401
1402 id = vq->free_head;
1403 BUG_ON(id == vq->packed.vring.num);
1404
1405 curr = id;
1406 c = 0;
1407 for (n = 0; n < out_sgs + in_sgs; n++) {
1408 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1409 dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1410 DMA_TO_DEVICE : DMA_FROM_DEVICE);
1411 if (vring_mapping_error(vq, addr))
1412 goto unmap_release;
1413
1414 flags = cpu_to_le16(vq->packed.avail_used_flags |
1415 (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1416 (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1417 if (i == head)
1418 head_flags = flags;
1419 else
1420 desc[i].flags = flags;
1421
1422 desc[i].addr = cpu_to_le64(addr);
1423 desc[i].len = cpu_to_le32(sg->length);
1424 desc[i].id = cpu_to_le16(id);
1425
1426 if (unlikely(vq->use_dma_api)) {
1427 vq->packed.desc_extra[curr].addr = addr;
1428 vq->packed.desc_extra[curr].len = sg->length;
1429 vq->packed.desc_extra[curr].flags =
1430 le16_to_cpu(flags);
1431 }
1432 prev = curr;
1433 curr = vq->packed.desc_extra[curr].next;
1434
1435 if ((unlikely(++i >= vq->packed.vring.num))) {
1436 i = 0;
1437 vq->packed.avail_used_flags ^=
1438 1 << VRING_PACKED_DESC_F_AVAIL |
1439 1 << VRING_PACKED_DESC_F_USED;
1440 }
1441 }
1442 }
1443
1444 if (i < head)
1445 vq->packed.avail_wrap_counter ^= 1;
1446
1447 /* We're using some buffers from the free list. */
1448 vq->vq.num_free -= descs_used;
1449
1450 /* Update free pointer */
1451 vq->packed.next_avail_idx = i;
1452 vq->free_head = curr;
1453
1454 /* Store token. */
1455 vq->packed.desc_state[id].num = descs_used;
1456 vq->packed.desc_state[id].data = data;
1457 vq->packed.desc_state[id].indir_desc = ctx;
1458 vq->packed.desc_state[id].last = prev;
1459
1460 /*
1461 * A driver MUST NOT make the first descriptor in the list
1462 * available before all subsequent descriptors comprising
1463 * the list are made available.
1464 */
1465 virtio_wmb(vq->weak_barriers);
1466 vq->packed.vring.desc[head].flags = head_flags;
1467 vq->num_added += descs_used;
1468
1469 pr_debug("Added buffer head %i to %p\n", head, vq);
1470 END_USE(vq);
1471
1472 return 0;
1473
1474unmap_release:
1475 err_idx = i;
1476 i = head;
1477 curr = vq->free_head;
1478
1479 vq->packed.avail_used_flags = avail_used_flags;
1480
1481 for (n = 0; n < total_sg; n++) {
1482 if (i == err_idx)
1483 break;
1484 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1485 curr = vq->packed.desc_extra[curr].next;
1486 i++;
1487 if (i >= vq->packed.vring.num)
1488 i = 0;
1489 }
1490
1491 END_USE(vq);
1492 return -EIO;
1493}
1494
1495static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1496{
1497 struct vring_virtqueue *vq = to_vvq(_vq);
1498 u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1499 bool needs_kick;
1500 union {
1501 struct {
1502 __le16 off_wrap;
1503 __le16 flags;
1504 };
1505 u32 u32;
1506 } snapshot;
1507
1508 START_USE(vq);
1509
1510 /*
1511 * We need to expose the new flags value before checking notification
1512 * suppressions.
1513 */
1514 virtio_mb(vq->weak_barriers);
1515
1516 old = vq->packed.next_avail_idx - vq->num_added;
1517 new = vq->packed.next_avail_idx;
1518 vq->num_added = 0;
1519
1520 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1521 flags = le16_to_cpu(snapshot.flags);
1522
1523 LAST_ADD_TIME_CHECK(vq);
1524 LAST_ADD_TIME_INVALID(vq);
1525
1526 if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1527 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1528 goto out;
1529 }
1530
1531 off_wrap = le16_to_cpu(snapshot.off_wrap);
1532
1533 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1534 event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1535 if (wrap_counter != vq->packed.avail_wrap_counter)
1536 event_idx -= vq->packed.vring.num;
1537
1538 needs_kick = vring_need_event(event_idx, new, old);
1539out:
1540 END_USE(vq);
1541 return needs_kick;
1542}
1543
1544static void detach_buf_packed(struct vring_virtqueue *vq,
1545 unsigned int id, void **ctx)
1546{
1547 struct vring_desc_state_packed *state = NULL;
1548 struct vring_packed_desc *desc;
1549 unsigned int i, curr;
1550
1551 state = &vq->packed.desc_state[id];
1552
1553 /* Clear data ptr. */
1554 state->data = NULL;
1555
1556 vq->packed.desc_extra[state->last].next = vq->free_head;
1557 vq->free_head = id;
1558 vq->vq.num_free += state->num;
1559
1560 if (unlikely(vq->use_dma_api)) {
1561 curr = id;
1562 for (i = 0; i < state->num; i++) {
1563 vring_unmap_extra_packed(vq,
1564 &vq->packed.desc_extra[curr]);
1565 curr = vq->packed.desc_extra[curr].next;
1566 }
1567 }
1568
1569 if (vq->indirect) {
1570 u32 len;
1571
1572 /* Free the indirect table, if any, now that it's unmapped. */
1573 desc = state->indir_desc;
1574 if (!desc)
1575 return;
1576
1577 if (vq->use_dma_api) {
1578 len = vq->packed.desc_extra[id].len;
1579 for (i = 0; i < len / sizeof(struct vring_packed_desc);
1580 i++)
1581 vring_unmap_desc_packed(vq, &desc[i]);
1582 }
1583 kfree(desc);
1584 state->indir_desc = NULL;
1585 } else if (ctx) {
1586 *ctx = state->indir_desc;
1587 }
1588}
1589
1590static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1591 u16 idx, bool used_wrap_counter)
1592{
1593 bool avail, used;
1594 u16 flags;
1595
1596 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1597 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1598 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1599
1600 return avail == used && used == used_wrap_counter;
1601}
1602
1603static inline bool more_used_packed(const struct vring_virtqueue *vq)
1604{
1605 u16 last_used;
1606 u16 last_used_idx;
1607 bool used_wrap_counter;
1608
1609 last_used_idx = READ_ONCE(vq->last_used_idx);
1610 last_used = packed_last_used(last_used_idx);
1611 used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1612 return is_used_desc_packed(vq, last_used, used_wrap_counter);
1613}
1614
1615static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1616 unsigned int *len,
1617 void **ctx)
1618{
1619 struct vring_virtqueue *vq = to_vvq(_vq);
1620 u16 last_used, id, last_used_idx;
1621 bool used_wrap_counter;
1622 void *ret;
1623
1624 START_USE(vq);
1625
1626 if (unlikely(vq->broken)) {
1627 END_USE(vq);
1628 return NULL;
1629 }
1630
1631 if (!more_used_packed(vq)) {
1632 pr_debug("No more buffers in queue\n");
1633 END_USE(vq);
1634 return NULL;
1635 }
1636
1637 /* Only get used elements after they have been exposed by host. */
1638 virtio_rmb(vq->weak_barriers);
1639
1640 last_used_idx = READ_ONCE(vq->last_used_idx);
1641 used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1642 last_used = packed_last_used(last_used_idx);
1643 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1644 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1645
1646 if (unlikely(id >= vq->packed.vring.num)) {
1647 BAD_RING(vq, "id %u out of range\n", id);
1648 return NULL;
1649 }
1650 if (unlikely(!vq->packed.desc_state[id].data)) {
1651 BAD_RING(vq, "id %u is not a head!\n", id);
1652 return NULL;
1653 }
1654
1655 /* detach_buf_packed clears data, so grab it now. */
1656 ret = vq->packed.desc_state[id].data;
1657 detach_buf_packed(vq, id, ctx);
1658
1659 last_used += vq->packed.desc_state[id].num;
1660 if (unlikely(last_used >= vq->packed.vring.num)) {
1661 last_used -= vq->packed.vring.num;
1662 used_wrap_counter ^= 1;
1663 }
1664
1665 last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1666 WRITE_ONCE(vq->last_used_idx, last_used);
1667
1668 /*
1669 * If we expect an interrupt for the next entry, tell host
1670 * by writing event index and flush out the write before
1671 * the read in the next get_buf call.
1672 */
1673 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1674 virtio_store_mb(vq->weak_barriers,
1675 &vq->packed.vring.driver->off_wrap,
1676 cpu_to_le16(vq->last_used_idx));
1677
1678 LAST_ADD_TIME_INVALID(vq);
1679
1680 END_USE(vq);
1681 return ret;
1682}
1683
1684static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1685{
1686 struct vring_virtqueue *vq = to_vvq(_vq);
1687
1688 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1689 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1690 vq->packed.vring.driver->flags =
1691 cpu_to_le16(vq->packed.event_flags_shadow);
1692 }
1693}
1694
1695static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1696{
1697 struct vring_virtqueue *vq = to_vvq(_vq);
1698
1699 START_USE(vq);
1700
1701 /*
1702 * We optimistically turn back on interrupts, then check if there was
1703 * more to do.
1704 */
1705
1706 if (vq->event) {
1707 vq->packed.vring.driver->off_wrap =
1708 cpu_to_le16(vq->last_used_idx);
1709 /*
1710 * We need to update event offset and event wrap
1711 * counter first before updating event flags.
1712 */
1713 virtio_wmb(vq->weak_barriers);
1714 }
1715
1716 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1717 vq->packed.event_flags_shadow = vq->event ?
1718 VRING_PACKED_EVENT_FLAG_DESC :
1719 VRING_PACKED_EVENT_FLAG_ENABLE;
1720 vq->packed.vring.driver->flags =
1721 cpu_to_le16(vq->packed.event_flags_shadow);
1722 }
1723
1724 END_USE(vq);
1725 return vq->last_used_idx;
1726}
1727
1728static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1729{
1730 struct vring_virtqueue *vq = to_vvq(_vq);
1731 bool wrap_counter;
1732 u16 used_idx;
1733
1734 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1735 used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1736
1737 return is_used_desc_packed(vq, used_idx, wrap_counter);
1738}
1739
1740static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1741{
1742 struct vring_virtqueue *vq = to_vvq(_vq);
1743 u16 used_idx, wrap_counter, last_used_idx;
1744 u16 bufs;
1745
1746 START_USE(vq);
1747
1748 /*
1749 * We optimistically turn back on interrupts, then check if there was
1750 * more to do.
1751 */
1752
1753 if (vq->event) {
1754 /* TODO: tune this threshold */
1755 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1756 last_used_idx = READ_ONCE(vq->last_used_idx);
1757 wrap_counter = packed_used_wrap_counter(last_used_idx);
1758
1759 used_idx = packed_last_used(last_used_idx) + bufs;
1760 if (used_idx >= vq->packed.vring.num) {
1761 used_idx -= vq->packed.vring.num;
1762 wrap_counter ^= 1;
1763 }
1764
1765 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1766 (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1767
1768 /*
1769 * We need to update event offset and event wrap
1770 * counter first before updating event flags.
1771 */
1772 virtio_wmb(vq->weak_barriers);
1773 }
1774
1775 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1776 vq->packed.event_flags_shadow = vq->event ?
1777 VRING_PACKED_EVENT_FLAG_DESC :
1778 VRING_PACKED_EVENT_FLAG_ENABLE;
1779 vq->packed.vring.driver->flags =
1780 cpu_to_le16(vq->packed.event_flags_shadow);
1781 }
1782
1783 /*
1784 * We need to update event suppression structure first
1785 * before re-checking for more used buffers.
1786 */
1787 virtio_mb(vq->weak_barriers);
1788
1789 last_used_idx = READ_ONCE(vq->last_used_idx);
1790 wrap_counter = packed_used_wrap_counter(last_used_idx);
1791 used_idx = packed_last_used(last_used_idx);
1792 if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
1793 END_USE(vq);
1794 return false;
1795 }
1796
1797 END_USE(vq);
1798 return true;
1799}
1800
1801static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1802{
1803 struct vring_virtqueue *vq = to_vvq(_vq);
1804 unsigned int i;
1805 void *buf;
1806
1807 START_USE(vq);
1808
1809 for (i = 0; i < vq->packed.vring.num; i++) {
1810 if (!vq->packed.desc_state[i].data)
1811 continue;
1812 /* detach_buf clears data, so grab it now. */
1813 buf = vq->packed.desc_state[i].data;
1814 detach_buf_packed(vq, i, NULL);
1815 END_USE(vq);
1816 return buf;
1817 }
1818 /* That should have freed everything. */
1819 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1820
1821 END_USE(vq);
1822 return NULL;
1823}
1824
1825static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
1826{
1827 struct vring_desc_extra *desc_extra;
1828 unsigned int i;
1829
1830 desc_extra = kmalloc_array(num, sizeof(struct vring_desc_extra),
1831 GFP_KERNEL);
1832 if (!desc_extra)
1833 return NULL;
1834
1835 memset(desc_extra, 0, num * sizeof(struct vring_desc_extra));
1836
1837 for (i = 0; i < num - 1; i++)
1838 desc_extra[i].next = i + 1;
1839
1840 return desc_extra;
1841}
1842
1843static void vring_free_packed(struct vring_virtqueue_packed *vring_packed,
1844 struct virtio_device *vdev)
1845{
1846 if (vring_packed->vring.desc)
1847 vring_free_queue(vdev, vring_packed->ring_size_in_bytes,
1848 vring_packed->vring.desc,
1849 vring_packed->ring_dma_addr);
1850
1851 if (vring_packed->vring.driver)
1852 vring_free_queue(vdev, vring_packed->event_size_in_bytes,
1853 vring_packed->vring.driver,
1854 vring_packed->driver_event_dma_addr);
1855
1856 if (vring_packed->vring.device)
1857 vring_free_queue(vdev, vring_packed->event_size_in_bytes,
1858 vring_packed->vring.device,
1859 vring_packed->device_event_dma_addr);
1860
1861 kfree(vring_packed->desc_state);
1862 kfree(vring_packed->desc_extra);
1863}
1864
1865static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed,
1866 struct virtio_device *vdev,
1867 u32 num)
1868{
1869 struct vring_packed_desc *ring;
1870 struct vring_packed_desc_event *driver, *device;
1871 dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1872 size_t ring_size_in_bytes, event_size_in_bytes;
1873
1874 ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1875
1876 ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1877 &ring_dma_addr,
1878 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
1879 if (!ring)
1880 goto err;
1881
1882 vring_packed->vring.desc = ring;
1883 vring_packed->ring_dma_addr = ring_dma_addr;
1884 vring_packed->ring_size_in_bytes = ring_size_in_bytes;
1885
1886 event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1887
1888 driver = vring_alloc_queue(vdev, event_size_in_bytes,
1889 &driver_event_dma_addr,
1890 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
1891 if (!driver)
1892 goto err;
1893
1894 vring_packed->vring.driver = driver;
1895 vring_packed->event_size_in_bytes = event_size_in_bytes;
1896 vring_packed->driver_event_dma_addr = driver_event_dma_addr;
1897
1898 device = vring_alloc_queue(vdev, event_size_in_bytes,
1899 &device_event_dma_addr,
1900 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
1901 if (!device)
1902 goto err;
1903
1904 vring_packed->vring.device = device;
1905 vring_packed->device_event_dma_addr = device_event_dma_addr;
1906
1907 vring_packed->vring.num = num;
1908
1909 return 0;
1910
1911err:
1912 vring_free_packed(vring_packed, vdev);
1913 return -ENOMEM;
1914}
1915
1916static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed)
1917{
1918 struct vring_desc_state_packed *state;
1919 struct vring_desc_extra *extra;
1920 u32 num = vring_packed->vring.num;
1921
1922 state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL);
1923 if (!state)
1924 goto err_desc_state;
1925
1926 memset(state, 0, num * sizeof(struct vring_desc_state_packed));
1927
1928 extra = vring_alloc_desc_extra(num);
1929 if (!extra)
1930 goto err_desc_extra;
1931
1932 vring_packed->desc_state = state;
1933 vring_packed->desc_extra = extra;
1934
1935 return 0;
1936
1937err_desc_extra:
1938 kfree(state);
1939err_desc_state:
1940 return -ENOMEM;
1941}
1942
1943static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
1944 bool callback)
1945{
1946 vring_packed->next_avail_idx = 0;
1947 vring_packed->avail_wrap_counter = 1;
1948 vring_packed->event_flags_shadow = 0;
1949 vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1950
1951 /* No callback? Tell other side not to bother us. */
1952 if (!callback) {
1953 vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1954 vring_packed->vring.driver->flags =
1955 cpu_to_le16(vring_packed->event_flags_shadow);
1956 }
1957}
1958
1959static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
1960 struct vring_virtqueue_packed *vring_packed)
1961{
1962 vq->packed = *vring_packed;
1963
1964 /* Put everything in free lists. */
1965 vq->free_head = 0;
1966}
1967
1968static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
1969{
1970 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
1971 memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
1972
1973 /* we need to reset the desc.flags. For more, see is_used_desc_packed() */
1974 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
1975
1976 virtqueue_init(vq, vq->packed.vring.num);
1977 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
1978}
1979
1980static struct virtqueue *vring_create_virtqueue_packed(
1981 unsigned int index,
1982 unsigned int num,
1983 unsigned int vring_align,
1984 struct virtio_device *vdev,
1985 bool weak_barriers,
1986 bool may_reduce_num,
1987 bool context,
1988 bool (*notify)(struct virtqueue *),
1989 void (*callback)(struct virtqueue *),
1990 const char *name)
1991{
1992 struct vring_virtqueue_packed vring_packed = {};
1993 struct vring_virtqueue *vq;
1994 int err;
1995
1996 if (vring_alloc_queue_packed(&vring_packed, vdev, num))
1997 goto err_ring;
1998
1999 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2000 if (!vq)
2001 goto err_vq;
2002
2003 vq->vq.callback = callback;
2004 vq->vq.vdev = vdev;
2005 vq->vq.name = name;
2006 vq->vq.index = index;
2007 vq->vq.reset = false;
2008 vq->we_own_ring = true;
2009 vq->notify = notify;
2010 vq->weak_barriers = weak_barriers;
2011#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2012 vq->broken = true;
2013#else
2014 vq->broken = false;
2015#endif
2016 vq->packed_ring = true;
2017 vq->use_dma_api = vring_use_dma_api(vdev);
2018
2019 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2020 !context;
2021 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2022
2023 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2024 vq->weak_barriers = false;
2025
2026 err = vring_alloc_state_extra_packed(&vring_packed);
2027 if (err)
2028 goto err_state_extra;
2029
2030 virtqueue_vring_init_packed(&vring_packed, !!callback);
2031
2032 virtqueue_init(vq, num);
2033 virtqueue_vring_attach_packed(vq, &vring_packed);
2034
2035 spin_lock(&vdev->vqs_list_lock);
2036 list_add_tail(&vq->vq.list, &vdev->vqs);
2037 spin_unlock(&vdev->vqs_list_lock);
2038 return &vq->vq;
2039
2040err_state_extra:
2041 kfree(vq);
2042err_vq:
2043 vring_free_packed(&vring_packed, vdev);
2044err_ring:
2045 return NULL;
2046}
2047
2048static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
2049{
2050 struct vring_virtqueue_packed vring_packed = {};
2051 struct vring_virtqueue *vq = to_vvq(_vq);
2052 struct virtio_device *vdev = _vq->vdev;
2053 int err;
2054
2055 if (vring_alloc_queue_packed(&vring_packed, vdev, num))
2056 goto err_ring;
2057
2058 err = vring_alloc_state_extra_packed(&vring_packed);
2059 if (err)
2060 goto err_state_extra;
2061
2062 vring_free(&vq->vq);
2063
2064 virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback);
2065
2066 virtqueue_init(vq, vring_packed.vring.num);
2067 virtqueue_vring_attach_packed(vq, &vring_packed);
2068
2069 return 0;
2070
2071err_state_extra:
2072 vring_free_packed(&vring_packed, vdev);
2073err_ring:
2074 virtqueue_reinit_packed(vq);
2075 return -ENOMEM;
2076}
2077
2078
2079/*
2080 * Generic functions and exported symbols.
2081 */
2082
2083static inline int virtqueue_add(struct virtqueue *_vq,
2084 struct scatterlist *sgs[],
2085 unsigned int total_sg,
2086 unsigned int out_sgs,
2087 unsigned int in_sgs,
2088 void *data,
2089 void *ctx,
2090 gfp_t gfp)
2091{
2092 struct vring_virtqueue *vq = to_vvq(_vq);
2093
2094 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
2095 out_sgs, in_sgs, data, ctx, gfp) :
2096 virtqueue_add_split(_vq, sgs, total_sg,
2097 out_sgs, in_sgs, data, ctx, gfp);
2098}
2099
2100/**
2101 * virtqueue_add_sgs - expose buffers to other end
2102 * @_vq: the struct virtqueue we're talking about.
2103 * @sgs: array of terminated scatterlists.
2104 * @out_sgs: the number of scatterlists readable by other side
2105 * @in_sgs: the number of scatterlists which are writable (after readable ones)
2106 * @data: the token identifying the buffer.
2107 * @gfp: how to do memory allocations (if necessary).
2108 *
2109 * Caller must ensure we don't call this with other virtqueue operations
2110 * at the same time (except where noted).
2111 *
2112 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2113 */
2114int virtqueue_add_sgs(struct virtqueue *_vq,
2115 struct scatterlist *sgs[],
2116 unsigned int out_sgs,
2117 unsigned int in_sgs,
2118 void *data,
2119 gfp_t gfp)
2120{
2121 unsigned int i, total_sg = 0;
2122
2123 /* Count them first. */
2124 for (i = 0; i < out_sgs + in_sgs; i++) {
2125 struct scatterlist *sg;
2126
2127 for (sg = sgs[i]; sg; sg = sg_next(sg))
2128 total_sg++;
2129 }
2130 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
2131 data, NULL, gfp);
2132}
2133EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
2134
2135/**
2136 * virtqueue_add_outbuf - expose output buffers to other end
2137 * @vq: the struct virtqueue we're talking about.
2138 * @sg: scatterlist (must be well-formed and terminated!)
2139 * @num: the number of entries in @sg readable by other side
2140 * @data: the token identifying the buffer.
2141 * @gfp: how to do memory allocations (if necessary).
2142 *
2143 * Caller must ensure we don't call this with other virtqueue operations
2144 * at the same time (except where noted).
2145 *
2146 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2147 */
2148int virtqueue_add_outbuf(struct virtqueue *vq,
2149 struct scatterlist *sg, unsigned int num,
2150 void *data,
2151 gfp_t gfp)
2152{
2153 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
2154}
2155EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
2156
2157/**
2158 * virtqueue_add_inbuf - expose input buffers to other end
2159 * @vq: the struct virtqueue we're talking about.
2160 * @sg: scatterlist (must be well-formed and terminated!)
2161 * @num: the number of entries in @sg writable by other side
2162 * @data: the token identifying the buffer.
2163 * @gfp: how to do memory allocations (if necessary).
2164 *
2165 * Caller must ensure we don't call this with other virtqueue operations
2166 * at the same time (except where noted).
2167 *
2168 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2169 */
2170int virtqueue_add_inbuf(struct virtqueue *vq,
2171 struct scatterlist *sg, unsigned int num,
2172 void *data,
2173 gfp_t gfp)
2174{
2175 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
2176}
2177EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
2178
2179/**
2180 * virtqueue_add_inbuf_ctx - expose input buffers to other end
2181 * @vq: the struct virtqueue we're talking about.
2182 * @sg: scatterlist (must be well-formed and terminated!)
2183 * @num: the number of entries in @sg writable by other side
2184 * @data: the token identifying the buffer.
2185 * @ctx: extra context for the token
2186 * @gfp: how to do memory allocations (if necessary).
2187 *
2188 * Caller must ensure we don't call this with other virtqueue operations
2189 * at the same time (except where noted).
2190 *
2191 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2192 */
2193int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
2194 struct scatterlist *sg, unsigned int num,
2195 void *data,
2196 void *ctx,
2197 gfp_t gfp)
2198{
2199 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
2200}
2201EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
2202
2203/**
2204 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
2205 * @_vq: the struct virtqueue
2206 *
2207 * Instead of virtqueue_kick(), you can do:
2208 * if (virtqueue_kick_prepare(vq))
2209 * virtqueue_notify(vq);
2210 *
2211 * This is sometimes useful because the virtqueue_kick_prepare() needs
2212 * to be serialized, but the actual virtqueue_notify() call does not.
2213 */
2214bool virtqueue_kick_prepare(struct virtqueue *_vq)
2215{
2216 struct vring_virtqueue *vq = to_vvq(_vq);
2217
2218 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
2219 virtqueue_kick_prepare_split(_vq);
2220}
2221EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
2222
2223/**
2224 * virtqueue_notify - second half of split virtqueue_kick call.
2225 * @_vq: the struct virtqueue
2226 *
2227 * This does not need to be serialized.
2228 *
2229 * Returns false if host notify failed or queue is broken, otherwise true.
2230 */
2231bool virtqueue_notify(struct virtqueue *_vq)
2232{
2233 struct vring_virtqueue *vq = to_vvq(_vq);
2234
2235 if (unlikely(vq->broken))
2236 return false;
2237
2238 /* Prod other side to tell it about changes. */
2239 if (!vq->notify(_vq)) {
2240 vq->broken = true;
2241 return false;
2242 }
2243 return true;
2244}
2245EXPORT_SYMBOL_GPL(virtqueue_notify);
2246
2247/**
2248 * virtqueue_kick - update after add_buf
2249 * @vq: the struct virtqueue
2250 *
2251 * After one or more virtqueue_add_* calls, invoke this to kick
2252 * the other side.
2253 *
2254 * Caller must ensure we don't call this with other virtqueue
2255 * operations at the same time (except where noted).
2256 *
2257 * Returns false if kick failed, otherwise true.
2258 */
2259bool virtqueue_kick(struct virtqueue *vq)
2260{
2261 if (virtqueue_kick_prepare(vq))
2262 return virtqueue_notify(vq);
2263 return true;
2264}
2265EXPORT_SYMBOL_GPL(virtqueue_kick);
2266
2267/**
2268 * virtqueue_get_buf_ctx - get the next used buffer
2269 * @_vq: the struct virtqueue we're talking about.
2270 * @len: the length written into the buffer
2271 * @ctx: extra context for the token
2272 *
2273 * If the device wrote data into the buffer, @len will be set to the
2274 * amount written. This means you don't need to clear the buffer
2275 * beforehand to ensure there's no data leakage in the case of short
2276 * writes.
2277 *
2278 * Caller must ensure we don't call this with other virtqueue
2279 * operations at the same time (except where noted).
2280 *
2281 * Returns NULL if there are no used buffers, or the "data" token
2282 * handed to virtqueue_add_*().
2283 */
2284void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
2285 void **ctx)
2286{
2287 struct vring_virtqueue *vq = to_vvq(_vq);
2288
2289 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
2290 virtqueue_get_buf_ctx_split(_vq, len, ctx);
2291}
2292EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
2293
2294void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
2295{
2296 return virtqueue_get_buf_ctx(_vq, len, NULL);
2297}
2298EXPORT_SYMBOL_GPL(virtqueue_get_buf);
2299/**
2300 * virtqueue_disable_cb - disable callbacks
2301 * @_vq: the struct virtqueue we're talking about.
2302 *
2303 * Note that this is not necessarily synchronous, hence unreliable and only
2304 * useful as an optimization.
2305 *
2306 * Unlike other operations, this need not be serialized.
2307 */
2308void virtqueue_disable_cb(struct virtqueue *_vq)
2309{
2310 struct vring_virtqueue *vq = to_vvq(_vq);
2311
2312 /* If device triggered an event already it won't trigger one again:
2313 * no need to disable.
2314 */
2315 if (vq->event_triggered)
2316 return;
2317
2318 if (vq->packed_ring)
2319 virtqueue_disable_cb_packed(_vq);
2320 else
2321 virtqueue_disable_cb_split(_vq);
2322}
2323EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
2324
2325/**
2326 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
2327 * @_vq: the struct virtqueue we're talking about.
2328 *
2329 * This re-enables callbacks; it returns current queue state
2330 * in an opaque unsigned value. This value should be later tested by
2331 * virtqueue_poll, to detect a possible race between the driver checking for
2332 * more work, and enabling callbacks.
2333 *
2334 * Caller must ensure we don't call this with other virtqueue
2335 * operations at the same time (except where noted).
2336 */
2337unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
2338{
2339 struct vring_virtqueue *vq = to_vvq(_vq);
2340
2341 if (vq->event_triggered)
2342 vq->event_triggered = false;
2343
2344 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
2345 virtqueue_enable_cb_prepare_split(_vq);
2346}
2347EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
2348
2349/**
2350 * virtqueue_poll - query pending used buffers
2351 * @_vq: the struct virtqueue we're talking about.
2352 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
2353 *
2354 * Returns "true" if there are pending used buffers in the queue.
2355 *
2356 * This does not need to be serialized.
2357 */
2358bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
2359{
2360 struct vring_virtqueue *vq = to_vvq(_vq);
2361
2362 if (unlikely(vq->broken))
2363 return false;
2364
2365 virtio_mb(vq->weak_barriers);
2366 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
2367 virtqueue_poll_split(_vq, last_used_idx);
2368}
2369EXPORT_SYMBOL_GPL(virtqueue_poll);
2370
2371/**
2372 * virtqueue_enable_cb - restart callbacks after disable_cb.
2373 * @_vq: the struct virtqueue we're talking about.
2374 *
2375 * This re-enables callbacks; it returns "false" if there are pending
2376 * buffers in the queue, to detect a possible race between the driver
2377 * checking for more work, and enabling callbacks.
2378 *
2379 * Caller must ensure we don't call this with other virtqueue
2380 * operations at the same time (except where noted).
2381 */
2382bool virtqueue_enable_cb(struct virtqueue *_vq)
2383{
2384 unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);
2385
2386 return !virtqueue_poll(_vq, last_used_idx);
2387}
2388EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
2389
2390/**
2391 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
2392 * @_vq: the struct virtqueue we're talking about.
2393 *
2394 * This re-enables callbacks but hints to the other side to delay
2395 * interrupts until most of the available buffers have been processed;
2396 * it returns "false" if there are many pending buffers in the queue,
2397 * to detect a possible race between the driver checking for more work,
2398 * and enabling callbacks.
2399 *
2400 * Caller must ensure we don't call this with other virtqueue
2401 * operations at the same time (except where noted).
2402 */
2403bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2404{
2405 struct vring_virtqueue *vq = to_vvq(_vq);
2406
2407 if (vq->event_triggered)
2408 vq->event_triggered = false;
2409
2410 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2411 virtqueue_enable_cb_delayed_split(_vq);
2412}
2413EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2414
2415/**
2416 * virtqueue_detach_unused_buf - detach first unused buffer
2417 * @_vq: the struct virtqueue we're talking about.
2418 *
2419 * Returns NULL or the "data" token handed to virtqueue_add_*().
2420 * This is not valid on an active queue; it is useful for device
2421 * shutdown or the reset queue.
2422 */
2423void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2424{
2425 struct vring_virtqueue *vq = to_vvq(_vq);
2426
2427 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2428 virtqueue_detach_unused_buf_split(_vq);
2429}
2430EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2431
2432static inline bool more_used(const struct vring_virtqueue *vq)
2433{
2434 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2435}
2436
2437/**
2438 * vring_interrupt - notify a virtqueue on an interrupt
2439 * @irq: the IRQ number (ignored)
2440 * @_vq: the struct virtqueue to notify
2441 *
2442 * Calls the callback function of @_vq to process the virtqueue
2443 * notification.
2444 */
2445irqreturn_t vring_interrupt(int irq, void *_vq)
2446{
2447 struct vring_virtqueue *vq = to_vvq(_vq);
2448
2449 if (!more_used(vq)) {
2450 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2451 return IRQ_NONE;
2452 }
2453
2454 if (unlikely(vq->broken)) {
2455#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2456 dev_warn_once(&vq->vq.vdev->dev,
2457 "virtio vring IRQ raised before DRIVER_OK");
2458 return IRQ_NONE;
2459#else
2460 return IRQ_HANDLED;
2461#endif
2462 }
2463
2464 /* Just a hint for performance: so it's ok that this can be racy! */
2465 if (vq->event)
2466 vq->event_triggered = true;
2467
2468 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2469 if (vq->vq.callback)
2470 vq->vq.callback(&vq->vq);
2471
2472 return IRQ_HANDLED;
2473}
2474EXPORT_SYMBOL_GPL(vring_interrupt);
2475
2476/* Only available for split ring */
2477static struct virtqueue *__vring_new_virtqueue(unsigned int index,
2478 struct vring_virtqueue_split *vring_split,
2479 struct virtio_device *vdev,
2480 bool weak_barriers,
2481 bool context,
2482 bool (*notify)(struct virtqueue *),
2483 void (*callback)(struct virtqueue *),
2484 const char *name)
2485{
2486 struct vring_virtqueue *vq;
2487 int err;
2488
2489 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2490 return NULL;
2491
2492 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2493 if (!vq)
2494 return NULL;
2495
2496 vq->packed_ring = false;
2497 vq->vq.callback = callback;
2498 vq->vq.vdev = vdev;
2499 vq->vq.name = name;
2500 vq->vq.index = index;
2501 vq->vq.reset = false;
2502 vq->we_own_ring = false;
2503 vq->notify = notify;
2504 vq->weak_barriers = weak_barriers;
2505#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2506 vq->broken = true;
2507#else
2508 vq->broken = false;
2509#endif
2510 vq->use_dma_api = vring_use_dma_api(vdev);
2511
2512 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2513 !context;
2514 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2515
2516 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2517 vq->weak_barriers = false;
2518
2519 err = vring_alloc_state_extra_split(vring_split);
2520 if (err) {
2521 kfree(vq);
2522 return NULL;
2523 }
2524
2525 virtqueue_vring_init_split(vring_split, vq);
2526
2527 virtqueue_init(vq, vring_split->vring.num);
2528 virtqueue_vring_attach_split(vq, vring_split);
2529
2530 spin_lock(&vdev->vqs_list_lock);
2531 list_add_tail(&vq->vq.list, &vdev->vqs);
2532 spin_unlock(&vdev->vqs_list_lock);
2533 return &vq->vq;
2534}
2535
2536struct virtqueue *vring_create_virtqueue(
2537 unsigned int index,
2538 unsigned int num,
2539 unsigned int vring_align,
2540 struct virtio_device *vdev,
2541 bool weak_barriers,
2542 bool may_reduce_num,
2543 bool context,
2544 bool (*notify)(struct virtqueue *),
2545 void (*callback)(struct virtqueue *),
2546 const char *name)
2547{
2548
2549 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2550 return vring_create_virtqueue_packed(index, num, vring_align,
2551 vdev, weak_barriers, may_reduce_num,
2552 context, notify, callback, name);
2553
2554 return vring_create_virtqueue_split(index, num, vring_align,
2555 vdev, weak_barriers, may_reduce_num,
2556 context, notify, callback, name);
2557}
2558EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2559
2560/**
2561 * virtqueue_resize - resize the vring of vq
2562 * @_vq: the struct virtqueue we're talking about.
2563 * @num: new ring num
2564 * @recycle: callback for recycle the useless buffer
2565 *
2566 * When it is really necessary to create a new vring, it will set the current vq
2567 * into the reset state. Then call the passed callback to recycle the buffer
2568 * that is no longer used. Only after the new vring is successfully created, the
2569 * old vring will be released.
2570 *
2571 * Caller must ensure we don't call this with other virtqueue operations
2572 * at the same time (except where noted).
2573 *
2574 * Returns zero or a negative error.
2575 * 0: success.
2576 * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size.
2577 * vq can still work normally
2578 * -EBUSY: Failed to sync with device, vq may not work properly
2579 * -ENOENT: Transport or device not supported
2580 * -E2BIG/-EINVAL: num error
2581 * -EPERM: Operation not permitted
2582 *
2583 */
2584int virtqueue_resize(struct virtqueue *_vq, u32 num,
2585 void (*recycle)(struct virtqueue *vq, void *buf))
2586{
2587 struct vring_virtqueue *vq = to_vvq(_vq);
2588 struct virtio_device *vdev = vq->vq.vdev;
2589 void *buf;
2590 int err;
2591
2592 if (!vq->we_own_ring)
2593 return -EPERM;
2594
2595 if (num > vq->vq.num_max)
2596 return -E2BIG;
2597
2598 if (!num)
2599 return -EINVAL;
2600
2601 if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
2602 return 0;
2603
2604 if (!vdev->config->disable_vq_and_reset)
2605 return -ENOENT;
2606
2607 if (!vdev->config->enable_vq_after_reset)
2608 return -ENOENT;
2609
2610 err = vdev->config->disable_vq_and_reset(_vq);
2611 if (err)
2612 return err;
2613
2614 while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
2615 recycle(_vq, buf);
2616
2617 if (vq->packed_ring)
2618 err = virtqueue_resize_packed(_vq, num);
2619 else
2620 err = virtqueue_resize_split(_vq, num);
2621
2622 if (vdev->config->enable_vq_after_reset(_vq))
2623 return -EBUSY;
2624
2625 return err;
2626}
2627EXPORT_SYMBOL_GPL(virtqueue_resize);
2628
2629/* Only available for split ring */
2630struct virtqueue *vring_new_virtqueue(unsigned int index,
2631 unsigned int num,
2632 unsigned int vring_align,
2633 struct virtio_device *vdev,
2634 bool weak_barriers,
2635 bool context,
2636 void *pages,
2637 bool (*notify)(struct virtqueue *vq),
2638 void (*callback)(struct virtqueue *vq),
2639 const char *name)
2640{
2641 struct vring_virtqueue_split vring_split = {};
2642
2643 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2644 return NULL;
2645
2646 vring_init(&vring_split.vring, num, pages, vring_align);
2647 return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
2648 context, notify, callback, name);
2649}
2650EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2651
2652static void vring_free(struct virtqueue *_vq)
2653{
2654 struct vring_virtqueue *vq = to_vvq(_vq);
2655
2656 if (vq->we_own_ring) {
2657 if (vq->packed_ring) {
2658 vring_free_queue(vq->vq.vdev,
2659 vq->packed.ring_size_in_bytes,
2660 vq->packed.vring.desc,
2661 vq->packed.ring_dma_addr);
2662
2663 vring_free_queue(vq->vq.vdev,
2664 vq->packed.event_size_in_bytes,
2665 vq->packed.vring.driver,
2666 vq->packed.driver_event_dma_addr);
2667
2668 vring_free_queue(vq->vq.vdev,
2669 vq->packed.event_size_in_bytes,
2670 vq->packed.vring.device,
2671 vq->packed.device_event_dma_addr);
2672
2673 kfree(vq->packed.desc_state);
2674 kfree(vq->packed.desc_extra);
2675 } else {
2676 vring_free_queue(vq->vq.vdev,
2677 vq->split.queue_size_in_bytes,
2678 vq->split.vring.desc,
2679 vq->split.queue_dma_addr);
2680 }
2681 }
2682 if (!vq->packed_ring) {
2683 kfree(vq->split.desc_state);
2684 kfree(vq->split.desc_extra);
2685 }
2686}
2687
2688void vring_del_virtqueue(struct virtqueue *_vq)
2689{
2690 struct vring_virtqueue *vq = to_vvq(_vq);
2691
2692 spin_lock(&vq->vq.vdev->vqs_list_lock);
2693 list_del(&_vq->list);
2694 spin_unlock(&vq->vq.vdev->vqs_list_lock);
2695
2696 vring_free(_vq);
2697
2698 kfree(vq);
2699}
2700EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2701
2702/* Manipulates transport-specific feature bits. */
2703void vring_transport_features(struct virtio_device *vdev)
2704{
2705 unsigned int i;
2706
2707 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2708 switch (i) {
2709 case VIRTIO_RING_F_INDIRECT_DESC:
2710 break;
2711 case VIRTIO_RING_F_EVENT_IDX:
2712 break;
2713 case VIRTIO_F_VERSION_1:
2714 break;
2715 case VIRTIO_F_ACCESS_PLATFORM:
2716 break;
2717 case VIRTIO_F_RING_PACKED:
2718 break;
2719 case VIRTIO_F_ORDER_PLATFORM:
2720 break;
2721 default:
2722 /* We don't understand this bit. */
2723 __virtio_clear_bit(vdev, i);
2724 }
2725 }
2726}
2727EXPORT_SYMBOL_GPL(vring_transport_features);
2728
2729/**
2730 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2731 * @_vq: the struct virtqueue containing the vring of interest.
2732 *
2733 * Returns the size of the vring. This is mainly used for boasting to
2734 * userspace. Unlike other operations, this need not be serialized.
2735 */
2736unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
2737{
2738
2739 struct vring_virtqueue *vq = to_vvq(_vq);
2740
2741 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2742}
2743EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2744
2745/*
2746 * This function should only be called by the core, not directly by the driver.
2747 */
2748void __virtqueue_break(struct virtqueue *_vq)
2749{
2750 struct vring_virtqueue *vq = to_vvq(_vq);
2751
2752 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2753 WRITE_ONCE(vq->broken, true);
2754}
2755EXPORT_SYMBOL_GPL(__virtqueue_break);
2756
2757/*
2758 * This function should only be called by the core, not directly by the driver.
2759 */
2760void __virtqueue_unbreak(struct virtqueue *_vq)
2761{
2762 struct vring_virtqueue *vq = to_vvq(_vq);
2763
2764 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2765 WRITE_ONCE(vq->broken, false);
2766}
2767EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
2768
2769bool virtqueue_is_broken(struct virtqueue *_vq)
2770{
2771 struct vring_virtqueue *vq = to_vvq(_vq);
2772
2773 return READ_ONCE(vq->broken);
2774}
2775EXPORT_SYMBOL_GPL(virtqueue_is_broken);
2776
2777/*
2778 * This should prevent the device from being used, allowing drivers to
2779 * recover. You may need to grab appropriate locks to flush.
2780 */
2781void virtio_break_device(struct virtio_device *dev)
2782{
2783 struct virtqueue *_vq;
2784
2785 spin_lock(&dev->vqs_list_lock);
2786 list_for_each_entry(_vq, &dev->vqs, list) {
2787 struct vring_virtqueue *vq = to_vvq(_vq);
2788
2789 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2790 WRITE_ONCE(vq->broken, true);
2791 }
2792 spin_unlock(&dev->vqs_list_lock);
2793}
2794EXPORT_SYMBOL_GPL(virtio_break_device);
2795
2796/*
2797 * This should allow the device to be used by the driver. You may
2798 * need to grab appropriate locks to flush the write to
2799 * vq->broken. This should only be used in some specific case e.g
2800 * (probing and restoring). This function should only be called by the
2801 * core, not directly by the driver.
2802 */
2803void __virtio_unbreak_device(struct virtio_device *dev)
2804{
2805 struct virtqueue *_vq;
2806
2807 spin_lock(&dev->vqs_list_lock);
2808 list_for_each_entry(_vq, &dev->vqs, list) {
2809 struct vring_virtqueue *vq = to_vvq(_vq);
2810
2811 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2812 WRITE_ONCE(vq->broken, false);
2813 }
2814 spin_unlock(&dev->vqs_list_lock);
2815}
2816EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
2817
2818dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
2819{
2820 struct vring_virtqueue *vq = to_vvq(_vq);
2821
2822 BUG_ON(!vq->we_own_ring);
2823
2824 if (vq->packed_ring)
2825 return vq->packed.ring_dma_addr;
2826
2827 return vq->split.queue_dma_addr;
2828}
2829EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
2830
2831dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
2832{
2833 struct vring_virtqueue *vq = to_vvq(_vq);
2834
2835 BUG_ON(!vq->we_own_ring);
2836
2837 if (vq->packed_ring)
2838 return vq->packed.driver_event_dma_addr;
2839
2840 return vq->split.queue_dma_addr +
2841 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2842}
2843EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
2844
2845dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
2846{
2847 struct vring_virtqueue *vq = to_vvq(_vq);
2848
2849 BUG_ON(!vq->we_own_ring);
2850
2851 if (vq->packed_ring)
2852 return vq->packed.device_event_dma_addr;
2853
2854 return vq->split.queue_dma_addr +
2855 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2856}
2857EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
2858
2859/* Only available for split ring */
2860const struct vring *virtqueue_get_vring(struct virtqueue *vq)
2861{
2862 return &to_vvq(vq)->split.vring;
2863}
2864EXPORT_SYMBOL_GPL(virtqueue_get_vring);
2865
2866MODULE_LICENSE("GPL");