Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VMware VMCI Driver
4 *
5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 */
7
8#include <linux/vmw_vmci_defs.h>
9#include <linux/vmw_vmci_api.h>
10#include <linux/highmem.h>
11#include <linux/kernel.h>
12#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/pagemap.h>
16#include <linux/pci.h>
17#include <linux/sched.h>
18#include <linux/slab.h>
19#include <linux/uio.h>
20#include <linux/wait.h>
21#include <linux/vmalloc.h>
22#include <linux/skbuff.h>
23
24#include "vmci_handle_array.h"
25#include "vmci_queue_pair.h"
26#include "vmci_datagram.h"
27#include "vmci_resource.h"
28#include "vmci_context.h"
29#include "vmci_driver.h"
30#include "vmci_event.h"
31#include "vmci_route.h"
32
33/*
34 * In the following, we will distinguish between two kinds of VMX processes -
35 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
36 * VMCI page files in the VMX and supporting VM to VM communication and the
37 * newer ones that use the guest memory directly. We will in the following
38 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
39 * new-style VMX'en.
40 *
41 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
42 * removed for readability) - see below for more details on the transtions:
43 *
44 * -------------- NEW -------------
45 * | |
46 * \_/ \_/
47 * CREATED_NO_MEM <-----------------> CREATED_MEM
48 * | | |
49 * | o-----------------------o |
50 * | | |
51 * \_/ \_/ \_/
52 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
53 * | | |
54 * | o----------------------o |
55 * | | |
56 * \_/ \_/ \_/
57 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
58 * | |
59 * | |
60 * -------------> gone <-------------
61 *
62 * In more detail. When a VMCI queue pair is first created, it will be in the
63 * VMCIQPB_NEW state. It will then move into one of the following states:
64 *
65 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
66 *
67 * - the created was performed by a host endpoint, in which case there is
68 * no backing memory yet.
69 *
70 * - the create was initiated by an old-style VMX, that uses
71 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
72 * a later point in time. This state can be distinguished from the one
73 * above by the context ID of the creator. A host side is not allowed to
74 * attach until the page store has been set.
75 *
76 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
77 * is created by a VMX using the queue pair device backend that
78 * sets the UVAs of the queue pair immediately and stores the
79 * information for later attachers. At this point, it is ready for
80 * the host side to attach to it.
81 *
82 * Once the queue pair is in one of the created states (with the exception of
83 * the case mentioned for older VMX'en above), it is possible to attach to the
84 * queue pair. Again we have two new states possible:
85 *
86 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
87 * paths:
88 *
89 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
90 * pair, and attaches to a queue pair previously created by the host side.
91 *
92 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
93 * already created by a guest.
94 *
95 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
96 * vmci_qp_broker_set_page_store (see below).
97 *
98 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
99 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
100 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
101 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
102 * will be entered.
103 *
104 * From the attached queue pair, the queue pair can enter the shutdown states
105 * when either side of the queue pair detaches. If the guest side detaches
106 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
107 * the content of the queue pair will no longer be available. If the host
108 * side detaches first, the queue pair will either enter the
109 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
110 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
111 * (e.g., the host detaches while a guest is stunned).
112 *
113 * New-style VMX'en will also unmap guest memory, if the guest is
114 * quiesced, e.g., during a snapshot operation. In that case, the guest
115 * memory will no longer be available, and the queue pair will transition from
116 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
117 * in which case the queue pair will transition from the *_NO_MEM state at that
118 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
119 * since the peer may have either attached or detached in the meantime. The
120 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
121 * *_MEM state, and vice versa.
122 */
123
124/* The Kernel specific component of the struct vmci_queue structure. */
125struct vmci_queue_kern_if {
126 struct mutex __mutex; /* Protects the queue. */
127 struct mutex *mutex; /* Shared by producer and consumer queues. */
128 size_t num_pages; /* Number of pages incl. header. */
129 bool host; /* Host or guest? */
130 union {
131 struct {
132 dma_addr_t *pas;
133 void **vas;
134 } g; /* Used by the guest. */
135 struct {
136 struct page **page;
137 struct page **header_page;
138 } h; /* Used by the host. */
139 } u;
140};
141
142/*
143 * This structure is opaque to the clients.
144 */
145struct vmci_qp {
146 struct vmci_handle handle;
147 struct vmci_queue *produce_q;
148 struct vmci_queue *consume_q;
149 u64 produce_q_size;
150 u64 consume_q_size;
151 u32 peer;
152 u32 flags;
153 u32 priv_flags;
154 bool guest_endpoint;
155 unsigned int blocked;
156 unsigned int generation;
157 wait_queue_head_t event;
158};
159
160enum qp_broker_state {
161 VMCIQPB_NEW,
162 VMCIQPB_CREATED_NO_MEM,
163 VMCIQPB_CREATED_MEM,
164 VMCIQPB_ATTACHED_NO_MEM,
165 VMCIQPB_ATTACHED_MEM,
166 VMCIQPB_SHUTDOWN_NO_MEM,
167 VMCIQPB_SHUTDOWN_MEM,
168 VMCIQPB_GONE
169};
170
171#define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
172 _qpb->state == VMCIQPB_ATTACHED_MEM || \
173 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
174
175/*
176 * In the queue pair broker, we always use the guest point of view for
177 * the produce and consume queue values and references, e.g., the
178 * produce queue size stored is the guests produce queue size. The
179 * host endpoint will need to swap these around. The only exception is
180 * the local queue pairs on the host, in which case the host endpoint
181 * that creates the queue pair will have the right orientation, and
182 * the attaching host endpoint will need to swap.
183 */
184struct qp_entry {
185 struct list_head list_item;
186 struct vmci_handle handle;
187 u32 peer;
188 u32 flags;
189 u64 produce_size;
190 u64 consume_size;
191 u32 ref_count;
192};
193
194struct qp_broker_entry {
195 struct vmci_resource resource;
196 struct qp_entry qp;
197 u32 create_id;
198 u32 attach_id;
199 enum qp_broker_state state;
200 bool require_trusted_attach;
201 bool created_by_trusted;
202 bool vmci_page_files; /* Created by VMX using VMCI page files */
203 struct vmci_queue *produce_q;
204 struct vmci_queue *consume_q;
205 struct vmci_queue_header saved_produce_q;
206 struct vmci_queue_header saved_consume_q;
207 vmci_event_release_cb wakeup_cb;
208 void *client_data;
209 void *local_mem; /* Kernel memory for local queue pair */
210};
211
212struct qp_guest_endpoint {
213 struct vmci_resource resource;
214 struct qp_entry qp;
215 u64 num_ppns;
216 void *produce_q;
217 void *consume_q;
218 struct ppn_set ppn_set;
219};
220
221struct qp_list {
222 struct list_head head;
223 struct mutex mutex; /* Protect queue list. */
224};
225
226static struct qp_list qp_broker_list = {
227 .head = LIST_HEAD_INIT(qp_broker_list.head),
228 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
229};
230
231static struct qp_list qp_guest_endpoints = {
232 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
233 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
234};
235
236#define INVALID_VMCI_GUEST_MEM_ID 0
237#define QPE_NUM_PAGES(_QPE) ((u32) \
238 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
239 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
240#define QP_SIZES_ARE_VALID(_prod_qsize, _cons_qsize) \
241 ((_prod_qsize) + (_cons_qsize) >= max(_prod_qsize, _cons_qsize) && \
242 (_prod_qsize) + (_cons_qsize) <= VMCI_MAX_GUEST_QP_MEMORY)
243
244/*
245 * Frees kernel VA space for a given queue and its queue header, and
246 * frees physical data pages.
247 */
248static void qp_free_queue(void *q, u64 size)
249{
250 struct vmci_queue *queue = q;
251
252 if (queue) {
253 u64 i;
254
255 /* Given size does not include header, so add in a page here. */
256 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
257 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
258 queue->kernel_if->u.g.vas[i],
259 queue->kernel_if->u.g.pas[i]);
260 }
261
262 vfree(queue);
263 }
264}
265
266/*
267 * Allocates kernel queue pages of specified size with IOMMU mappings,
268 * plus space for the queue structure/kernel interface and the queue
269 * header.
270 */
271static void *qp_alloc_queue(u64 size, u32 flags)
272{
273 u64 i;
274 struct vmci_queue *queue;
275 size_t pas_size;
276 size_t vas_size;
277 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
278 u64 num_pages;
279
280 if (size > SIZE_MAX - PAGE_SIZE)
281 return NULL;
282 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
283 if (num_pages >
284 (SIZE_MAX - queue_size) /
285 (sizeof(*queue->kernel_if->u.g.pas) +
286 sizeof(*queue->kernel_if->u.g.vas)))
287 return NULL;
288
289 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
290 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
291 queue_size += pas_size + vas_size;
292
293 queue = vmalloc(queue_size);
294 if (!queue)
295 return NULL;
296
297 queue->q_header = NULL;
298 queue->saved_header = NULL;
299 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
300 queue->kernel_if->mutex = NULL;
301 queue->kernel_if->num_pages = num_pages;
302 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
303 queue->kernel_if->u.g.vas =
304 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
305 queue->kernel_if->host = false;
306
307 for (i = 0; i < num_pages; i++) {
308 queue->kernel_if->u.g.vas[i] =
309 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
310 &queue->kernel_if->u.g.pas[i],
311 GFP_KERNEL);
312 if (!queue->kernel_if->u.g.vas[i]) {
313 /* Size excl. the header. */
314 qp_free_queue(queue, i * PAGE_SIZE);
315 return NULL;
316 }
317 }
318
319 /* Queue header is the first page. */
320 queue->q_header = queue->kernel_if->u.g.vas[0];
321
322 return queue;
323}
324
325/*
326 * Copies from a given buffer or iovector to a VMCI Queue. Uses
327 * kmap_local_page() to dynamically map required portions of the queue
328 * by traversing the offset -> page translation structure for the queue.
329 * Assumes that offset + size does not wrap around in the queue.
330 */
331static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
332 u64 queue_offset,
333 struct iov_iter *from,
334 size_t size)
335{
336 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
337 size_t bytes_copied = 0;
338
339 while (bytes_copied < size) {
340 const u64 page_index =
341 (queue_offset + bytes_copied) / PAGE_SIZE;
342 const size_t page_offset =
343 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
344 void *va;
345 size_t to_copy;
346
347 if (kernel_if->host)
348 va = kmap_local_page(kernel_if->u.h.page[page_index]);
349 else
350 va = kernel_if->u.g.vas[page_index + 1];
351 /* Skip header. */
352
353 if (size - bytes_copied > PAGE_SIZE - page_offset)
354 /* Enough payload to fill up from this page. */
355 to_copy = PAGE_SIZE - page_offset;
356 else
357 to_copy = size - bytes_copied;
358
359 if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
360 from)) {
361 if (kernel_if->host)
362 kunmap_local(va);
363 return VMCI_ERROR_INVALID_ARGS;
364 }
365 bytes_copied += to_copy;
366 if (kernel_if->host)
367 kunmap_local(va);
368 }
369
370 return VMCI_SUCCESS;
371}
372
373/*
374 * Copies to a given buffer or iovector from a VMCI Queue. Uses
375 * kmap_local_page() to dynamically map required portions of the queue
376 * by traversing the offset -> page translation structure for the queue.
377 * Assumes that offset + size does not wrap around in the queue.
378 */
379static int qp_memcpy_from_queue_iter(struct iov_iter *to,
380 const struct vmci_queue *queue,
381 u64 queue_offset, size_t size)
382{
383 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
384 size_t bytes_copied = 0;
385
386 while (bytes_copied < size) {
387 const u64 page_index =
388 (queue_offset + bytes_copied) / PAGE_SIZE;
389 const size_t page_offset =
390 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
391 void *va;
392 size_t to_copy;
393 int err;
394
395 if (kernel_if->host)
396 va = kmap_local_page(kernel_if->u.h.page[page_index]);
397 else
398 va = kernel_if->u.g.vas[page_index + 1];
399 /* Skip header. */
400
401 if (size - bytes_copied > PAGE_SIZE - page_offset)
402 /* Enough payload to fill up this page. */
403 to_copy = PAGE_SIZE - page_offset;
404 else
405 to_copy = size - bytes_copied;
406
407 err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
408 if (err != to_copy) {
409 if (kernel_if->host)
410 kunmap_local(va);
411 return VMCI_ERROR_INVALID_ARGS;
412 }
413 bytes_copied += to_copy;
414 if (kernel_if->host)
415 kunmap_local(va);
416 }
417
418 return VMCI_SUCCESS;
419}
420
421/*
422 * Allocates two list of PPNs --- one for the pages in the produce queue,
423 * and the other for the pages in the consume queue. Intializes the list
424 * of PPNs with the page frame numbers of the KVA for the two queues (and
425 * the queue headers).
426 */
427static int qp_alloc_ppn_set(void *prod_q,
428 u64 num_produce_pages,
429 void *cons_q,
430 u64 num_consume_pages, struct ppn_set *ppn_set)
431{
432 u64 *produce_ppns;
433 u64 *consume_ppns;
434 struct vmci_queue *produce_q = prod_q;
435 struct vmci_queue *consume_q = cons_q;
436 u64 i;
437
438 if (!produce_q || !num_produce_pages || !consume_q ||
439 !num_consume_pages || !ppn_set)
440 return VMCI_ERROR_INVALID_ARGS;
441
442 if (ppn_set->initialized)
443 return VMCI_ERROR_ALREADY_EXISTS;
444
445 produce_ppns =
446 kmalloc_array(num_produce_pages, sizeof(*produce_ppns),
447 GFP_KERNEL);
448 if (!produce_ppns)
449 return VMCI_ERROR_NO_MEM;
450
451 consume_ppns =
452 kmalloc_array(num_consume_pages, sizeof(*consume_ppns),
453 GFP_KERNEL);
454 if (!consume_ppns) {
455 kfree(produce_ppns);
456 return VMCI_ERROR_NO_MEM;
457 }
458
459 for (i = 0; i < num_produce_pages; i++)
460 produce_ppns[i] =
461 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
462
463 for (i = 0; i < num_consume_pages; i++)
464 consume_ppns[i] =
465 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
466
467 ppn_set->num_produce_pages = num_produce_pages;
468 ppn_set->num_consume_pages = num_consume_pages;
469 ppn_set->produce_ppns = produce_ppns;
470 ppn_set->consume_ppns = consume_ppns;
471 ppn_set->initialized = true;
472 return VMCI_SUCCESS;
473}
474
475/*
476 * Frees the two list of PPNs for a queue pair.
477 */
478static void qp_free_ppn_set(struct ppn_set *ppn_set)
479{
480 if (ppn_set->initialized) {
481 /* Do not call these functions on NULL inputs. */
482 kfree(ppn_set->produce_ppns);
483 kfree(ppn_set->consume_ppns);
484 }
485 memset(ppn_set, 0, sizeof(*ppn_set));
486}
487
488/*
489 * Populates the list of PPNs in the hypercall structure with the PPNS
490 * of the produce queue and the consume queue.
491 */
492static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
493{
494 if (vmci_use_ppn64()) {
495 memcpy(call_buf, ppn_set->produce_ppns,
496 ppn_set->num_produce_pages *
497 sizeof(*ppn_set->produce_ppns));
498 memcpy(call_buf +
499 ppn_set->num_produce_pages *
500 sizeof(*ppn_set->produce_ppns),
501 ppn_set->consume_ppns,
502 ppn_set->num_consume_pages *
503 sizeof(*ppn_set->consume_ppns));
504 } else {
505 int i;
506 u32 *ppns = (u32 *) call_buf;
507
508 for (i = 0; i < ppn_set->num_produce_pages; i++)
509 ppns[i] = (u32) ppn_set->produce_ppns[i];
510
511 ppns = &ppns[ppn_set->num_produce_pages];
512
513 for (i = 0; i < ppn_set->num_consume_pages; i++)
514 ppns[i] = (u32) ppn_set->consume_ppns[i];
515 }
516
517 return VMCI_SUCCESS;
518}
519
520/*
521 * Allocates kernel VA space of specified size plus space for the queue
522 * and kernel interface. This is different from the guest queue allocator,
523 * because we do not allocate our own queue header/data pages here but
524 * share those of the guest.
525 */
526static struct vmci_queue *qp_host_alloc_queue(u64 size)
527{
528 struct vmci_queue *queue;
529 size_t queue_page_size;
530 u64 num_pages;
531 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
532
533 if (size > min_t(size_t, VMCI_MAX_GUEST_QP_MEMORY, SIZE_MAX - PAGE_SIZE))
534 return NULL;
535 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
536 if (num_pages > (SIZE_MAX - queue_size) /
537 sizeof(*queue->kernel_if->u.h.page))
538 return NULL;
539
540 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
541
542 if (queue_size + queue_page_size > KMALLOC_MAX_SIZE)
543 return NULL;
544
545 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
546 if (queue) {
547 queue->q_header = NULL;
548 queue->saved_header = NULL;
549 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
550 queue->kernel_if->host = true;
551 queue->kernel_if->mutex = NULL;
552 queue->kernel_if->num_pages = num_pages;
553 queue->kernel_if->u.h.header_page =
554 (struct page **)((u8 *)queue + queue_size);
555 queue->kernel_if->u.h.page =
556 &queue->kernel_if->u.h.header_page[1];
557 }
558
559 return queue;
560}
561
562/*
563 * Frees kernel memory for a given queue (header plus translation
564 * structure).
565 */
566static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
567{
568 kfree(queue);
569}
570
571/*
572 * Initialize the mutex for the pair of queues. This mutex is used to
573 * protect the q_header and the buffer from changing out from under any
574 * users of either queue. Of course, it's only any good if the mutexes
575 * are actually acquired. Queue structure must lie on non-paged memory
576 * or we cannot guarantee access to the mutex.
577 */
578static void qp_init_queue_mutex(struct vmci_queue *produce_q,
579 struct vmci_queue *consume_q)
580{
581 /*
582 * Only the host queue has shared state - the guest queues do not
583 * need to synchronize access using a queue mutex.
584 */
585
586 if (produce_q->kernel_if->host) {
587 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
588 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
589 mutex_init(produce_q->kernel_if->mutex);
590 }
591}
592
593/*
594 * Cleans up the mutex for the pair of queues.
595 */
596static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
597 struct vmci_queue *consume_q)
598{
599 if (produce_q->kernel_if->host) {
600 produce_q->kernel_if->mutex = NULL;
601 consume_q->kernel_if->mutex = NULL;
602 }
603}
604
605/*
606 * Acquire the mutex for the queue. Note that the produce_q and
607 * the consume_q share a mutex. So, only one of the two need to
608 * be passed in to this routine. Either will work just fine.
609 */
610static void qp_acquire_queue_mutex(struct vmci_queue *queue)
611{
612 if (queue->kernel_if->host)
613 mutex_lock(queue->kernel_if->mutex);
614}
615
616/*
617 * Release the mutex for the queue. Note that the produce_q and
618 * the consume_q share a mutex. So, only one of the two need to
619 * be passed in to this routine. Either will work just fine.
620 */
621static void qp_release_queue_mutex(struct vmci_queue *queue)
622{
623 if (queue->kernel_if->host)
624 mutex_unlock(queue->kernel_if->mutex);
625}
626
627/*
628 * Helper function to release pages in the PageStoreAttachInfo
629 * previously obtained using get_user_pages.
630 */
631static void qp_release_pages(struct page **pages,
632 u64 num_pages, bool dirty)
633{
634 int i;
635
636 for (i = 0; i < num_pages; i++) {
637 if (dirty)
638 set_page_dirty_lock(pages[i]);
639
640 put_page(pages[i]);
641 pages[i] = NULL;
642 }
643}
644
645/*
646 * Lock the user pages referenced by the {produce,consume}Buffer
647 * struct into memory and populate the {produce,consume}Pages
648 * arrays in the attach structure with them.
649 */
650static int qp_host_get_user_memory(u64 produce_uva,
651 u64 consume_uva,
652 struct vmci_queue *produce_q,
653 struct vmci_queue *consume_q)
654{
655 int retval;
656 int err = VMCI_SUCCESS;
657
658 retval = get_user_pages_fast((uintptr_t) produce_uva,
659 produce_q->kernel_if->num_pages,
660 FOLL_WRITE,
661 produce_q->kernel_if->u.h.header_page);
662 if (retval < (int)produce_q->kernel_if->num_pages) {
663 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
664 retval);
665 if (retval > 0)
666 qp_release_pages(produce_q->kernel_if->u.h.header_page,
667 retval, false);
668 err = VMCI_ERROR_NO_MEM;
669 goto out;
670 }
671
672 retval = get_user_pages_fast((uintptr_t) consume_uva,
673 consume_q->kernel_if->num_pages,
674 FOLL_WRITE,
675 consume_q->kernel_if->u.h.header_page);
676 if (retval < (int)consume_q->kernel_if->num_pages) {
677 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
678 retval);
679 if (retval > 0)
680 qp_release_pages(consume_q->kernel_if->u.h.header_page,
681 retval, false);
682 qp_release_pages(produce_q->kernel_if->u.h.header_page,
683 produce_q->kernel_if->num_pages, false);
684 err = VMCI_ERROR_NO_MEM;
685 }
686
687 out:
688 return err;
689}
690
691/*
692 * Registers the specification of the user pages used for backing a queue
693 * pair. Enough information to map in pages is stored in the OS specific
694 * part of the struct vmci_queue structure.
695 */
696static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
697 struct vmci_queue *produce_q,
698 struct vmci_queue *consume_q)
699{
700 u64 produce_uva;
701 u64 consume_uva;
702
703 /*
704 * The new style and the old style mapping only differs in
705 * that we either get a single or two UVAs, so we split the
706 * single UVA range at the appropriate spot.
707 */
708 produce_uva = page_store->pages;
709 consume_uva = page_store->pages +
710 produce_q->kernel_if->num_pages * PAGE_SIZE;
711 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
712 consume_q);
713}
714
715/*
716 * Releases and removes the references to user pages stored in the attach
717 * struct. Pages are released from the page cache and may become
718 * swappable again.
719 */
720static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
721 struct vmci_queue *consume_q)
722{
723 qp_release_pages(produce_q->kernel_if->u.h.header_page,
724 produce_q->kernel_if->num_pages, true);
725 memset(produce_q->kernel_if->u.h.header_page, 0,
726 sizeof(*produce_q->kernel_if->u.h.header_page) *
727 produce_q->kernel_if->num_pages);
728 qp_release_pages(consume_q->kernel_if->u.h.header_page,
729 consume_q->kernel_if->num_pages, true);
730 memset(consume_q->kernel_if->u.h.header_page, 0,
731 sizeof(*consume_q->kernel_if->u.h.header_page) *
732 consume_q->kernel_if->num_pages);
733}
734
735/*
736 * Once qp_host_register_user_memory has been performed on a
737 * queue, the queue pair headers can be mapped into the
738 * kernel. Once mapped, they must be unmapped with
739 * qp_host_unmap_queues prior to calling
740 * qp_host_unregister_user_memory.
741 * Pages are pinned.
742 */
743static int qp_host_map_queues(struct vmci_queue *produce_q,
744 struct vmci_queue *consume_q)
745{
746 int result;
747
748 if (!produce_q->q_header || !consume_q->q_header) {
749 struct page *headers[2];
750
751 if (produce_q->q_header != consume_q->q_header)
752 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
753
754 if (produce_q->kernel_if->u.h.header_page == NULL ||
755 *produce_q->kernel_if->u.h.header_page == NULL)
756 return VMCI_ERROR_UNAVAILABLE;
757
758 headers[0] = *produce_q->kernel_if->u.h.header_page;
759 headers[1] = *consume_q->kernel_if->u.h.header_page;
760
761 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
762 if (produce_q->q_header != NULL) {
763 consume_q->q_header =
764 (struct vmci_queue_header *)((u8 *)
765 produce_q->q_header +
766 PAGE_SIZE);
767 result = VMCI_SUCCESS;
768 } else {
769 pr_warn("vmap failed\n");
770 result = VMCI_ERROR_NO_MEM;
771 }
772 } else {
773 result = VMCI_SUCCESS;
774 }
775
776 return result;
777}
778
779/*
780 * Unmaps previously mapped queue pair headers from the kernel.
781 * Pages are unpinned.
782 */
783static int qp_host_unmap_queues(u32 gid,
784 struct vmci_queue *produce_q,
785 struct vmci_queue *consume_q)
786{
787 if (produce_q->q_header) {
788 if (produce_q->q_header < consume_q->q_header)
789 vunmap(produce_q->q_header);
790 else
791 vunmap(consume_q->q_header);
792
793 produce_q->q_header = NULL;
794 consume_q->q_header = NULL;
795 }
796
797 return VMCI_SUCCESS;
798}
799
800/*
801 * Finds the entry in the list corresponding to a given handle. Assumes
802 * that the list is locked.
803 */
804static struct qp_entry *qp_list_find(struct qp_list *qp_list,
805 struct vmci_handle handle)
806{
807 struct qp_entry *entry;
808
809 if (vmci_handle_is_invalid(handle))
810 return NULL;
811
812 list_for_each_entry(entry, &qp_list->head, list_item) {
813 if (vmci_handle_is_equal(entry->handle, handle))
814 return entry;
815 }
816
817 return NULL;
818}
819
820/*
821 * Finds the entry in the list corresponding to a given handle.
822 */
823static struct qp_guest_endpoint *
824qp_guest_handle_to_entry(struct vmci_handle handle)
825{
826 struct qp_guest_endpoint *entry;
827 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
828
829 entry = qp ? container_of(
830 qp, struct qp_guest_endpoint, qp) : NULL;
831 return entry;
832}
833
834/*
835 * Finds the entry in the list corresponding to a given handle.
836 */
837static struct qp_broker_entry *
838qp_broker_handle_to_entry(struct vmci_handle handle)
839{
840 struct qp_broker_entry *entry;
841 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
842
843 entry = qp ? container_of(
844 qp, struct qp_broker_entry, qp) : NULL;
845 return entry;
846}
847
848/*
849 * Dispatches a queue pair event message directly into the local event
850 * queue.
851 */
852static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
853{
854 u32 context_id = vmci_get_context_id();
855 struct vmci_event_qp ev;
856
857 memset(&ev, 0, sizeof(ev));
858 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
859 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
860 VMCI_CONTEXT_RESOURCE_ID);
861 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
862 ev.msg.event_data.event =
863 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
864 ev.payload.peer_id = context_id;
865 ev.payload.handle = handle;
866
867 return vmci_event_dispatch(&ev.msg.hdr);
868}
869
870/*
871 * Allocates and initializes a qp_guest_endpoint structure.
872 * Allocates a queue_pair rid (and handle) iff the given entry has
873 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
874 * are reserved handles. Assumes that the QP list mutex is held
875 * by the caller.
876 */
877static struct qp_guest_endpoint *
878qp_guest_endpoint_create(struct vmci_handle handle,
879 u32 peer,
880 u32 flags,
881 u64 produce_size,
882 u64 consume_size,
883 void *produce_q,
884 void *consume_q)
885{
886 int result;
887 struct qp_guest_endpoint *entry;
888 /* One page each for the queue headers. */
889 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
890 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
891
892 if (vmci_handle_is_invalid(handle)) {
893 u32 context_id = vmci_get_context_id();
894
895 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
896 }
897
898 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
899 if (entry) {
900 entry->qp.peer = peer;
901 entry->qp.flags = flags;
902 entry->qp.produce_size = produce_size;
903 entry->qp.consume_size = consume_size;
904 entry->qp.ref_count = 0;
905 entry->num_ppns = num_ppns;
906 entry->produce_q = produce_q;
907 entry->consume_q = consume_q;
908 INIT_LIST_HEAD(&entry->qp.list_item);
909
910 /* Add resource obj */
911 result = vmci_resource_add(&entry->resource,
912 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
913 handle);
914 entry->qp.handle = vmci_resource_handle(&entry->resource);
915 if ((result != VMCI_SUCCESS) ||
916 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
917 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
918 handle.context, handle.resource, result);
919 kfree(entry);
920 entry = NULL;
921 }
922 }
923 return entry;
924}
925
926/*
927 * Frees a qp_guest_endpoint structure.
928 */
929static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
930{
931 qp_free_ppn_set(&entry->ppn_set);
932 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
933 qp_free_queue(entry->produce_q, entry->qp.produce_size);
934 qp_free_queue(entry->consume_q, entry->qp.consume_size);
935 /* Unlink from resource hash table and free callback */
936 vmci_resource_remove(&entry->resource);
937
938 kfree(entry);
939}
940
941/*
942 * Helper to make a queue_pairAlloc hypercall when the driver is
943 * supporting a guest device.
944 */
945static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
946{
947 struct vmci_qp_alloc_msg *alloc_msg;
948 size_t msg_size;
949 size_t ppn_size;
950 int result;
951
952 if (!entry || entry->num_ppns <= 2)
953 return VMCI_ERROR_INVALID_ARGS;
954
955 ppn_size = vmci_use_ppn64() ? sizeof(u64) : sizeof(u32);
956 msg_size = sizeof(*alloc_msg) +
957 (size_t) entry->num_ppns * ppn_size;
958 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
959 if (!alloc_msg)
960 return VMCI_ERROR_NO_MEM;
961
962 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
963 VMCI_QUEUEPAIR_ALLOC);
964 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
965 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
966 alloc_msg->handle = entry->qp.handle;
967 alloc_msg->peer = entry->qp.peer;
968 alloc_msg->flags = entry->qp.flags;
969 alloc_msg->produce_size = entry->qp.produce_size;
970 alloc_msg->consume_size = entry->qp.consume_size;
971 alloc_msg->num_ppns = entry->num_ppns;
972
973 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
974 &entry->ppn_set);
975 if (result == VMCI_SUCCESS)
976 result = vmci_send_datagram(&alloc_msg->hdr);
977
978 kfree(alloc_msg);
979
980 return result;
981}
982
983/*
984 * Helper to make a queue_pairDetach hypercall when the driver is
985 * supporting a guest device.
986 */
987static int qp_detatch_hypercall(struct vmci_handle handle)
988{
989 struct vmci_qp_detach_msg detach_msg;
990
991 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
992 VMCI_QUEUEPAIR_DETACH);
993 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
994 detach_msg.hdr.payload_size = sizeof(handle);
995 detach_msg.handle = handle;
996
997 return vmci_send_datagram(&detach_msg.hdr);
998}
999
1000/*
1001 * Adds the given entry to the list. Assumes that the list is locked.
1002 */
1003static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
1004{
1005 if (entry)
1006 list_add(&entry->list_item, &qp_list->head);
1007}
1008
1009/*
1010 * Removes the given entry from the list. Assumes that the list is locked.
1011 */
1012static void qp_list_remove_entry(struct qp_list *qp_list,
1013 struct qp_entry *entry)
1014{
1015 if (entry)
1016 list_del(&entry->list_item);
1017}
1018
1019/*
1020 * Helper for VMCI queue_pair detach interface. Frees the physical
1021 * pages for the queue pair.
1022 */
1023static int qp_detatch_guest_work(struct vmci_handle handle)
1024{
1025 int result;
1026 struct qp_guest_endpoint *entry;
1027 u32 ref_count = ~0; /* To avoid compiler warning below */
1028
1029 mutex_lock(&qp_guest_endpoints.mutex);
1030
1031 entry = qp_guest_handle_to_entry(handle);
1032 if (!entry) {
1033 mutex_unlock(&qp_guest_endpoints.mutex);
1034 return VMCI_ERROR_NOT_FOUND;
1035 }
1036
1037 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1038 result = VMCI_SUCCESS;
1039
1040 if (entry->qp.ref_count > 1) {
1041 result = qp_notify_peer_local(false, handle);
1042 /*
1043 * We can fail to notify a local queuepair
1044 * because we can't allocate. We still want
1045 * to release the entry if that happens, so
1046 * don't bail out yet.
1047 */
1048 }
1049 } else {
1050 result = qp_detatch_hypercall(handle);
1051 if (result < VMCI_SUCCESS) {
1052 /*
1053 * We failed to notify a non-local queuepair.
1054 * That other queuepair might still be
1055 * accessing the shared memory, so don't
1056 * release the entry yet. It will get cleaned
1057 * up by VMCIqueue_pair_Exit() if necessary
1058 * (assuming we are going away, otherwise why
1059 * did this fail?).
1060 */
1061
1062 mutex_unlock(&qp_guest_endpoints.mutex);
1063 return result;
1064 }
1065 }
1066
1067 /*
1068 * If we get here then we either failed to notify a local queuepair, or
1069 * we succeeded in all cases. Release the entry if required.
1070 */
1071
1072 entry->qp.ref_count--;
1073 if (entry->qp.ref_count == 0)
1074 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1075
1076 /* If we didn't remove the entry, this could change once we unlock. */
1077 if (entry)
1078 ref_count = entry->qp.ref_count;
1079
1080 mutex_unlock(&qp_guest_endpoints.mutex);
1081
1082 if (ref_count == 0)
1083 qp_guest_endpoint_destroy(entry);
1084
1085 return result;
1086}
1087
1088/*
1089 * This functions handles the actual allocation of a VMCI queue
1090 * pair guest endpoint. Allocates physical pages for the queue
1091 * pair. It makes OS dependent calls through generic wrappers.
1092 */
1093static int qp_alloc_guest_work(struct vmci_handle *handle,
1094 struct vmci_queue **produce_q,
1095 u64 produce_size,
1096 struct vmci_queue **consume_q,
1097 u64 consume_size,
1098 u32 peer,
1099 u32 flags,
1100 u32 priv_flags)
1101{
1102 const u64 num_produce_pages =
1103 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1104 const u64 num_consume_pages =
1105 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1106 void *my_produce_q = NULL;
1107 void *my_consume_q = NULL;
1108 int result;
1109 struct qp_guest_endpoint *queue_pair_entry = NULL;
1110
1111 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1112 return VMCI_ERROR_NO_ACCESS;
1113
1114 mutex_lock(&qp_guest_endpoints.mutex);
1115
1116 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1117 if (queue_pair_entry) {
1118 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1119 /* Local attach case. */
1120 if (queue_pair_entry->qp.ref_count > 1) {
1121 pr_devel("Error attempting to attach more than once\n");
1122 result = VMCI_ERROR_UNAVAILABLE;
1123 goto error_keep_entry;
1124 }
1125
1126 if (queue_pair_entry->qp.produce_size != consume_size ||
1127 queue_pair_entry->qp.consume_size !=
1128 produce_size ||
1129 queue_pair_entry->qp.flags !=
1130 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1131 pr_devel("Error mismatched queue pair in local attach\n");
1132 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1133 goto error_keep_entry;
1134 }
1135
1136 /*
1137 * Do a local attach. We swap the consume and
1138 * produce queues for the attacher and deliver
1139 * an attach event.
1140 */
1141 result = qp_notify_peer_local(true, *handle);
1142 if (result < VMCI_SUCCESS)
1143 goto error_keep_entry;
1144
1145 my_produce_q = queue_pair_entry->consume_q;
1146 my_consume_q = queue_pair_entry->produce_q;
1147 goto out;
1148 }
1149
1150 result = VMCI_ERROR_ALREADY_EXISTS;
1151 goto error_keep_entry;
1152 }
1153
1154 my_produce_q = qp_alloc_queue(produce_size, flags);
1155 if (!my_produce_q) {
1156 pr_warn("Error allocating pages for produce queue\n");
1157 result = VMCI_ERROR_NO_MEM;
1158 goto error;
1159 }
1160
1161 my_consume_q = qp_alloc_queue(consume_size, flags);
1162 if (!my_consume_q) {
1163 pr_warn("Error allocating pages for consume queue\n");
1164 result = VMCI_ERROR_NO_MEM;
1165 goto error;
1166 }
1167
1168 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1169 produce_size, consume_size,
1170 my_produce_q, my_consume_q);
1171 if (!queue_pair_entry) {
1172 pr_warn("Error allocating memory in %s\n", __func__);
1173 result = VMCI_ERROR_NO_MEM;
1174 goto error;
1175 }
1176
1177 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1178 num_consume_pages,
1179 &queue_pair_entry->ppn_set);
1180 if (result < VMCI_SUCCESS) {
1181 pr_warn("qp_alloc_ppn_set failed\n");
1182 goto error;
1183 }
1184
1185 /*
1186 * It's only necessary to notify the host if this queue pair will be
1187 * attached to from another context.
1188 */
1189 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1190 /* Local create case. */
1191 u32 context_id = vmci_get_context_id();
1192
1193 /*
1194 * Enforce similar checks on local queue pairs as we
1195 * do for regular ones. The handle's context must
1196 * match the creator or attacher context id (here they
1197 * are both the current context id) and the
1198 * attach-only flag cannot exist during create. We
1199 * also ensure specified peer is this context or an
1200 * invalid one.
1201 */
1202 if (queue_pair_entry->qp.handle.context != context_id ||
1203 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1204 queue_pair_entry->qp.peer != context_id)) {
1205 result = VMCI_ERROR_NO_ACCESS;
1206 goto error;
1207 }
1208
1209 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1210 result = VMCI_ERROR_NOT_FOUND;
1211 goto error;
1212 }
1213 } else {
1214 result = qp_alloc_hypercall(queue_pair_entry);
1215 if (result < VMCI_SUCCESS) {
1216 pr_devel("qp_alloc_hypercall result = %d\n", result);
1217 goto error;
1218 }
1219 }
1220
1221 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1222 (struct vmci_queue *)my_consume_q);
1223
1224 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1225
1226 out:
1227 queue_pair_entry->qp.ref_count++;
1228 *handle = queue_pair_entry->qp.handle;
1229 *produce_q = (struct vmci_queue *)my_produce_q;
1230 *consume_q = (struct vmci_queue *)my_consume_q;
1231
1232 /*
1233 * We should initialize the queue pair header pages on a local
1234 * queue pair create. For non-local queue pairs, the
1235 * hypervisor initializes the header pages in the create step.
1236 */
1237 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1238 queue_pair_entry->qp.ref_count == 1) {
1239 vmci_q_header_init((*produce_q)->q_header, *handle);
1240 vmci_q_header_init((*consume_q)->q_header, *handle);
1241 }
1242
1243 mutex_unlock(&qp_guest_endpoints.mutex);
1244
1245 return VMCI_SUCCESS;
1246
1247 error:
1248 mutex_unlock(&qp_guest_endpoints.mutex);
1249 if (queue_pair_entry) {
1250 /* The queues will be freed inside the destroy routine. */
1251 qp_guest_endpoint_destroy(queue_pair_entry);
1252 } else {
1253 qp_free_queue(my_produce_q, produce_size);
1254 qp_free_queue(my_consume_q, consume_size);
1255 }
1256 return result;
1257
1258 error_keep_entry:
1259 /* This path should only be used when an existing entry was found. */
1260 mutex_unlock(&qp_guest_endpoints.mutex);
1261 return result;
1262}
1263
1264/*
1265 * The first endpoint issuing a queue pair allocation will create the state
1266 * of the queue pair in the queue pair broker.
1267 *
1268 * If the creator is a guest, it will associate a VMX virtual address range
1269 * with the queue pair as specified by the page_store. For compatibility with
1270 * older VMX'en, that would use a separate step to set the VMX virtual
1271 * address range, the virtual address range can be registered later using
1272 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1273 * used.
1274 *
1275 * If the creator is the host, a page_store of NULL should be used as well,
1276 * since the host is not able to supply a page store for the queue pair.
1277 *
1278 * For older VMX and host callers, the queue pair will be created in the
1279 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1280 * created in VMCOQPB_CREATED_MEM state.
1281 */
1282static int qp_broker_create(struct vmci_handle handle,
1283 u32 peer,
1284 u32 flags,
1285 u32 priv_flags,
1286 u64 produce_size,
1287 u64 consume_size,
1288 struct vmci_qp_page_store *page_store,
1289 struct vmci_ctx *context,
1290 vmci_event_release_cb wakeup_cb,
1291 void *client_data, struct qp_broker_entry **ent)
1292{
1293 struct qp_broker_entry *entry = NULL;
1294 const u32 context_id = vmci_ctx_get_id(context);
1295 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1296 int result;
1297 u64 guest_produce_size;
1298 u64 guest_consume_size;
1299
1300 /* Do not create if the caller asked not to. */
1301 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1302 return VMCI_ERROR_NOT_FOUND;
1303
1304 /*
1305 * Creator's context ID should match handle's context ID or the creator
1306 * must allow the context in handle's context ID as the "peer".
1307 */
1308 if (handle.context != context_id && handle.context != peer)
1309 return VMCI_ERROR_NO_ACCESS;
1310
1311 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1312 return VMCI_ERROR_DST_UNREACHABLE;
1313
1314 /*
1315 * Creator's context ID for local queue pairs should match the
1316 * peer, if a peer is specified.
1317 */
1318 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1319 return VMCI_ERROR_NO_ACCESS;
1320
1321 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1322 if (!entry)
1323 return VMCI_ERROR_NO_MEM;
1324
1325 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1326 /*
1327 * The queue pair broker entry stores values from the guest
1328 * point of view, so a creating host side endpoint should swap
1329 * produce and consume values -- unless it is a local queue
1330 * pair, in which case no swapping is necessary, since the local
1331 * attacher will swap queues.
1332 */
1333
1334 guest_produce_size = consume_size;
1335 guest_consume_size = produce_size;
1336 } else {
1337 guest_produce_size = produce_size;
1338 guest_consume_size = consume_size;
1339 }
1340
1341 entry->qp.handle = handle;
1342 entry->qp.peer = peer;
1343 entry->qp.flags = flags;
1344 entry->qp.produce_size = guest_produce_size;
1345 entry->qp.consume_size = guest_consume_size;
1346 entry->qp.ref_count = 1;
1347 entry->create_id = context_id;
1348 entry->attach_id = VMCI_INVALID_ID;
1349 entry->state = VMCIQPB_NEW;
1350 entry->require_trusted_attach =
1351 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1352 entry->created_by_trusted =
1353 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1354 entry->vmci_page_files = false;
1355 entry->wakeup_cb = wakeup_cb;
1356 entry->client_data = client_data;
1357 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1358 if (entry->produce_q == NULL) {
1359 result = VMCI_ERROR_NO_MEM;
1360 goto error;
1361 }
1362 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1363 if (entry->consume_q == NULL) {
1364 result = VMCI_ERROR_NO_MEM;
1365 goto error;
1366 }
1367
1368 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1369
1370 INIT_LIST_HEAD(&entry->qp.list_item);
1371
1372 if (is_local) {
1373 u8 *tmp;
1374
1375 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1376 PAGE_SIZE, GFP_KERNEL);
1377 if (entry->local_mem == NULL) {
1378 result = VMCI_ERROR_NO_MEM;
1379 goto error;
1380 }
1381 entry->state = VMCIQPB_CREATED_MEM;
1382 entry->produce_q->q_header = entry->local_mem;
1383 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1384 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1385 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1386 } else if (page_store) {
1387 /*
1388 * The VMX already initialized the queue pair headers, so no
1389 * need for the kernel side to do that.
1390 */
1391 result = qp_host_register_user_memory(page_store,
1392 entry->produce_q,
1393 entry->consume_q);
1394 if (result < VMCI_SUCCESS)
1395 goto error;
1396
1397 entry->state = VMCIQPB_CREATED_MEM;
1398 } else {
1399 /*
1400 * A create without a page_store may be either a host
1401 * side create (in which case we are waiting for the
1402 * guest side to supply the memory) or an old style
1403 * queue pair create (in which case we will expect a
1404 * set page store call as the next step).
1405 */
1406 entry->state = VMCIQPB_CREATED_NO_MEM;
1407 }
1408
1409 qp_list_add_entry(&qp_broker_list, &entry->qp);
1410 if (ent != NULL)
1411 *ent = entry;
1412
1413 /* Add to resource obj */
1414 result = vmci_resource_add(&entry->resource,
1415 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1416 handle);
1417 if (result != VMCI_SUCCESS) {
1418 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1419 handle.context, handle.resource, result);
1420 goto error;
1421 }
1422
1423 entry->qp.handle = vmci_resource_handle(&entry->resource);
1424 if (is_local) {
1425 vmci_q_header_init(entry->produce_q->q_header,
1426 entry->qp.handle);
1427 vmci_q_header_init(entry->consume_q->q_header,
1428 entry->qp.handle);
1429 }
1430
1431 vmci_ctx_qp_create(context, entry->qp.handle);
1432
1433 return VMCI_SUCCESS;
1434
1435 error:
1436 if (entry != NULL) {
1437 qp_host_free_queue(entry->produce_q, guest_produce_size);
1438 qp_host_free_queue(entry->consume_q, guest_consume_size);
1439 kfree(entry);
1440 }
1441
1442 return result;
1443}
1444
1445/*
1446 * Enqueues an event datagram to notify the peer VM attached to
1447 * the given queue pair handle about attach/detach event by the
1448 * given VM. Returns Payload size of datagram enqueued on
1449 * success, error code otherwise.
1450 */
1451static int qp_notify_peer(bool attach,
1452 struct vmci_handle handle,
1453 u32 my_id,
1454 u32 peer_id)
1455{
1456 int rv;
1457 struct vmci_event_qp ev;
1458
1459 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1460 peer_id == VMCI_INVALID_ID)
1461 return VMCI_ERROR_INVALID_ARGS;
1462
1463 /*
1464 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1465 * number of pending events from the hypervisor to a given VM
1466 * otherwise a rogue VM could do an arbitrary number of attach
1467 * and detach operations causing memory pressure in the host
1468 * kernel.
1469 */
1470
1471 memset(&ev, 0, sizeof(ev));
1472 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1473 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1474 VMCI_CONTEXT_RESOURCE_ID);
1475 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1476 ev.msg.event_data.event = attach ?
1477 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1478 ev.payload.handle = handle;
1479 ev.payload.peer_id = my_id;
1480
1481 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1482 &ev.msg.hdr, false);
1483 if (rv < VMCI_SUCCESS)
1484 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1485 attach ? "ATTACH" : "DETACH", peer_id);
1486
1487 return rv;
1488}
1489
1490/*
1491 * The second endpoint issuing a queue pair allocation will attach to
1492 * the queue pair registered with the queue pair broker.
1493 *
1494 * If the attacher is a guest, it will associate a VMX virtual address
1495 * range with the queue pair as specified by the page_store. At this
1496 * point, the already attach host endpoint may start using the queue
1497 * pair, and an attach event is sent to it. For compatibility with
1498 * older VMX'en, that used a separate step to set the VMX virtual
1499 * address range, the virtual address range can be registered later
1500 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1501 * NULL should be used, and the attach event will be generated once
1502 * the actual page store has been set.
1503 *
1504 * If the attacher is the host, a page_store of NULL should be used as
1505 * well, since the page store information is already set by the guest.
1506 *
1507 * For new VMX and host callers, the queue pair will be moved to the
1508 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1509 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1510 */
1511static int qp_broker_attach(struct qp_broker_entry *entry,
1512 u32 peer,
1513 u32 flags,
1514 u32 priv_flags,
1515 u64 produce_size,
1516 u64 consume_size,
1517 struct vmci_qp_page_store *page_store,
1518 struct vmci_ctx *context,
1519 vmci_event_release_cb wakeup_cb,
1520 void *client_data,
1521 struct qp_broker_entry **ent)
1522{
1523 const u32 context_id = vmci_ctx_get_id(context);
1524 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1525 int result;
1526
1527 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1528 entry->state != VMCIQPB_CREATED_MEM)
1529 return VMCI_ERROR_UNAVAILABLE;
1530
1531 if (is_local) {
1532 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1533 context_id != entry->create_id) {
1534 return VMCI_ERROR_INVALID_ARGS;
1535 }
1536 } else if (context_id == entry->create_id ||
1537 context_id == entry->attach_id) {
1538 return VMCI_ERROR_ALREADY_EXISTS;
1539 }
1540
1541 if (VMCI_CONTEXT_IS_VM(context_id) &&
1542 VMCI_CONTEXT_IS_VM(entry->create_id))
1543 return VMCI_ERROR_DST_UNREACHABLE;
1544
1545 /*
1546 * If we are attaching from a restricted context then the queuepair
1547 * must have been created by a trusted endpoint.
1548 */
1549 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1550 !entry->created_by_trusted)
1551 return VMCI_ERROR_NO_ACCESS;
1552
1553 /*
1554 * If we are attaching to a queuepair that was created by a restricted
1555 * context then we must be trusted.
1556 */
1557 if (entry->require_trusted_attach &&
1558 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1559 return VMCI_ERROR_NO_ACCESS;
1560
1561 /*
1562 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1563 * control check is not performed.
1564 */
1565 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1566 return VMCI_ERROR_NO_ACCESS;
1567
1568 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1569 /*
1570 * Do not attach if the caller doesn't support Host Queue Pairs
1571 * and a host created this queue pair.
1572 */
1573
1574 if (!vmci_ctx_supports_host_qp(context))
1575 return VMCI_ERROR_INVALID_RESOURCE;
1576
1577 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1578 struct vmci_ctx *create_context;
1579 bool supports_host_qp;
1580
1581 /*
1582 * Do not attach a host to a user created queue pair if that
1583 * user doesn't support host queue pair end points.
1584 */
1585
1586 create_context = vmci_ctx_get(entry->create_id);
1587 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1588 vmci_ctx_put(create_context);
1589
1590 if (!supports_host_qp)
1591 return VMCI_ERROR_INVALID_RESOURCE;
1592 }
1593
1594 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1595 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1596
1597 if (context_id != VMCI_HOST_CONTEXT_ID) {
1598 /*
1599 * The queue pair broker entry stores values from the guest
1600 * point of view, so an attaching guest should match the values
1601 * stored in the entry.
1602 */
1603
1604 if (entry->qp.produce_size != produce_size ||
1605 entry->qp.consume_size != consume_size) {
1606 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1607 }
1608 } else if (entry->qp.produce_size != consume_size ||
1609 entry->qp.consume_size != produce_size) {
1610 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1611 }
1612
1613 if (context_id != VMCI_HOST_CONTEXT_ID) {
1614 /*
1615 * If a guest attached to a queue pair, it will supply
1616 * the backing memory. If this is a pre NOVMVM vmx,
1617 * the backing memory will be supplied by calling
1618 * vmci_qp_broker_set_page_store() following the
1619 * return of the vmci_qp_broker_alloc() call. If it is
1620 * a vmx of version NOVMVM or later, the page store
1621 * must be supplied as part of the
1622 * vmci_qp_broker_alloc call. Under all circumstances
1623 * must the initially created queue pair not have any
1624 * memory associated with it already.
1625 */
1626
1627 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1628 return VMCI_ERROR_INVALID_ARGS;
1629
1630 if (page_store != NULL) {
1631 /*
1632 * Patch up host state to point to guest
1633 * supplied memory. The VMX already
1634 * initialized the queue pair headers, so no
1635 * need for the kernel side to do that.
1636 */
1637
1638 result = qp_host_register_user_memory(page_store,
1639 entry->produce_q,
1640 entry->consume_q);
1641 if (result < VMCI_SUCCESS)
1642 return result;
1643
1644 entry->state = VMCIQPB_ATTACHED_MEM;
1645 } else {
1646 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1647 }
1648 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1649 /*
1650 * The host side is attempting to attach to a queue
1651 * pair that doesn't have any memory associated with
1652 * it. This must be a pre NOVMVM vmx that hasn't set
1653 * the page store information yet, or a quiesced VM.
1654 */
1655
1656 return VMCI_ERROR_UNAVAILABLE;
1657 } else {
1658 /* The host side has successfully attached to a queue pair. */
1659 entry->state = VMCIQPB_ATTACHED_MEM;
1660 }
1661
1662 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1663 result =
1664 qp_notify_peer(true, entry->qp.handle, context_id,
1665 entry->create_id);
1666 if (result < VMCI_SUCCESS)
1667 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1668 entry->create_id, entry->qp.handle.context,
1669 entry->qp.handle.resource);
1670 }
1671
1672 entry->attach_id = context_id;
1673 entry->qp.ref_count++;
1674 if (wakeup_cb) {
1675 entry->wakeup_cb = wakeup_cb;
1676 entry->client_data = client_data;
1677 }
1678
1679 /*
1680 * When attaching to local queue pairs, the context already has
1681 * an entry tracking the queue pair, so don't add another one.
1682 */
1683 if (!is_local)
1684 vmci_ctx_qp_create(context, entry->qp.handle);
1685
1686 if (ent != NULL)
1687 *ent = entry;
1688
1689 return VMCI_SUCCESS;
1690}
1691
1692/*
1693 * queue_pair_Alloc for use when setting up queue pair endpoints
1694 * on the host.
1695 */
1696static int qp_broker_alloc(struct vmci_handle handle,
1697 u32 peer,
1698 u32 flags,
1699 u32 priv_flags,
1700 u64 produce_size,
1701 u64 consume_size,
1702 struct vmci_qp_page_store *page_store,
1703 struct vmci_ctx *context,
1704 vmci_event_release_cb wakeup_cb,
1705 void *client_data,
1706 struct qp_broker_entry **ent,
1707 bool *swap)
1708{
1709 const u32 context_id = vmci_ctx_get_id(context);
1710 bool create;
1711 struct qp_broker_entry *entry = NULL;
1712 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1713 int result;
1714
1715 if (vmci_handle_is_invalid(handle) ||
1716 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1717 !(produce_size || consume_size) ||
1718 !context || context_id == VMCI_INVALID_ID ||
1719 handle.context == VMCI_INVALID_ID) {
1720 return VMCI_ERROR_INVALID_ARGS;
1721 }
1722
1723 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1724 return VMCI_ERROR_INVALID_ARGS;
1725
1726 /*
1727 * In the initial argument check, we ensure that non-vmkernel hosts
1728 * are not allowed to create local queue pairs.
1729 */
1730
1731 mutex_lock(&qp_broker_list.mutex);
1732
1733 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1734 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1735 context_id, handle.context, handle.resource);
1736 mutex_unlock(&qp_broker_list.mutex);
1737 return VMCI_ERROR_ALREADY_EXISTS;
1738 }
1739
1740 if (handle.resource != VMCI_INVALID_ID)
1741 entry = qp_broker_handle_to_entry(handle);
1742
1743 if (!entry) {
1744 create = true;
1745 result =
1746 qp_broker_create(handle, peer, flags, priv_flags,
1747 produce_size, consume_size, page_store,
1748 context, wakeup_cb, client_data, ent);
1749 } else {
1750 create = false;
1751 result =
1752 qp_broker_attach(entry, peer, flags, priv_flags,
1753 produce_size, consume_size, page_store,
1754 context, wakeup_cb, client_data, ent);
1755 }
1756
1757 mutex_unlock(&qp_broker_list.mutex);
1758
1759 if (swap)
1760 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1761 !(create && is_local);
1762
1763 return result;
1764}
1765
1766/*
1767 * This function implements the kernel API for allocating a queue
1768 * pair.
1769 */
1770static int qp_alloc_host_work(struct vmci_handle *handle,
1771 struct vmci_queue **produce_q,
1772 u64 produce_size,
1773 struct vmci_queue **consume_q,
1774 u64 consume_size,
1775 u32 peer,
1776 u32 flags,
1777 u32 priv_flags,
1778 vmci_event_release_cb wakeup_cb,
1779 void *client_data)
1780{
1781 struct vmci_handle new_handle;
1782 struct vmci_ctx *context;
1783 struct qp_broker_entry *entry;
1784 int result;
1785 bool swap;
1786
1787 if (vmci_handle_is_invalid(*handle)) {
1788 new_handle = vmci_make_handle(
1789 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1790 } else
1791 new_handle = *handle;
1792
1793 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1794 entry = NULL;
1795 result =
1796 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1797 produce_size, consume_size, NULL, context,
1798 wakeup_cb, client_data, &entry, &swap);
1799 if (result == VMCI_SUCCESS) {
1800 if (swap) {
1801 /*
1802 * If this is a local queue pair, the attacher
1803 * will swap around produce and consume
1804 * queues.
1805 */
1806
1807 *produce_q = entry->consume_q;
1808 *consume_q = entry->produce_q;
1809 } else {
1810 *produce_q = entry->produce_q;
1811 *consume_q = entry->consume_q;
1812 }
1813
1814 *handle = vmci_resource_handle(&entry->resource);
1815 } else {
1816 *handle = VMCI_INVALID_HANDLE;
1817 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1818 result);
1819 }
1820 vmci_ctx_put(context);
1821 return result;
1822}
1823
1824/*
1825 * Allocates a VMCI queue_pair. Only checks validity of input
1826 * arguments. The real work is done in the host or guest
1827 * specific function.
1828 */
1829int vmci_qp_alloc(struct vmci_handle *handle,
1830 struct vmci_queue **produce_q,
1831 u64 produce_size,
1832 struct vmci_queue **consume_q,
1833 u64 consume_size,
1834 u32 peer,
1835 u32 flags,
1836 u32 priv_flags,
1837 bool guest_endpoint,
1838 vmci_event_release_cb wakeup_cb,
1839 void *client_data)
1840{
1841 if (!handle || !produce_q || !consume_q ||
1842 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1843 return VMCI_ERROR_INVALID_ARGS;
1844
1845 if (guest_endpoint) {
1846 return qp_alloc_guest_work(handle, produce_q,
1847 produce_size, consume_q,
1848 consume_size, peer,
1849 flags, priv_flags);
1850 } else {
1851 return qp_alloc_host_work(handle, produce_q,
1852 produce_size, consume_q,
1853 consume_size, peer, flags,
1854 priv_flags, wakeup_cb, client_data);
1855 }
1856}
1857
1858/*
1859 * This function implements the host kernel API for detaching from
1860 * a queue pair.
1861 */
1862static int qp_detatch_host_work(struct vmci_handle handle)
1863{
1864 int result;
1865 struct vmci_ctx *context;
1866
1867 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1868
1869 result = vmci_qp_broker_detach(handle, context);
1870
1871 vmci_ctx_put(context);
1872 return result;
1873}
1874
1875/*
1876 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1877 * Real work is done in the host or guest specific function.
1878 */
1879static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1880{
1881 if (vmci_handle_is_invalid(handle))
1882 return VMCI_ERROR_INVALID_ARGS;
1883
1884 if (guest_endpoint)
1885 return qp_detatch_guest_work(handle);
1886 else
1887 return qp_detatch_host_work(handle);
1888}
1889
1890/*
1891 * Returns the entry from the head of the list. Assumes that the list is
1892 * locked.
1893 */
1894static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1895{
1896 if (!list_empty(&qp_list->head)) {
1897 struct qp_entry *entry =
1898 list_first_entry(&qp_list->head, struct qp_entry,
1899 list_item);
1900 return entry;
1901 }
1902
1903 return NULL;
1904}
1905
1906void vmci_qp_broker_exit(void)
1907{
1908 struct qp_entry *entry;
1909 struct qp_broker_entry *be;
1910
1911 mutex_lock(&qp_broker_list.mutex);
1912
1913 while ((entry = qp_list_get_head(&qp_broker_list))) {
1914 be = (struct qp_broker_entry *)entry;
1915
1916 qp_list_remove_entry(&qp_broker_list, entry);
1917 kfree(be);
1918 }
1919
1920 mutex_unlock(&qp_broker_list.mutex);
1921}
1922
1923/*
1924 * Requests that a queue pair be allocated with the VMCI queue
1925 * pair broker. Allocates a queue pair entry if one does not
1926 * exist. Attaches to one if it exists, and retrieves the page
1927 * files backing that queue_pair. Assumes that the queue pair
1928 * broker lock is held.
1929 */
1930int vmci_qp_broker_alloc(struct vmci_handle handle,
1931 u32 peer,
1932 u32 flags,
1933 u32 priv_flags,
1934 u64 produce_size,
1935 u64 consume_size,
1936 struct vmci_qp_page_store *page_store,
1937 struct vmci_ctx *context)
1938{
1939 if (!QP_SIZES_ARE_VALID(produce_size, consume_size))
1940 return VMCI_ERROR_NO_RESOURCES;
1941
1942 return qp_broker_alloc(handle, peer, flags, priv_flags,
1943 produce_size, consume_size,
1944 page_store, context, NULL, NULL, NULL, NULL);
1945}
1946
1947/*
1948 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
1949 * step to add the UVAs of the VMX mapping of the queue pair. This function
1950 * provides backwards compatibility with such VMX'en, and takes care of
1951 * registering the page store for a queue pair previously allocated by the
1952 * VMX during create or attach. This function will move the queue pair state
1953 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
1954 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
1955 * attached state with memory, the queue pair is ready to be used by the
1956 * host peer, and an attached event will be generated.
1957 *
1958 * Assumes that the queue pair broker lock is held.
1959 *
1960 * This function is only used by the hosted platform, since there is no
1961 * issue with backwards compatibility for vmkernel.
1962 */
1963int vmci_qp_broker_set_page_store(struct vmci_handle handle,
1964 u64 produce_uva,
1965 u64 consume_uva,
1966 struct vmci_ctx *context)
1967{
1968 struct qp_broker_entry *entry;
1969 int result;
1970 const u32 context_id = vmci_ctx_get_id(context);
1971
1972 if (vmci_handle_is_invalid(handle) || !context ||
1973 context_id == VMCI_INVALID_ID)
1974 return VMCI_ERROR_INVALID_ARGS;
1975
1976 /*
1977 * We only support guest to host queue pairs, so the VMX must
1978 * supply UVAs for the mapped page files.
1979 */
1980
1981 if (produce_uva == 0 || consume_uva == 0)
1982 return VMCI_ERROR_INVALID_ARGS;
1983
1984 mutex_lock(&qp_broker_list.mutex);
1985
1986 if (!vmci_ctx_qp_exists(context, handle)) {
1987 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
1988 context_id, handle.context, handle.resource);
1989 result = VMCI_ERROR_NOT_FOUND;
1990 goto out;
1991 }
1992
1993 entry = qp_broker_handle_to_entry(handle);
1994 if (!entry) {
1995 result = VMCI_ERROR_NOT_FOUND;
1996 goto out;
1997 }
1998
1999 /*
2000 * If I'm the owner then I can set the page store.
2001 *
2002 * Or, if a host created the queue_pair and I'm the attached peer
2003 * then I can set the page store.
2004 */
2005 if (entry->create_id != context_id &&
2006 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
2007 entry->attach_id != context_id)) {
2008 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
2009 goto out;
2010 }
2011
2012 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2013 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2014 result = VMCI_ERROR_UNAVAILABLE;
2015 goto out;
2016 }
2017
2018 result = qp_host_get_user_memory(produce_uva, consume_uva,
2019 entry->produce_q, entry->consume_q);
2020 if (result < VMCI_SUCCESS)
2021 goto out;
2022
2023 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2024 if (result < VMCI_SUCCESS) {
2025 qp_host_unregister_user_memory(entry->produce_q,
2026 entry->consume_q);
2027 goto out;
2028 }
2029
2030 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2031 entry->state = VMCIQPB_CREATED_MEM;
2032 else
2033 entry->state = VMCIQPB_ATTACHED_MEM;
2034
2035 entry->vmci_page_files = true;
2036
2037 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2038 result =
2039 qp_notify_peer(true, handle, context_id, entry->create_id);
2040 if (result < VMCI_SUCCESS) {
2041 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2042 entry->create_id, entry->qp.handle.context,
2043 entry->qp.handle.resource);
2044 }
2045 }
2046
2047 result = VMCI_SUCCESS;
2048 out:
2049 mutex_unlock(&qp_broker_list.mutex);
2050 return result;
2051}
2052
2053/*
2054 * Resets saved queue headers for the given QP broker
2055 * entry. Should be used when guest memory becomes available
2056 * again, or the guest detaches.
2057 */
2058static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2059{
2060 entry->produce_q->saved_header = NULL;
2061 entry->consume_q->saved_header = NULL;
2062}
2063
2064/*
2065 * The main entry point for detaching from a queue pair registered with the
2066 * queue pair broker. If more than one endpoint is attached to the queue
2067 * pair, the first endpoint will mainly decrement a reference count and
2068 * generate a notification to its peer. The last endpoint will clean up
2069 * the queue pair state registered with the broker.
2070 *
2071 * When a guest endpoint detaches, it will unmap and unregister the guest
2072 * memory backing the queue pair. If the host is still attached, it will
2073 * no longer be able to access the queue pair content.
2074 *
2075 * If the queue pair is already in a state where there is no memory
2076 * registered for the queue pair (any *_NO_MEM state), it will transition to
2077 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2078 * endpoint is the first of two endpoints to detach. If the host endpoint is
2079 * the first out of two to detach, the queue pair will move to the
2080 * VMCIQPB_SHUTDOWN_MEM state.
2081 */
2082int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2083{
2084 struct qp_broker_entry *entry;
2085 const u32 context_id = vmci_ctx_get_id(context);
2086 u32 peer_id;
2087 bool is_local = false;
2088 int result;
2089
2090 if (vmci_handle_is_invalid(handle) || !context ||
2091 context_id == VMCI_INVALID_ID) {
2092 return VMCI_ERROR_INVALID_ARGS;
2093 }
2094
2095 mutex_lock(&qp_broker_list.mutex);
2096
2097 if (!vmci_ctx_qp_exists(context, handle)) {
2098 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2099 context_id, handle.context, handle.resource);
2100 result = VMCI_ERROR_NOT_FOUND;
2101 goto out;
2102 }
2103
2104 entry = qp_broker_handle_to_entry(handle);
2105 if (!entry) {
2106 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2107 context_id, handle.context, handle.resource);
2108 result = VMCI_ERROR_NOT_FOUND;
2109 goto out;
2110 }
2111
2112 if (context_id != entry->create_id && context_id != entry->attach_id) {
2113 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2114 goto out;
2115 }
2116
2117 if (context_id == entry->create_id) {
2118 peer_id = entry->attach_id;
2119 entry->create_id = VMCI_INVALID_ID;
2120 } else {
2121 peer_id = entry->create_id;
2122 entry->attach_id = VMCI_INVALID_ID;
2123 }
2124 entry->qp.ref_count--;
2125
2126 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2127
2128 if (context_id != VMCI_HOST_CONTEXT_ID) {
2129 bool headers_mapped;
2130
2131 /*
2132 * Pre NOVMVM vmx'en may detach from a queue pair
2133 * before setting the page store, and in that case
2134 * there is no user memory to detach from. Also, more
2135 * recent VMX'en may detach from a queue pair in the
2136 * quiesced state.
2137 */
2138
2139 qp_acquire_queue_mutex(entry->produce_q);
2140 headers_mapped = entry->produce_q->q_header ||
2141 entry->consume_q->q_header;
2142 if (QPBROKERSTATE_HAS_MEM(entry)) {
2143 result =
2144 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2145 entry->produce_q,
2146 entry->consume_q);
2147 if (result < VMCI_SUCCESS)
2148 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2149 handle.context, handle.resource,
2150 result);
2151
2152 qp_host_unregister_user_memory(entry->produce_q,
2153 entry->consume_q);
2154
2155 }
2156
2157 if (!headers_mapped)
2158 qp_reset_saved_headers(entry);
2159
2160 qp_release_queue_mutex(entry->produce_q);
2161
2162 if (!headers_mapped && entry->wakeup_cb)
2163 entry->wakeup_cb(entry->client_data);
2164
2165 } else {
2166 if (entry->wakeup_cb) {
2167 entry->wakeup_cb = NULL;
2168 entry->client_data = NULL;
2169 }
2170 }
2171
2172 if (entry->qp.ref_count == 0) {
2173 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2174
2175 if (is_local)
2176 kfree(entry->local_mem);
2177
2178 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2179 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2180 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2181 /* Unlink from resource hash table and free callback */
2182 vmci_resource_remove(&entry->resource);
2183
2184 kfree(entry);
2185
2186 vmci_ctx_qp_destroy(context, handle);
2187 } else {
2188 qp_notify_peer(false, handle, context_id, peer_id);
2189 if (context_id == VMCI_HOST_CONTEXT_ID &&
2190 QPBROKERSTATE_HAS_MEM(entry)) {
2191 entry->state = VMCIQPB_SHUTDOWN_MEM;
2192 } else {
2193 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2194 }
2195
2196 if (!is_local)
2197 vmci_ctx_qp_destroy(context, handle);
2198
2199 }
2200 result = VMCI_SUCCESS;
2201 out:
2202 mutex_unlock(&qp_broker_list.mutex);
2203 return result;
2204}
2205
2206/*
2207 * Establishes the necessary mappings for a queue pair given a
2208 * reference to the queue pair guest memory. This is usually
2209 * called when a guest is unquiesced and the VMX is allowed to
2210 * map guest memory once again.
2211 */
2212int vmci_qp_broker_map(struct vmci_handle handle,
2213 struct vmci_ctx *context,
2214 u64 guest_mem)
2215{
2216 struct qp_broker_entry *entry;
2217 const u32 context_id = vmci_ctx_get_id(context);
2218 int result;
2219
2220 if (vmci_handle_is_invalid(handle) || !context ||
2221 context_id == VMCI_INVALID_ID)
2222 return VMCI_ERROR_INVALID_ARGS;
2223
2224 mutex_lock(&qp_broker_list.mutex);
2225
2226 if (!vmci_ctx_qp_exists(context, handle)) {
2227 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2228 context_id, handle.context, handle.resource);
2229 result = VMCI_ERROR_NOT_FOUND;
2230 goto out;
2231 }
2232
2233 entry = qp_broker_handle_to_entry(handle);
2234 if (!entry) {
2235 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2236 context_id, handle.context, handle.resource);
2237 result = VMCI_ERROR_NOT_FOUND;
2238 goto out;
2239 }
2240
2241 if (context_id != entry->create_id && context_id != entry->attach_id) {
2242 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2243 goto out;
2244 }
2245
2246 result = VMCI_SUCCESS;
2247
2248 if (context_id != VMCI_HOST_CONTEXT_ID &&
2249 !QPBROKERSTATE_HAS_MEM(entry)) {
2250 struct vmci_qp_page_store page_store;
2251
2252 page_store.pages = guest_mem;
2253 page_store.len = QPE_NUM_PAGES(entry->qp);
2254
2255 qp_acquire_queue_mutex(entry->produce_q);
2256 qp_reset_saved_headers(entry);
2257 result =
2258 qp_host_register_user_memory(&page_store,
2259 entry->produce_q,
2260 entry->consume_q);
2261 qp_release_queue_mutex(entry->produce_q);
2262 if (result == VMCI_SUCCESS) {
2263 /* Move state from *_NO_MEM to *_MEM */
2264
2265 entry->state++;
2266
2267 if (entry->wakeup_cb)
2268 entry->wakeup_cb(entry->client_data);
2269 }
2270 }
2271
2272 out:
2273 mutex_unlock(&qp_broker_list.mutex);
2274 return result;
2275}
2276
2277/*
2278 * Saves a snapshot of the queue headers for the given QP broker
2279 * entry. Should be used when guest memory is unmapped.
2280 * Results:
2281 * VMCI_SUCCESS on success, appropriate error code if guest memory
2282 * can't be accessed..
2283 */
2284static int qp_save_headers(struct qp_broker_entry *entry)
2285{
2286 int result;
2287
2288 if (entry->produce_q->saved_header != NULL &&
2289 entry->consume_q->saved_header != NULL) {
2290 /*
2291 * If the headers have already been saved, we don't need to do
2292 * it again, and we don't want to map in the headers
2293 * unnecessarily.
2294 */
2295
2296 return VMCI_SUCCESS;
2297 }
2298
2299 if (NULL == entry->produce_q->q_header ||
2300 NULL == entry->consume_q->q_header) {
2301 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2302 if (result < VMCI_SUCCESS)
2303 return result;
2304 }
2305
2306 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2307 sizeof(entry->saved_produce_q));
2308 entry->produce_q->saved_header = &entry->saved_produce_q;
2309 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2310 sizeof(entry->saved_consume_q));
2311 entry->consume_q->saved_header = &entry->saved_consume_q;
2312
2313 return VMCI_SUCCESS;
2314}
2315
2316/*
2317 * Removes all references to the guest memory of a given queue pair, and
2318 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2319 * called when a VM is being quiesced where access to guest memory should
2320 * avoided.
2321 */
2322int vmci_qp_broker_unmap(struct vmci_handle handle,
2323 struct vmci_ctx *context,
2324 u32 gid)
2325{
2326 struct qp_broker_entry *entry;
2327 const u32 context_id = vmci_ctx_get_id(context);
2328 int result;
2329
2330 if (vmci_handle_is_invalid(handle) || !context ||
2331 context_id == VMCI_INVALID_ID)
2332 return VMCI_ERROR_INVALID_ARGS;
2333
2334 mutex_lock(&qp_broker_list.mutex);
2335
2336 if (!vmci_ctx_qp_exists(context, handle)) {
2337 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2338 context_id, handle.context, handle.resource);
2339 result = VMCI_ERROR_NOT_FOUND;
2340 goto out;
2341 }
2342
2343 entry = qp_broker_handle_to_entry(handle);
2344 if (!entry) {
2345 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2346 context_id, handle.context, handle.resource);
2347 result = VMCI_ERROR_NOT_FOUND;
2348 goto out;
2349 }
2350
2351 if (context_id != entry->create_id && context_id != entry->attach_id) {
2352 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2353 goto out;
2354 }
2355
2356 if (context_id != VMCI_HOST_CONTEXT_ID &&
2357 QPBROKERSTATE_HAS_MEM(entry)) {
2358 qp_acquire_queue_mutex(entry->produce_q);
2359 result = qp_save_headers(entry);
2360 if (result < VMCI_SUCCESS)
2361 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2362 handle.context, handle.resource, result);
2363
2364 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2365
2366 /*
2367 * On hosted, when we unmap queue pairs, the VMX will also
2368 * unmap the guest memory, so we invalidate the previously
2369 * registered memory. If the queue pair is mapped again at a
2370 * later point in time, we will need to reregister the user
2371 * memory with a possibly new user VA.
2372 */
2373 qp_host_unregister_user_memory(entry->produce_q,
2374 entry->consume_q);
2375
2376 /*
2377 * Move state from *_MEM to *_NO_MEM.
2378 */
2379 entry->state--;
2380
2381 qp_release_queue_mutex(entry->produce_q);
2382 }
2383
2384 result = VMCI_SUCCESS;
2385
2386 out:
2387 mutex_unlock(&qp_broker_list.mutex);
2388 return result;
2389}
2390
2391/*
2392 * Destroys all guest queue pair endpoints. If active guest queue
2393 * pairs still exist, hypercalls to attempt detach from these
2394 * queue pairs will be made. Any failure to detach is silently
2395 * ignored.
2396 */
2397void vmci_qp_guest_endpoints_exit(void)
2398{
2399 struct qp_entry *entry;
2400 struct qp_guest_endpoint *ep;
2401
2402 mutex_lock(&qp_guest_endpoints.mutex);
2403
2404 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2405 ep = (struct qp_guest_endpoint *)entry;
2406
2407 /* Don't make a hypercall for local queue_pairs. */
2408 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2409 qp_detatch_hypercall(entry->handle);
2410
2411 /* We cannot fail the exit, so let's reset ref_count. */
2412 entry->ref_count = 0;
2413 qp_list_remove_entry(&qp_guest_endpoints, entry);
2414
2415 qp_guest_endpoint_destroy(ep);
2416 }
2417
2418 mutex_unlock(&qp_guest_endpoints.mutex);
2419}
2420
2421/*
2422 * Helper routine that will lock the queue pair before subsequent
2423 * operations.
2424 * Note: Non-blocking on the host side is currently only implemented in ESX.
2425 * Since non-blocking isn't yet implemented on the host personality we
2426 * have no reason to acquire a spin lock. So to avoid the use of an
2427 * unnecessary lock only acquire the mutex if we can block.
2428 */
2429static void qp_lock(const struct vmci_qp *qpair)
2430{
2431 qp_acquire_queue_mutex(qpair->produce_q);
2432}
2433
2434/*
2435 * Helper routine that unlocks the queue pair after calling
2436 * qp_lock.
2437 */
2438static void qp_unlock(const struct vmci_qp *qpair)
2439{
2440 qp_release_queue_mutex(qpair->produce_q);
2441}
2442
2443/*
2444 * The queue headers may not be mapped at all times. If a queue is
2445 * currently not mapped, it will be attempted to do so.
2446 */
2447static int qp_map_queue_headers(struct vmci_queue *produce_q,
2448 struct vmci_queue *consume_q)
2449{
2450 int result;
2451
2452 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2453 result = qp_host_map_queues(produce_q, consume_q);
2454 if (result < VMCI_SUCCESS)
2455 return (produce_q->saved_header &&
2456 consume_q->saved_header) ?
2457 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2458 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2459 }
2460
2461 return VMCI_SUCCESS;
2462}
2463
2464/*
2465 * Helper routine that will retrieve the produce and consume
2466 * headers of a given queue pair. If the guest memory of the
2467 * queue pair is currently not available, the saved queue headers
2468 * will be returned, if these are available.
2469 */
2470static int qp_get_queue_headers(const struct vmci_qp *qpair,
2471 struct vmci_queue_header **produce_q_header,
2472 struct vmci_queue_header **consume_q_header)
2473{
2474 int result;
2475
2476 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2477 if (result == VMCI_SUCCESS) {
2478 *produce_q_header = qpair->produce_q->q_header;
2479 *consume_q_header = qpair->consume_q->q_header;
2480 } else if (qpair->produce_q->saved_header &&
2481 qpair->consume_q->saved_header) {
2482 *produce_q_header = qpair->produce_q->saved_header;
2483 *consume_q_header = qpair->consume_q->saved_header;
2484 result = VMCI_SUCCESS;
2485 }
2486
2487 return result;
2488}
2489
2490/*
2491 * Callback from VMCI queue pair broker indicating that a queue
2492 * pair that was previously not ready, now either is ready or
2493 * gone forever.
2494 */
2495static int qp_wakeup_cb(void *client_data)
2496{
2497 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2498
2499 qp_lock(qpair);
2500 while (qpair->blocked > 0) {
2501 qpair->blocked--;
2502 qpair->generation++;
2503 wake_up(&qpair->event);
2504 }
2505 qp_unlock(qpair);
2506
2507 return VMCI_SUCCESS;
2508}
2509
2510/*
2511 * Makes the calling thread wait for the queue pair to become
2512 * ready for host side access. Returns true when thread is
2513 * woken up after queue pair state change, false otherwise.
2514 */
2515static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2516{
2517 unsigned int generation;
2518
2519 qpair->blocked++;
2520 generation = qpair->generation;
2521 qp_unlock(qpair);
2522 wait_event(qpair->event, generation != qpair->generation);
2523 qp_lock(qpair);
2524
2525 return true;
2526}
2527
2528/*
2529 * Enqueues a given buffer to the produce queue using the provided
2530 * function. As many bytes as possible (space available in the queue)
2531 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2532 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2533 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2534 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2535 * an error occured when accessing the buffer,
2536 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2537 * available. Otherwise, the number of bytes written to the queue is
2538 * returned. Updates the tail pointer of the produce queue.
2539 */
2540static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2541 struct vmci_queue *consume_q,
2542 const u64 produce_q_size,
2543 struct iov_iter *from)
2544{
2545 s64 free_space;
2546 u64 tail;
2547 size_t buf_size = iov_iter_count(from);
2548 size_t written;
2549 ssize_t result;
2550
2551 result = qp_map_queue_headers(produce_q, consume_q);
2552 if (unlikely(result != VMCI_SUCCESS))
2553 return result;
2554
2555 free_space = vmci_q_header_free_space(produce_q->q_header,
2556 consume_q->q_header,
2557 produce_q_size);
2558 if (free_space == 0)
2559 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2560
2561 if (free_space < VMCI_SUCCESS)
2562 return (ssize_t) free_space;
2563
2564 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2565 tail = vmci_q_header_producer_tail(produce_q->q_header);
2566 if (likely(tail + written < produce_q_size)) {
2567 result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2568 } else {
2569 /* Tail pointer wraps around. */
2570
2571 const size_t tmp = (size_t) (produce_q_size - tail);
2572
2573 result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2574 if (result >= VMCI_SUCCESS)
2575 result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2576 written - tmp);
2577 }
2578
2579 if (result < VMCI_SUCCESS)
2580 return result;
2581
2582 /*
2583 * This virt_wmb() ensures that data written to the queue
2584 * is observable before the new producer_tail is.
2585 */
2586 virt_wmb();
2587
2588 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2589 produce_q_size);
2590 return written;
2591}
2592
2593/*
2594 * Dequeues data (if available) from the given consume queue. Writes data
2595 * to the user provided buffer using the provided function.
2596 * Assumes the queue->mutex has been acquired.
2597 * Results:
2598 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2599 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2600 * (as defined by the queue size).
2601 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2602 * Otherwise the number of bytes dequeued is returned.
2603 * Side effects:
2604 * Updates the head pointer of the consume queue.
2605 */
2606static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2607 struct vmci_queue *consume_q,
2608 const u64 consume_q_size,
2609 struct iov_iter *to,
2610 bool update_consumer)
2611{
2612 size_t buf_size = iov_iter_count(to);
2613 s64 buf_ready;
2614 u64 head;
2615 size_t read;
2616 ssize_t result;
2617
2618 result = qp_map_queue_headers(produce_q, consume_q);
2619 if (unlikely(result != VMCI_SUCCESS))
2620 return result;
2621
2622 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2623 produce_q->q_header,
2624 consume_q_size);
2625 if (buf_ready == 0)
2626 return VMCI_ERROR_QUEUEPAIR_NODATA;
2627
2628 if (buf_ready < VMCI_SUCCESS)
2629 return (ssize_t) buf_ready;
2630
2631 /*
2632 * This virt_rmb() ensures that data from the queue will be read
2633 * after we have determined how much is ready to be consumed.
2634 */
2635 virt_rmb();
2636
2637 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2638 head = vmci_q_header_consumer_head(produce_q->q_header);
2639 if (likely(head + read < consume_q_size)) {
2640 result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
2641 } else {
2642 /* Head pointer wraps around. */
2643
2644 const size_t tmp = (size_t) (consume_q_size - head);
2645
2646 result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
2647 if (result >= VMCI_SUCCESS)
2648 result = qp_memcpy_from_queue_iter(to, consume_q, 0,
2649 read - tmp);
2650
2651 }
2652
2653 if (result < VMCI_SUCCESS)
2654 return result;
2655
2656 if (update_consumer)
2657 vmci_q_header_add_consumer_head(produce_q->q_header,
2658 read, consume_q_size);
2659
2660 return read;
2661}
2662
2663/*
2664 * vmci_qpair_alloc() - Allocates a queue pair.
2665 * @qpair: Pointer for the new vmci_qp struct.
2666 * @handle: Handle to track the resource.
2667 * @produce_qsize: Desired size of the producer queue.
2668 * @consume_qsize: Desired size of the consumer queue.
2669 * @peer: ContextID of the peer.
2670 * @flags: VMCI flags.
2671 * @priv_flags: VMCI priviledge flags.
2672 *
2673 * This is the client interface for allocating the memory for a
2674 * vmci_qp structure and then attaching to the underlying
2675 * queue. If an error occurs allocating the memory for the
2676 * vmci_qp structure no attempt is made to attach. If an
2677 * error occurs attaching, then the structure is freed.
2678 */
2679int vmci_qpair_alloc(struct vmci_qp **qpair,
2680 struct vmci_handle *handle,
2681 u64 produce_qsize,
2682 u64 consume_qsize,
2683 u32 peer,
2684 u32 flags,
2685 u32 priv_flags)
2686{
2687 struct vmci_qp *my_qpair;
2688 int retval;
2689 struct vmci_handle src = VMCI_INVALID_HANDLE;
2690 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2691 enum vmci_route route;
2692 vmci_event_release_cb wakeup_cb;
2693 void *client_data;
2694
2695 /*
2696 * Restrict the size of a queuepair. The device already
2697 * enforces a limit on the total amount of memory that can be
2698 * allocated to queuepairs for a guest. However, we try to
2699 * allocate this memory before we make the queuepair
2700 * allocation hypercall. On Linux, we allocate each page
2701 * separately, which means rather than fail, the guest will
2702 * thrash while it tries to allocate, and will become
2703 * increasingly unresponsive to the point where it appears to
2704 * be hung. So we place a limit on the size of an individual
2705 * queuepair here, and leave the device to enforce the
2706 * restriction on total queuepair memory. (Note that this
2707 * doesn't prevent all cases; a user with only this much
2708 * physical memory could still get into trouble.) The error
2709 * used by the device is NO_RESOURCES, so use that here too.
2710 */
2711
2712 if (!QP_SIZES_ARE_VALID(produce_qsize, consume_qsize))
2713 return VMCI_ERROR_NO_RESOURCES;
2714
2715 retval = vmci_route(&src, &dst, false, &route);
2716 if (retval < VMCI_SUCCESS)
2717 route = vmci_guest_code_active() ?
2718 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2719
2720 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2721 pr_devel("NONBLOCK OR PINNED set");
2722 return VMCI_ERROR_INVALID_ARGS;
2723 }
2724
2725 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2726 if (!my_qpair)
2727 return VMCI_ERROR_NO_MEM;
2728
2729 my_qpair->produce_q_size = produce_qsize;
2730 my_qpair->consume_q_size = consume_qsize;
2731 my_qpair->peer = peer;
2732 my_qpair->flags = flags;
2733 my_qpair->priv_flags = priv_flags;
2734
2735 wakeup_cb = NULL;
2736 client_data = NULL;
2737
2738 if (VMCI_ROUTE_AS_HOST == route) {
2739 my_qpair->guest_endpoint = false;
2740 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2741 my_qpair->blocked = 0;
2742 my_qpair->generation = 0;
2743 init_waitqueue_head(&my_qpair->event);
2744 wakeup_cb = qp_wakeup_cb;
2745 client_data = (void *)my_qpair;
2746 }
2747 } else {
2748 my_qpair->guest_endpoint = true;
2749 }
2750
2751 retval = vmci_qp_alloc(handle,
2752 &my_qpair->produce_q,
2753 my_qpair->produce_q_size,
2754 &my_qpair->consume_q,
2755 my_qpair->consume_q_size,
2756 my_qpair->peer,
2757 my_qpair->flags,
2758 my_qpair->priv_flags,
2759 my_qpair->guest_endpoint,
2760 wakeup_cb, client_data);
2761
2762 if (retval < VMCI_SUCCESS) {
2763 kfree(my_qpair);
2764 return retval;
2765 }
2766
2767 *qpair = my_qpair;
2768 my_qpair->handle = *handle;
2769
2770 return retval;
2771}
2772EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2773
2774/*
2775 * vmci_qpair_detach() - Detatches the client from a queue pair.
2776 * @qpair: Reference of a pointer to the qpair struct.
2777 *
2778 * This is the client interface for detaching from a VMCIQPair.
2779 * Note that this routine will free the memory allocated for the
2780 * vmci_qp structure too.
2781 */
2782int vmci_qpair_detach(struct vmci_qp **qpair)
2783{
2784 int result;
2785 struct vmci_qp *old_qpair;
2786
2787 if (!qpair || !(*qpair))
2788 return VMCI_ERROR_INVALID_ARGS;
2789
2790 old_qpair = *qpair;
2791 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2792
2793 /*
2794 * The guest can fail to detach for a number of reasons, and
2795 * if it does so, it will cleanup the entry (if there is one).
2796 * The host can fail too, but it won't cleanup the entry
2797 * immediately, it will do that later when the context is
2798 * freed. Either way, we need to release the qpair struct
2799 * here; there isn't much the caller can do, and we don't want
2800 * to leak.
2801 */
2802
2803 memset(old_qpair, 0, sizeof(*old_qpair));
2804 old_qpair->handle = VMCI_INVALID_HANDLE;
2805 old_qpair->peer = VMCI_INVALID_ID;
2806 kfree(old_qpair);
2807 *qpair = NULL;
2808
2809 return result;
2810}
2811EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2812
2813/*
2814 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2815 * @qpair: Pointer to the queue pair struct.
2816 * @producer_tail: Reference used for storing producer tail index.
2817 * @consumer_head: Reference used for storing the consumer head index.
2818 *
2819 * This is the client interface for getting the current indexes of the
2820 * QPair from the point of the view of the caller as the producer.
2821 */
2822int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2823 u64 *producer_tail,
2824 u64 *consumer_head)
2825{
2826 struct vmci_queue_header *produce_q_header;
2827 struct vmci_queue_header *consume_q_header;
2828 int result;
2829
2830 if (!qpair)
2831 return VMCI_ERROR_INVALID_ARGS;
2832
2833 qp_lock(qpair);
2834 result =
2835 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2836 if (result == VMCI_SUCCESS)
2837 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2838 producer_tail, consumer_head);
2839 qp_unlock(qpair);
2840
2841 if (result == VMCI_SUCCESS &&
2842 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2843 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2844 return VMCI_ERROR_INVALID_SIZE;
2845
2846 return result;
2847}
2848EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2849
2850/*
2851 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
2852 * @qpair: Pointer to the queue pair struct.
2853 * @consumer_tail: Reference used for storing consumer tail index.
2854 * @producer_head: Reference used for storing the producer head index.
2855 *
2856 * This is the client interface for getting the current indexes of the
2857 * QPair from the point of the view of the caller as the consumer.
2858 */
2859int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2860 u64 *consumer_tail,
2861 u64 *producer_head)
2862{
2863 struct vmci_queue_header *produce_q_header;
2864 struct vmci_queue_header *consume_q_header;
2865 int result;
2866
2867 if (!qpair)
2868 return VMCI_ERROR_INVALID_ARGS;
2869
2870 qp_lock(qpair);
2871 result =
2872 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2873 if (result == VMCI_SUCCESS)
2874 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2875 consumer_tail, producer_head);
2876 qp_unlock(qpair);
2877
2878 if (result == VMCI_SUCCESS &&
2879 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2880 (producer_head && *producer_head >= qpair->consume_q_size)))
2881 return VMCI_ERROR_INVALID_SIZE;
2882
2883 return result;
2884}
2885EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2886
2887/*
2888 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2889 * @qpair: Pointer to the queue pair struct.
2890 *
2891 * This is the client interface for getting the amount of free
2892 * space in the QPair from the point of the view of the caller as
2893 * the producer which is the common case. Returns < 0 if err, else
2894 * available bytes into which data can be enqueued if > 0.
2895 */
2896s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2897{
2898 struct vmci_queue_header *produce_q_header;
2899 struct vmci_queue_header *consume_q_header;
2900 s64 result;
2901
2902 if (!qpair)
2903 return VMCI_ERROR_INVALID_ARGS;
2904
2905 qp_lock(qpair);
2906 result =
2907 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2908 if (result == VMCI_SUCCESS)
2909 result = vmci_q_header_free_space(produce_q_header,
2910 consume_q_header,
2911 qpair->produce_q_size);
2912 else
2913 result = 0;
2914
2915 qp_unlock(qpair);
2916
2917 return result;
2918}
2919EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
2920
2921/*
2922 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
2923 * @qpair: Pointer to the queue pair struct.
2924 *
2925 * This is the client interface for getting the amount of free
2926 * space in the QPair from the point of the view of the caller as
2927 * the consumer which is not the common case. Returns < 0 if err, else
2928 * available bytes into which data can be enqueued if > 0.
2929 */
2930s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
2931{
2932 struct vmci_queue_header *produce_q_header;
2933 struct vmci_queue_header *consume_q_header;
2934 s64 result;
2935
2936 if (!qpair)
2937 return VMCI_ERROR_INVALID_ARGS;
2938
2939 qp_lock(qpair);
2940 result =
2941 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2942 if (result == VMCI_SUCCESS)
2943 result = vmci_q_header_free_space(consume_q_header,
2944 produce_q_header,
2945 qpair->consume_q_size);
2946 else
2947 result = 0;
2948
2949 qp_unlock(qpair);
2950
2951 return result;
2952}
2953EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
2954
2955/*
2956 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
2957 * producer queue.
2958 * @qpair: Pointer to the queue pair struct.
2959 *
2960 * This is the client interface for getting the amount of
2961 * enqueued data in the QPair from the point of the view of the
2962 * caller as the producer which is not the common case. Returns < 0 if err,
2963 * else available bytes that may be read.
2964 */
2965s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
2966{
2967 struct vmci_queue_header *produce_q_header;
2968 struct vmci_queue_header *consume_q_header;
2969 s64 result;
2970
2971 if (!qpair)
2972 return VMCI_ERROR_INVALID_ARGS;
2973
2974 qp_lock(qpair);
2975 result =
2976 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2977 if (result == VMCI_SUCCESS)
2978 result = vmci_q_header_buf_ready(produce_q_header,
2979 consume_q_header,
2980 qpair->produce_q_size);
2981 else
2982 result = 0;
2983
2984 qp_unlock(qpair);
2985
2986 return result;
2987}
2988EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
2989
2990/*
2991 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
2992 * consumer queue.
2993 * @qpair: Pointer to the queue pair struct.
2994 *
2995 * This is the client interface for getting the amount of
2996 * enqueued data in the QPair from the point of the view of the
2997 * caller as the consumer which is the normal case. Returns < 0 if err,
2998 * else available bytes that may be read.
2999 */
3000s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
3001{
3002 struct vmci_queue_header *produce_q_header;
3003 struct vmci_queue_header *consume_q_header;
3004 s64 result;
3005
3006 if (!qpair)
3007 return VMCI_ERROR_INVALID_ARGS;
3008
3009 qp_lock(qpair);
3010 result =
3011 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3012 if (result == VMCI_SUCCESS)
3013 result = vmci_q_header_buf_ready(consume_q_header,
3014 produce_q_header,
3015 qpair->consume_q_size);
3016 else
3017 result = 0;
3018
3019 qp_unlock(qpair);
3020
3021 return result;
3022}
3023EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
3024
3025/*
3026 * vmci_qpair_enqueue() - Throw data on the queue.
3027 * @qpair: Pointer to the queue pair struct.
3028 * @buf: Pointer to buffer containing data
3029 * @buf_size: Length of buffer.
3030 * @buf_type: Buffer type (Unused).
3031 *
3032 * This is the client interface for enqueueing data into the queue.
3033 * Returns number of bytes enqueued or < 0 on error.
3034 */
3035ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3036 const void *buf,
3037 size_t buf_size,
3038 int buf_type)
3039{
3040 ssize_t result;
3041 struct iov_iter from;
3042 struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
3043
3044 if (!qpair || !buf)
3045 return VMCI_ERROR_INVALID_ARGS;
3046
3047 iov_iter_kvec(&from, ITER_SOURCE, &v, 1, buf_size);
3048
3049 qp_lock(qpair);
3050
3051 do {
3052 result = qp_enqueue_locked(qpair->produce_q,
3053 qpair->consume_q,
3054 qpair->produce_q_size,
3055 &from);
3056
3057 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3058 !qp_wait_for_ready_queue(qpair))
3059 result = VMCI_ERROR_WOULD_BLOCK;
3060
3061 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3062
3063 qp_unlock(qpair);
3064
3065 return result;
3066}
3067EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3068
3069/*
3070 * vmci_qpair_dequeue() - Get data from the queue.
3071 * @qpair: Pointer to the queue pair struct.
3072 * @buf: Pointer to buffer for the data
3073 * @buf_size: Length of buffer.
3074 * @buf_type: Buffer type (Unused).
3075 *
3076 * This is the client interface for dequeueing data from the queue.
3077 * Returns number of bytes dequeued or < 0 on error.
3078 */
3079ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3080 void *buf,
3081 size_t buf_size,
3082 int buf_type)
3083{
3084 ssize_t result;
3085 struct iov_iter to;
3086 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3087
3088 if (!qpair || !buf)
3089 return VMCI_ERROR_INVALID_ARGS;
3090
3091 iov_iter_kvec(&to, ITER_DEST, &v, 1, buf_size);
3092
3093 qp_lock(qpair);
3094
3095 do {
3096 result = qp_dequeue_locked(qpair->produce_q,
3097 qpair->consume_q,
3098 qpair->consume_q_size,
3099 &to, true);
3100
3101 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3102 !qp_wait_for_ready_queue(qpair))
3103 result = VMCI_ERROR_WOULD_BLOCK;
3104
3105 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3106
3107 qp_unlock(qpair);
3108
3109 return result;
3110}
3111EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3112
3113/*
3114 * vmci_qpair_peek() - Peek at the data in the queue.
3115 * @qpair: Pointer to the queue pair struct.
3116 * @buf: Pointer to buffer for the data
3117 * @buf_size: Length of buffer.
3118 * @buf_type: Buffer type (Unused on Linux).
3119 *
3120 * This is the client interface for peeking into a queue. (I.e.,
3121 * copy data from the queue without updating the head pointer.)
3122 * Returns number of bytes dequeued or < 0 on error.
3123 */
3124ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3125 void *buf,
3126 size_t buf_size,
3127 int buf_type)
3128{
3129 struct iov_iter to;
3130 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3131 ssize_t result;
3132
3133 if (!qpair || !buf)
3134 return VMCI_ERROR_INVALID_ARGS;
3135
3136 iov_iter_kvec(&to, ITER_DEST, &v, 1, buf_size);
3137
3138 qp_lock(qpair);
3139
3140 do {
3141 result = qp_dequeue_locked(qpair->produce_q,
3142 qpair->consume_q,
3143 qpair->consume_q_size,
3144 &to, false);
3145
3146 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3147 !qp_wait_for_ready_queue(qpair))
3148 result = VMCI_ERROR_WOULD_BLOCK;
3149
3150 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3151
3152 qp_unlock(qpair);
3153
3154 return result;
3155}
3156EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3157
3158/*
3159 * vmci_qpair_enquev() - Throw data on the queue using iov.
3160 * @qpair: Pointer to the queue pair struct.
3161 * @iov: Pointer to buffer containing data
3162 * @iov_size: Length of buffer.
3163 * @buf_type: Buffer type (Unused).
3164 *
3165 * This is the client interface for enqueueing data into the queue.
3166 * This function uses IO vectors to handle the work. Returns number
3167 * of bytes enqueued or < 0 on error.
3168 */
3169ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3170 struct msghdr *msg,
3171 size_t iov_size,
3172 int buf_type)
3173{
3174 ssize_t result;
3175
3176 if (!qpair)
3177 return VMCI_ERROR_INVALID_ARGS;
3178
3179 qp_lock(qpair);
3180
3181 do {
3182 result = qp_enqueue_locked(qpair->produce_q,
3183 qpair->consume_q,
3184 qpair->produce_q_size,
3185 &msg->msg_iter);
3186
3187 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3188 !qp_wait_for_ready_queue(qpair))
3189 result = VMCI_ERROR_WOULD_BLOCK;
3190
3191 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3192
3193 qp_unlock(qpair);
3194
3195 return result;
3196}
3197EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3198
3199/*
3200 * vmci_qpair_dequev() - Get data from the queue using iov.
3201 * @qpair: Pointer to the queue pair struct.
3202 * @iov: Pointer to buffer for the data
3203 * @iov_size: Length of buffer.
3204 * @buf_type: Buffer type (Unused).
3205 *
3206 * This is the client interface for dequeueing data from the queue.
3207 * This function uses IO vectors to handle the work. Returns number
3208 * of bytes dequeued or < 0 on error.
3209 */
3210ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3211 struct msghdr *msg,
3212 size_t iov_size,
3213 int buf_type)
3214{
3215 ssize_t result;
3216
3217 if (!qpair)
3218 return VMCI_ERROR_INVALID_ARGS;
3219
3220 qp_lock(qpair);
3221
3222 do {
3223 result = qp_dequeue_locked(qpair->produce_q,
3224 qpair->consume_q,
3225 qpair->consume_q_size,
3226 &msg->msg_iter, true);
3227
3228 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3229 !qp_wait_for_ready_queue(qpair))
3230 result = VMCI_ERROR_WOULD_BLOCK;
3231
3232 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3233
3234 qp_unlock(qpair);
3235
3236 return result;
3237}
3238EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3239
3240/*
3241 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3242 * @qpair: Pointer to the queue pair struct.
3243 * @iov: Pointer to buffer for the data
3244 * @iov_size: Length of buffer.
3245 * @buf_type: Buffer type (Unused on Linux).
3246 *
3247 * This is the client interface for peeking into a queue. (I.e.,
3248 * copy data from the queue without updating the head pointer.)
3249 * This function uses IO vectors to handle the work. Returns number
3250 * of bytes peeked or < 0 on error.
3251 */
3252ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3253 struct msghdr *msg,
3254 size_t iov_size,
3255 int buf_type)
3256{
3257 ssize_t result;
3258
3259 if (!qpair)
3260 return VMCI_ERROR_INVALID_ARGS;
3261
3262 qp_lock(qpair);
3263
3264 do {
3265 result = qp_dequeue_locked(qpair->produce_q,
3266 qpair->consume_q,
3267 qpair->consume_q_size,
3268 &msg->msg_iter, false);
3269
3270 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3271 !qp_wait_for_ready_queue(qpair))
3272 result = VMCI_ERROR_WOULD_BLOCK;
3273
3274 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3275
3276 qp_unlock(qpair);
3277 return result;
3278}
3279EXPORT_SYMBOL_GPL(vmci_qpair_peekv);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VMware VMCI Driver
4 *
5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 */
7
8#include <linux/vmw_vmci_defs.h>
9#include <linux/vmw_vmci_api.h>
10#include <linux/highmem.h>
11#include <linux/kernel.h>
12#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/pagemap.h>
16#include <linux/pci.h>
17#include <linux/sched.h>
18#include <linux/slab.h>
19#include <linux/uio.h>
20#include <linux/wait.h>
21#include <linux/vmalloc.h>
22#include <linux/skbuff.h>
23
24#include "vmci_handle_array.h"
25#include "vmci_queue_pair.h"
26#include "vmci_datagram.h"
27#include "vmci_resource.h"
28#include "vmci_context.h"
29#include "vmci_driver.h"
30#include "vmci_event.h"
31#include "vmci_route.h"
32
33/*
34 * In the following, we will distinguish between two kinds of VMX processes -
35 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
36 * VMCI page files in the VMX and supporting VM to VM communication and the
37 * newer ones that use the guest memory directly. We will in the following
38 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
39 * new-style VMX'en.
40 *
41 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
42 * removed for readability) - see below for more details on the transtions:
43 *
44 * -------------- NEW -------------
45 * | |
46 * \_/ \_/
47 * CREATED_NO_MEM <-----------------> CREATED_MEM
48 * | | |
49 * | o-----------------------o |
50 * | | |
51 * \_/ \_/ \_/
52 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
53 * | | |
54 * | o----------------------o |
55 * | | |
56 * \_/ \_/ \_/
57 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
58 * | |
59 * | |
60 * -------------> gone <-------------
61 *
62 * In more detail. When a VMCI queue pair is first created, it will be in the
63 * VMCIQPB_NEW state. It will then move into one of the following states:
64 *
65 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
66 *
67 * - the created was performed by a host endpoint, in which case there is
68 * no backing memory yet.
69 *
70 * - the create was initiated by an old-style VMX, that uses
71 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
72 * a later point in time. This state can be distinguished from the one
73 * above by the context ID of the creator. A host side is not allowed to
74 * attach until the page store has been set.
75 *
76 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
77 * is created by a VMX using the queue pair device backend that
78 * sets the UVAs of the queue pair immediately and stores the
79 * information for later attachers. At this point, it is ready for
80 * the host side to attach to it.
81 *
82 * Once the queue pair is in one of the created states (with the exception of
83 * the case mentioned for older VMX'en above), it is possible to attach to the
84 * queue pair. Again we have two new states possible:
85 *
86 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
87 * paths:
88 *
89 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
90 * pair, and attaches to a queue pair previously created by the host side.
91 *
92 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
93 * already created by a guest.
94 *
95 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
96 * vmci_qp_broker_set_page_store (see below).
97 *
98 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
99 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
100 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
101 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
102 * will be entered.
103 *
104 * From the attached queue pair, the queue pair can enter the shutdown states
105 * when either side of the queue pair detaches. If the guest side detaches
106 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
107 * the content of the queue pair will no longer be available. If the host
108 * side detaches first, the queue pair will either enter the
109 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
110 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
111 * (e.g., the host detaches while a guest is stunned).
112 *
113 * New-style VMX'en will also unmap guest memory, if the guest is
114 * quiesced, e.g., during a snapshot operation. In that case, the guest
115 * memory will no longer be available, and the queue pair will transition from
116 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
117 * in which case the queue pair will transition from the *_NO_MEM state at that
118 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
119 * since the peer may have either attached or detached in the meantime. The
120 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
121 * *_MEM state, and vice versa.
122 */
123
124/* The Kernel specific component of the struct vmci_queue structure. */
125struct vmci_queue_kern_if {
126 struct mutex __mutex; /* Protects the queue. */
127 struct mutex *mutex; /* Shared by producer and consumer queues. */
128 size_t num_pages; /* Number of pages incl. header. */
129 bool host; /* Host or guest? */
130 union {
131 struct {
132 dma_addr_t *pas;
133 void **vas;
134 } g; /* Used by the guest. */
135 struct {
136 struct page **page;
137 struct page **header_page;
138 } h; /* Used by the host. */
139 } u;
140};
141
142/*
143 * This structure is opaque to the clients.
144 */
145struct vmci_qp {
146 struct vmci_handle handle;
147 struct vmci_queue *produce_q;
148 struct vmci_queue *consume_q;
149 u64 produce_q_size;
150 u64 consume_q_size;
151 u32 peer;
152 u32 flags;
153 u32 priv_flags;
154 bool guest_endpoint;
155 unsigned int blocked;
156 unsigned int generation;
157 wait_queue_head_t event;
158};
159
160enum qp_broker_state {
161 VMCIQPB_NEW,
162 VMCIQPB_CREATED_NO_MEM,
163 VMCIQPB_CREATED_MEM,
164 VMCIQPB_ATTACHED_NO_MEM,
165 VMCIQPB_ATTACHED_MEM,
166 VMCIQPB_SHUTDOWN_NO_MEM,
167 VMCIQPB_SHUTDOWN_MEM,
168 VMCIQPB_GONE
169};
170
171#define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
172 _qpb->state == VMCIQPB_ATTACHED_MEM || \
173 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
174
175/*
176 * In the queue pair broker, we always use the guest point of view for
177 * the produce and consume queue values and references, e.g., the
178 * produce queue size stored is the guests produce queue size. The
179 * host endpoint will need to swap these around. The only exception is
180 * the local queue pairs on the host, in which case the host endpoint
181 * that creates the queue pair will have the right orientation, and
182 * the attaching host endpoint will need to swap.
183 */
184struct qp_entry {
185 struct list_head list_item;
186 struct vmci_handle handle;
187 u32 peer;
188 u32 flags;
189 u64 produce_size;
190 u64 consume_size;
191 u32 ref_count;
192};
193
194struct qp_broker_entry {
195 struct vmci_resource resource;
196 struct qp_entry qp;
197 u32 create_id;
198 u32 attach_id;
199 enum qp_broker_state state;
200 bool require_trusted_attach;
201 bool created_by_trusted;
202 bool vmci_page_files; /* Created by VMX using VMCI page files */
203 struct vmci_queue *produce_q;
204 struct vmci_queue *consume_q;
205 struct vmci_queue_header saved_produce_q;
206 struct vmci_queue_header saved_consume_q;
207 vmci_event_release_cb wakeup_cb;
208 void *client_data;
209 void *local_mem; /* Kernel memory for local queue pair */
210};
211
212struct qp_guest_endpoint {
213 struct vmci_resource resource;
214 struct qp_entry qp;
215 u64 num_ppns;
216 void *produce_q;
217 void *consume_q;
218 struct ppn_set ppn_set;
219};
220
221struct qp_list {
222 struct list_head head;
223 struct mutex mutex; /* Protect queue list. */
224};
225
226static struct qp_list qp_broker_list = {
227 .head = LIST_HEAD_INIT(qp_broker_list.head),
228 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
229};
230
231static struct qp_list qp_guest_endpoints = {
232 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
233 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
234};
235
236#define INVALID_VMCI_GUEST_MEM_ID 0
237#define QPE_NUM_PAGES(_QPE) ((u32) \
238 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
239 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
240
241
242/*
243 * Frees kernel VA space for a given queue and its queue header, and
244 * frees physical data pages.
245 */
246static void qp_free_queue(void *q, u64 size)
247{
248 struct vmci_queue *queue = q;
249
250 if (queue) {
251 u64 i;
252
253 /* Given size does not include header, so add in a page here. */
254 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
255 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
256 queue->kernel_if->u.g.vas[i],
257 queue->kernel_if->u.g.pas[i]);
258 }
259
260 vfree(queue);
261 }
262}
263
264/*
265 * Allocates kernel queue pages of specified size with IOMMU mappings,
266 * plus space for the queue structure/kernel interface and the queue
267 * header.
268 */
269static void *qp_alloc_queue(u64 size, u32 flags)
270{
271 u64 i;
272 struct vmci_queue *queue;
273 size_t pas_size;
274 size_t vas_size;
275 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
276 u64 num_pages;
277
278 if (size > SIZE_MAX - PAGE_SIZE)
279 return NULL;
280 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
281 if (num_pages >
282 (SIZE_MAX - queue_size) /
283 (sizeof(*queue->kernel_if->u.g.pas) +
284 sizeof(*queue->kernel_if->u.g.vas)))
285 return NULL;
286
287 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
288 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
289 queue_size += pas_size + vas_size;
290
291 queue = vmalloc(queue_size);
292 if (!queue)
293 return NULL;
294
295 queue->q_header = NULL;
296 queue->saved_header = NULL;
297 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
298 queue->kernel_if->mutex = NULL;
299 queue->kernel_if->num_pages = num_pages;
300 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
301 queue->kernel_if->u.g.vas =
302 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
303 queue->kernel_if->host = false;
304
305 for (i = 0; i < num_pages; i++) {
306 queue->kernel_if->u.g.vas[i] =
307 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
308 &queue->kernel_if->u.g.pas[i],
309 GFP_KERNEL);
310 if (!queue->kernel_if->u.g.vas[i]) {
311 /* Size excl. the header. */
312 qp_free_queue(queue, i * PAGE_SIZE);
313 return NULL;
314 }
315 }
316
317 /* Queue header is the first page. */
318 queue->q_header = queue->kernel_if->u.g.vas[0];
319
320 return queue;
321}
322
323/*
324 * Copies from a given buffer or iovector to a VMCI Queue. Uses
325 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
326 * by traversing the offset -> page translation structure for the queue.
327 * Assumes that offset + size does not wrap around in the queue.
328 */
329static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
330 u64 queue_offset,
331 struct iov_iter *from,
332 size_t size)
333{
334 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
335 size_t bytes_copied = 0;
336
337 while (bytes_copied < size) {
338 const u64 page_index =
339 (queue_offset + bytes_copied) / PAGE_SIZE;
340 const size_t page_offset =
341 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
342 void *va;
343 size_t to_copy;
344
345 if (kernel_if->host)
346 va = kmap(kernel_if->u.h.page[page_index]);
347 else
348 va = kernel_if->u.g.vas[page_index + 1];
349 /* Skip header. */
350
351 if (size - bytes_copied > PAGE_SIZE - page_offset)
352 /* Enough payload to fill up from this page. */
353 to_copy = PAGE_SIZE - page_offset;
354 else
355 to_copy = size - bytes_copied;
356
357 if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
358 from)) {
359 if (kernel_if->host)
360 kunmap(kernel_if->u.h.page[page_index]);
361 return VMCI_ERROR_INVALID_ARGS;
362 }
363 bytes_copied += to_copy;
364 if (kernel_if->host)
365 kunmap(kernel_if->u.h.page[page_index]);
366 }
367
368 return VMCI_SUCCESS;
369}
370
371/*
372 * Copies to a given buffer or iovector from a VMCI Queue. Uses
373 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
374 * by traversing the offset -> page translation structure for the queue.
375 * Assumes that offset + size does not wrap around in the queue.
376 */
377static int qp_memcpy_from_queue_iter(struct iov_iter *to,
378 const struct vmci_queue *queue,
379 u64 queue_offset, size_t size)
380{
381 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
382 size_t bytes_copied = 0;
383
384 while (bytes_copied < size) {
385 const u64 page_index =
386 (queue_offset + bytes_copied) / PAGE_SIZE;
387 const size_t page_offset =
388 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
389 void *va;
390 size_t to_copy;
391 int err;
392
393 if (kernel_if->host)
394 va = kmap(kernel_if->u.h.page[page_index]);
395 else
396 va = kernel_if->u.g.vas[page_index + 1];
397 /* Skip header. */
398
399 if (size - bytes_copied > PAGE_SIZE - page_offset)
400 /* Enough payload to fill up this page. */
401 to_copy = PAGE_SIZE - page_offset;
402 else
403 to_copy = size - bytes_copied;
404
405 err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
406 if (err != to_copy) {
407 if (kernel_if->host)
408 kunmap(kernel_if->u.h.page[page_index]);
409 return VMCI_ERROR_INVALID_ARGS;
410 }
411 bytes_copied += to_copy;
412 if (kernel_if->host)
413 kunmap(kernel_if->u.h.page[page_index]);
414 }
415
416 return VMCI_SUCCESS;
417}
418
419/*
420 * Allocates two list of PPNs --- one for the pages in the produce queue,
421 * and the other for the pages in the consume queue. Intializes the list
422 * of PPNs with the page frame numbers of the KVA for the two queues (and
423 * the queue headers).
424 */
425static int qp_alloc_ppn_set(void *prod_q,
426 u64 num_produce_pages,
427 void *cons_q,
428 u64 num_consume_pages, struct ppn_set *ppn_set)
429{
430 u64 *produce_ppns;
431 u64 *consume_ppns;
432 struct vmci_queue *produce_q = prod_q;
433 struct vmci_queue *consume_q = cons_q;
434 u64 i;
435
436 if (!produce_q || !num_produce_pages || !consume_q ||
437 !num_consume_pages || !ppn_set)
438 return VMCI_ERROR_INVALID_ARGS;
439
440 if (ppn_set->initialized)
441 return VMCI_ERROR_ALREADY_EXISTS;
442
443 produce_ppns =
444 kmalloc_array(num_produce_pages, sizeof(*produce_ppns),
445 GFP_KERNEL);
446 if (!produce_ppns)
447 return VMCI_ERROR_NO_MEM;
448
449 consume_ppns =
450 kmalloc_array(num_consume_pages, sizeof(*consume_ppns),
451 GFP_KERNEL);
452 if (!consume_ppns) {
453 kfree(produce_ppns);
454 return VMCI_ERROR_NO_MEM;
455 }
456
457 for (i = 0; i < num_produce_pages; i++)
458 produce_ppns[i] =
459 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
460
461 for (i = 0; i < num_consume_pages; i++)
462 consume_ppns[i] =
463 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
464
465 ppn_set->num_produce_pages = num_produce_pages;
466 ppn_set->num_consume_pages = num_consume_pages;
467 ppn_set->produce_ppns = produce_ppns;
468 ppn_set->consume_ppns = consume_ppns;
469 ppn_set->initialized = true;
470 return VMCI_SUCCESS;
471}
472
473/*
474 * Frees the two list of PPNs for a queue pair.
475 */
476static void qp_free_ppn_set(struct ppn_set *ppn_set)
477{
478 if (ppn_set->initialized) {
479 /* Do not call these functions on NULL inputs. */
480 kfree(ppn_set->produce_ppns);
481 kfree(ppn_set->consume_ppns);
482 }
483 memset(ppn_set, 0, sizeof(*ppn_set));
484}
485
486/*
487 * Populates the list of PPNs in the hypercall structure with the PPNS
488 * of the produce queue and the consume queue.
489 */
490static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
491{
492 if (vmci_use_ppn64()) {
493 memcpy(call_buf, ppn_set->produce_ppns,
494 ppn_set->num_produce_pages *
495 sizeof(*ppn_set->produce_ppns));
496 memcpy(call_buf +
497 ppn_set->num_produce_pages *
498 sizeof(*ppn_set->produce_ppns),
499 ppn_set->consume_ppns,
500 ppn_set->num_consume_pages *
501 sizeof(*ppn_set->consume_ppns));
502 } else {
503 int i;
504 u32 *ppns = (u32 *) call_buf;
505
506 for (i = 0; i < ppn_set->num_produce_pages; i++)
507 ppns[i] = (u32) ppn_set->produce_ppns[i];
508
509 ppns = &ppns[ppn_set->num_produce_pages];
510
511 for (i = 0; i < ppn_set->num_consume_pages; i++)
512 ppns[i] = (u32) ppn_set->consume_ppns[i];
513 }
514
515 return VMCI_SUCCESS;
516}
517
518/*
519 * Allocates kernel VA space of specified size plus space for the queue
520 * and kernel interface. This is different from the guest queue allocator,
521 * because we do not allocate our own queue header/data pages here but
522 * share those of the guest.
523 */
524static struct vmci_queue *qp_host_alloc_queue(u64 size)
525{
526 struct vmci_queue *queue;
527 size_t queue_page_size;
528 u64 num_pages;
529 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
530
531 if (size > SIZE_MAX - PAGE_SIZE)
532 return NULL;
533 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
534 if (num_pages > (SIZE_MAX - queue_size) /
535 sizeof(*queue->kernel_if->u.h.page))
536 return NULL;
537
538 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
539
540 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
541 if (queue) {
542 queue->q_header = NULL;
543 queue->saved_header = NULL;
544 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
545 queue->kernel_if->host = true;
546 queue->kernel_if->mutex = NULL;
547 queue->kernel_if->num_pages = num_pages;
548 queue->kernel_if->u.h.header_page =
549 (struct page **)((u8 *)queue + queue_size);
550 queue->kernel_if->u.h.page =
551 &queue->kernel_if->u.h.header_page[1];
552 }
553
554 return queue;
555}
556
557/*
558 * Frees kernel memory for a given queue (header plus translation
559 * structure).
560 */
561static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
562{
563 kfree(queue);
564}
565
566/*
567 * Initialize the mutex for the pair of queues. This mutex is used to
568 * protect the q_header and the buffer from changing out from under any
569 * users of either queue. Of course, it's only any good if the mutexes
570 * are actually acquired. Queue structure must lie on non-paged memory
571 * or we cannot guarantee access to the mutex.
572 */
573static void qp_init_queue_mutex(struct vmci_queue *produce_q,
574 struct vmci_queue *consume_q)
575{
576 /*
577 * Only the host queue has shared state - the guest queues do not
578 * need to synchronize access using a queue mutex.
579 */
580
581 if (produce_q->kernel_if->host) {
582 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
583 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
584 mutex_init(produce_q->kernel_if->mutex);
585 }
586}
587
588/*
589 * Cleans up the mutex for the pair of queues.
590 */
591static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
592 struct vmci_queue *consume_q)
593{
594 if (produce_q->kernel_if->host) {
595 produce_q->kernel_if->mutex = NULL;
596 consume_q->kernel_if->mutex = NULL;
597 }
598}
599
600/*
601 * Acquire the mutex for the queue. Note that the produce_q and
602 * the consume_q share a mutex. So, only one of the two need to
603 * be passed in to this routine. Either will work just fine.
604 */
605static void qp_acquire_queue_mutex(struct vmci_queue *queue)
606{
607 if (queue->kernel_if->host)
608 mutex_lock(queue->kernel_if->mutex);
609}
610
611/*
612 * Release the mutex for the queue. Note that the produce_q and
613 * the consume_q share a mutex. So, only one of the two need to
614 * be passed in to this routine. Either will work just fine.
615 */
616static void qp_release_queue_mutex(struct vmci_queue *queue)
617{
618 if (queue->kernel_if->host)
619 mutex_unlock(queue->kernel_if->mutex);
620}
621
622/*
623 * Helper function to release pages in the PageStoreAttachInfo
624 * previously obtained using get_user_pages.
625 */
626static void qp_release_pages(struct page **pages,
627 u64 num_pages, bool dirty)
628{
629 int i;
630
631 for (i = 0; i < num_pages; i++) {
632 if (dirty)
633 set_page_dirty(pages[i]);
634
635 put_page(pages[i]);
636 pages[i] = NULL;
637 }
638}
639
640/*
641 * Lock the user pages referenced by the {produce,consume}Buffer
642 * struct into memory and populate the {produce,consume}Pages
643 * arrays in the attach structure with them.
644 */
645static int qp_host_get_user_memory(u64 produce_uva,
646 u64 consume_uva,
647 struct vmci_queue *produce_q,
648 struct vmci_queue *consume_q)
649{
650 int retval;
651 int err = VMCI_SUCCESS;
652
653 retval = get_user_pages_fast((uintptr_t) produce_uva,
654 produce_q->kernel_if->num_pages,
655 FOLL_WRITE,
656 produce_q->kernel_if->u.h.header_page);
657 if (retval < (int)produce_q->kernel_if->num_pages) {
658 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
659 retval);
660 qp_release_pages(produce_q->kernel_if->u.h.header_page,
661 retval, false);
662 err = VMCI_ERROR_NO_MEM;
663 goto out;
664 }
665
666 retval = get_user_pages_fast((uintptr_t) consume_uva,
667 consume_q->kernel_if->num_pages,
668 FOLL_WRITE,
669 consume_q->kernel_if->u.h.header_page);
670 if (retval < (int)consume_q->kernel_if->num_pages) {
671 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
672 retval);
673 qp_release_pages(consume_q->kernel_if->u.h.header_page,
674 retval, false);
675 qp_release_pages(produce_q->kernel_if->u.h.header_page,
676 produce_q->kernel_if->num_pages, false);
677 err = VMCI_ERROR_NO_MEM;
678 }
679
680 out:
681 return err;
682}
683
684/*
685 * Registers the specification of the user pages used for backing a queue
686 * pair. Enough information to map in pages is stored in the OS specific
687 * part of the struct vmci_queue structure.
688 */
689static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
690 struct vmci_queue *produce_q,
691 struct vmci_queue *consume_q)
692{
693 u64 produce_uva;
694 u64 consume_uva;
695
696 /*
697 * The new style and the old style mapping only differs in
698 * that we either get a single or two UVAs, so we split the
699 * single UVA range at the appropriate spot.
700 */
701 produce_uva = page_store->pages;
702 consume_uva = page_store->pages +
703 produce_q->kernel_if->num_pages * PAGE_SIZE;
704 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
705 consume_q);
706}
707
708/*
709 * Releases and removes the references to user pages stored in the attach
710 * struct. Pages are released from the page cache and may become
711 * swappable again.
712 */
713static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
714 struct vmci_queue *consume_q)
715{
716 qp_release_pages(produce_q->kernel_if->u.h.header_page,
717 produce_q->kernel_if->num_pages, true);
718 memset(produce_q->kernel_if->u.h.header_page, 0,
719 sizeof(*produce_q->kernel_if->u.h.header_page) *
720 produce_q->kernel_if->num_pages);
721 qp_release_pages(consume_q->kernel_if->u.h.header_page,
722 consume_q->kernel_if->num_pages, true);
723 memset(consume_q->kernel_if->u.h.header_page, 0,
724 sizeof(*consume_q->kernel_if->u.h.header_page) *
725 consume_q->kernel_if->num_pages);
726}
727
728/*
729 * Once qp_host_register_user_memory has been performed on a
730 * queue, the queue pair headers can be mapped into the
731 * kernel. Once mapped, they must be unmapped with
732 * qp_host_unmap_queues prior to calling
733 * qp_host_unregister_user_memory.
734 * Pages are pinned.
735 */
736static int qp_host_map_queues(struct vmci_queue *produce_q,
737 struct vmci_queue *consume_q)
738{
739 int result;
740
741 if (!produce_q->q_header || !consume_q->q_header) {
742 struct page *headers[2];
743
744 if (produce_q->q_header != consume_q->q_header)
745 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
746
747 if (produce_q->kernel_if->u.h.header_page == NULL ||
748 *produce_q->kernel_if->u.h.header_page == NULL)
749 return VMCI_ERROR_UNAVAILABLE;
750
751 headers[0] = *produce_q->kernel_if->u.h.header_page;
752 headers[1] = *consume_q->kernel_if->u.h.header_page;
753
754 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
755 if (produce_q->q_header != NULL) {
756 consume_q->q_header =
757 (struct vmci_queue_header *)((u8 *)
758 produce_q->q_header +
759 PAGE_SIZE);
760 result = VMCI_SUCCESS;
761 } else {
762 pr_warn("vmap failed\n");
763 result = VMCI_ERROR_NO_MEM;
764 }
765 } else {
766 result = VMCI_SUCCESS;
767 }
768
769 return result;
770}
771
772/*
773 * Unmaps previously mapped queue pair headers from the kernel.
774 * Pages are unpinned.
775 */
776static int qp_host_unmap_queues(u32 gid,
777 struct vmci_queue *produce_q,
778 struct vmci_queue *consume_q)
779{
780 if (produce_q->q_header) {
781 if (produce_q->q_header < consume_q->q_header)
782 vunmap(produce_q->q_header);
783 else
784 vunmap(consume_q->q_header);
785
786 produce_q->q_header = NULL;
787 consume_q->q_header = NULL;
788 }
789
790 return VMCI_SUCCESS;
791}
792
793/*
794 * Finds the entry in the list corresponding to a given handle. Assumes
795 * that the list is locked.
796 */
797static struct qp_entry *qp_list_find(struct qp_list *qp_list,
798 struct vmci_handle handle)
799{
800 struct qp_entry *entry;
801
802 if (vmci_handle_is_invalid(handle))
803 return NULL;
804
805 list_for_each_entry(entry, &qp_list->head, list_item) {
806 if (vmci_handle_is_equal(entry->handle, handle))
807 return entry;
808 }
809
810 return NULL;
811}
812
813/*
814 * Finds the entry in the list corresponding to a given handle.
815 */
816static struct qp_guest_endpoint *
817qp_guest_handle_to_entry(struct vmci_handle handle)
818{
819 struct qp_guest_endpoint *entry;
820 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
821
822 entry = qp ? container_of(
823 qp, struct qp_guest_endpoint, qp) : NULL;
824 return entry;
825}
826
827/*
828 * Finds the entry in the list corresponding to a given handle.
829 */
830static struct qp_broker_entry *
831qp_broker_handle_to_entry(struct vmci_handle handle)
832{
833 struct qp_broker_entry *entry;
834 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
835
836 entry = qp ? container_of(
837 qp, struct qp_broker_entry, qp) : NULL;
838 return entry;
839}
840
841/*
842 * Dispatches a queue pair event message directly into the local event
843 * queue.
844 */
845static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
846{
847 u32 context_id = vmci_get_context_id();
848 struct vmci_event_qp ev;
849
850 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
851 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
852 VMCI_CONTEXT_RESOURCE_ID);
853 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
854 ev.msg.event_data.event =
855 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
856 ev.payload.peer_id = context_id;
857 ev.payload.handle = handle;
858
859 return vmci_event_dispatch(&ev.msg.hdr);
860}
861
862/*
863 * Allocates and initializes a qp_guest_endpoint structure.
864 * Allocates a queue_pair rid (and handle) iff the given entry has
865 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
866 * are reserved handles. Assumes that the QP list mutex is held
867 * by the caller.
868 */
869static struct qp_guest_endpoint *
870qp_guest_endpoint_create(struct vmci_handle handle,
871 u32 peer,
872 u32 flags,
873 u64 produce_size,
874 u64 consume_size,
875 void *produce_q,
876 void *consume_q)
877{
878 int result;
879 struct qp_guest_endpoint *entry;
880 /* One page each for the queue headers. */
881 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
882 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
883
884 if (vmci_handle_is_invalid(handle)) {
885 u32 context_id = vmci_get_context_id();
886
887 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
888 }
889
890 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
891 if (entry) {
892 entry->qp.peer = peer;
893 entry->qp.flags = flags;
894 entry->qp.produce_size = produce_size;
895 entry->qp.consume_size = consume_size;
896 entry->qp.ref_count = 0;
897 entry->num_ppns = num_ppns;
898 entry->produce_q = produce_q;
899 entry->consume_q = consume_q;
900 INIT_LIST_HEAD(&entry->qp.list_item);
901
902 /* Add resource obj */
903 result = vmci_resource_add(&entry->resource,
904 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
905 handle);
906 entry->qp.handle = vmci_resource_handle(&entry->resource);
907 if ((result != VMCI_SUCCESS) ||
908 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
909 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
910 handle.context, handle.resource, result);
911 kfree(entry);
912 entry = NULL;
913 }
914 }
915 return entry;
916}
917
918/*
919 * Frees a qp_guest_endpoint structure.
920 */
921static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
922{
923 qp_free_ppn_set(&entry->ppn_set);
924 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
925 qp_free_queue(entry->produce_q, entry->qp.produce_size);
926 qp_free_queue(entry->consume_q, entry->qp.consume_size);
927 /* Unlink from resource hash table and free callback */
928 vmci_resource_remove(&entry->resource);
929
930 kfree(entry);
931}
932
933/*
934 * Helper to make a queue_pairAlloc hypercall when the driver is
935 * supporting a guest device.
936 */
937static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
938{
939 struct vmci_qp_alloc_msg *alloc_msg;
940 size_t msg_size;
941 size_t ppn_size;
942 int result;
943
944 if (!entry || entry->num_ppns <= 2)
945 return VMCI_ERROR_INVALID_ARGS;
946
947 ppn_size = vmci_use_ppn64() ? sizeof(u64) : sizeof(u32);
948 msg_size = sizeof(*alloc_msg) +
949 (size_t) entry->num_ppns * ppn_size;
950 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
951 if (!alloc_msg)
952 return VMCI_ERROR_NO_MEM;
953
954 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
955 VMCI_QUEUEPAIR_ALLOC);
956 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
957 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
958 alloc_msg->handle = entry->qp.handle;
959 alloc_msg->peer = entry->qp.peer;
960 alloc_msg->flags = entry->qp.flags;
961 alloc_msg->produce_size = entry->qp.produce_size;
962 alloc_msg->consume_size = entry->qp.consume_size;
963 alloc_msg->num_ppns = entry->num_ppns;
964
965 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
966 &entry->ppn_set);
967 if (result == VMCI_SUCCESS)
968 result = vmci_send_datagram(&alloc_msg->hdr);
969
970 kfree(alloc_msg);
971
972 return result;
973}
974
975/*
976 * Helper to make a queue_pairDetach hypercall when the driver is
977 * supporting a guest device.
978 */
979static int qp_detatch_hypercall(struct vmci_handle handle)
980{
981 struct vmci_qp_detach_msg detach_msg;
982
983 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
984 VMCI_QUEUEPAIR_DETACH);
985 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
986 detach_msg.hdr.payload_size = sizeof(handle);
987 detach_msg.handle = handle;
988
989 return vmci_send_datagram(&detach_msg.hdr);
990}
991
992/*
993 * Adds the given entry to the list. Assumes that the list is locked.
994 */
995static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
996{
997 if (entry)
998 list_add(&entry->list_item, &qp_list->head);
999}
1000
1001/*
1002 * Removes the given entry from the list. Assumes that the list is locked.
1003 */
1004static void qp_list_remove_entry(struct qp_list *qp_list,
1005 struct qp_entry *entry)
1006{
1007 if (entry)
1008 list_del(&entry->list_item);
1009}
1010
1011/*
1012 * Helper for VMCI queue_pair detach interface. Frees the physical
1013 * pages for the queue pair.
1014 */
1015static int qp_detatch_guest_work(struct vmci_handle handle)
1016{
1017 int result;
1018 struct qp_guest_endpoint *entry;
1019 u32 ref_count = ~0; /* To avoid compiler warning below */
1020
1021 mutex_lock(&qp_guest_endpoints.mutex);
1022
1023 entry = qp_guest_handle_to_entry(handle);
1024 if (!entry) {
1025 mutex_unlock(&qp_guest_endpoints.mutex);
1026 return VMCI_ERROR_NOT_FOUND;
1027 }
1028
1029 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1030 result = VMCI_SUCCESS;
1031
1032 if (entry->qp.ref_count > 1) {
1033 result = qp_notify_peer_local(false, handle);
1034 /*
1035 * We can fail to notify a local queuepair
1036 * because we can't allocate. We still want
1037 * to release the entry if that happens, so
1038 * don't bail out yet.
1039 */
1040 }
1041 } else {
1042 result = qp_detatch_hypercall(handle);
1043 if (result < VMCI_SUCCESS) {
1044 /*
1045 * We failed to notify a non-local queuepair.
1046 * That other queuepair might still be
1047 * accessing the shared memory, so don't
1048 * release the entry yet. It will get cleaned
1049 * up by VMCIqueue_pair_Exit() if necessary
1050 * (assuming we are going away, otherwise why
1051 * did this fail?).
1052 */
1053
1054 mutex_unlock(&qp_guest_endpoints.mutex);
1055 return result;
1056 }
1057 }
1058
1059 /*
1060 * If we get here then we either failed to notify a local queuepair, or
1061 * we succeeded in all cases. Release the entry if required.
1062 */
1063
1064 entry->qp.ref_count--;
1065 if (entry->qp.ref_count == 0)
1066 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1067
1068 /* If we didn't remove the entry, this could change once we unlock. */
1069 if (entry)
1070 ref_count = entry->qp.ref_count;
1071
1072 mutex_unlock(&qp_guest_endpoints.mutex);
1073
1074 if (ref_count == 0)
1075 qp_guest_endpoint_destroy(entry);
1076
1077 return result;
1078}
1079
1080/*
1081 * This functions handles the actual allocation of a VMCI queue
1082 * pair guest endpoint. Allocates physical pages for the queue
1083 * pair. It makes OS dependent calls through generic wrappers.
1084 */
1085static int qp_alloc_guest_work(struct vmci_handle *handle,
1086 struct vmci_queue **produce_q,
1087 u64 produce_size,
1088 struct vmci_queue **consume_q,
1089 u64 consume_size,
1090 u32 peer,
1091 u32 flags,
1092 u32 priv_flags)
1093{
1094 const u64 num_produce_pages =
1095 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1096 const u64 num_consume_pages =
1097 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1098 void *my_produce_q = NULL;
1099 void *my_consume_q = NULL;
1100 int result;
1101 struct qp_guest_endpoint *queue_pair_entry = NULL;
1102
1103 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1104 return VMCI_ERROR_NO_ACCESS;
1105
1106 mutex_lock(&qp_guest_endpoints.mutex);
1107
1108 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1109 if (queue_pair_entry) {
1110 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1111 /* Local attach case. */
1112 if (queue_pair_entry->qp.ref_count > 1) {
1113 pr_devel("Error attempting to attach more than once\n");
1114 result = VMCI_ERROR_UNAVAILABLE;
1115 goto error_keep_entry;
1116 }
1117
1118 if (queue_pair_entry->qp.produce_size != consume_size ||
1119 queue_pair_entry->qp.consume_size !=
1120 produce_size ||
1121 queue_pair_entry->qp.flags !=
1122 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1123 pr_devel("Error mismatched queue pair in local attach\n");
1124 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1125 goto error_keep_entry;
1126 }
1127
1128 /*
1129 * Do a local attach. We swap the consume and
1130 * produce queues for the attacher and deliver
1131 * an attach event.
1132 */
1133 result = qp_notify_peer_local(true, *handle);
1134 if (result < VMCI_SUCCESS)
1135 goto error_keep_entry;
1136
1137 my_produce_q = queue_pair_entry->consume_q;
1138 my_consume_q = queue_pair_entry->produce_q;
1139 goto out;
1140 }
1141
1142 result = VMCI_ERROR_ALREADY_EXISTS;
1143 goto error_keep_entry;
1144 }
1145
1146 my_produce_q = qp_alloc_queue(produce_size, flags);
1147 if (!my_produce_q) {
1148 pr_warn("Error allocating pages for produce queue\n");
1149 result = VMCI_ERROR_NO_MEM;
1150 goto error;
1151 }
1152
1153 my_consume_q = qp_alloc_queue(consume_size, flags);
1154 if (!my_consume_q) {
1155 pr_warn("Error allocating pages for consume queue\n");
1156 result = VMCI_ERROR_NO_MEM;
1157 goto error;
1158 }
1159
1160 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1161 produce_size, consume_size,
1162 my_produce_q, my_consume_q);
1163 if (!queue_pair_entry) {
1164 pr_warn("Error allocating memory in %s\n", __func__);
1165 result = VMCI_ERROR_NO_MEM;
1166 goto error;
1167 }
1168
1169 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1170 num_consume_pages,
1171 &queue_pair_entry->ppn_set);
1172 if (result < VMCI_SUCCESS) {
1173 pr_warn("qp_alloc_ppn_set failed\n");
1174 goto error;
1175 }
1176
1177 /*
1178 * It's only necessary to notify the host if this queue pair will be
1179 * attached to from another context.
1180 */
1181 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1182 /* Local create case. */
1183 u32 context_id = vmci_get_context_id();
1184
1185 /*
1186 * Enforce similar checks on local queue pairs as we
1187 * do for regular ones. The handle's context must
1188 * match the creator or attacher context id (here they
1189 * are both the current context id) and the
1190 * attach-only flag cannot exist during create. We
1191 * also ensure specified peer is this context or an
1192 * invalid one.
1193 */
1194 if (queue_pair_entry->qp.handle.context != context_id ||
1195 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1196 queue_pair_entry->qp.peer != context_id)) {
1197 result = VMCI_ERROR_NO_ACCESS;
1198 goto error;
1199 }
1200
1201 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1202 result = VMCI_ERROR_NOT_FOUND;
1203 goto error;
1204 }
1205 } else {
1206 result = qp_alloc_hypercall(queue_pair_entry);
1207 if (result < VMCI_SUCCESS) {
1208 pr_warn("qp_alloc_hypercall result = %d\n", result);
1209 goto error;
1210 }
1211 }
1212
1213 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1214 (struct vmci_queue *)my_consume_q);
1215
1216 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1217
1218 out:
1219 queue_pair_entry->qp.ref_count++;
1220 *handle = queue_pair_entry->qp.handle;
1221 *produce_q = (struct vmci_queue *)my_produce_q;
1222 *consume_q = (struct vmci_queue *)my_consume_q;
1223
1224 /*
1225 * We should initialize the queue pair header pages on a local
1226 * queue pair create. For non-local queue pairs, the
1227 * hypervisor initializes the header pages in the create step.
1228 */
1229 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1230 queue_pair_entry->qp.ref_count == 1) {
1231 vmci_q_header_init((*produce_q)->q_header, *handle);
1232 vmci_q_header_init((*consume_q)->q_header, *handle);
1233 }
1234
1235 mutex_unlock(&qp_guest_endpoints.mutex);
1236
1237 return VMCI_SUCCESS;
1238
1239 error:
1240 mutex_unlock(&qp_guest_endpoints.mutex);
1241 if (queue_pair_entry) {
1242 /* The queues will be freed inside the destroy routine. */
1243 qp_guest_endpoint_destroy(queue_pair_entry);
1244 } else {
1245 qp_free_queue(my_produce_q, produce_size);
1246 qp_free_queue(my_consume_q, consume_size);
1247 }
1248 return result;
1249
1250 error_keep_entry:
1251 /* This path should only be used when an existing entry was found. */
1252 mutex_unlock(&qp_guest_endpoints.mutex);
1253 return result;
1254}
1255
1256/*
1257 * The first endpoint issuing a queue pair allocation will create the state
1258 * of the queue pair in the queue pair broker.
1259 *
1260 * If the creator is a guest, it will associate a VMX virtual address range
1261 * with the queue pair as specified by the page_store. For compatibility with
1262 * older VMX'en, that would use a separate step to set the VMX virtual
1263 * address range, the virtual address range can be registered later using
1264 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1265 * used.
1266 *
1267 * If the creator is the host, a page_store of NULL should be used as well,
1268 * since the host is not able to supply a page store for the queue pair.
1269 *
1270 * For older VMX and host callers, the queue pair will be created in the
1271 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1272 * created in VMCOQPB_CREATED_MEM state.
1273 */
1274static int qp_broker_create(struct vmci_handle handle,
1275 u32 peer,
1276 u32 flags,
1277 u32 priv_flags,
1278 u64 produce_size,
1279 u64 consume_size,
1280 struct vmci_qp_page_store *page_store,
1281 struct vmci_ctx *context,
1282 vmci_event_release_cb wakeup_cb,
1283 void *client_data, struct qp_broker_entry **ent)
1284{
1285 struct qp_broker_entry *entry = NULL;
1286 const u32 context_id = vmci_ctx_get_id(context);
1287 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1288 int result;
1289 u64 guest_produce_size;
1290 u64 guest_consume_size;
1291
1292 /* Do not create if the caller asked not to. */
1293 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1294 return VMCI_ERROR_NOT_FOUND;
1295
1296 /*
1297 * Creator's context ID should match handle's context ID or the creator
1298 * must allow the context in handle's context ID as the "peer".
1299 */
1300 if (handle.context != context_id && handle.context != peer)
1301 return VMCI_ERROR_NO_ACCESS;
1302
1303 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1304 return VMCI_ERROR_DST_UNREACHABLE;
1305
1306 /*
1307 * Creator's context ID for local queue pairs should match the
1308 * peer, if a peer is specified.
1309 */
1310 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1311 return VMCI_ERROR_NO_ACCESS;
1312
1313 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1314 if (!entry)
1315 return VMCI_ERROR_NO_MEM;
1316
1317 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1318 /*
1319 * The queue pair broker entry stores values from the guest
1320 * point of view, so a creating host side endpoint should swap
1321 * produce and consume values -- unless it is a local queue
1322 * pair, in which case no swapping is necessary, since the local
1323 * attacher will swap queues.
1324 */
1325
1326 guest_produce_size = consume_size;
1327 guest_consume_size = produce_size;
1328 } else {
1329 guest_produce_size = produce_size;
1330 guest_consume_size = consume_size;
1331 }
1332
1333 entry->qp.handle = handle;
1334 entry->qp.peer = peer;
1335 entry->qp.flags = flags;
1336 entry->qp.produce_size = guest_produce_size;
1337 entry->qp.consume_size = guest_consume_size;
1338 entry->qp.ref_count = 1;
1339 entry->create_id = context_id;
1340 entry->attach_id = VMCI_INVALID_ID;
1341 entry->state = VMCIQPB_NEW;
1342 entry->require_trusted_attach =
1343 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1344 entry->created_by_trusted =
1345 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1346 entry->vmci_page_files = false;
1347 entry->wakeup_cb = wakeup_cb;
1348 entry->client_data = client_data;
1349 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1350 if (entry->produce_q == NULL) {
1351 result = VMCI_ERROR_NO_MEM;
1352 goto error;
1353 }
1354 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1355 if (entry->consume_q == NULL) {
1356 result = VMCI_ERROR_NO_MEM;
1357 goto error;
1358 }
1359
1360 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1361
1362 INIT_LIST_HEAD(&entry->qp.list_item);
1363
1364 if (is_local) {
1365 u8 *tmp;
1366
1367 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1368 PAGE_SIZE, GFP_KERNEL);
1369 if (entry->local_mem == NULL) {
1370 result = VMCI_ERROR_NO_MEM;
1371 goto error;
1372 }
1373 entry->state = VMCIQPB_CREATED_MEM;
1374 entry->produce_q->q_header = entry->local_mem;
1375 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1376 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1377 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1378 } else if (page_store) {
1379 /*
1380 * The VMX already initialized the queue pair headers, so no
1381 * need for the kernel side to do that.
1382 */
1383 result = qp_host_register_user_memory(page_store,
1384 entry->produce_q,
1385 entry->consume_q);
1386 if (result < VMCI_SUCCESS)
1387 goto error;
1388
1389 entry->state = VMCIQPB_CREATED_MEM;
1390 } else {
1391 /*
1392 * A create without a page_store may be either a host
1393 * side create (in which case we are waiting for the
1394 * guest side to supply the memory) or an old style
1395 * queue pair create (in which case we will expect a
1396 * set page store call as the next step).
1397 */
1398 entry->state = VMCIQPB_CREATED_NO_MEM;
1399 }
1400
1401 qp_list_add_entry(&qp_broker_list, &entry->qp);
1402 if (ent != NULL)
1403 *ent = entry;
1404
1405 /* Add to resource obj */
1406 result = vmci_resource_add(&entry->resource,
1407 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1408 handle);
1409 if (result != VMCI_SUCCESS) {
1410 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1411 handle.context, handle.resource, result);
1412 goto error;
1413 }
1414
1415 entry->qp.handle = vmci_resource_handle(&entry->resource);
1416 if (is_local) {
1417 vmci_q_header_init(entry->produce_q->q_header,
1418 entry->qp.handle);
1419 vmci_q_header_init(entry->consume_q->q_header,
1420 entry->qp.handle);
1421 }
1422
1423 vmci_ctx_qp_create(context, entry->qp.handle);
1424
1425 return VMCI_SUCCESS;
1426
1427 error:
1428 if (entry != NULL) {
1429 qp_host_free_queue(entry->produce_q, guest_produce_size);
1430 qp_host_free_queue(entry->consume_q, guest_consume_size);
1431 kfree(entry);
1432 }
1433
1434 return result;
1435}
1436
1437/*
1438 * Enqueues an event datagram to notify the peer VM attached to
1439 * the given queue pair handle about attach/detach event by the
1440 * given VM. Returns Payload size of datagram enqueued on
1441 * success, error code otherwise.
1442 */
1443static int qp_notify_peer(bool attach,
1444 struct vmci_handle handle,
1445 u32 my_id,
1446 u32 peer_id)
1447{
1448 int rv;
1449 struct vmci_event_qp ev;
1450
1451 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1452 peer_id == VMCI_INVALID_ID)
1453 return VMCI_ERROR_INVALID_ARGS;
1454
1455 /*
1456 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1457 * number of pending events from the hypervisor to a given VM
1458 * otherwise a rogue VM could do an arbitrary number of attach
1459 * and detach operations causing memory pressure in the host
1460 * kernel.
1461 */
1462
1463 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1464 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1465 VMCI_CONTEXT_RESOURCE_ID);
1466 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1467 ev.msg.event_data.event = attach ?
1468 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1469 ev.payload.handle = handle;
1470 ev.payload.peer_id = my_id;
1471
1472 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1473 &ev.msg.hdr, false);
1474 if (rv < VMCI_SUCCESS)
1475 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1476 attach ? "ATTACH" : "DETACH", peer_id);
1477
1478 return rv;
1479}
1480
1481/*
1482 * The second endpoint issuing a queue pair allocation will attach to
1483 * the queue pair registered with the queue pair broker.
1484 *
1485 * If the attacher is a guest, it will associate a VMX virtual address
1486 * range with the queue pair as specified by the page_store. At this
1487 * point, the already attach host endpoint may start using the queue
1488 * pair, and an attach event is sent to it. For compatibility with
1489 * older VMX'en, that used a separate step to set the VMX virtual
1490 * address range, the virtual address range can be registered later
1491 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1492 * NULL should be used, and the attach event will be generated once
1493 * the actual page store has been set.
1494 *
1495 * If the attacher is the host, a page_store of NULL should be used as
1496 * well, since the page store information is already set by the guest.
1497 *
1498 * For new VMX and host callers, the queue pair will be moved to the
1499 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1500 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1501 */
1502static int qp_broker_attach(struct qp_broker_entry *entry,
1503 u32 peer,
1504 u32 flags,
1505 u32 priv_flags,
1506 u64 produce_size,
1507 u64 consume_size,
1508 struct vmci_qp_page_store *page_store,
1509 struct vmci_ctx *context,
1510 vmci_event_release_cb wakeup_cb,
1511 void *client_data,
1512 struct qp_broker_entry **ent)
1513{
1514 const u32 context_id = vmci_ctx_get_id(context);
1515 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1516 int result;
1517
1518 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1519 entry->state != VMCIQPB_CREATED_MEM)
1520 return VMCI_ERROR_UNAVAILABLE;
1521
1522 if (is_local) {
1523 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1524 context_id != entry->create_id) {
1525 return VMCI_ERROR_INVALID_ARGS;
1526 }
1527 } else if (context_id == entry->create_id ||
1528 context_id == entry->attach_id) {
1529 return VMCI_ERROR_ALREADY_EXISTS;
1530 }
1531
1532 if (VMCI_CONTEXT_IS_VM(context_id) &&
1533 VMCI_CONTEXT_IS_VM(entry->create_id))
1534 return VMCI_ERROR_DST_UNREACHABLE;
1535
1536 /*
1537 * If we are attaching from a restricted context then the queuepair
1538 * must have been created by a trusted endpoint.
1539 */
1540 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1541 !entry->created_by_trusted)
1542 return VMCI_ERROR_NO_ACCESS;
1543
1544 /*
1545 * If we are attaching to a queuepair that was created by a restricted
1546 * context then we must be trusted.
1547 */
1548 if (entry->require_trusted_attach &&
1549 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1550 return VMCI_ERROR_NO_ACCESS;
1551
1552 /*
1553 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1554 * control check is not performed.
1555 */
1556 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1557 return VMCI_ERROR_NO_ACCESS;
1558
1559 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1560 /*
1561 * Do not attach if the caller doesn't support Host Queue Pairs
1562 * and a host created this queue pair.
1563 */
1564
1565 if (!vmci_ctx_supports_host_qp(context))
1566 return VMCI_ERROR_INVALID_RESOURCE;
1567
1568 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1569 struct vmci_ctx *create_context;
1570 bool supports_host_qp;
1571
1572 /*
1573 * Do not attach a host to a user created queue pair if that
1574 * user doesn't support host queue pair end points.
1575 */
1576
1577 create_context = vmci_ctx_get(entry->create_id);
1578 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1579 vmci_ctx_put(create_context);
1580
1581 if (!supports_host_qp)
1582 return VMCI_ERROR_INVALID_RESOURCE;
1583 }
1584
1585 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1586 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1587
1588 if (context_id != VMCI_HOST_CONTEXT_ID) {
1589 /*
1590 * The queue pair broker entry stores values from the guest
1591 * point of view, so an attaching guest should match the values
1592 * stored in the entry.
1593 */
1594
1595 if (entry->qp.produce_size != produce_size ||
1596 entry->qp.consume_size != consume_size) {
1597 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1598 }
1599 } else if (entry->qp.produce_size != consume_size ||
1600 entry->qp.consume_size != produce_size) {
1601 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1602 }
1603
1604 if (context_id != VMCI_HOST_CONTEXT_ID) {
1605 /*
1606 * If a guest attached to a queue pair, it will supply
1607 * the backing memory. If this is a pre NOVMVM vmx,
1608 * the backing memory will be supplied by calling
1609 * vmci_qp_broker_set_page_store() following the
1610 * return of the vmci_qp_broker_alloc() call. If it is
1611 * a vmx of version NOVMVM or later, the page store
1612 * must be supplied as part of the
1613 * vmci_qp_broker_alloc call. Under all circumstances
1614 * must the initially created queue pair not have any
1615 * memory associated with it already.
1616 */
1617
1618 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1619 return VMCI_ERROR_INVALID_ARGS;
1620
1621 if (page_store != NULL) {
1622 /*
1623 * Patch up host state to point to guest
1624 * supplied memory. The VMX already
1625 * initialized the queue pair headers, so no
1626 * need for the kernel side to do that.
1627 */
1628
1629 result = qp_host_register_user_memory(page_store,
1630 entry->produce_q,
1631 entry->consume_q);
1632 if (result < VMCI_SUCCESS)
1633 return result;
1634
1635 entry->state = VMCIQPB_ATTACHED_MEM;
1636 } else {
1637 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1638 }
1639 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1640 /*
1641 * The host side is attempting to attach to a queue
1642 * pair that doesn't have any memory associated with
1643 * it. This must be a pre NOVMVM vmx that hasn't set
1644 * the page store information yet, or a quiesced VM.
1645 */
1646
1647 return VMCI_ERROR_UNAVAILABLE;
1648 } else {
1649 /* The host side has successfully attached to a queue pair. */
1650 entry->state = VMCIQPB_ATTACHED_MEM;
1651 }
1652
1653 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1654 result =
1655 qp_notify_peer(true, entry->qp.handle, context_id,
1656 entry->create_id);
1657 if (result < VMCI_SUCCESS)
1658 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1659 entry->create_id, entry->qp.handle.context,
1660 entry->qp.handle.resource);
1661 }
1662
1663 entry->attach_id = context_id;
1664 entry->qp.ref_count++;
1665 if (wakeup_cb) {
1666 entry->wakeup_cb = wakeup_cb;
1667 entry->client_data = client_data;
1668 }
1669
1670 /*
1671 * When attaching to local queue pairs, the context already has
1672 * an entry tracking the queue pair, so don't add another one.
1673 */
1674 if (!is_local)
1675 vmci_ctx_qp_create(context, entry->qp.handle);
1676
1677 if (ent != NULL)
1678 *ent = entry;
1679
1680 return VMCI_SUCCESS;
1681}
1682
1683/*
1684 * queue_pair_Alloc for use when setting up queue pair endpoints
1685 * on the host.
1686 */
1687static int qp_broker_alloc(struct vmci_handle handle,
1688 u32 peer,
1689 u32 flags,
1690 u32 priv_flags,
1691 u64 produce_size,
1692 u64 consume_size,
1693 struct vmci_qp_page_store *page_store,
1694 struct vmci_ctx *context,
1695 vmci_event_release_cb wakeup_cb,
1696 void *client_data,
1697 struct qp_broker_entry **ent,
1698 bool *swap)
1699{
1700 const u32 context_id = vmci_ctx_get_id(context);
1701 bool create;
1702 struct qp_broker_entry *entry = NULL;
1703 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1704 int result;
1705
1706 if (vmci_handle_is_invalid(handle) ||
1707 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1708 !(produce_size || consume_size) ||
1709 !context || context_id == VMCI_INVALID_ID ||
1710 handle.context == VMCI_INVALID_ID) {
1711 return VMCI_ERROR_INVALID_ARGS;
1712 }
1713
1714 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1715 return VMCI_ERROR_INVALID_ARGS;
1716
1717 /*
1718 * In the initial argument check, we ensure that non-vmkernel hosts
1719 * are not allowed to create local queue pairs.
1720 */
1721
1722 mutex_lock(&qp_broker_list.mutex);
1723
1724 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1725 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1726 context_id, handle.context, handle.resource);
1727 mutex_unlock(&qp_broker_list.mutex);
1728 return VMCI_ERROR_ALREADY_EXISTS;
1729 }
1730
1731 if (handle.resource != VMCI_INVALID_ID)
1732 entry = qp_broker_handle_to_entry(handle);
1733
1734 if (!entry) {
1735 create = true;
1736 result =
1737 qp_broker_create(handle, peer, flags, priv_flags,
1738 produce_size, consume_size, page_store,
1739 context, wakeup_cb, client_data, ent);
1740 } else {
1741 create = false;
1742 result =
1743 qp_broker_attach(entry, peer, flags, priv_flags,
1744 produce_size, consume_size, page_store,
1745 context, wakeup_cb, client_data, ent);
1746 }
1747
1748 mutex_unlock(&qp_broker_list.mutex);
1749
1750 if (swap)
1751 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1752 !(create && is_local);
1753
1754 return result;
1755}
1756
1757/*
1758 * This function implements the kernel API for allocating a queue
1759 * pair.
1760 */
1761static int qp_alloc_host_work(struct vmci_handle *handle,
1762 struct vmci_queue **produce_q,
1763 u64 produce_size,
1764 struct vmci_queue **consume_q,
1765 u64 consume_size,
1766 u32 peer,
1767 u32 flags,
1768 u32 priv_flags,
1769 vmci_event_release_cb wakeup_cb,
1770 void *client_data)
1771{
1772 struct vmci_handle new_handle;
1773 struct vmci_ctx *context;
1774 struct qp_broker_entry *entry;
1775 int result;
1776 bool swap;
1777
1778 if (vmci_handle_is_invalid(*handle)) {
1779 new_handle = vmci_make_handle(
1780 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1781 } else
1782 new_handle = *handle;
1783
1784 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1785 entry = NULL;
1786 result =
1787 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1788 produce_size, consume_size, NULL, context,
1789 wakeup_cb, client_data, &entry, &swap);
1790 if (result == VMCI_SUCCESS) {
1791 if (swap) {
1792 /*
1793 * If this is a local queue pair, the attacher
1794 * will swap around produce and consume
1795 * queues.
1796 */
1797
1798 *produce_q = entry->consume_q;
1799 *consume_q = entry->produce_q;
1800 } else {
1801 *produce_q = entry->produce_q;
1802 *consume_q = entry->consume_q;
1803 }
1804
1805 *handle = vmci_resource_handle(&entry->resource);
1806 } else {
1807 *handle = VMCI_INVALID_HANDLE;
1808 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1809 result);
1810 }
1811 vmci_ctx_put(context);
1812 return result;
1813}
1814
1815/*
1816 * Allocates a VMCI queue_pair. Only checks validity of input
1817 * arguments. The real work is done in the host or guest
1818 * specific function.
1819 */
1820int vmci_qp_alloc(struct vmci_handle *handle,
1821 struct vmci_queue **produce_q,
1822 u64 produce_size,
1823 struct vmci_queue **consume_q,
1824 u64 consume_size,
1825 u32 peer,
1826 u32 flags,
1827 u32 priv_flags,
1828 bool guest_endpoint,
1829 vmci_event_release_cb wakeup_cb,
1830 void *client_data)
1831{
1832 if (!handle || !produce_q || !consume_q ||
1833 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1834 return VMCI_ERROR_INVALID_ARGS;
1835
1836 if (guest_endpoint) {
1837 return qp_alloc_guest_work(handle, produce_q,
1838 produce_size, consume_q,
1839 consume_size, peer,
1840 flags, priv_flags);
1841 } else {
1842 return qp_alloc_host_work(handle, produce_q,
1843 produce_size, consume_q,
1844 consume_size, peer, flags,
1845 priv_flags, wakeup_cb, client_data);
1846 }
1847}
1848
1849/*
1850 * This function implements the host kernel API for detaching from
1851 * a queue pair.
1852 */
1853static int qp_detatch_host_work(struct vmci_handle handle)
1854{
1855 int result;
1856 struct vmci_ctx *context;
1857
1858 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1859
1860 result = vmci_qp_broker_detach(handle, context);
1861
1862 vmci_ctx_put(context);
1863 return result;
1864}
1865
1866/*
1867 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1868 * Real work is done in the host or guest specific function.
1869 */
1870static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1871{
1872 if (vmci_handle_is_invalid(handle))
1873 return VMCI_ERROR_INVALID_ARGS;
1874
1875 if (guest_endpoint)
1876 return qp_detatch_guest_work(handle);
1877 else
1878 return qp_detatch_host_work(handle);
1879}
1880
1881/*
1882 * Returns the entry from the head of the list. Assumes that the list is
1883 * locked.
1884 */
1885static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1886{
1887 if (!list_empty(&qp_list->head)) {
1888 struct qp_entry *entry =
1889 list_first_entry(&qp_list->head, struct qp_entry,
1890 list_item);
1891 return entry;
1892 }
1893
1894 return NULL;
1895}
1896
1897void vmci_qp_broker_exit(void)
1898{
1899 struct qp_entry *entry;
1900 struct qp_broker_entry *be;
1901
1902 mutex_lock(&qp_broker_list.mutex);
1903
1904 while ((entry = qp_list_get_head(&qp_broker_list))) {
1905 be = (struct qp_broker_entry *)entry;
1906
1907 qp_list_remove_entry(&qp_broker_list, entry);
1908 kfree(be);
1909 }
1910
1911 mutex_unlock(&qp_broker_list.mutex);
1912}
1913
1914/*
1915 * Requests that a queue pair be allocated with the VMCI queue
1916 * pair broker. Allocates a queue pair entry if one does not
1917 * exist. Attaches to one if it exists, and retrieves the page
1918 * files backing that queue_pair. Assumes that the queue pair
1919 * broker lock is held.
1920 */
1921int vmci_qp_broker_alloc(struct vmci_handle handle,
1922 u32 peer,
1923 u32 flags,
1924 u32 priv_flags,
1925 u64 produce_size,
1926 u64 consume_size,
1927 struct vmci_qp_page_store *page_store,
1928 struct vmci_ctx *context)
1929{
1930 return qp_broker_alloc(handle, peer, flags, priv_flags,
1931 produce_size, consume_size,
1932 page_store, context, NULL, NULL, NULL, NULL);
1933}
1934
1935/*
1936 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
1937 * step to add the UVAs of the VMX mapping of the queue pair. This function
1938 * provides backwards compatibility with such VMX'en, and takes care of
1939 * registering the page store for a queue pair previously allocated by the
1940 * VMX during create or attach. This function will move the queue pair state
1941 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
1942 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
1943 * attached state with memory, the queue pair is ready to be used by the
1944 * host peer, and an attached event will be generated.
1945 *
1946 * Assumes that the queue pair broker lock is held.
1947 *
1948 * This function is only used by the hosted platform, since there is no
1949 * issue with backwards compatibility for vmkernel.
1950 */
1951int vmci_qp_broker_set_page_store(struct vmci_handle handle,
1952 u64 produce_uva,
1953 u64 consume_uva,
1954 struct vmci_ctx *context)
1955{
1956 struct qp_broker_entry *entry;
1957 int result;
1958 const u32 context_id = vmci_ctx_get_id(context);
1959
1960 if (vmci_handle_is_invalid(handle) || !context ||
1961 context_id == VMCI_INVALID_ID)
1962 return VMCI_ERROR_INVALID_ARGS;
1963
1964 /*
1965 * We only support guest to host queue pairs, so the VMX must
1966 * supply UVAs for the mapped page files.
1967 */
1968
1969 if (produce_uva == 0 || consume_uva == 0)
1970 return VMCI_ERROR_INVALID_ARGS;
1971
1972 mutex_lock(&qp_broker_list.mutex);
1973
1974 if (!vmci_ctx_qp_exists(context, handle)) {
1975 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
1976 context_id, handle.context, handle.resource);
1977 result = VMCI_ERROR_NOT_FOUND;
1978 goto out;
1979 }
1980
1981 entry = qp_broker_handle_to_entry(handle);
1982 if (!entry) {
1983 result = VMCI_ERROR_NOT_FOUND;
1984 goto out;
1985 }
1986
1987 /*
1988 * If I'm the owner then I can set the page store.
1989 *
1990 * Or, if a host created the queue_pair and I'm the attached peer
1991 * then I can set the page store.
1992 */
1993 if (entry->create_id != context_id &&
1994 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
1995 entry->attach_id != context_id)) {
1996 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
1997 goto out;
1998 }
1999
2000 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2001 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2002 result = VMCI_ERROR_UNAVAILABLE;
2003 goto out;
2004 }
2005
2006 result = qp_host_get_user_memory(produce_uva, consume_uva,
2007 entry->produce_q, entry->consume_q);
2008 if (result < VMCI_SUCCESS)
2009 goto out;
2010
2011 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2012 if (result < VMCI_SUCCESS) {
2013 qp_host_unregister_user_memory(entry->produce_q,
2014 entry->consume_q);
2015 goto out;
2016 }
2017
2018 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2019 entry->state = VMCIQPB_CREATED_MEM;
2020 else
2021 entry->state = VMCIQPB_ATTACHED_MEM;
2022
2023 entry->vmci_page_files = true;
2024
2025 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2026 result =
2027 qp_notify_peer(true, handle, context_id, entry->create_id);
2028 if (result < VMCI_SUCCESS) {
2029 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2030 entry->create_id, entry->qp.handle.context,
2031 entry->qp.handle.resource);
2032 }
2033 }
2034
2035 result = VMCI_SUCCESS;
2036 out:
2037 mutex_unlock(&qp_broker_list.mutex);
2038 return result;
2039}
2040
2041/*
2042 * Resets saved queue headers for the given QP broker
2043 * entry. Should be used when guest memory becomes available
2044 * again, or the guest detaches.
2045 */
2046static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2047{
2048 entry->produce_q->saved_header = NULL;
2049 entry->consume_q->saved_header = NULL;
2050}
2051
2052/*
2053 * The main entry point for detaching from a queue pair registered with the
2054 * queue pair broker. If more than one endpoint is attached to the queue
2055 * pair, the first endpoint will mainly decrement a reference count and
2056 * generate a notification to its peer. The last endpoint will clean up
2057 * the queue pair state registered with the broker.
2058 *
2059 * When a guest endpoint detaches, it will unmap and unregister the guest
2060 * memory backing the queue pair. If the host is still attached, it will
2061 * no longer be able to access the queue pair content.
2062 *
2063 * If the queue pair is already in a state where there is no memory
2064 * registered for the queue pair (any *_NO_MEM state), it will transition to
2065 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2066 * endpoint is the first of two endpoints to detach. If the host endpoint is
2067 * the first out of two to detach, the queue pair will move to the
2068 * VMCIQPB_SHUTDOWN_MEM state.
2069 */
2070int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2071{
2072 struct qp_broker_entry *entry;
2073 const u32 context_id = vmci_ctx_get_id(context);
2074 u32 peer_id;
2075 bool is_local = false;
2076 int result;
2077
2078 if (vmci_handle_is_invalid(handle) || !context ||
2079 context_id == VMCI_INVALID_ID) {
2080 return VMCI_ERROR_INVALID_ARGS;
2081 }
2082
2083 mutex_lock(&qp_broker_list.mutex);
2084
2085 if (!vmci_ctx_qp_exists(context, handle)) {
2086 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2087 context_id, handle.context, handle.resource);
2088 result = VMCI_ERROR_NOT_FOUND;
2089 goto out;
2090 }
2091
2092 entry = qp_broker_handle_to_entry(handle);
2093 if (!entry) {
2094 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2095 context_id, handle.context, handle.resource);
2096 result = VMCI_ERROR_NOT_FOUND;
2097 goto out;
2098 }
2099
2100 if (context_id != entry->create_id && context_id != entry->attach_id) {
2101 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2102 goto out;
2103 }
2104
2105 if (context_id == entry->create_id) {
2106 peer_id = entry->attach_id;
2107 entry->create_id = VMCI_INVALID_ID;
2108 } else {
2109 peer_id = entry->create_id;
2110 entry->attach_id = VMCI_INVALID_ID;
2111 }
2112 entry->qp.ref_count--;
2113
2114 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2115
2116 if (context_id != VMCI_HOST_CONTEXT_ID) {
2117 bool headers_mapped;
2118
2119 /*
2120 * Pre NOVMVM vmx'en may detach from a queue pair
2121 * before setting the page store, and in that case
2122 * there is no user memory to detach from. Also, more
2123 * recent VMX'en may detach from a queue pair in the
2124 * quiesced state.
2125 */
2126
2127 qp_acquire_queue_mutex(entry->produce_q);
2128 headers_mapped = entry->produce_q->q_header ||
2129 entry->consume_q->q_header;
2130 if (QPBROKERSTATE_HAS_MEM(entry)) {
2131 result =
2132 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2133 entry->produce_q,
2134 entry->consume_q);
2135 if (result < VMCI_SUCCESS)
2136 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2137 handle.context, handle.resource,
2138 result);
2139
2140 qp_host_unregister_user_memory(entry->produce_q,
2141 entry->consume_q);
2142
2143 }
2144
2145 if (!headers_mapped)
2146 qp_reset_saved_headers(entry);
2147
2148 qp_release_queue_mutex(entry->produce_q);
2149
2150 if (!headers_mapped && entry->wakeup_cb)
2151 entry->wakeup_cb(entry->client_data);
2152
2153 } else {
2154 if (entry->wakeup_cb) {
2155 entry->wakeup_cb = NULL;
2156 entry->client_data = NULL;
2157 }
2158 }
2159
2160 if (entry->qp.ref_count == 0) {
2161 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2162
2163 if (is_local)
2164 kfree(entry->local_mem);
2165
2166 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2167 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2168 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2169 /* Unlink from resource hash table and free callback */
2170 vmci_resource_remove(&entry->resource);
2171
2172 kfree(entry);
2173
2174 vmci_ctx_qp_destroy(context, handle);
2175 } else {
2176 qp_notify_peer(false, handle, context_id, peer_id);
2177 if (context_id == VMCI_HOST_CONTEXT_ID &&
2178 QPBROKERSTATE_HAS_MEM(entry)) {
2179 entry->state = VMCIQPB_SHUTDOWN_MEM;
2180 } else {
2181 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2182 }
2183
2184 if (!is_local)
2185 vmci_ctx_qp_destroy(context, handle);
2186
2187 }
2188 result = VMCI_SUCCESS;
2189 out:
2190 mutex_unlock(&qp_broker_list.mutex);
2191 return result;
2192}
2193
2194/*
2195 * Establishes the necessary mappings for a queue pair given a
2196 * reference to the queue pair guest memory. This is usually
2197 * called when a guest is unquiesced and the VMX is allowed to
2198 * map guest memory once again.
2199 */
2200int vmci_qp_broker_map(struct vmci_handle handle,
2201 struct vmci_ctx *context,
2202 u64 guest_mem)
2203{
2204 struct qp_broker_entry *entry;
2205 const u32 context_id = vmci_ctx_get_id(context);
2206 int result;
2207
2208 if (vmci_handle_is_invalid(handle) || !context ||
2209 context_id == VMCI_INVALID_ID)
2210 return VMCI_ERROR_INVALID_ARGS;
2211
2212 mutex_lock(&qp_broker_list.mutex);
2213
2214 if (!vmci_ctx_qp_exists(context, handle)) {
2215 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2216 context_id, handle.context, handle.resource);
2217 result = VMCI_ERROR_NOT_FOUND;
2218 goto out;
2219 }
2220
2221 entry = qp_broker_handle_to_entry(handle);
2222 if (!entry) {
2223 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2224 context_id, handle.context, handle.resource);
2225 result = VMCI_ERROR_NOT_FOUND;
2226 goto out;
2227 }
2228
2229 if (context_id != entry->create_id && context_id != entry->attach_id) {
2230 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2231 goto out;
2232 }
2233
2234 result = VMCI_SUCCESS;
2235
2236 if (context_id != VMCI_HOST_CONTEXT_ID) {
2237 struct vmci_qp_page_store page_store;
2238
2239 page_store.pages = guest_mem;
2240 page_store.len = QPE_NUM_PAGES(entry->qp);
2241
2242 qp_acquire_queue_mutex(entry->produce_q);
2243 qp_reset_saved_headers(entry);
2244 result =
2245 qp_host_register_user_memory(&page_store,
2246 entry->produce_q,
2247 entry->consume_q);
2248 qp_release_queue_mutex(entry->produce_q);
2249 if (result == VMCI_SUCCESS) {
2250 /* Move state from *_NO_MEM to *_MEM */
2251
2252 entry->state++;
2253
2254 if (entry->wakeup_cb)
2255 entry->wakeup_cb(entry->client_data);
2256 }
2257 }
2258
2259 out:
2260 mutex_unlock(&qp_broker_list.mutex);
2261 return result;
2262}
2263
2264/*
2265 * Saves a snapshot of the queue headers for the given QP broker
2266 * entry. Should be used when guest memory is unmapped.
2267 * Results:
2268 * VMCI_SUCCESS on success, appropriate error code if guest memory
2269 * can't be accessed..
2270 */
2271static int qp_save_headers(struct qp_broker_entry *entry)
2272{
2273 int result;
2274
2275 if (entry->produce_q->saved_header != NULL &&
2276 entry->consume_q->saved_header != NULL) {
2277 /*
2278 * If the headers have already been saved, we don't need to do
2279 * it again, and we don't want to map in the headers
2280 * unnecessarily.
2281 */
2282
2283 return VMCI_SUCCESS;
2284 }
2285
2286 if (NULL == entry->produce_q->q_header ||
2287 NULL == entry->consume_q->q_header) {
2288 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2289 if (result < VMCI_SUCCESS)
2290 return result;
2291 }
2292
2293 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2294 sizeof(entry->saved_produce_q));
2295 entry->produce_q->saved_header = &entry->saved_produce_q;
2296 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2297 sizeof(entry->saved_consume_q));
2298 entry->consume_q->saved_header = &entry->saved_consume_q;
2299
2300 return VMCI_SUCCESS;
2301}
2302
2303/*
2304 * Removes all references to the guest memory of a given queue pair, and
2305 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2306 * called when a VM is being quiesced where access to guest memory should
2307 * avoided.
2308 */
2309int vmci_qp_broker_unmap(struct vmci_handle handle,
2310 struct vmci_ctx *context,
2311 u32 gid)
2312{
2313 struct qp_broker_entry *entry;
2314 const u32 context_id = vmci_ctx_get_id(context);
2315 int result;
2316
2317 if (vmci_handle_is_invalid(handle) || !context ||
2318 context_id == VMCI_INVALID_ID)
2319 return VMCI_ERROR_INVALID_ARGS;
2320
2321 mutex_lock(&qp_broker_list.mutex);
2322
2323 if (!vmci_ctx_qp_exists(context, handle)) {
2324 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2325 context_id, handle.context, handle.resource);
2326 result = VMCI_ERROR_NOT_FOUND;
2327 goto out;
2328 }
2329
2330 entry = qp_broker_handle_to_entry(handle);
2331 if (!entry) {
2332 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2333 context_id, handle.context, handle.resource);
2334 result = VMCI_ERROR_NOT_FOUND;
2335 goto out;
2336 }
2337
2338 if (context_id != entry->create_id && context_id != entry->attach_id) {
2339 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2340 goto out;
2341 }
2342
2343 if (context_id != VMCI_HOST_CONTEXT_ID) {
2344 qp_acquire_queue_mutex(entry->produce_q);
2345 result = qp_save_headers(entry);
2346 if (result < VMCI_SUCCESS)
2347 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2348 handle.context, handle.resource, result);
2349
2350 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2351
2352 /*
2353 * On hosted, when we unmap queue pairs, the VMX will also
2354 * unmap the guest memory, so we invalidate the previously
2355 * registered memory. If the queue pair is mapped again at a
2356 * later point in time, we will need to reregister the user
2357 * memory with a possibly new user VA.
2358 */
2359 qp_host_unregister_user_memory(entry->produce_q,
2360 entry->consume_q);
2361
2362 /*
2363 * Move state from *_MEM to *_NO_MEM.
2364 */
2365 entry->state--;
2366
2367 qp_release_queue_mutex(entry->produce_q);
2368 }
2369
2370 result = VMCI_SUCCESS;
2371
2372 out:
2373 mutex_unlock(&qp_broker_list.mutex);
2374 return result;
2375}
2376
2377/*
2378 * Destroys all guest queue pair endpoints. If active guest queue
2379 * pairs still exist, hypercalls to attempt detach from these
2380 * queue pairs will be made. Any failure to detach is silently
2381 * ignored.
2382 */
2383void vmci_qp_guest_endpoints_exit(void)
2384{
2385 struct qp_entry *entry;
2386 struct qp_guest_endpoint *ep;
2387
2388 mutex_lock(&qp_guest_endpoints.mutex);
2389
2390 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2391 ep = (struct qp_guest_endpoint *)entry;
2392
2393 /* Don't make a hypercall for local queue_pairs. */
2394 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2395 qp_detatch_hypercall(entry->handle);
2396
2397 /* We cannot fail the exit, so let's reset ref_count. */
2398 entry->ref_count = 0;
2399 qp_list_remove_entry(&qp_guest_endpoints, entry);
2400
2401 qp_guest_endpoint_destroy(ep);
2402 }
2403
2404 mutex_unlock(&qp_guest_endpoints.mutex);
2405}
2406
2407/*
2408 * Helper routine that will lock the queue pair before subsequent
2409 * operations.
2410 * Note: Non-blocking on the host side is currently only implemented in ESX.
2411 * Since non-blocking isn't yet implemented on the host personality we
2412 * have no reason to acquire a spin lock. So to avoid the use of an
2413 * unnecessary lock only acquire the mutex if we can block.
2414 */
2415static void qp_lock(const struct vmci_qp *qpair)
2416{
2417 qp_acquire_queue_mutex(qpair->produce_q);
2418}
2419
2420/*
2421 * Helper routine that unlocks the queue pair after calling
2422 * qp_lock.
2423 */
2424static void qp_unlock(const struct vmci_qp *qpair)
2425{
2426 qp_release_queue_mutex(qpair->produce_q);
2427}
2428
2429/*
2430 * The queue headers may not be mapped at all times. If a queue is
2431 * currently not mapped, it will be attempted to do so.
2432 */
2433static int qp_map_queue_headers(struct vmci_queue *produce_q,
2434 struct vmci_queue *consume_q)
2435{
2436 int result;
2437
2438 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2439 result = qp_host_map_queues(produce_q, consume_q);
2440 if (result < VMCI_SUCCESS)
2441 return (produce_q->saved_header &&
2442 consume_q->saved_header) ?
2443 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2444 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2445 }
2446
2447 return VMCI_SUCCESS;
2448}
2449
2450/*
2451 * Helper routine that will retrieve the produce and consume
2452 * headers of a given queue pair. If the guest memory of the
2453 * queue pair is currently not available, the saved queue headers
2454 * will be returned, if these are available.
2455 */
2456static int qp_get_queue_headers(const struct vmci_qp *qpair,
2457 struct vmci_queue_header **produce_q_header,
2458 struct vmci_queue_header **consume_q_header)
2459{
2460 int result;
2461
2462 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2463 if (result == VMCI_SUCCESS) {
2464 *produce_q_header = qpair->produce_q->q_header;
2465 *consume_q_header = qpair->consume_q->q_header;
2466 } else if (qpair->produce_q->saved_header &&
2467 qpair->consume_q->saved_header) {
2468 *produce_q_header = qpair->produce_q->saved_header;
2469 *consume_q_header = qpair->consume_q->saved_header;
2470 result = VMCI_SUCCESS;
2471 }
2472
2473 return result;
2474}
2475
2476/*
2477 * Callback from VMCI queue pair broker indicating that a queue
2478 * pair that was previously not ready, now either is ready or
2479 * gone forever.
2480 */
2481static int qp_wakeup_cb(void *client_data)
2482{
2483 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2484
2485 qp_lock(qpair);
2486 while (qpair->blocked > 0) {
2487 qpair->blocked--;
2488 qpair->generation++;
2489 wake_up(&qpair->event);
2490 }
2491 qp_unlock(qpair);
2492
2493 return VMCI_SUCCESS;
2494}
2495
2496/*
2497 * Makes the calling thread wait for the queue pair to become
2498 * ready for host side access. Returns true when thread is
2499 * woken up after queue pair state change, false otherwise.
2500 */
2501static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2502{
2503 unsigned int generation;
2504
2505 qpair->blocked++;
2506 generation = qpair->generation;
2507 qp_unlock(qpair);
2508 wait_event(qpair->event, generation != qpair->generation);
2509 qp_lock(qpair);
2510
2511 return true;
2512}
2513
2514/*
2515 * Enqueues a given buffer to the produce queue using the provided
2516 * function. As many bytes as possible (space available in the queue)
2517 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2518 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2519 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2520 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2521 * an error occured when accessing the buffer,
2522 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2523 * available. Otherwise, the number of bytes written to the queue is
2524 * returned. Updates the tail pointer of the produce queue.
2525 */
2526static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2527 struct vmci_queue *consume_q,
2528 const u64 produce_q_size,
2529 struct iov_iter *from)
2530{
2531 s64 free_space;
2532 u64 tail;
2533 size_t buf_size = iov_iter_count(from);
2534 size_t written;
2535 ssize_t result;
2536
2537 result = qp_map_queue_headers(produce_q, consume_q);
2538 if (unlikely(result != VMCI_SUCCESS))
2539 return result;
2540
2541 free_space = vmci_q_header_free_space(produce_q->q_header,
2542 consume_q->q_header,
2543 produce_q_size);
2544 if (free_space == 0)
2545 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2546
2547 if (free_space < VMCI_SUCCESS)
2548 return (ssize_t) free_space;
2549
2550 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2551 tail = vmci_q_header_producer_tail(produce_q->q_header);
2552 if (likely(tail + written < produce_q_size)) {
2553 result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2554 } else {
2555 /* Tail pointer wraps around. */
2556
2557 const size_t tmp = (size_t) (produce_q_size - tail);
2558
2559 result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2560 if (result >= VMCI_SUCCESS)
2561 result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2562 written - tmp);
2563 }
2564
2565 if (result < VMCI_SUCCESS)
2566 return result;
2567
2568 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2569 produce_q_size);
2570 return written;
2571}
2572
2573/*
2574 * Dequeues data (if available) from the given consume queue. Writes data
2575 * to the user provided buffer using the provided function.
2576 * Assumes the queue->mutex has been acquired.
2577 * Results:
2578 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2579 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2580 * (as defined by the queue size).
2581 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2582 * Otherwise the number of bytes dequeued is returned.
2583 * Side effects:
2584 * Updates the head pointer of the consume queue.
2585 */
2586static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2587 struct vmci_queue *consume_q,
2588 const u64 consume_q_size,
2589 struct iov_iter *to,
2590 bool update_consumer)
2591{
2592 size_t buf_size = iov_iter_count(to);
2593 s64 buf_ready;
2594 u64 head;
2595 size_t read;
2596 ssize_t result;
2597
2598 result = qp_map_queue_headers(produce_q, consume_q);
2599 if (unlikely(result != VMCI_SUCCESS))
2600 return result;
2601
2602 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2603 produce_q->q_header,
2604 consume_q_size);
2605 if (buf_ready == 0)
2606 return VMCI_ERROR_QUEUEPAIR_NODATA;
2607
2608 if (buf_ready < VMCI_SUCCESS)
2609 return (ssize_t) buf_ready;
2610
2611 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2612 head = vmci_q_header_consumer_head(produce_q->q_header);
2613 if (likely(head + read < consume_q_size)) {
2614 result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
2615 } else {
2616 /* Head pointer wraps around. */
2617
2618 const size_t tmp = (size_t) (consume_q_size - head);
2619
2620 result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
2621 if (result >= VMCI_SUCCESS)
2622 result = qp_memcpy_from_queue_iter(to, consume_q, 0,
2623 read - tmp);
2624
2625 }
2626
2627 if (result < VMCI_SUCCESS)
2628 return result;
2629
2630 if (update_consumer)
2631 vmci_q_header_add_consumer_head(produce_q->q_header,
2632 read, consume_q_size);
2633
2634 return read;
2635}
2636
2637/*
2638 * vmci_qpair_alloc() - Allocates a queue pair.
2639 * @qpair: Pointer for the new vmci_qp struct.
2640 * @handle: Handle to track the resource.
2641 * @produce_qsize: Desired size of the producer queue.
2642 * @consume_qsize: Desired size of the consumer queue.
2643 * @peer: ContextID of the peer.
2644 * @flags: VMCI flags.
2645 * @priv_flags: VMCI priviledge flags.
2646 *
2647 * This is the client interface for allocating the memory for a
2648 * vmci_qp structure and then attaching to the underlying
2649 * queue. If an error occurs allocating the memory for the
2650 * vmci_qp structure no attempt is made to attach. If an
2651 * error occurs attaching, then the structure is freed.
2652 */
2653int vmci_qpair_alloc(struct vmci_qp **qpair,
2654 struct vmci_handle *handle,
2655 u64 produce_qsize,
2656 u64 consume_qsize,
2657 u32 peer,
2658 u32 flags,
2659 u32 priv_flags)
2660{
2661 struct vmci_qp *my_qpair;
2662 int retval;
2663 struct vmci_handle src = VMCI_INVALID_HANDLE;
2664 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2665 enum vmci_route route;
2666 vmci_event_release_cb wakeup_cb;
2667 void *client_data;
2668
2669 /*
2670 * Restrict the size of a queuepair. The device already
2671 * enforces a limit on the total amount of memory that can be
2672 * allocated to queuepairs for a guest. However, we try to
2673 * allocate this memory before we make the queuepair
2674 * allocation hypercall. On Linux, we allocate each page
2675 * separately, which means rather than fail, the guest will
2676 * thrash while it tries to allocate, and will become
2677 * increasingly unresponsive to the point where it appears to
2678 * be hung. So we place a limit on the size of an individual
2679 * queuepair here, and leave the device to enforce the
2680 * restriction on total queuepair memory. (Note that this
2681 * doesn't prevent all cases; a user with only this much
2682 * physical memory could still get into trouble.) The error
2683 * used by the device is NO_RESOURCES, so use that here too.
2684 */
2685
2686 if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) ||
2687 produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY)
2688 return VMCI_ERROR_NO_RESOURCES;
2689
2690 retval = vmci_route(&src, &dst, false, &route);
2691 if (retval < VMCI_SUCCESS)
2692 route = vmci_guest_code_active() ?
2693 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2694
2695 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2696 pr_devel("NONBLOCK OR PINNED set");
2697 return VMCI_ERROR_INVALID_ARGS;
2698 }
2699
2700 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2701 if (!my_qpair)
2702 return VMCI_ERROR_NO_MEM;
2703
2704 my_qpair->produce_q_size = produce_qsize;
2705 my_qpair->consume_q_size = consume_qsize;
2706 my_qpair->peer = peer;
2707 my_qpair->flags = flags;
2708 my_qpair->priv_flags = priv_flags;
2709
2710 wakeup_cb = NULL;
2711 client_data = NULL;
2712
2713 if (VMCI_ROUTE_AS_HOST == route) {
2714 my_qpair->guest_endpoint = false;
2715 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2716 my_qpair->blocked = 0;
2717 my_qpair->generation = 0;
2718 init_waitqueue_head(&my_qpair->event);
2719 wakeup_cb = qp_wakeup_cb;
2720 client_data = (void *)my_qpair;
2721 }
2722 } else {
2723 my_qpair->guest_endpoint = true;
2724 }
2725
2726 retval = vmci_qp_alloc(handle,
2727 &my_qpair->produce_q,
2728 my_qpair->produce_q_size,
2729 &my_qpair->consume_q,
2730 my_qpair->consume_q_size,
2731 my_qpair->peer,
2732 my_qpair->flags,
2733 my_qpair->priv_flags,
2734 my_qpair->guest_endpoint,
2735 wakeup_cb, client_data);
2736
2737 if (retval < VMCI_SUCCESS) {
2738 kfree(my_qpair);
2739 return retval;
2740 }
2741
2742 *qpair = my_qpair;
2743 my_qpair->handle = *handle;
2744
2745 return retval;
2746}
2747EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2748
2749/*
2750 * vmci_qpair_detach() - Detatches the client from a queue pair.
2751 * @qpair: Reference of a pointer to the qpair struct.
2752 *
2753 * This is the client interface for detaching from a VMCIQPair.
2754 * Note that this routine will free the memory allocated for the
2755 * vmci_qp structure too.
2756 */
2757int vmci_qpair_detach(struct vmci_qp **qpair)
2758{
2759 int result;
2760 struct vmci_qp *old_qpair;
2761
2762 if (!qpair || !(*qpair))
2763 return VMCI_ERROR_INVALID_ARGS;
2764
2765 old_qpair = *qpair;
2766 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2767
2768 /*
2769 * The guest can fail to detach for a number of reasons, and
2770 * if it does so, it will cleanup the entry (if there is one).
2771 * The host can fail too, but it won't cleanup the entry
2772 * immediately, it will do that later when the context is
2773 * freed. Either way, we need to release the qpair struct
2774 * here; there isn't much the caller can do, and we don't want
2775 * to leak.
2776 */
2777
2778 memset(old_qpair, 0, sizeof(*old_qpair));
2779 old_qpair->handle = VMCI_INVALID_HANDLE;
2780 old_qpair->peer = VMCI_INVALID_ID;
2781 kfree(old_qpair);
2782 *qpair = NULL;
2783
2784 return result;
2785}
2786EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2787
2788/*
2789 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2790 * @qpair: Pointer to the queue pair struct.
2791 * @producer_tail: Reference used for storing producer tail index.
2792 * @consumer_head: Reference used for storing the consumer head index.
2793 *
2794 * This is the client interface for getting the current indexes of the
2795 * QPair from the point of the view of the caller as the producer.
2796 */
2797int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2798 u64 *producer_tail,
2799 u64 *consumer_head)
2800{
2801 struct vmci_queue_header *produce_q_header;
2802 struct vmci_queue_header *consume_q_header;
2803 int result;
2804
2805 if (!qpair)
2806 return VMCI_ERROR_INVALID_ARGS;
2807
2808 qp_lock(qpair);
2809 result =
2810 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2811 if (result == VMCI_SUCCESS)
2812 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2813 producer_tail, consumer_head);
2814 qp_unlock(qpair);
2815
2816 if (result == VMCI_SUCCESS &&
2817 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2818 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2819 return VMCI_ERROR_INVALID_SIZE;
2820
2821 return result;
2822}
2823EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2824
2825/*
2826 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
2827 * @qpair: Pointer to the queue pair struct.
2828 * @consumer_tail: Reference used for storing consumer tail index.
2829 * @producer_head: Reference used for storing the producer head index.
2830 *
2831 * This is the client interface for getting the current indexes of the
2832 * QPair from the point of the view of the caller as the consumer.
2833 */
2834int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2835 u64 *consumer_tail,
2836 u64 *producer_head)
2837{
2838 struct vmci_queue_header *produce_q_header;
2839 struct vmci_queue_header *consume_q_header;
2840 int result;
2841
2842 if (!qpair)
2843 return VMCI_ERROR_INVALID_ARGS;
2844
2845 qp_lock(qpair);
2846 result =
2847 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2848 if (result == VMCI_SUCCESS)
2849 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2850 consumer_tail, producer_head);
2851 qp_unlock(qpair);
2852
2853 if (result == VMCI_SUCCESS &&
2854 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2855 (producer_head && *producer_head >= qpair->consume_q_size)))
2856 return VMCI_ERROR_INVALID_SIZE;
2857
2858 return result;
2859}
2860EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2861
2862/*
2863 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2864 * @qpair: Pointer to the queue pair struct.
2865 *
2866 * This is the client interface for getting the amount of free
2867 * space in the QPair from the point of the view of the caller as
2868 * the producer which is the common case. Returns < 0 if err, else
2869 * available bytes into which data can be enqueued if > 0.
2870 */
2871s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2872{
2873 struct vmci_queue_header *produce_q_header;
2874 struct vmci_queue_header *consume_q_header;
2875 s64 result;
2876
2877 if (!qpair)
2878 return VMCI_ERROR_INVALID_ARGS;
2879
2880 qp_lock(qpair);
2881 result =
2882 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2883 if (result == VMCI_SUCCESS)
2884 result = vmci_q_header_free_space(produce_q_header,
2885 consume_q_header,
2886 qpair->produce_q_size);
2887 else
2888 result = 0;
2889
2890 qp_unlock(qpair);
2891
2892 return result;
2893}
2894EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
2895
2896/*
2897 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
2898 * @qpair: Pointer to the queue pair struct.
2899 *
2900 * This is the client interface for getting the amount of free
2901 * space in the QPair from the point of the view of the caller as
2902 * the consumer which is not the common case. Returns < 0 if err, else
2903 * available bytes into which data can be enqueued if > 0.
2904 */
2905s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
2906{
2907 struct vmci_queue_header *produce_q_header;
2908 struct vmci_queue_header *consume_q_header;
2909 s64 result;
2910
2911 if (!qpair)
2912 return VMCI_ERROR_INVALID_ARGS;
2913
2914 qp_lock(qpair);
2915 result =
2916 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2917 if (result == VMCI_SUCCESS)
2918 result = vmci_q_header_free_space(consume_q_header,
2919 produce_q_header,
2920 qpair->consume_q_size);
2921 else
2922 result = 0;
2923
2924 qp_unlock(qpair);
2925
2926 return result;
2927}
2928EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
2929
2930/*
2931 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
2932 * producer queue.
2933 * @qpair: Pointer to the queue pair struct.
2934 *
2935 * This is the client interface for getting the amount of
2936 * enqueued data in the QPair from the point of the view of the
2937 * caller as the producer which is not the common case. Returns < 0 if err,
2938 * else available bytes that may be read.
2939 */
2940s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
2941{
2942 struct vmci_queue_header *produce_q_header;
2943 struct vmci_queue_header *consume_q_header;
2944 s64 result;
2945
2946 if (!qpair)
2947 return VMCI_ERROR_INVALID_ARGS;
2948
2949 qp_lock(qpair);
2950 result =
2951 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2952 if (result == VMCI_SUCCESS)
2953 result = vmci_q_header_buf_ready(produce_q_header,
2954 consume_q_header,
2955 qpair->produce_q_size);
2956 else
2957 result = 0;
2958
2959 qp_unlock(qpair);
2960
2961 return result;
2962}
2963EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
2964
2965/*
2966 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
2967 * consumer queue.
2968 * @qpair: Pointer to the queue pair struct.
2969 *
2970 * This is the client interface for getting the amount of
2971 * enqueued data in the QPair from the point of the view of the
2972 * caller as the consumer which is the normal case. Returns < 0 if err,
2973 * else available bytes that may be read.
2974 */
2975s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
2976{
2977 struct vmci_queue_header *produce_q_header;
2978 struct vmci_queue_header *consume_q_header;
2979 s64 result;
2980
2981 if (!qpair)
2982 return VMCI_ERROR_INVALID_ARGS;
2983
2984 qp_lock(qpair);
2985 result =
2986 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2987 if (result == VMCI_SUCCESS)
2988 result = vmci_q_header_buf_ready(consume_q_header,
2989 produce_q_header,
2990 qpair->consume_q_size);
2991 else
2992 result = 0;
2993
2994 qp_unlock(qpair);
2995
2996 return result;
2997}
2998EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
2999
3000/*
3001 * vmci_qpair_enqueue() - Throw data on the queue.
3002 * @qpair: Pointer to the queue pair struct.
3003 * @buf: Pointer to buffer containing data
3004 * @buf_size: Length of buffer.
3005 * @buf_type: Buffer type (Unused).
3006 *
3007 * This is the client interface for enqueueing data into the queue.
3008 * Returns number of bytes enqueued or < 0 on error.
3009 */
3010ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3011 const void *buf,
3012 size_t buf_size,
3013 int buf_type)
3014{
3015 ssize_t result;
3016 struct iov_iter from;
3017 struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
3018
3019 if (!qpair || !buf)
3020 return VMCI_ERROR_INVALID_ARGS;
3021
3022 iov_iter_kvec(&from, WRITE, &v, 1, buf_size);
3023
3024 qp_lock(qpair);
3025
3026 do {
3027 result = qp_enqueue_locked(qpair->produce_q,
3028 qpair->consume_q,
3029 qpair->produce_q_size,
3030 &from);
3031
3032 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3033 !qp_wait_for_ready_queue(qpair))
3034 result = VMCI_ERROR_WOULD_BLOCK;
3035
3036 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3037
3038 qp_unlock(qpair);
3039
3040 return result;
3041}
3042EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3043
3044/*
3045 * vmci_qpair_dequeue() - Get data from the queue.
3046 * @qpair: Pointer to the queue pair struct.
3047 * @buf: Pointer to buffer for the data
3048 * @buf_size: Length of buffer.
3049 * @buf_type: Buffer type (Unused).
3050 *
3051 * This is the client interface for dequeueing data from the queue.
3052 * Returns number of bytes dequeued or < 0 on error.
3053 */
3054ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3055 void *buf,
3056 size_t buf_size,
3057 int buf_type)
3058{
3059 ssize_t result;
3060 struct iov_iter to;
3061 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3062
3063 if (!qpair || !buf)
3064 return VMCI_ERROR_INVALID_ARGS;
3065
3066 iov_iter_kvec(&to, READ, &v, 1, buf_size);
3067
3068 qp_lock(qpair);
3069
3070 do {
3071 result = qp_dequeue_locked(qpair->produce_q,
3072 qpair->consume_q,
3073 qpair->consume_q_size,
3074 &to, true);
3075
3076 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3077 !qp_wait_for_ready_queue(qpair))
3078 result = VMCI_ERROR_WOULD_BLOCK;
3079
3080 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3081
3082 qp_unlock(qpair);
3083
3084 return result;
3085}
3086EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3087
3088/*
3089 * vmci_qpair_peek() - Peek at the data in the queue.
3090 * @qpair: Pointer to the queue pair struct.
3091 * @buf: Pointer to buffer for the data
3092 * @buf_size: Length of buffer.
3093 * @buf_type: Buffer type (Unused on Linux).
3094 *
3095 * This is the client interface for peeking into a queue. (I.e.,
3096 * copy data from the queue without updating the head pointer.)
3097 * Returns number of bytes dequeued or < 0 on error.
3098 */
3099ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3100 void *buf,
3101 size_t buf_size,
3102 int buf_type)
3103{
3104 struct iov_iter to;
3105 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3106 ssize_t result;
3107
3108 if (!qpair || !buf)
3109 return VMCI_ERROR_INVALID_ARGS;
3110
3111 iov_iter_kvec(&to, READ, &v, 1, buf_size);
3112
3113 qp_lock(qpair);
3114
3115 do {
3116 result = qp_dequeue_locked(qpair->produce_q,
3117 qpair->consume_q,
3118 qpair->consume_q_size,
3119 &to, false);
3120
3121 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3122 !qp_wait_for_ready_queue(qpair))
3123 result = VMCI_ERROR_WOULD_BLOCK;
3124
3125 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3126
3127 qp_unlock(qpair);
3128
3129 return result;
3130}
3131EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3132
3133/*
3134 * vmci_qpair_enquev() - Throw data on the queue using iov.
3135 * @qpair: Pointer to the queue pair struct.
3136 * @iov: Pointer to buffer containing data
3137 * @iov_size: Length of buffer.
3138 * @buf_type: Buffer type (Unused).
3139 *
3140 * This is the client interface for enqueueing data into the queue.
3141 * This function uses IO vectors to handle the work. Returns number
3142 * of bytes enqueued or < 0 on error.
3143 */
3144ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3145 struct msghdr *msg,
3146 size_t iov_size,
3147 int buf_type)
3148{
3149 ssize_t result;
3150
3151 if (!qpair)
3152 return VMCI_ERROR_INVALID_ARGS;
3153
3154 qp_lock(qpair);
3155
3156 do {
3157 result = qp_enqueue_locked(qpair->produce_q,
3158 qpair->consume_q,
3159 qpair->produce_q_size,
3160 &msg->msg_iter);
3161
3162 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3163 !qp_wait_for_ready_queue(qpair))
3164 result = VMCI_ERROR_WOULD_BLOCK;
3165
3166 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3167
3168 qp_unlock(qpair);
3169
3170 return result;
3171}
3172EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3173
3174/*
3175 * vmci_qpair_dequev() - Get data from the queue using iov.
3176 * @qpair: Pointer to the queue pair struct.
3177 * @iov: Pointer to buffer for the data
3178 * @iov_size: Length of buffer.
3179 * @buf_type: Buffer type (Unused).
3180 *
3181 * This is the client interface for dequeueing data from the queue.
3182 * This function uses IO vectors to handle the work. Returns number
3183 * of bytes dequeued or < 0 on error.
3184 */
3185ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3186 struct msghdr *msg,
3187 size_t iov_size,
3188 int buf_type)
3189{
3190 ssize_t result;
3191
3192 if (!qpair)
3193 return VMCI_ERROR_INVALID_ARGS;
3194
3195 qp_lock(qpair);
3196
3197 do {
3198 result = qp_dequeue_locked(qpair->produce_q,
3199 qpair->consume_q,
3200 qpair->consume_q_size,
3201 &msg->msg_iter, true);
3202
3203 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3204 !qp_wait_for_ready_queue(qpair))
3205 result = VMCI_ERROR_WOULD_BLOCK;
3206
3207 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3208
3209 qp_unlock(qpair);
3210
3211 return result;
3212}
3213EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3214
3215/*
3216 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3217 * @qpair: Pointer to the queue pair struct.
3218 * @iov: Pointer to buffer for the data
3219 * @iov_size: Length of buffer.
3220 * @buf_type: Buffer type (Unused on Linux).
3221 *
3222 * This is the client interface for peeking into a queue. (I.e.,
3223 * copy data from the queue without updating the head pointer.)
3224 * This function uses IO vectors to handle the work. Returns number
3225 * of bytes peeked or < 0 on error.
3226 */
3227ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3228 struct msghdr *msg,
3229 size_t iov_size,
3230 int buf_type)
3231{
3232 ssize_t result;
3233
3234 if (!qpair)
3235 return VMCI_ERROR_INVALID_ARGS;
3236
3237 qp_lock(qpair);
3238
3239 do {
3240 result = qp_dequeue_locked(qpair->produce_q,
3241 qpair->consume_q,
3242 qpair->consume_q_size,
3243 &msg->msg_iter, false);
3244
3245 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3246 !qp_wait_for_ready_queue(qpair))
3247 result = VMCI_ERROR_WOULD_BLOCK;
3248
3249 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3250
3251 qp_unlock(qpair);
3252 return result;
3253}
3254EXPORT_SYMBOL_GPL(vmci_qpair_peekv);