Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VMware VMCI Driver
4 *
5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 */
7
8#include <linux/vmw_vmci_defs.h>
9#include <linux/vmw_vmci_api.h>
10#include <linux/highmem.h>
11#include <linux/kernel.h>
12#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/pagemap.h>
16#include <linux/pci.h>
17#include <linux/sched.h>
18#include <linux/slab.h>
19#include <linux/uio.h>
20#include <linux/wait.h>
21#include <linux/vmalloc.h>
22#include <linux/skbuff.h>
23
24#include "vmci_handle_array.h"
25#include "vmci_queue_pair.h"
26#include "vmci_datagram.h"
27#include "vmci_resource.h"
28#include "vmci_context.h"
29#include "vmci_driver.h"
30#include "vmci_event.h"
31#include "vmci_route.h"
32
33/*
34 * In the following, we will distinguish between two kinds of VMX processes -
35 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
36 * VMCI page files in the VMX and supporting VM to VM communication and the
37 * newer ones that use the guest memory directly. We will in the following
38 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
39 * new-style VMX'en.
40 *
41 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
42 * removed for readability) - see below for more details on the transtions:
43 *
44 * -------------- NEW -------------
45 * | |
46 * \_/ \_/
47 * CREATED_NO_MEM <-----------------> CREATED_MEM
48 * | | |
49 * | o-----------------------o |
50 * | | |
51 * \_/ \_/ \_/
52 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
53 * | | |
54 * | o----------------------o |
55 * | | |
56 * \_/ \_/ \_/
57 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
58 * | |
59 * | |
60 * -------------> gone <-------------
61 *
62 * In more detail. When a VMCI queue pair is first created, it will be in the
63 * VMCIQPB_NEW state. It will then move into one of the following states:
64 *
65 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
66 *
67 * - the created was performed by a host endpoint, in which case there is
68 * no backing memory yet.
69 *
70 * - the create was initiated by an old-style VMX, that uses
71 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
72 * a later point in time. This state can be distinguished from the one
73 * above by the context ID of the creator. A host side is not allowed to
74 * attach until the page store has been set.
75 *
76 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
77 * is created by a VMX using the queue pair device backend that
78 * sets the UVAs of the queue pair immediately and stores the
79 * information for later attachers. At this point, it is ready for
80 * the host side to attach to it.
81 *
82 * Once the queue pair is in one of the created states (with the exception of
83 * the case mentioned for older VMX'en above), it is possible to attach to the
84 * queue pair. Again we have two new states possible:
85 *
86 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
87 * paths:
88 *
89 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
90 * pair, and attaches to a queue pair previously created by the host side.
91 *
92 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
93 * already created by a guest.
94 *
95 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
96 * vmci_qp_broker_set_page_store (see below).
97 *
98 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
99 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
100 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
101 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
102 * will be entered.
103 *
104 * From the attached queue pair, the queue pair can enter the shutdown states
105 * when either side of the queue pair detaches. If the guest side detaches
106 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
107 * the content of the queue pair will no longer be available. If the host
108 * side detaches first, the queue pair will either enter the
109 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
110 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
111 * (e.g., the host detaches while a guest is stunned).
112 *
113 * New-style VMX'en will also unmap guest memory, if the guest is
114 * quiesced, e.g., during a snapshot operation. In that case, the guest
115 * memory will no longer be available, and the queue pair will transition from
116 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
117 * in which case the queue pair will transition from the *_NO_MEM state at that
118 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
119 * since the peer may have either attached or detached in the meantime. The
120 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
121 * *_MEM state, and vice versa.
122 */
123
124/* The Kernel specific component of the struct vmci_queue structure. */
125struct vmci_queue_kern_if {
126 struct mutex __mutex; /* Protects the queue. */
127 struct mutex *mutex; /* Shared by producer and consumer queues. */
128 size_t num_pages; /* Number of pages incl. header. */
129 bool host; /* Host or guest? */
130 union {
131 struct {
132 dma_addr_t *pas;
133 void **vas;
134 } g; /* Used by the guest. */
135 struct {
136 struct page **page;
137 struct page **header_page;
138 } h; /* Used by the host. */
139 } u;
140};
141
142/*
143 * This structure is opaque to the clients.
144 */
145struct vmci_qp {
146 struct vmci_handle handle;
147 struct vmci_queue *produce_q;
148 struct vmci_queue *consume_q;
149 u64 produce_q_size;
150 u64 consume_q_size;
151 u32 peer;
152 u32 flags;
153 u32 priv_flags;
154 bool guest_endpoint;
155 unsigned int blocked;
156 unsigned int generation;
157 wait_queue_head_t event;
158};
159
160enum qp_broker_state {
161 VMCIQPB_NEW,
162 VMCIQPB_CREATED_NO_MEM,
163 VMCIQPB_CREATED_MEM,
164 VMCIQPB_ATTACHED_NO_MEM,
165 VMCIQPB_ATTACHED_MEM,
166 VMCIQPB_SHUTDOWN_NO_MEM,
167 VMCIQPB_SHUTDOWN_MEM,
168 VMCIQPB_GONE
169};
170
171#define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
172 _qpb->state == VMCIQPB_ATTACHED_MEM || \
173 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
174
175/*
176 * In the queue pair broker, we always use the guest point of view for
177 * the produce and consume queue values and references, e.g., the
178 * produce queue size stored is the guests produce queue size. The
179 * host endpoint will need to swap these around. The only exception is
180 * the local queue pairs on the host, in which case the host endpoint
181 * that creates the queue pair will have the right orientation, and
182 * the attaching host endpoint will need to swap.
183 */
184struct qp_entry {
185 struct list_head list_item;
186 struct vmci_handle handle;
187 u32 peer;
188 u32 flags;
189 u64 produce_size;
190 u64 consume_size;
191 u32 ref_count;
192};
193
194struct qp_broker_entry {
195 struct vmci_resource resource;
196 struct qp_entry qp;
197 u32 create_id;
198 u32 attach_id;
199 enum qp_broker_state state;
200 bool require_trusted_attach;
201 bool created_by_trusted;
202 bool vmci_page_files; /* Created by VMX using VMCI page files */
203 struct vmci_queue *produce_q;
204 struct vmci_queue *consume_q;
205 struct vmci_queue_header saved_produce_q;
206 struct vmci_queue_header saved_consume_q;
207 vmci_event_release_cb wakeup_cb;
208 void *client_data;
209 void *local_mem; /* Kernel memory for local queue pair */
210};
211
212struct qp_guest_endpoint {
213 struct vmci_resource resource;
214 struct qp_entry qp;
215 u64 num_ppns;
216 void *produce_q;
217 void *consume_q;
218 struct ppn_set ppn_set;
219};
220
221struct qp_list {
222 struct list_head head;
223 struct mutex mutex; /* Protect queue list. */
224};
225
226static struct qp_list qp_broker_list = {
227 .head = LIST_HEAD_INIT(qp_broker_list.head),
228 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
229};
230
231static struct qp_list qp_guest_endpoints = {
232 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
233 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
234};
235
236#define INVALID_VMCI_GUEST_MEM_ID 0
237#define QPE_NUM_PAGES(_QPE) ((u32) \
238 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
239 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
240#define QP_SIZES_ARE_VALID(_prod_qsize, _cons_qsize) \
241 ((_prod_qsize) + (_cons_qsize) >= max(_prod_qsize, _cons_qsize) && \
242 (_prod_qsize) + (_cons_qsize) <= VMCI_MAX_GUEST_QP_MEMORY)
243
244/*
245 * Frees kernel VA space for a given queue and its queue header, and
246 * frees physical data pages.
247 */
248static void qp_free_queue(void *q, u64 size)
249{
250 struct vmci_queue *queue = q;
251
252 if (queue) {
253 u64 i;
254
255 /* Given size does not include header, so add in a page here. */
256 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
257 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
258 queue->kernel_if->u.g.vas[i],
259 queue->kernel_if->u.g.pas[i]);
260 }
261
262 vfree(queue);
263 }
264}
265
266/*
267 * Allocates kernel queue pages of specified size with IOMMU mappings,
268 * plus space for the queue structure/kernel interface and the queue
269 * header.
270 */
271static void *qp_alloc_queue(u64 size, u32 flags)
272{
273 u64 i;
274 struct vmci_queue *queue;
275 size_t pas_size;
276 size_t vas_size;
277 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
278 u64 num_pages;
279
280 if (size > SIZE_MAX - PAGE_SIZE)
281 return NULL;
282 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
283 if (num_pages >
284 (SIZE_MAX - queue_size) /
285 (sizeof(*queue->kernel_if->u.g.pas) +
286 sizeof(*queue->kernel_if->u.g.vas)))
287 return NULL;
288
289 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
290 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
291 queue_size += pas_size + vas_size;
292
293 queue = vmalloc(queue_size);
294 if (!queue)
295 return NULL;
296
297 queue->q_header = NULL;
298 queue->saved_header = NULL;
299 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
300 queue->kernel_if->mutex = NULL;
301 queue->kernel_if->num_pages = num_pages;
302 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
303 queue->kernel_if->u.g.vas =
304 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
305 queue->kernel_if->host = false;
306
307 for (i = 0; i < num_pages; i++) {
308 queue->kernel_if->u.g.vas[i] =
309 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
310 &queue->kernel_if->u.g.pas[i],
311 GFP_KERNEL);
312 if (!queue->kernel_if->u.g.vas[i]) {
313 /* Size excl. the header. */
314 qp_free_queue(queue, i * PAGE_SIZE);
315 return NULL;
316 }
317 }
318
319 /* Queue header is the first page. */
320 queue->q_header = queue->kernel_if->u.g.vas[0];
321
322 return queue;
323}
324
325/*
326 * Copies from a given buffer or iovector to a VMCI Queue. Uses
327 * kmap_local_page() to dynamically map required portions of the queue
328 * by traversing the offset -> page translation structure for the queue.
329 * Assumes that offset + size does not wrap around in the queue.
330 */
331static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
332 u64 queue_offset,
333 struct iov_iter *from,
334 size_t size)
335{
336 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
337 size_t bytes_copied = 0;
338
339 while (bytes_copied < size) {
340 const u64 page_index =
341 (queue_offset + bytes_copied) / PAGE_SIZE;
342 const size_t page_offset =
343 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
344 void *va;
345 size_t to_copy;
346
347 if (kernel_if->host)
348 va = kmap_local_page(kernel_if->u.h.page[page_index]);
349 else
350 va = kernel_if->u.g.vas[page_index + 1];
351 /* Skip header. */
352
353 if (size - bytes_copied > PAGE_SIZE - page_offset)
354 /* Enough payload to fill up from this page. */
355 to_copy = PAGE_SIZE - page_offset;
356 else
357 to_copy = size - bytes_copied;
358
359 if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
360 from)) {
361 if (kernel_if->host)
362 kunmap_local(va);
363 return VMCI_ERROR_INVALID_ARGS;
364 }
365 bytes_copied += to_copy;
366 if (kernel_if->host)
367 kunmap_local(va);
368 }
369
370 return VMCI_SUCCESS;
371}
372
373/*
374 * Copies to a given buffer or iovector from a VMCI Queue. Uses
375 * kmap_local_page() to dynamically map required portions of the queue
376 * by traversing the offset -> page translation structure for the queue.
377 * Assumes that offset + size does not wrap around in the queue.
378 */
379static int qp_memcpy_from_queue_iter(struct iov_iter *to,
380 const struct vmci_queue *queue,
381 u64 queue_offset, size_t size)
382{
383 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
384 size_t bytes_copied = 0;
385
386 while (bytes_copied < size) {
387 const u64 page_index =
388 (queue_offset + bytes_copied) / PAGE_SIZE;
389 const size_t page_offset =
390 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
391 void *va;
392 size_t to_copy;
393 int err;
394
395 if (kernel_if->host)
396 va = kmap_local_page(kernel_if->u.h.page[page_index]);
397 else
398 va = kernel_if->u.g.vas[page_index + 1];
399 /* Skip header. */
400
401 if (size - bytes_copied > PAGE_SIZE - page_offset)
402 /* Enough payload to fill up this page. */
403 to_copy = PAGE_SIZE - page_offset;
404 else
405 to_copy = size - bytes_copied;
406
407 err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
408 if (err != to_copy) {
409 if (kernel_if->host)
410 kunmap_local(va);
411 return VMCI_ERROR_INVALID_ARGS;
412 }
413 bytes_copied += to_copy;
414 if (kernel_if->host)
415 kunmap_local(va);
416 }
417
418 return VMCI_SUCCESS;
419}
420
421/*
422 * Allocates two list of PPNs --- one for the pages in the produce queue,
423 * and the other for the pages in the consume queue. Intializes the list
424 * of PPNs with the page frame numbers of the KVA for the two queues (and
425 * the queue headers).
426 */
427static int qp_alloc_ppn_set(void *prod_q,
428 u64 num_produce_pages,
429 void *cons_q,
430 u64 num_consume_pages, struct ppn_set *ppn_set)
431{
432 u64 *produce_ppns;
433 u64 *consume_ppns;
434 struct vmci_queue *produce_q = prod_q;
435 struct vmci_queue *consume_q = cons_q;
436 u64 i;
437
438 if (!produce_q || !num_produce_pages || !consume_q ||
439 !num_consume_pages || !ppn_set)
440 return VMCI_ERROR_INVALID_ARGS;
441
442 if (ppn_set->initialized)
443 return VMCI_ERROR_ALREADY_EXISTS;
444
445 produce_ppns =
446 kmalloc_array(num_produce_pages, sizeof(*produce_ppns),
447 GFP_KERNEL);
448 if (!produce_ppns)
449 return VMCI_ERROR_NO_MEM;
450
451 consume_ppns =
452 kmalloc_array(num_consume_pages, sizeof(*consume_ppns),
453 GFP_KERNEL);
454 if (!consume_ppns) {
455 kfree(produce_ppns);
456 return VMCI_ERROR_NO_MEM;
457 }
458
459 for (i = 0; i < num_produce_pages; i++)
460 produce_ppns[i] =
461 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
462
463 for (i = 0; i < num_consume_pages; i++)
464 consume_ppns[i] =
465 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
466
467 ppn_set->num_produce_pages = num_produce_pages;
468 ppn_set->num_consume_pages = num_consume_pages;
469 ppn_set->produce_ppns = produce_ppns;
470 ppn_set->consume_ppns = consume_ppns;
471 ppn_set->initialized = true;
472 return VMCI_SUCCESS;
473}
474
475/*
476 * Frees the two list of PPNs for a queue pair.
477 */
478static void qp_free_ppn_set(struct ppn_set *ppn_set)
479{
480 if (ppn_set->initialized) {
481 /* Do not call these functions on NULL inputs. */
482 kfree(ppn_set->produce_ppns);
483 kfree(ppn_set->consume_ppns);
484 }
485 memset(ppn_set, 0, sizeof(*ppn_set));
486}
487
488/*
489 * Populates the list of PPNs in the hypercall structure with the PPNS
490 * of the produce queue and the consume queue.
491 */
492static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
493{
494 if (vmci_use_ppn64()) {
495 memcpy(call_buf, ppn_set->produce_ppns,
496 ppn_set->num_produce_pages *
497 sizeof(*ppn_set->produce_ppns));
498 memcpy(call_buf +
499 ppn_set->num_produce_pages *
500 sizeof(*ppn_set->produce_ppns),
501 ppn_set->consume_ppns,
502 ppn_set->num_consume_pages *
503 sizeof(*ppn_set->consume_ppns));
504 } else {
505 int i;
506 u32 *ppns = (u32 *) call_buf;
507
508 for (i = 0; i < ppn_set->num_produce_pages; i++)
509 ppns[i] = (u32) ppn_set->produce_ppns[i];
510
511 ppns = &ppns[ppn_set->num_produce_pages];
512
513 for (i = 0; i < ppn_set->num_consume_pages; i++)
514 ppns[i] = (u32) ppn_set->consume_ppns[i];
515 }
516
517 return VMCI_SUCCESS;
518}
519
520/*
521 * Allocates kernel VA space of specified size plus space for the queue
522 * and kernel interface. This is different from the guest queue allocator,
523 * because we do not allocate our own queue header/data pages here but
524 * share those of the guest.
525 */
526static struct vmci_queue *qp_host_alloc_queue(u64 size)
527{
528 struct vmci_queue *queue;
529 size_t queue_page_size;
530 u64 num_pages;
531 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
532
533 if (size > min_t(size_t, VMCI_MAX_GUEST_QP_MEMORY, SIZE_MAX - PAGE_SIZE))
534 return NULL;
535 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
536 if (num_pages > (SIZE_MAX - queue_size) /
537 sizeof(*queue->kernel_if->u.h.page))
538 return NULL;
539
540 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
541
542 if (queue_size + queue_page_size > KMALLOC_MAX_SIZE)
543 return NULL;
544
545 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
546 if (queue) {
547 queue->q_header = NULL;
548 queue->saved_header = NULL;
549 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
550 queue->kernel_if->host = true;
551 queue->kernel_if->mutex = NULL;
552 queue->kernel_if->num_pages = num_pages;
553 queue->kernel_if->u.h.header_page =
554 (struct page **)((u8 *)queue + queue_size);
555 queue->kernel_if->u.h.page =
556 &queue->kernel_if->u.h.header_page[1];
557 }
558
559 return queue;
560}
561
562/*
563 * Frees kernel memory for a given queue (header plus translation
564 * structure).
565 */
566static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
567{
568 kfree(queue);
569}
570
571/*
572 * Initialize the mutex for the pair of queues. This mutex is used to
573 * protect the q_header and the buffer from changing out from under any
574 * users of either queue. Of course, it's only any good if the mutexes
575 * are actually acquired. Queue structure must lie on non-paged memory
576 * or we cannot guarantee access to the mutex.
577 */
578static void qp_init_queue_mutex(struct vmci_queue *produce_q,
579 struct vmci_queue *consume_q)
580{
581 /*
582 * Only the host queue has shared state - the guest queues do not
583 * need to synchronize access using a queue mutex.
584 */
585
586 if (produce_q->kernel_if->host) {
587 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
588 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
589 mutex_init(produce_q->kernel_if->mutex);
590 }
591}
592
593/*
594 * Cleans up the mutex for the pair of queues.
595 */
596static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
597 struct vmci_queue *consume_q)
598{
599 if (produce_q->kernel_if->host) {
600 produce_q->kernel_if->mutex = NULL;
601 consume_q->kernel_if->mutex = NULL;
602 }
603}
604
605/*
606 * Acquire the mutex for the queue. Note that the produce_q and
607 * the consume_q share a mutex. So, only one of the two need to
608 * be passed in to this routine. Either will work just fine.
609 */
610static void qp_acquire_queue_mutex(struct vmci_queue *queue)
611{
612 if (queue->kernel_if->host)
613 mutex_lock(queue->kernel_if->mutex);
614}
615
616/*
617 * Release the mutex for the queue. Note that the produce_q and
618 * the consume_q share a mutex. So, only one of the two need to
619 * be passed in to this routine. Either will work just fine.
620 */
621static void qp_release_queue_mutex(struct vmci_queue *queue)
622{
623 if (queue->kernel_if->host)
624 mutex_unlock(queue->kernel_if->mutex);
625}
626
627/*
628 * Helper function to release pages in the PageStoreAttachInfo
629 * previously obtained using get_user_pages.
630 */
631static void qp_release_pages(struct page **pages,
632 u64 num_pages, bool dirty)
633{
634 int i;
635
636 for (i = 0; i < num_pages; i++) {
637 if (dirty)
638 set_page_dirty_lock(pages[i]);
639
640 put_page(pages[i]);
641 pages[i] = NULL;
642 }
643}
644
645/*
646 * Lock the user pages referenced by the {produce,consume}Buffer
647 * struct into memory and populate the {produce,consume}Pages
648 * arrays in the attach structure with them.
649 */
650static int qp_host_get_user_memory(u64 produce_uva,
651 u64 consume_uva,
652 struct vmci_queue *produce_q,
653 struct vmci_queue *consume_q)
654{
655 int retval;
656 int err = VMCI_SUCCESS;
657
658 retval = get_user_pages_fast((uintptr_t) produce_uva,
659 produce_q->kernel_if->num_pages,
660 FOLL_WRITE,
661 produce_q->kernel_if->u.h.header_page);
662 if (retval < (int)produce_q->kernel_if->num_pages) {
663 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
664 retval);
665 if (retval > 0)
666 qp_release_pages(produce_q->kernel_if->u.h.header_page,
667 retval, false);
668 err = VMCI_ERROR_NO_MEM;
669 goto out;
670 }
671
672 retval = get_user_pages_fast((uintptr_t) consume_uva,
673 consume_q->kernel_if->num_pages,
674 FOLL_WRITE,
675 consume_q->kernel_if->u.h.header_page);
676 if (retval < (int)consume_q->kernel_if->num_pages) {
677 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
678 retval);
679 if (retval > 0)
680 qp_release_pages(consume_q->kernel_if->u.h.header_page,
681 retval, false);
682 qp_release_pages(produce_q->kernel_if->u.h.header_page,
683 produce_q->kernel_if->num_pages, false);
684 err = VMCI_ERROR_NO_MEM;
685 }
686
687 out:
688 return err;
689}
690
691/*
692 * Registers the specification of the user pages used for backing a queue
693 * pair. Enough information to map in pages is stored in the OS specific
694 * part of the struct vmci_queue structure.
695 */
696static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
697 struct vmci_queue *produce_q,
698 struct vmci_queue *consume_q)
699{
700 u64 produce_uva;
701 u64 consume_uva;
702
703 /*
704 * The new style and the old style mapping only differs in
705 * that we either get a single or two UVAs, so we split the
706 * single UVA range at the appropriate spot.
707 */
708 produce_uva = page_store->pages;
709 consume_uva = page_store->pages +
710 produce_q->kernel_if->num_pages * PAGE_SIZE;
711 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
712 consume_q);
713}
714
715/*
716 * Releases and removes the references to user pages stored in the attach
717 * struct. Pages are released from the page cache and may become
718 * swappable again.
719 */
720static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
721 struct vmci_queue *consume_q)
722{
723 qp_release_pages(produce_q->kernel_if->u.h.header_page,
724 produce_q->kernel_if->num_pages, true);
725 memset(produce_q->kernel_if->u.h.header_page, 0,
726 sizeof(*produce_q->kernel_if->u.h.header_page) *
727 produce_q->kernel_if->num_pages);
728 qp_release_pages(consume_q->kernel_if->u.h.header_page,
729 consume_q->kernel_if->num_pages, true);
730 memset(consume_q->kernel_if->u.h.header_page, 0,
731 sizeof(*consume_q->kernel_if->u.h.header_page) *
732 consume_q->kernel_if->num_pages);
733}
734
735/*
736 * Once qp_host_register_user_memory has been performed on a
737 * queue, the queue pair headers can be mapped into the
738 * kernel. Once mapped, they must be unmapped with
739 * qp_host_unmap_queues prior to calling
740 * qp_host_unregister_user_memory.
741 * Pages are pinned.
742 */
743static int qp_host_map_queues(struct vmci_queue *produce_q,
744 struct vmci_queue *consume_q)
745{
746 int result;
747
748 if (!produce_q->q_header || !consume_q->q_header) {
749 struct page *headers[2];
750
751 if (produce_q->q_header != consume_q->q_header)
752 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
753
754 if (produce_q->kernel_if->u.h.header_page == NULL ||
755 *produce_q->kernel_if->u.h.header_page == NULL)
756 return VMCI_ERROR_UNAVAILABLE;
757
758 headers[0] = *produce_q->kernel_if->u.h.header_page;
759 headers[1] = *consume_q->kernel_if->u.h.header_page;
760
761 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
762 if (produce_q->q_header != NULL) {
763 consume_q->q_header =
764 (struct vmci_queue_header *)((u8 *)
765 produce_q->q_header +
766 PAGE_SIZE);
767 result = VMCI_SUCCESS;
768 } else {
769 pr_warn("vmap failed\n");
770 result = VMCI_ERROR_NO_MEM;
771 }
772 } else {
773 result = VMCI_SUCCESS;
774 }
775
776 return result;
777}
778
779/*
780 * Unmaps previously mapped queue pair headers from the kernel.
781 * Pages are unpinned.
782 */
783static int qp_host_unmap_queues(u32 gid,
784 struct vmci_queue *produce_q,
785 struct vmci_queue *consume_q)
786{
787 if (produce_q->q_header) {
788 if (produce_q->q_header < consume_q->q_header)
789 vunmap(produce_q->q_header);
790 else
791 vunmap(consume_q->q_header);
792
793 produce_q->q_header = NULL;
794 consume_q->q_header = NULL;
795 }
796
797 return VMCI_SUCCESS;
798}
799
800/*
801 * Finds the entry in the list corresponding to a given handle. Assumes
802 * that the list is locked.
803 */
804static struct qp_entry *qp_list_find(struct qp_list *qp_list,
805 struct vmci_handle handle)
806{
807 struct qp_entry *entry;
808
809 if (vmci_handle_is_invalid(handle))
810 return NULL;
811
812 list_for_each_entry(entry, &qp_list->head, list_item) {
813 if (vmci_handle_is_equal(entry->handle, handle))
814 return entry;
815 }
816
817 return NULL;
818}
819
820/*
821 * Finds the entry in the list corresponding to a given handle.
822 */
823static struct qp_guest_endpoint *
824qp_guest_handle_to_entry(struct vmci_handle handle)
825{
826 struct qp_guest_endpoint *entry;
827 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
828
829 entry = qp ? container_of(
830 qp, struct qp_guest_endpoint, qp) : NULL;
831 return entry;
832}
833
834/*
835 * Finds the entry in the list corresponding to a given handle.
836 */
837static struct qp_broker_entry *
838qp_broker_handle_to_entry(struct vmci_handle handle)
839{
840 struct qp_broker_entry *entry;
841 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
842
843 entry = qp ? container_of(
844 qp, struct qp_broker_entry, qp) : NULL;
845 return entry;
846}
847
848/*
849 * Dispatches a queue pair event message directly into the local event
850 * queue.
851 */
852static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
853{
854 u32 context_id = vmci_get_context_id();
855 struct vmci_event_qp ev;
856
857 memset(&ev, 0, sizeof(ev));
858 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
859 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
860 VMCI_CONTEXT_RESOURCE_ID);
861 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
862 ev.msg.event_data.event =
863 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
864 ev.payload.peer_id = context_id;
865 ev.payload.handle = handle;
866
867 return vmci_event_dispatch(&ev.msg.hdr);
868}
869
870/*
871 * Allocates and initializes a qp_guest_endpoint structure.
872 * Allocates a queue_pair rid (and handle) iff the given entry has
873 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
874 * are reserved handles. Assumes that the QP list mutex is held
875 * by the caller.
876 */
877static struct qp_guest_endpoint *
878qp_guest_endpoint_create(struct vmci_handle handle,
879 u32 peer,
880 u32 flags,
881 u64 produce_size,
882 u64 consume_size,
883 void *produce_q,
884 void *consume_q)
885{
886 int result;
887 struct qp_guest_endpoint *entry;
888 /* One page each for the queue headers. */
889 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
890 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
891
892 if (vmci_handle_is_invalid(handle)) {
893 u32 context_id = vmci_get_context_id();
894
895 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
896 }
897
898 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
899 if (entry) {
900 entry->qp.peer = peer;
901 entry->qp.flags = flags;
902 entry->qp.produce_size = produce_size;
903 entry->qp.consume_size = consume_size;
904 entry->qp.ref_count = 0;
905 entry->num_ppns = num_ppns;
906 entry->produce_q = produce_q;
907 entry->consume_q = consume_q;
908 INIT_LIST_HEAD(&entry->qp.list_item);
909
910 /* Add resource obj */
911 result = vmci_resource_add(&entry->resource,
912 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
913 handle);
914 entry->qp.handle = vmci_resource_handle(&entry->resource);
915 if ((result != VMCI_SUCCESS) ||
916 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
917 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
918 handle.context, handle.resource, result);
919 kfree(entry);
920 entry = NULL;
921 }
922 }
923 return entry;
924}
925
926/*
927 * Frees a qp_guest_endpoint structure.
928 */
929static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
930{
931 qp_free_ppn_set(&entry->ppn_set);
932 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
933 qp_free_queue(entry->produce_q, entry->qp.produce_size);
934 qp_free_queue(entry->consume_q, entry->qp.consume_size);
935 /* Unlink from resource hash table and free callback */
936 vmci_resource_remove(&entry->resource);
937
938 kfree(entry);
939}
940
941/*
942 * Helper to make a queue_pairAlloc hypercall when the driver is
943 * supporting a guest device.
944 */
945static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
946{
947 struct vmci_qp_alloc_msg *alloc_msg;
948 size_t msg_size;
949 size_t ppn_size;
950 int result;
951
952 if (!entry || entry->num_ppns <= 2)
953 return VMCI_ERROR_INVALID_ARGS;
954
955 ppn_size = vmci_use_ppn64() ? sizeof(u64) : sizeof(u32);
956 msg_size = sizeof(*alloc_msg) +
957 (size_t) entry->num_ppns * ppn_size;
958 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
959 if (!alloc_msg)
960 return VMCI_ERROR_NO_MEM;
961
962 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
963 VMCI_QUEUEPAIR_ALLOC);
964 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
965 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
966 alloc_msg->handle = entry->qp.handle;
967 alloc_msg->peer = entry->qp.peer;
968 alloc_msg->flags = entry->qp.flags;
969 alloc_msg->produce_size = entry->qp.produce_size;
970 alloc_msg->consume_size = entry->qp.consume_size;
971 alloc_msg->num_ppns = entry->num_ppns;
972
973 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
974 &entry->ppn_set);
975 if (result == VMCI_SUCCESS)
976 result = vmci_send_datagram(&alloc_msg->hdr);
977
978 kfree(alloc_msg);
979
980 return result;
981}
982
983/*
984 * Helper to make a queue_pairDetach hypercall when the driver is
985 * supporting a guest device.
986 */
987static int qp_detatch_hypercall(struct vmci_handle handle)
988{
989 struct vmci_qp_detach_msg detach_msg;
990
991 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
992 VMCI_QUEUEPAIR_DETACH);
993 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
994 detach_msg.hdr.payload_size = sizeof(handle);
995 detach_msg.handle = handle;
996
997 return vmci_send_datagram(&detach_msg.hdr);
998}
999
1000/*
1001 * Adds the given entry to the list. Assumes that the list is locked.
1002 */
1003static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
1004{
1005 if (entry)
1006 list_add(&entry->list_item, &qp_list->head);
1007}
1008
1009/*
1010 * Removes the given entry from the list. Assumes that the list is locked.
1011 */
1012static void qp_list_remove_entry(struct qp_list *qp_list,
1013 struct qp_entry *entry)
1014{
1015 if (entry)
1016 list_del(&entry->list_item);
1017}
1018
1019/*
1020 * Helper for VMCI queue_pair detach interface. Frees the physical
1021 * pages for the queue pair.
1022 */
1023static int qp_detatch_guest_work(struct vmci_handle handle)
1024{
1025 int result;
1026 struct qp_guest_endpoint *entry;
1027 u32 ref_count = ~0; /* To avoid compiler warning below */
1028
1029 mutex_lock(&qp_guest_endpoints.mutex);
1030
1031 entry = qp_guest_handle_to_entry(handle);
1032 if (!entry) {
1033 mutex_unlock(&qp_guest_endpoints.mutex);
1034 return VMCI_ERROR_NOT_FOUND;
1035 }
1036
1037 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1038 result = VMCI_SUCCESS;
1039
1040 if (entry->qp.ref_count > 1) {
1041 result = qp_notify_peer_local(false, handle);
1042 /*
1043 * We can fail to notify a local queuepair
1044 * because we can't allocate. We still want
1045 * to release the entry if that happens, so
1046 * don't bail out yet.
1047 */
1048 }
1049 } else {
1050 result = qp_detatch_hypercall(handle);
1051 if (result < VMCI_SUCCESS) {
1052 /*
1053 * We failed to notify a non-local queuepair.
1054 * That other queuepair might still be
1055 * accessing the shared memory, so don't
1056 * release the entry yet. It will get cleaned
1057 * up by VMCIqueue_pair_Exit() if necessary
1058 * (assuming we are going away, otherwise why
1059 * did this fail?).
1060 */
1061
1062 mutex_unlock(&qp_guest_endpoints.mutex);
1063 return result;
1064 }
1065 }
1066
1067 /*
1068 * If we get here then we either failed to notify a local queuepair, or
1069 * we succeeded in all cases. Release the entry if required.
1070 */
1071
1072 entry->qp.ref_count--;
1073 if (entry->qp.ref_count == 0)
1074 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1075
1076 /* If we didn't remove the entry, this could change once we unlock. */
1077 if (entry)
1078 ref_count = entry->qp.ref_count;
1079
1080 mutex_unlock(&qp_guest_endpoints.mutex);
1081
1082 if (ref_count == 0)
1083 qp_guest_endpoint_destroy(entry);
1084
1085 return result;
1086}
1087
1088/*
1089 * This functions handles the actual allocation of a VMCI queue
1090 * pair guest endpoint. Allocates physical pages for the queue
1091 * pair. It makes OS dependent calls through generic wrappers.
1092 */
1093static int qp_alloc_guest_work(struct vmci_handle *handle,
1094 struct vmci_queue **produce_q,
1095 u64 produce_size,
1096 struct vmci_queue **consume_q,
1097 u64 consume_size,
1098 u32 peer,
1099 u32 flags,
1100 u32 priv_flags)
1101{
1102 const u64 num_produce_pages =
1103 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1104 const u64 num_consume_pages =
1105 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1106 void *my_produce_q = NULL;
1107 void *my_consume_q = NULL;
1108 int result;
1109 struct qp_guest_endpoint *queue_pair_entry = NULL;
1110
1111 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1112 return VMCI_ERROR_NO_ACCESS;
1113
1114 mutex_lock(&qp_guest_endpoints.mutex);
1115
1116 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1117 if (queue_pair_entry) {
1118 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1119 /* Local attach case. */
1120 if (queue_pair_entry->qp.ref_count > 1) {
1121 pr_devel("Error attempting to attach more than once\n");
1122 result = VMCI_ERROR_UNAVAILABLE;
1123 goto error_keep_entry;
1124 }
1125
1126 if (queue_pair_entry->qp.produce_size != consume_size ||
1127 queue_pair_entry->qp.consume_size !=
1128 produce_size ||
1129 queue_pair_entry->qp.flags !=
1130 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1131 pr_devel("Error mismatched queue pair in local attach\n");
1132 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1133 goto error_keep_entry;
1134 }
1135
1136 /*
1137 * Do a local attach. We swap the consume and
1138 * produce queues for the attacher and deliver
1139 * an attach event.
1140 */
1141 result = qp_notify_peer_local(true, *handle);
1142 if (result < VMCI_SUCCESS)
1143 goto error_keep_entry;
1144
1145 my_produce_q = queue_pair_entry->consume_q;
1146 my_consume_q = queue_pair_entry->produce_q;
1147 goto out;
1148 }
1149
1150 result = VMCI_ERROR_ALREADY_EXISTS;
1151 goto error_keep_entry;
1152 }
1153
1154 my_produce_q = qp_alloc_queue(produce_size, flags);
1155 if (!my_produce_q) {
1156 pr_warn("Error allocating pages for produce queue\n");
1157 result = VMCI_ERROR_NO_MEM;
1158 goto error;
1159 }
1160
1161 my_consume_q = qp_alloc_queue(consume_size, flags);
1162 if (!my_consume_q) {
1163 pr_warn("Error allocating pages for consume queue\n");
1164 result = VMCI_ERROR_NO_MEM;
1165 goto error;
1166 }
1167
1168 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1169 produce_size, consume_size,
1170 my_produce_q, my_consume_q);
1171 if (!queue_pair_entry) {
1172 pr_warn("Error allocating memory in %s\n", __func__);
1173 result = VMCI_ERROR_NO_MEM;
1174 goto error;
1175 }
1176
1177 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1178 num_consume_pages,
1179 &queue_pair_entry->ppn_set);
1180 if (result < VMCI_SUCCESS) {
1181 pr_warn("qp_alloc_ppn_set failed\n");
1182 goto error;
1183 }
1184
1185 /*
1186 * It's only necessary to notify the host if this queue pair will be
1187 * attached to from another context.
1188 */
1189 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1190 /* Local create case. */
1191 u32 context_id = vmci_get_context_id();
1192
1193 /*
1194 * Enforce similar checks on local queue pairs as we
1195 * do for regular ones. The handle's context must
1196 * match the creator or attacher context id (here they
1197 * are both the current context id) and the
1198 * attach-only flag cannot exist during create. We
1199 * also ensure specified peer is this context or an
1200 * invalid one.
1201 */
1202 if (queue_pair_entry->qp.handle.context != context_id ||
1203 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1204 queue_pair_entry->qp.peer != context_id)) {
1205 result = VMCI_ERROR_NO_ACCESS;
1206 goto error;
1207 }
1208
1209 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1210 result = VMCI_ERROR_NOT_FOUND;
1211 goto error;
1212 }
1213 } else {
1214 result = qp_alloc_hypercall(queue_pair_entry);
1215 if (result < VMCI_SUCCESS) {
1216 pr_devel("qp_alloc_hypercall result = %d\n", result);
1217 goto error;
1218 }
1219 }
1220
1221 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1222 (struct vmci_queue *)my_consume_q);
1223
1224 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1225
1226 out:
1227 queue_pair_entry->qp.ref_count++;
1228 *handle = queue_pair_entry->qp.handle;
1229 *produce_q = (struct vmci_queue *)my_produce_q;
1230 *consume_q = (struct vmci_queue *)my_consume_q;
1231
1232 /*
1233 * We should initialize the queue pair header pages on a local
1234 * queue pair create. For non-local queue pairs, the
1235 * hypervisor initializes the header pages in the create step.
1236 */
1237 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1238 queue_pair_entry->qp.ref_count == 1) {
1239 vmci_q_header_init((*produce_q)->q_header, *handle);
1240 vmci_q_header_init((*consume_q)->q_header, *handle);
1241 }
1242
1243 mutex_unlock(&qp_guest_endpoints.mutex);
1244
1245 return VMCI_SUCCESS;
1246
1247 error:
1248 mutex_unlock(&qp_guest_endpoints.mutex);
1249 if (queue_pair_entry) {
1250 /* The queues will be freed inside the destroy routine. */
1251 qp_guest_endpoint_destroy(queue_pair_entry);
1252 } else {
1253 qp_free_queue(my_produce_q, produce_size);
1254 qp_free_queue(my_consume_q, consume_size);
1255 }
1256 return result;
1257
1258 error_keep_entry:
1259 /* This path should only be used when an existing entry was found. */
1260 mutex_unlock(&qp_guest_endpoints.mutex);
1261 return result;
1262}
1263
1264/*
1265 * The first endpoint issuing a queue pair allocation will create the state
1266 * of the queue pair in the queue pair broker.
1267 *
1268 * If the creator is a guest, it will associate a VMX virtual address range
1269 * with the queue pair as specified by the page_store. For compatibility with
1270 * older VMX'en, that would use a separate step to set the VMX virtual
1271 * address range, the virtual address range can be registered later using
1272 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1273 * used.
1274 *
1275 * If the creator is the host, a page_store of NULL should be used as well,
1276 * since the host is not able to supply a page store for the queue pair.
1277 *
1278 * For older VMX and host callers, the queue pair will be created in the
1279 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1280 * created in VMCOQPB_CREATED_MEM state.
1281 */
1282static int qp_broker_create(struct vmci_handle handle,
1283 u32 peer,
1284 u32 flags,
1285 u32 priv_flags,
1286 u64 produce_size,
1287 u64 consume_size,
1288 struct vmci_qp_page_store *page_store,
1289 struct vmci_ctx *context,
1290 vmci_event_release_cb wakeup_cb,
1291 void *client_data, struct qp_broker_entry **ent)
1292{
1293 struct qp_broker_entry *entry = NULL;
1294 const u32 context_id = vmci_ctx_get_id(context);
1295 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1296 int result;
1297 u64 guest_produce_size;
1298 u64 guest_consume_size;
1299
1300 /* Do not create if the caller asked not to. */
1301 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1302 return VMCI_ERROR_NOT_FOUND;
1303
1304 /*
1305 * Creator's context ID should match handle's context ID or the creator
1306 * must allow the context in handle's context ID as the "peer".
1307 */
1308 if (handle.context != context_id && handle.context != peer)
1309 return VMCI_ERROR_NO_ACCESS;
1310
1311 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1312 return VMCI_ERROR_DST_UNREACHABLE;
1313
1314 /*
1315 * Creator's context ID for local queue pairs should match the
1316 * peer, if a peer is specified.
1317 */
1318 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1319 return VMCI_ERROR_NO_ACCESS;
1320
1321 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1322 if (!entry)
1323 return VMCI_ERROR_NO_MEM;
1324
1325 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1326 /*
1327 * The queue pair broker entry stores values from the guest
1328 * point of view, so a creating host side endpoint should swap
1329 * produce and consume values -- unless it is a local queue
1330 * pair, in which case no swapping is necessary, since the local
1331 * attacher will swap queues.
1332 */
1333
1334 guest_produce_size = consume_size;
1335 guest_consume_size = produce_size;
1336 } else {
1337 guest_produce_size = produce_size;
1338 guest_consume_size = consume_size;
1339 }
1340
1341 entry->qp.handle = handle;
1342 entry->qp.peer = peer;
1343 entry->qp.flags = flags;
1344 entry->qp.produce_size = guest_produce_size;
1345 entry->qp.consume_size = guest_consume_size;
1346 entry->qp.ref_count = 1;
1347 entry->create_id = context_id;
1348 entry->attach_id = VMCI_INVALID_ID;
1349 entry->state = VMCIQPB_NEW;
1350 entry->require_trusted_attach =
1351 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1352 entry->created_by_trusted =
1353 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1354 entry->vmci_page_files = false;
1355 entry->wakeup_cb = wakeup_cb;
1356 entry->client_data = client_data;
1357 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1358 if (entry->produce_q == NULL) {
1359 result = VMCI_ERROR_NO_MEM;
1360 goto error;
1361 }
1362 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1363 if (entry->consume_q == NULL) {
1364 result = VMCI_ERROR_NO_MEM;
1365 goto error;
1366 }
1367
1368 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1369
1370 INIT_LIST_HEAD(&entry->qp.list_item);
1371
1372 if (is_local) {
1373 u8 *tmp;
1374
1375 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1376 PAGE_SIZE, GFP_KERNEL);
1377 if (entry->local_mem == NULL) {
1378 result = VMCI_ERROR_NO_MEM;
1379 goto error;
1380 }
1381 entry->state = VMCIQPB_CREATED_MEM;
1382 entry->produce_q->q_header = entry->local_mem;
1383 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1384 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1385 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1386 } else if (page_store) {
1387 /*
1388 * The VMX already initialized the queue pair headers, so no
1389 * need for the kernel side to do that.
1390 */
1391 result = qp_host_register_user_memory(page_store,
1392 entry->produce_q,
1393 entry->consume_q);
1394 if (result < VMCI_SUCCESS)
1395 goto error;
1396
1397 entry->state = VMCIQPB_CREATED_MEM;
1398 } else {
1399 /*
1400 * A create without a page_store may be either a host
1401 * side create (in which case we are waiting for the
1402 * guest side to supply the memory) or an old style
1403 * queue pair create (in which case we will expect a
1404 * set page store call as the next step).
1405 */
1406 entry->state = VMCIQPB_CREATED_NO_MEM;
1407 }
1408
1409 qp_list_add_entry(&qp_broker_list, &entry->qp);
1410 if (ent != NULL)
1411 *ent = entry;
1412
1413 /* Add to resource obj */
1414 result = vmci_resource_add(&entry->resource,
1415 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1416 handle);
1417 if (result != VMCI_SUCCESS) {
1418 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1419 handle.context, handle.resource, result);
1420 goto error;
1421 }
1422
1423 entry->qp.handle = vmci_resource_handle(&entry->resource);
1424 if (is_local) {
1425 vmci_q_header_init(entry->produce_q->q_header,
1426 entry->qp.handle);
1427 vmci_q_header_init(entry->consume_q->q_header,
1428 entry->qp.handle);
1429 }
1430
1431 vmci_ctx_qp_create(context, entry->qp.handle);
1432
1433 return VMCI_SUCCESS;
1434
1435 error:
1436 if (entry != NULL) {
1437 qp_host_free_queue(entry->produce_q, guest_produce_size);
1438 qp_host_free_queue(entry->consume_q, guest_consume_size);
1439 kfree(entry);
1440 }
1441
1442 return result;
1443}
1444
1445/*
1446 * Enqueues an event datagram to notify the peer VM attached to
1447 * the given queue pair handle about attach/detach event by the
1448 * given VM. Returns Payload size of datagram enqueued on
1449 * success, error code otherwise.
1450 */
1451static int qp_notify_peer(bool attach,
1452 struct vmci_handle handle,
1453 u32 my_id,
1454 u32 peer_id)
1455{
1456 int rv;
1457 struct vmci_event_qp ev;
1458
1459 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1460 peer_id == VMCI_INVALID_ID)
1461 return VMCI_ERROR_INVALID_ARGS;
1462
1463 /*
1464 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1465 * number of pending events from the hypervisor to a given VM
1466 * otherwise a rogue VM could do an arbitrary number of attach
1467 * and detach operations causing memory pressure in the host
1468 * kernel.
1469 */
1470
1471 memset(&ev, 0, sizeof(ev));
1472 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1473 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1474 VMCI_CONTEXT_RESOURCE_ID);
1475 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1476 ev.msg.event_data.event = attach ?
1477 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1478 ev.payload.handle = handle;
1479 ev.payload.peer_id = my_id;
1480
1481 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1482 &ev.msg.hdr, false);
1483 if (rv < VMCI_SUCCESS)
1484 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1485 attach ? "ATTACH" : "DETACH", peer_id);
1486
1487 return rv;
1488}
1489
1490/*
1491 * The second endpoint issuing a queue pair allocation will attach to
1492 * the queue pair registered with the queue pair broker.
1493 *
1494 * If the attacher is a guest, it will associate a VMX virtual address
1495 * range with the queue pair as specified by the page_store. At this
1496 * point, the already attach host endpoint may start using the queue
1497 * pair, and an attach event is sent to it. For compatibility with
1498 * older VMX'en, that used a separate step to set the VMX virtual
1499 * address range, the virtual address range can be registered later
1500 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1501 * NULL should be used, and the attach event will be generated once
1502 * the actual page store has been set.
1503 *
1504 * If the attacher is the host, a page_store of NULL should be used as
1505 * well, since the page store information is already set by the guest.
1506 *
1507 * For new VMX and host callers, the queue pair will be moved to the
1508 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1509 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1510 */
1511static int qp_broker_attach(struct qp_broker_entry *entry,
1512 u32 peer,
1513 u32 flags,
1514 u32 priv_flags,
1515 u64 produce_size,
1516 u64 consume_size,
1517 struct vmci_qp_page_store *page_store,
1518 struct vmci_ctx *context,
1519 vmci_event_release_cb wakeup_cb,
1520 void *client_data,
1521 struct qp_broker_entry **ent)
1522{
1523 const u32 context_id = vmci_ctx_get_id(context);
1524 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1525 int result;
1526
1527 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1528 entry->state != VMCIQPB_CREATED_MEM)
1529 return VMCI_ERROR_UNAVAILABLE;
1530
1531 if (is_local) {
1532 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1533 context_id != entry->create_id) {
1534 return VMCI_ERROR_INVALID_ARGS;
1535 }
1536 } else if (context_id == entry->create_id ||
1537 context_id == entry->attach_id) {
1538 return VMCI_ERROR_ALREADY_EXISTS;
1539 }
1540
1541 if (VMCI_CONTEXT_IS_VM(context_id) &&
1542 VMCI_CONTEXT_IS_VM(entry->create_id))
1543 return VMCI_ERROR_DST_UNREACHABLE;
1544
1545 /*
1546 * If we are attaching from a restricted context then the queuepair
1547 * must have been created by a trusted endpoint.
1548 */
1549 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1550 !entry->created_by_trusted)
1551 return VMCI_ERROR_NO_ACCESS;
1552
1553 /*
1554 * If we are attaching to a queuepair that was created by a restricted
1555 * context then we must be trusted.
1556 */
1557 if (entry->require_trusted_attach &&
1558 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1559 return VMCI_ERROR_NO_ACCESS;
1560
1561 /*
1562 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1563 * control check is not performed.
1564 */
1565 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1566 return VMCI_ERROR_NO_ACCESS;
1567
1568 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1569 /*
1570 * Do not attach if the caller doesn't support Host Queue Pairs
1571 * and a host created this queue pair.
1572 */
1573
1574 if (!vmci_ctx_supports_host_qp(context))
1575 return VMCI_ERROR_INVALID_RESOURCE;
1576
1577 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1578 struct vmci_ctx *create_context;
1579 bool supports_host_qp;
1580
1581 /*
1582 * Do not attach a host to a user created queue pair if that
1583 * user doesn't support host queue pair end points.
1584 */
1585
1586 create_context = vmci_ctx_get(entry->create_id);
1587 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1588 vmci_ctx_put(create_context);
1589
1590 if (!supports_host_qp)
1591 return VMCI_ERROR_INVALID_RESOURCE;
1592 }
1593
1594 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1595 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1596
1597 if (context_id != VMCI_HOST_CONTEXT_ID) {
1598 /*
1599 * The queue pair broker entry stores values from the guest
1600 * point of view, so an attaching guest should match the values
1601 * stored in the entry.
1602 */
1603
1604 if (entry->qp.produce_size != produce_size ||
1605 entry->qp.consume_size != consume_size) {
1606 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1607 }
1608 } else if (entry->qp.produce_size != consume_size ||
1609 entry->qp.consume_size != produce_size) {
1610 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1611 }
1612
1613 if (context_id != VMCI_HOST_CONTEXT_ID) {
1614 /*
1615 * If a guest attached to a queue pair, it will supply
1616 * the backing memory. If this is a pre NOVMVM vmx,
1617 * the backing memory will be supplied by calling
1618 * vmci_qp_broker_set_page_store() following the
1619 * return of the vmci_qp_broker_alloc() call. If it is
1620 * a vmx of version NOVMVM or later, the page store
1621 * must be supplied as part of the
1622 * vmci_qp_broker_alloc call. Under all circumstances
1623 * must the initially created queue pair not have any
1624 * memory associated with it already.
1625 */
1626
1627 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1628 return VMCI_ERROR_INVALID_ARGS;
1629
1630 if (page_store != NULL) {
1631 /*
1632 * Patch up host state to point to guest
1633 * supplied memory. The VMX already
1634 * initialized the queue pair headers, so no
1635 * need for the kernel side to do that.
1636 */
1637
1638 result = qp_host_register_user_memory(page_store,
1639 entry->produce_q,
1640 entry->consume_q);
1641 if (result < VMCI_SUCCESS)
1642 return result;
1643
1644 entry->state = VMCIQPB_ATTACHED_MEM;
1645 } else {
1646 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1647 }
1648 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1649 /*
1650 * The host side is attempting to attach to a queue
1651 * pair that doesn't have any memory associated with
1652 * it. This must be a pre NOVMVM vmx that hasn't set
1653 * the page store information yet, or a quiesced VM.
1654 */
1655
1656 return VMCI_ERROR_UNAVAILABLE;
1657 } else {
1658 /* The host side has successfully attached to a queue pair. */
1659 entry->state = VMCIQPB_ATTACHED_MEM;
1660 }
1661
1662 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1663 result =
1664 qp_notify_peer(true, entry->qp.handle, context_id,
1665 entry->create_id);
1666 if (result < VMCI_SUCCESS)
1667 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1668 entry->create_id, entry->qp.handle.context,
1669 entry->qp.handle.resource);
1670 }
1671
1672 entry->attach_id = context_id;
1673 entry->qp.ref_count++;
1674 if (wakeup_cb) {
1675 entry->wakeup_cb = wakeup_cb;
1676 entry->client_data = client_data;
1677 }
1678
1679 /*
1680 * When attaching to local queue pairs, the context already has
1681 * an entry tracking the queue pair, so don't add another one.
1682 */
1683 if (!is_local)
1684 vmci_ctx_qp_create(context, entry->qp.handle);
1685
1686 if (ent != NULL)
1687 *ent = entry;
1688
1689 return VMCI_SUCCESS;
1690}
1691
1692/*
1693 * queue_pair_Alloc for use when setting up queue pair endpoints
1694 * on the host.
1695 */
1696static int qp_broker_alloc(struct vmci_handle handle,
1697 u32 peer,
1698 u32 flags,
1699 u32 priv_flags,
1700 u64 produce_size,
1701 u64 consume_size,
1702 struct vmci_qp_page_store *page_store,
1703 struct vmci_ctx *context,
1704 vmci_event_release_cb wakeup_cb,
1705 void *client_data,
1706 struct qp_broker_entry **ent,
1707 bool *swap)
1708{
1709 const u32 context_id = vmci_ctx_get_id(context);
1710 bool create;
1711 struct qp_broker_entry *entry = NULL;
1712 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1713 int result;
1714
1715 if (vmci_handle_is_invalid(handle) ||
1716 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1717 !(produce_size || consume_size) ||
1718 !context || context_id == VMCI_INVALID_ID ||
1719 handle.context == VMCI_INVALID_ID) {
1720 return VMCI_ERROR_INVALID_ARGS;
1721 }
1722
1723 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1724 return VMCI_ERROR_INVALID_ARGS;
1725
1726 /*
1727 * In the initial argument check, we ensure that non-vmkernel hosts
1728 * are not allowed to create local queue pairs.
1729 */
1730
1731 mutex_lock(&qp_broker_list.mutex);
1732
1733 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1734 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1735 context_id, handle.context, handle.resource);
1736 mutex_unlock(&qp_broker_list.mutex);
1737 return VMCI_ERROR_ALREADY_EXISTS;
1738 }
1739
1740 if (handle.resource != VMCI_INVALID_ID)
1741 entry = qp_broker_handle_to_entry(handle);
1742
1743 if (!entry) {
1744 create = true;
1745 result =
1746 qp_broker_create(handle, peer, flags, priv_flags,
1747 produce_size, consume_size, page_store,
1748 context, wakeup_cb, client_data, ent);
1749 } else {
1750 create = false;
1751 result =
1752 qp_broker_attach(entry, peer, flags, priv_flags,
1753 produce_size, consume_size, page_store,
1754 context, wakeup_cb, client_data, ent);
1755 }
1756
1757 mutex_unlock(&qp_broker_list.mutex);
1758
1759 if (swap)
1760 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1761 !(create && is_local);
1762
1763 return result;
1764}
1765
1766/*
1767 * This function implements the kernel API for allocating a queue
1768 * pair.
1769 */
1770static int qp_alloc_host_work(struct vmci_handle *handle,
1771 struct vmci_queue **produce_q,
1772 u64 produce_size,
1773 struct vmci_queue **consume_q,
1774 u64 consume_size,
1775 u32 peer,
1776 u32 flags,
1777 u32 priv_flags,
1778 vmci_event_release_cb wakeup_cb,
1779 void *client_data)
1780{
1781 struct vmci_handle new_handle;
1782 struct vmci_ctx *context;
1783 struct qp_broker_entry *entry;
1784 int result;
1785 bool swap;
1786
1787 if (vmci_handle_is_invalid(*handle)) {
1788 new_handle = vmci_make_handle(
1789 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1790 } else
1791 new_handle = *handle;
1792
1793 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1794 entry = NULL;
1795 result =
1796 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1797 produce_size, consume_size, NULL, context,
1798 wakeup_cb, client_data, &entry, &swap);
1799 if (result == VMCI_SUCCESS) {
1800 if (swap) {
1801 /*
1802 * If this is a local queue pair, the attacher
1803 * will swap around produce and consume
1804 * queues.
1805 */
1806
1807 *produce_q = entry->consume_q;
1808 *consume_q = entry->produce_q;
1809 } else {
1810 *produce_q = entry->produce_q;
1811 *consume_q = entry->consume_q;
1812 }
1813
1814 *handle = vmci_resource_handle(&entry->resource);
1815 } else {
1816 *handle = VMCI_INVALID_HANDLE;
1817 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1818 result);
1819 }
1820 vmci_ctx_put(context);
1821 return result;
1822}
1823
1824/*
1825 * Allocates a VMCI queue_pair. Only checks validity of input
1826 * arguments. The real work is done in the host or guest
1827 * specific function.
1828 */
1829int vmci_qp_alloc(struct vmci_handle *handle,
1830 struct vmci_queue **produce_q,
1831 u64 produce_size,
1832 struct vmci_queue **consume_q,
1833 u64 consume_size,
1834 u32 peer,
1835 u32 flags,
1836 u32 priv_flags,
1837 bool guest_endpoint,
1838 vmci_event_release_cb wakeup_cb,
1839 void *client_data)
1840{
1841 if (!handle || !produce_q || !consume_q ||
1842 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1843 return VMCI_ERROR_INVALID_ARGS;
1844
1845 if (guest_endpoint) {
1846 return qp_alloc_guest_work(handle, produce_q,
1847 produce_size, consume_q,
1848 consume_size, peer,
1849 flags, priv_flags);
1850 } else {
1851 return qp_alloc_host_work(handle, produce_q,
1852 produce_size, consume_q,
1853 consume_size, peer, flags,
1854 priv_flags, wakeup_cb, client_data);
1855 }
1856}
1857
1858/*
1859 * This function implements the host kernel API for detaching from
1860 * a queue pair.
1861 */
1862static int qp_detatch_host_work(struct vmci_handle handle)
1863{
1864 int result;
1865 struct vmci_ctx *context;
1866
1867 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1868
1869 result = vmci_qp_broker_detach(handle, context);
1870
1871 vmci_ctx_put(context);
1872 return result;
1873}
1874
1875/*
1876 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1877 * Real work is done in the host or guest specific function.
1878 */
1879static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1880{
1881 if (vmci_handle_is_invalid(handle))
1882 return VMCI_ERROR_INVALID_ARGS;
1883
1884 if (guest_endpoint)
1885 return qp_detatch_guest_work(handle);
1886 else
1887 return qp_detatch_host_work(handle);
1888}
1889
1890/*
1891 * Returns the entry from the head of the list. Assumes that the list is
1892 * locked.
1893 */
1894static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1895{
1896 if (!list_empty(&qp_list->head)) {
1897 struct qp_entry *entry =
1898 list_first_entry(&qp_list->head, struct qp_entry,
1899 list_item);
1900 return entry;
1901 }
1902
1903 return NULL;
1904}
1905
1906void vmci_qp_broker_exit(void)
1907{
1908 struct qp_entry *entry;
1909 struct qp_broker_entry *be;
1910
1911 mutex_lock(&qp_broker_list.mutex);
1912
1913 while ((entry = qp_list_get_head(&qp_broker_list))) {
1914 be = (struct qp_broker_entry *)entry;
1915
1916 qp_list_remove_entry(&qp_broker_list, entry);
1917 kfree(be);
1918 }
1919
1920 mutex_unlock(&qp_broker_list.mutex);
1921}
1922
1923/*
1924 * Requests that a queue pair be allocated with the VMCI queue
1925 * pair broker. Allocates a queue pair entry if one does not
1926 * exist. Attaches to one if it exists, and retrieves the page
1927 * files backing that queue_pair. Assumes that the queue pair
1928 * broker lock is held.
1929 */
1930int vmci_qp_broker_alloc(struct vmci_handle handle,
1931 u32 peer,
1932 u32 flags,
1933 u32 priv_flags,
1934 u64 produce_size,
1935 u64 consume_size,
1936 struct vmci_qp_page_store *page_store,
1937 struct vmci_ctx *context)
1938{
1939 if (!QP_SIZES_ARE_VALID(produce_size, consume_size))
1940 return VMCI_ERROR_NO_RESOURCES;
1941
1942 return qp_broker_alloc(handle, peer, flags, priv_flags,
1943 produce_size, consume_size,
1944 page_store, context, NULL, NULL, NULL, NULL);
1945}
1946
1947/*
1948 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
1949 * step to add the UVAs of the VMX mapping of the queue pair. This function
1950 * provides backwards compatibility with such VMX'en, and takes care of
1951 * registering the page store for a queue pair previously allocated by the
1952 * VMX during create or attach. This function will move the queue pair state
1953 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
1954 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
1955 * attached state with memory, the queue pair is ready to be used by the
1956 * host peer, and an attached event will be generated.
1957 *
1958 * Assumes that the queue pair broker lock is held.
1959 *
1960 * This function is only used by the hosted platform, since there is no
1961 * issue with backwards compatibility for vmkernel.
1962 */
1963int vmci_qp_broker_set_page_store(struct vmci_handle handle,
1964 u64 produce_uva,
1965 u64 consume_uva,
1966 struct vmci_ctx *context)
1967{
1968 struct qp_broker_entry *entry;
1969 int result;
1970 const u32 context_id = vmci_ctx_get_id(context);
1971
1972 if (vmci_handle_is_invalid(handle) || !context ||
1973 context_id == VMCI_INVALID_ID)
1974 return VMCI_ERROR_INVALID_ARGS;
1975
1976 /*
1977 * We only support guest to host queue pairs, so the VMX must
1978 * supply UVAs for the mapped page files.
1979 */
1980
1981 if (produce_uva == 0 || consume_uva == 0)
1982 return VMCI_ERROR_INVALID_ARGS;
1983
1984 mutex_lock(&qp_broker_list.mutex);
1985
1986 if (!vmci_ctx_qp_exists(context, handle)) {
1987 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
1988 context_id, handle.context, handle.resource);
1989 result = VMCI_ERROR_NOT_FOUND;
1990 goto out;
1991 }
1992
1993 entry = qp_broker_handle_to_entry(handle);
1994 if (!entry) {
1995 result = VMCI_ERROR_NOT_FOUND;
1996 goto out;
1997 }
1998
1999 /*
2000 * If I'm the owner then I can set the page store.
2001 *
2002 * Or, if a host created the queue_pair and I'm the attached peer
2003 * then I can set the page store.
2004 */
2005 if (entry->create_id != context_id &&
2006 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
2007 entry->attach_id != context_id)) {
2008 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
2009 goto out;
2010 }
2011
2012 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2013 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2014 result = VMCI_ERROR_UNAVAILABLE;
2015 goto out;
2016 }
2017
2018 result = qp_host_get_user_memory(produce_uva, consume_uva,
2019 entry->produce_q, entry->consume_q);
2020 if (result < VMCI_SUCCESS)
2021 goto out;
2022
2023 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2024 if (result < VMCI_SUCCESS) {
2025 qp_host_unregister_user_memory(entry->produce_q,
2026 entry->consume_q);
2027 goto out;
2028 }
2029
2030 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2031 entry->state = VMCIQPB_CREATED_MEM;
2032 else
2033 entry->state = VMCIQPB_ATTACHED_MEM;
2034
2035 entry->vmci_page_files = true;
2036
2037 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2038 result =
2039 qp_notify_peer(true, handle, context_id, entry->create_id);
2040 if (result < VMCI_SUCCESS) {
2041 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2042 entry->create_id, entry->qp.handle.context,
2043 entry->qp.handle.resource);
2044 }
2045 }
2046
2047 result = VMCI_SUCCESS;
2048 out:
2049 mutex_unlock(&qp_broker_list.mutex);
2050 return result;
2051}
2052
2053/*
2054 * Resets saved queue headers for the given QP broker
2055 * entry. Should be used when guest memory becomes available
2056 * again, or the guest detaches.
2057 */
2058static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2059{
2060 entry->produce_q->saved_header = NULL;
2061 entry->consume_q->saved_header = NULL;
2062}
2063
2064/*
2065 * The main entry point for detaching from a queue pair registered with the
2066 * queue pair broker. If more than one endpoint is attached to the queue
2067 * pair, the first endpoint will mainly decrement a reference count and
2068 * generate a notification to its peer. The last endpoint will clean up
2069 * the queue pair state registered with the broker.
2070 *
2071 * When a guest endpoint detaches, it will unmap and unregister the guest
2072 * memory backing the queue pair. If the host is still attached, it will
2073 * no longer be able to access the queue pair content.
2074 *
2075 * If the queue pair is already in a state where there is no memory
2076 * registered for the queue pair (any *_NO_MEM state), it will transition to
2077 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2078 * endpoint is the first of two endpoints to detach. If the host endpoint is
2079 * the first out of two to detach, the queue pair will move to the
2080 * VMCIQPB_SHUTDOWN_MEM state.
2081 */
2082int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2083{
2084 struct qp_broker_entry *entry;
2085 const u32 context_id = vmci_ctx_get_id(context);
2086 u32 peer_id;
2087 bool is_local = false;
2088 int result;
2089
2090 if (vmci_handle_is_invalid(handle) || !context ||
2091 context_id == VMCI_INVALID_ID) {
2092 return VMCI_ERROR_INVALID_ARGS;
2093 }
2094
2095 mutex_lock(&qp_broker_list.mutex);
2096
2097 if (!vmci_ctx_qp_exists(context, handle)) {
2098 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2099 context_id, handle.context, handle.resource);
2100 result = VMCI_ERROR_NOT_FOUND;
2101 goto out;
2102 }
2103
2104 entry = qp_broker_handle_to_entry(handle);
2105 if (!entry) {
2106 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2107 context_id, handle.context, handle.resource);
2108 result = VMCI_ERROR_NOT_FOUND;
2109 goto out;
2110 }
2111
2112 if (context_id != entry->create_id && context_id != entry->attach_id) {
2113 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2114 goto out;
2115 }
2116
2117 if (context_id == entry->create_id) {
2118 peer_id = entry->attach_id;
2119 entry->create_id = VMCI_INVALID_ID;
2120 } else {
2121 peer_id = entry->create_id;
2122 entry->attach_id = VMCI_INVALID_ID;
2123 }
2124 entry->qp.ref_count--;
2125
2126 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2127
2128 if (context_id != VMCI_HOST_CONTEXT_ID) {
2129 bool headers_mapped;
2130
2131 /*
2132 * Pre NOVMVM vmx'en may detach from a queue pair
2133 * before setting the page store, and in that case
2134 * there is no user memory to detach from. Also, more
2135 * recent VMX'en may detach from a queue pair in the
2136 * quiesced state.
2137 */
2138
2139 qp_acquire_queue_mutex(entry->produce_q);
2140 headers_mapped = entry->produce_q->q_header ||
2141 entry->consume_q->q_header;
2142 if (QPBROKERSTATE_HAS_MEM(entry)) {
2143 result =
2144 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2145 entry->produce_q,
2146 entry->consume_q);
2147 if (result < VMCI_SUCCESS)
2148 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2149 handle.context, handle.resource,
2150 result);
2151
2152 qp_host_unregister_user_memory(entry->produce_q,
2153 entry->consume_q);
2154
2155 }
2156
2157 if (!headers_mapped)
2158 qp_reset_saved_headers(entry);
2159
2160 qp_release_queue_mutex(entry->produce_q);
2161
2162 if (!headers_mapped && entry->wakeup_cb)
2163 entry->wakeup_cb(entry->client_data);
2164
2165 } else {
2166 if (entry->wakeup_cb) {
2167 entry->wakeup_cb = NULL;
2168 entry->client_data = NULL;
2169 }
2170 }
2171
2172 if (entry->qp.ref_count == 0) {
2173 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2174
2175 if (is_local)
2176 kfree(entry->local_mem);
2177
2178 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2179 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2180 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2181 /* Unlink from resource hash table and free callback */
2182 vmci_resource_remove(&entry->resource);
2183
2184 kfree(entry);
2185
2186 vmci_ctx_qp_destroy(context, handle);
2187 } else {
2188 qp_notify_peer(false, handle, context_id, peer_id);
2189 if (context_id == VMCI_HOST_CONTEXT_ID &&
2190 QPBROKERSTATE_HAS_MEM(entry)) {
2191 entry->state = VMCIQPB_SHUTDOWN_MEM;
2192 } else {
2193 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2194 }
2195
2196 if (!is_local)
2197 vmci_ctx_qp_destroy(context, handle);
2198
2199 }
2200 result = VMCI_SUCCESS;
2201 out:
2202 mutex_unlock(&qp_broker_list.mutex);
2203 return result;
2204}
2205
2206/*
2207 * Establishes the necessary mappings for a queue pair given a
2208 * reference to the queue pair guest memory. This is usually
2209 * called when a guest is unquiesced and the VMX is allowed to
2210 * map guest memory once again.
2211 */
2212int vmci_qp_broker_map(struct vmci_handle handle,
2213 struct vmci_ctx *context,
2214 u64 guest_mem)
2215{
2216 struct qp_broker_entry *entry;
2217 const u32 context_id = vmci_ctx_get_id(context);
2218 int result;
2219
2220 if (vmci_handle_is_invalid(handle) || !context ||
2221 context_id == VMCI_INVALID_ID)
2222 return VMCI_ERROR_INVALID_ARGS;
2223
2224 mutex_lock(&qp_broker_list.mutex);
2225
2226 if (!vmci_ctx_qp_exists(context, handle)) {
2227 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2228 context_id, handle.context, handle.resource);
2229 result = VMCI_ERROR_NOT_FOUND;
2230 goto out;
2231 }
2232
2233 entry = qp_broker_handle_to_entry(handle);
2234 if (!entry) {
2235 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2236 context_id, handle.context, handle.resource);
2237 result = VMCI_ERROR_NOT_FOUND;
2238 goto out;
2239 }
2240
2241 if (context_id != entry->create_id && context_id != entry->attach_id) {
2242 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2243 goto out;
2244 }
2245
2246 result = VMCI_SUCCESS;
2247
2248 if (context_id != VMCI_HOST_CONTEXT_ID &&
2249 !QPBROKERSTATE_HAS_MEM(entry)) {
2250 struct vmci_qp_page_store page_store;
2251
2252 page_store.pages = guest_mem;
2253 page_store.len = QPE_NUM_PAGES(entry->qp);
2254
2255 qp_acquire_queue_mutex(entry->produce_q);
2256 qp_reset_saved_headers(entry);
2257 result =
2258 qp_host_register_user_memory(&page_store,
2259 entry->produce_q,
2260 entry->consume_q);
2261 qp_release_queue_mutex(entry->produce_q);
2262 if (result == VMCI_SUCCESS) {
2263 /* Move state from *_NO_MEM to *_MEM */
2264
2265 entry->state++;
2266
2267 if (entry->wakeup_cb)
2268 entry->wakeup_cb(entry->client_data);
2269 }
2270 }
2271
2272 out:
2273 mutex_unlock(&qp_broker_list.mutex);
2274 return result;
2275}
2276
2277/*
2278 * Saves a snapshot of the queue headers for the given QP broker
2279 * entry. Should be used when guest memory is unmapped.
2280 * Results:
2281 * VMCI_SUCCESS on success, appropriate error code if guest memory
2282 * can't be accessed..
2283 */
2284static int qp_save_headers(struct qp_broker_entry *entry)
2285{
2286 int result;
2287
2288 if (entry->produce_q->saved_header != NULL &&
2289 entry->consume_q->saved_header != NULL) {
2290 /*
2291 * If the headers have already been saved, we don't need to do
2292 * it again, and we don't want to map in the headers
2293 * unnecessarily.
2294 */
2295
2296 return VMCI_SUCCESS;
2297 }
2298
2299 if (NULL == entry->produce_q->q_header ||
2300 NULL == entry->consume_q->q_header) {
2301 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2302 if (result < VMCI_SUCCESS)
2303 return result;
2304 }
2305
2306 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2307 sizeof(entry->saved_produce_q));
2308 entry->produce_q->saved_header = &entry->saved_produce_q;
2309 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2310 sizeof(entry->saved_consume_q));
2311 entry->consume_q->saved_header = &entry->saved_consume_q;
2312
2313 return VMCI_SUCCESS;
2314}
2315
2316/*
2317 * Removes all references to the guest memory of a given queue pair, and
2318 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2319 * called when a VM is being quiesced where access to guest memory should
2320 * avoided.
2321 */
2322int vmci_qp_broker_unmap(struct vmci_handle handle,
2323 struct vmci_ctx *context,
2324 u32 gid)
2325{
2326 struct qp_broker_entry *entry;
2327 const u32 context_id = vmci_ctx_get_id(context);
2328 int result;
2329
2330 if (vmci_handle_is_invalid(handle) || !context ||
2331 context_id == VMCI_INVALID_ID)
2332 return VMCI_ERROR_INVALID_ARGS;
2333
2334 mutex_lock(&qp_broker_list.mutex);
2335
2336 if (!vmci_ctx_qp_exists(context, handle)) {
2337 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2338 context_id, handle.context, handle.resource);
2339 result = VMCI_ERROR_NOT_FOUND;
2340 goto out;
2341 }
2342
2343 entry = qp_broker_handle_to_entry(handle);
2344 if (!entry) {
2345 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2346 context_id, handle.context, handle.resource);
2347 result = VMCI_ERROR_NOT_FOUND;
2348 goto out;
2349 }
2350
2351 if (context_id != entry->create_id && context_id != entry->attach_id) {
2352 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2353 goto out;
2354 }
2355
2356 if (context_id != VMCI_HOST_CONTEXT_ID &&
2357 QPBROKERSTATE_HAS_MEM(entry)) {
2358 qp_acquire_queue_mutex(entry->produce_q);
2359 result = qp_save_headers(entry);
2360 if (result < VMCI_SUCCESS)
2361 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2362 handle.context, handle.resource, result);
2363
2364 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2365
2366 /*
2367 * On hosted, when we unmap queue pairs, the VMX will also
2368 * unmap the guest memory, so we invalidate the previously
2369 * registered memory. If the queue pair is mapped again at a
2370 * later point in time, we will need to reregister the user
2371 * memory with a possibly new user VA.
2372 */
2373 qp_host_unregister_user_memory(entry->produce_q,
2374 entry->consume_q);
2375
2376 /*
2377 * Move state from *_MEM to *_NO_MEM.
2378 */
2379 entry->state--;
2380
2381 qp_release_queue_mutex(entry->produce_q);
2382 }
2383
2384 result = VMCI_SUCCESS;
2385
2386 out:
2387 mutex_unlock(&qp_broker_list.mutex);
2388 return result;
2389}
2390
2391/*
2392 * Destroys all guest queue pair endpoints. If active guest queue
2393 * pairs still exist, hypercalls to attempt detach from these
2394 * queue pairs will be made. Any failure to detach is silently
2395 * ignored.
2396 */
2397void vmci_qp_guest_endpoints_exit(void)
2398{
2399 struct qp_entry *entry;
2400 struct qp_guest_endpoint *ep;
2401
2402 mutex_lock(&qp_guest_endpoints.mutex);
2403
2404 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2405 ep = (struct qp_guest_endpoint *)entry;
2406
2407 /* Don't make a hypercall for local queue_pairs. */
2408 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2409 qp_detatch_hypercall(entry->handle);
2410
2411 /* We cannot fail the exit, so let's reset ref_count. */
2412 entry->ref_count = 0;
2413 qp_list_remove_entry(&qp_guest_endpoints, entry);
2414
2415 qp_guest_endpoint_destroy(ep);
2416 }
2417
2418 mutex_unlock(&qp_guest_endpoints.mutex);
2419}
2420
2421/*
2422 * Helper routine that will lock the queue pair before subsequent
2423 * operations.
2424 * Note: Non-blocking on the host side is currently only implemented in ESX.
2425 * Since non-blocking isn't yet implemented on the host personality we
2426 * have no reason to acquire a spin lock. So to avoid the use of an
2427 * unnecessary lock only acquire the mutex if we can block.
2428 */
2429static void qp_lock(const struct vmci_qp *qpair)
2430{
2431 qp_acquire_queue_mutex(qpair->produce_q);
2432}
2433
2434/*
2435 * Helper routine that unlocks the queue pair after calling
2436 * qp_lock.
2437 */
2438static void qp_unlock(const struct vmci_qp *qpair)
2439{
2440 qp_release_queue_mutex(qpair->produce_q);
2441}
2442
2443/*
2444 * The queue headers may not be mapped at all times. If a queue is
2445 * currently not mapped, it will be attempted to do so.
2446 */
2447static int qp_map_queue_headers(struct vmci_queue *produce_q,
2448 struct vmci_queue *consume_q)
2449{
2450 int result;
2451
2452 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2453 result = qp_host_map_queues(produce_q, consume_q);
2454 if (result < VMCI_SUCCESS)
2455 return (produce_q->saved_header &&
2456 consume_q->saved_header) ?
2457 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2458 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2459 }
2460
2461 return VMCI_SUCCESS;
2462}
2463
2464/*
2465 * Helper routine that will retrieve the produce and consume
2466 * headers of a given queue pair. If the guest memory of the
2467 * queue pair is currently not available, the saved queue headers
2468 * will be returned, if these are available.
2469 */
2470static int qp_get_queue_headers(const struct vmci_qp *qpair,
2471 struct vmci_queue_header **produce_q_header,
2472 struct vmci_queue_header **consume_q_header)
2473{
2474 int result;
2475
2476 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2477 if (result == VMCI_SUCCESS) {
2478 *produce_q_header = qpair->produce_q->q_header;
2479 *consume_q_header = qpair->consume_q->q_header;
2480 } else if (qpair->produce_q->saved_header &&
2481 qpair->consume_q->saved_header) {
2482 *produce_q_header = qpair->produce_q->saved_header;
2483 *consume_q_header = qpair->consume_q->saved_header;
2484 result = VMCI_SUCCESS;
2485 }
2486
2487 return result;
2488}
2489
2490/*
2491 * Callback from VMCI queue pair broker indicating that a queue
2492 * pair that was previously not ready, now either is ready or
2493 * gone forever.
2494 */
2495static int qp_wakeup_cb(void *client_data)
2496{
2497 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2498
2499 qp_lock(qpair);
2500 while (qpair->blocked > 0) {
2501 qpair->blocked--;
2502 qpair->generation++;
2503 wake_up(&qpair->event);
2504 }
2505 qp_unlock(qpair);
2506
2507 return VMCI_SUCCESS;
2508}
2509
2510/*
2511 * Makes the calling thread wait for the queue pair to become
2512 * ready for host side access. Returns true when thread is
2513 * woken up after queue pair state change, false otherwise.
2514 */
2515static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2516{
2517 unsigned int generation;
2518
2519 qpair->blocked++;
2520 generation = qpair->generation;
2521 qp_unlock(qpair);
2522 wait_event(qpair->event, generation != qpair->generation);
2523 qp_lock(qpair);
2524
2525 return true;
2526}
2527
2528/*
2529 * Enqueues a given buffer to the produce queue using the provided
2530 * function. As many bytes as possible (space available in the queue)
2531 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2532 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2533 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2534 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2535 * an error occured when accessing the buffer,
2536 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2537 * available. Otherwise, the number of bytes written to the queue is
2538 * returned. Updates the tail pointer of the produce queue.
2539 */
2540static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2541 struct vmci_queue *consume_q,
2542 const u64 produce_q_size,
2543 struct iov_iter *from)
2544{
2545 s64 free_space;
2546 u64 tail;
2547 size_t buf_size = iov_iter_count(from);
2548 size_t written;
2549 ssize_t result;
2550
2551 result = qp_map_queue_headers(produce_q, consume_q);
2552 if (unlikely(result != VMCI_SUCCESS))
2553 return result;
2554
2555 free_space = vmci_q_header_free_space(produce_q->q_header,
2556 consume_q->q_header,
2557 produce_q_size);
2558 if (free_space == 0)
2559 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2560
2561 if (free_space < VMCI_SUCCESS)
2562 return (ssize_t) free_space;
2563
2564 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2565 tail = vmci_q_header_producer_tail(produce_q->q_header);
2566 if (likely(tail + written < produce_q_size)) {
2567 result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2568 } else {
2569 /* Tail pointer wraps around. */
2570
2571 const size_t tmp = (size_t) (produce_q_size - tail);
2572
2573 result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2574 if (result >= VMCI_SUCCESS)
2575 result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2576 written - tmp);
2577 }
2578
2579 if (result < VMCI_SUCCESS)
2580 return result;
2581
2582 /*
2583 * This virt_wmb() ensures that data written to the queue
2584 * is observable before the new producer_tail is.
2585 */
2586 virt_wmb();
2587
2588 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2589 produce_q_size);
2590 return written;
2591}
2592
2593/*
2594 * Dequeues data (if available) from the given consume queue. Writes data
2595 * to the user provided buffer using the provided function.
2596 * Assumes the queue->mutex has been acquired.
2597 * Results:
2598 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2599 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2600 * (as defined by the queue size).
2601 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2602 * Otherwise the number of bytes dequeued is returned.
2603 * Side effects:
2604 * Updates the head pointer of the consume queue.
2605 */
2606static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2607 struct vmci_queue *consume_q,
2608 const u64 consume_q_size,
2609 struct iov_iter *to,
2610 bool update_consumer)
2611{
2612 size_t buf_size = iov_iter_count(to);
2613 s64 buf_ready;
2614 u64 head;
2615 size_t read;
2616 ssize_t result;
2617
2618 result = qp_map_queue_headers(produce_q, consume_q);
2619 if (unlikely(result != VMCI_SUCCESS))
2620 return result;
2621
2622 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2623 produce_q->q_header,
2624 consume_q_size);
2625 if (buf_ready == 0)
2626 return VMCI_ERROR_QUEUEPAIR_NODATA;
2627
2628 if (buf_ready < VMCI_SUCCESS)
2629 return (ssize_t) buf_ready;
2630
2631 /*
2632 * This virt_rmb() ensures that data from the queue will be read
2633 * after we have determined how much is ready to be consumed.
2634 */
2635 virt_rmb();
2636
2637 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2638 head = vmci_q_header_consumer_head(produce_q->q_header);
2639 if (likely(head + read < consume_q_size)) {
2640 result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
2641 } else {
2642 /* Head pointer wraps around. */
2643
2644 const size_t tmp = (size_t) (consume_q_size - head);
2645
2646 result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
2647 if (result >= VMCI_SUCCESS)
2648 result = qp_memcpy_from_queue_iter(to, consume_q, 0,
2649 read - tmp);
2650
2651 }
2652
2653 if (result < VMCI_SUCCESS)
2654 return result;
2655
2656 if (update_consumer)
2657 vmci_q_header_add_consumer_head(produce_q->q_header,
2658 read, consume_q_size);
2659
2660 return read;
2661}
2662
2663/*
2664 * vmci_qpair_alloc() - Allocates a queue pair.
2665 * @qpair: Pointer for the new vmci_qp struct.
2666 * @handle: Handle to track the resource.
2667 * @produce_qsize: Desired size of the producer queue.
2668 * @consume_qsize: Desired size of the consumer queue.
2669 * @peer: ContextID of the peer.
2670 * @flags: VMCI flags.
2671 * @priv_flags: VMCI priviledge flags.
2672 *
2673 * This is the client interface for allocating the memory for a
2674 * vmci_qp structure and then attaching to the underlying
2675 * queue. If an error occurs allocating the memory for the
2676 * vmci_qp structure no attempt is made to attach. If an
2677 * error occurs attaching, then the structure is freed.
2678 */
2679int vmci_qpair_alloc(struct vmci_qp **qpair,
2680 struct vmci_handle *handle,
2681 u64 produce_qsize,
2682 u64 consume_qsize,
2683 u32 peer,
2684 u32 flags,
2685 u32 priv_flags)
2686{
2687 struct vmci_qp *my_qpair;
2688 int retval;
2689 struct vmci_handle src = VMCI_INVALID_HANDLE;
2690 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2691 enum vmci_route route;
2692 vmci_event_release_cb wakeup_cb;
2693 void *client_data;
2694
2695 /*
2696 * Restrict the size of a queuepair. The device already
2697 * enforces a limit on the total amount of memory that can be
2698 * allocated to queuepairs for a guest. However, we try to
2699 * allocate this memory before we make the queuepair
2700 * allocation hypercall. On Linux, we allocate each page
2701 * separately, which means rather than fail, the guest will
2702 * thrash while it tries to allocate, and will become
2703 * increasingly unresponsive to the point where it appears to
2704 * be hung. So we place a limit on the size of an individual
2705 * queuepair here, and leave the device to enforce the
2706 * restriction on total queuepair memory. (Note that this
2707 * doesn't prevent all cases; a user with only this much
2708 * physical memory could still get into trouble.) The error
2709 * used by the device is NO_RESOURCES, so use that here too.
2710 */
2711
2712 if (!QP_SIZES_ARE_VALID(produce_qsize, consume_qsize))
2713 return VMCI_ERROR_NO_RESOURCES;
2714
2715 retval = vmci_route(&src, &dst, false, &route);
2716 if (retval < VMCI_SUCCESS)
2717 route = vmci_guest_code_active() ?
2718 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2719
2720 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2721 pr_devel("NONBLOCK OR PINNED set");
2722 return VMCI_ERROR_INVALID_ARGS;
2723 }
2724
2725 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2726 if (!my_qpair)
2727 return VMCI_ERROR_NO_MEM;
2728
2729 my_qpair->produce_q_size = produce_qsize;
2730 my_qpair->consume_q_size = consume_qsize;
2731 my_qpair->peer = peer;
2732 my_qpair->flags = flags;
2733 my_qpair->priv_flags = priv_flags;
2734
2735 wakeup_cb = NULL;
2736 client_data = NULL;
2737
2738 if (VMCI_ROUTE_AS_HOST == route) {
2739 my_qpair->guest_endpoint = false;
2740 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2741 my_qpair->blocked = 0;
2742 my_qpair->generation = 0;
2743 init_waitqueue_head(&my_qpair->event);
2744 wakeup_cb = qp_wakeup_cb;
2745 client_data = (void *)my_qpair;
2746 }
2747 } else {
2748 my_qpair->guest_endpoint = true;
2749 }
2750
2751 retval = vmci_qp_alloc(handle,
2752 &my_qpair->produce_q,
2753 my_qpair->produce_q_size,
2754 &my_qpair->consume_q,
2755 my_qpair->consume_q_size,
2756 my_qpair->peer,
2757 my_qpair->flags,
2758 my_qpair->priv_flags,
2759 my_qpair->guest_endpoint,
2760 wakeup_cb, client_data);
2761
2762 if (retval < VMCI_SUCCESS) {
2763 kfree(my_qpair);
2764 return retval;
2765 }
2766
2767 *qpair = my_qpair;
2768 my_qpair->handle = *handle;
2769
2770 return retval;
2771}
2772EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2773
2774/*
2775 * vmci_qpair_detach() - Detatches the client from a queue pair.
2776 * @qpair: Reference of a pointer to the qpair struct.
2777 *
2778 * This is the client interface for detaching from a VMCIQPair.
2779 * Note that this routine will free the memory allocated for the
2780 * vmci_qp structure too.
2781 */
2782int vmci_qpair_detach(struct vmci_qp **qpair)
2783{
2784 int result;
2785 struct vmci_qp *old_qpair;
2786
2787 if (!qpair || !(*qpair))
2788 return VMCI_ERROR_INVALID_ARGS;
2789
2790 old_qpair = *qpair;
2791 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2792
2793 /*
2794 * The guest can fail to detach for a number of reasons, and
2795 * if it does so, it will cleanup the entry (if there is one).
2796 * The host can fail too, but it won't cleanup the entry
2797 * immediately, it will do that later when the context is
2798 * freed. Either way, we need to release the qpair struct
2799 * here; there isn't much the caller can do, and we don't want
2800 * to leak.
2801 */
2802
2803 memset(old_qpair, 0, sizeof(*old_qpair));
2804 old_qpair->handle = VMCI_INVALID_HANDLE;
2805 old_qpair->peer = VMCI_INVALID_ID;
2806 kfree(old_qpair);
2807 *qpair = NULL;
2808
2809 return result;
2810}
2811EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2812
2813/*
2814 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2815 * @qpair: Pointer to the queue pair struct.
2816 * @producer_tail: Reference used for storing producer tail index.
2817 * @consumer_head: Reference used for storing the consumer head index.
2818 *
2819 * This is the client interface for getting the current indexes of the
2820 * QPair from the point of the view of the caller as the producer.
2821 */
2822int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2823 u64 *producer_tail,
2824 u64 *consumer_head)
2825{
2826 struct vmci_queue_header *produce_q_header;
2827 struct vmci_queue_header *consume_q_header;
2828 int result;
2829
2830 if (!qpair)
2831 return VMCI_ERROR_INVALID_ARGS;
2832
2833 qp_lock(qpair);
2834 result =
2835 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2836 if (result == VMCI_SUCCESS)
2837 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2838 producer_tail, consumer_head);
2839 qp_unlock(qpair);
2840
2841 if (result == VMCI_SUCCESS &&
2842 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2843 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2844 return VMCI_ERROR_INVALID_SIZE;
2845
2846 return result;
2847}
2848EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2849
2850/*
2851 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
2852 * @qpair: Pointer to the queue pair struct.
2853 * @consumer_tail: Reference used for storing consumer tail index.
2854 * @producer_head: Reference used for storing the producer head index.
2855 *
2856 * This is the client interface for getting the current indexes of the
2857 * QPair from the point of the view of the caller as the consumer.
2858 */
2859int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2860 u64 *consumer_tail,
2861 u64 *producer_head)
2862{
2863 struct vmci_queue_header *produce_q_header;
2864 struct vmci_queue_header *consume_q_header;
2865 int result;
2866
2867 if (!qpair)
2868 return VMCI_ERROR_INVALID_ARGS;
2869
2870 qp_lock(qpair);
2871 result =
2872 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2873 if (result == VMCI_SUCCESS)
2874 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2875 consumer_tail, producer_head);
2876 qp_unlock(qpair);
2877
2878 if (result == VMCI_SUCCESS &&
2879 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2880 (producer_head && *producer_head >= qpair->consume_q_size)))
2881 return VMCI_ERROR_INVALID_SIZE;
2882
2883 return result;
2884}
2885EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2886
2887/*
2888 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2889 * @qpair: Pointer to the queue pair struct.
2890 *
2891 * This is the client interface for getting the amount of free
2892 * space in the QPair from the point of the view of the caller as
2893 * the producer which is the common case. Returns < 0 if err, else
2894 * available bytes into which data can be enqueued if > 0.
2895 */
2896s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2897{
2898 struct vmci_queue_header *produce_q_header;
2899 struct vmci_queue_header *consume_q_header;
2900 s64 result;
2901
2902 if (!qpair)
2903 return VMCI_ERROR_INVALID_ARGS;
2904
2905 qp_lock(qpair);
2906 result =
2907 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2908 if (result == VMCI_SUCCESS)
2909 result = vmci_q_header_free_space(produce_q_header,
2910 consume_q_header,
2911 qpair->produce_q_size);
2912 else
2913 result = 0;
2914
2915 qp_unlock(qpair);
2916
2917 return result;
2918}
2919EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
2920
2921/*
2922 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
2923 * @qpair: Pointer to the queue pair struct.
2924 *
2925 * This is the client interface for getting the amount of free
2926 * space in the QPair from the point of the view of the caller as
2927 * the consumer which is not the common case. Returns < 0 if err, else
2928 * available bytes into which data can be enqueued if > 0.
2929 */
2930s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
2931{
2932 struct vmci_queue_header *produce_q_header;
2933 struct vmci_queue_header *consume_q_header;
2934 s64 result;
2935
2936 if (!qpair)
2937 return VMCI_ERROR_INVALID_ARGS;
2938
2939 qp_lock(qpair);
2940 result =
2941 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2942 if (result == VMCI_SUCCESS)
2943 result = vmci_q_header_free_space(consume_q_header,
2944 produce_q_header,
2945 qpair->consume_q_size);
2946 else
2947 result = 0;
2948
2949 qp_unlock(qpair);
2950
2951 return result;
2952}
2953EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
2954
2955/*
2956 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
2957 * producer queue.
2958 * @qpair: Pointer to the queue pair struct.
2959 *
2960 * This is the client interface for getting the amount of
2961 * enqueued data in the QPair from the point of the view of the
2962 * caller as the producer which is not the common case. Returns < 0 if err,
2963 * else available bytes that may be read.
2964 */
2965s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
2966{
2967 struct vmci_queue_header *produce_q_header;
2968 struct vmci_queue_header *consume_q_header;
2969 s64 result;
2970
2971 if (!qpair)
2972 return VMCI_ERROR_INVALID_ARGS;
2973
2974 qp_lock(qpair);
2975 result =
2976 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2977 if (result == VMCI_SUCCESS)
2978 result = vmci_q_header_buf_ready(produce_q_header,
2979 consume_q_header,
2980 qpair->produce_q_size);
2981 else
2982 result = 0;
2983
2984 qp_unlock(qpair);
2985
2986 return result;
2987}
2988EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
2989
2990/*
2991 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
2992 * consumer queue.
2993 * @qpair: Pointer to the queue pair struct.
2994 *
2995 * This is the client interface for getting the amount of
2996 * enqueued data in the QPair from the point of the view of the
2997 * caller as the consumer which is the normal case. Returns < 0 if err,
2998 * else available bytes that may be read.
2999 */
3000s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
3001{
3002 struct vmci_queue_header *produce_q_header;
3003 struct vmci_queue_header *consume_q_header;
3004 s64 result;
3005
3006 if (!qpair)
3007 return VMCI_ERROR_INVALID_ARGS;
3008
3009 qp_lock(qpair);
3010 result =
3011 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3012 if (result == VMCI_SUCCESS)
3013 result = vmci_q_header_buf_ready(consume_q_header,
3014 produce_q_header,
3015 qpair->consume_q_size);
3016 else
3017 result = 0;
3018
3019 qp_unlock(qpair);
3020
3021 return result;
3022}
3023EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
3024
3025/*
3026 * vmci_qpair_enqueue() - Throw data on the queue.
3027 * @qpair: Pointer to the queue pair struct.
3028 * @buf: Pointer to buffer containing data
3029 * @buf_size: Length of buffer.
3030 * @buf_type: Buffer type (Unused).
3031 *
3032 * This is the client interface for enqueueing data into the queue.
3033 * Returns number of bytes enqueued or < 0 on error.
3034 */
3035ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3036 const void *buf,
3037 size_t buf_size,
3038 int buf_type)
3039{
3040 ssize_t result;
3041 struct iov_iter from;
3042 struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
3043
3044 if (!qpair || !buf)
3045 return VMCI_ERROR_INVALID_ARGS;
3046
3047 iov_iter_kvec(&from, ITER_SOURCE, &v, 1, buf_size);
3048
3049 qp_lock(qpair);
3050
3051 do {
3052 result = qp_enqueue_locked(qpair->produce_q,
3053 qpair->consume_q,
3054 qpair->produce_q_size,
3055 &from);
3056
3057 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3058 !qp_wait_for_ready_queue(qpair))
3059 result = VMCI_ERROR_WOULD_BLOCK;
3060
3061 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3062
3063 qp_unlock(qpair);
3064
3065 return result;
3066}
3067EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3068
3069/*
3070 * vmci_qpair_dequeue() - Get data from the queue.
3071 * @qpair: Pointer to the queue pair struct.
3072 * @buf: Pointer to buffer for the data
3073 * @buf_size: Length of buffer.
3074 * @buf_type: Buffer type (Unused).
3075 *
3076 * This is the client interface for dequeueing data from the queue.
3077 * Returns number of bytes dequeued or < 0 on error.
3078 */
3079ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3080 void *buf,
3081 size_t buf_size,
3082 int buf_type)
3083{
3084 ssize_t result;
3085 struct iov_iter to;
3086 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3087
3088 if (!qpair || !buf)
3089 return VMCI_ERROR_INVALID_ARGS;
3090
3091 iov_iter_kvec(&to, ITER_DEST, &v, 1, buf_size);
3092
3093 qp_lock(qpair);
3094
3095 do {
3096 result = qp_dequeue_locked(qpair->produce_q,
3097 qpair->consume_q,
3098 qpair->consume_q_size,
3099 &to, true);
3100
3101 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3102 !qp_wait_for_ready_queue(qpair))
3103 result = VMCI_ERROR_WOULD_BLOCK;
3104
3105 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3106
3107 qp_unlock(qpair);
3108
3109 return result;
3110}
3111EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3112
3113/*
3114 * vmci_qpair_peek() - Peek at the data in the queue.
3115 * @qpair: Pointer to the queue pair struct.
3116 * @buf: Pointer to buffer for the data
3117 * @buf_size: Length of buffer.
3118 * @buf_type: Buffer type (Unused on Linux).
3119 *
3120 * This is the client interface for peeking into a queue. (I.e.,
3121 * copy data from the queue without updating the head pointer.)
3122 * Returns number of bytes dequeued or < 0 on error.
3123 */
3124ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3125 void *buf,
3126 size_t buf_size,
3127 int buf_type)
3128{
3129 struct iov_iter to;
3130 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3131 ssize_t result;
3132
3133 if (!qpair || !buf)
3134 return VMCI_ERROR_INVALID_ARGS;
3135
3136 iov_iter_kvec(&to, ITER_DEST, &v, 1, buf_size);
3137
3138 qp_lock(qpair);
3139
3140 do {
3141 result = qp_dequeue_locked(qpair->produce_q,
3142 qpair->consume_q,
3143 qpair->consume_q_size,
3144 &to, false);
3145
3146 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3147 !qp_wait_for_ready_queue(qpair))
3148 result = VMCI_ERROR_WOULD_BLOCK;
3149
3150 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3151
3152 qp_unlock(qpair);
3153
3154 return result;
3155}
3156EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3157
3158/*
3159 * vmci_qpair_enquev() - Throw data on the queue using iov.
3160 * @qpair: Pointer to the queue pair struct.
3161 * @iov: Pointer to buffer containing data
3162 * @iov_size: Length of buffer.
3163 * @buf_type: Buffer type (Unused).
3164 *
3165 * This is the client interface for enqueueing data into the queue.
3166 * This function uses IO vectors to handle the work. Returns number
3167 * of bytes enqueued or < 0 on error.
3168 */
3169ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3170 struct msghdr *msg,
3171 size_t iov_size,
3172 int buf_type)
3173{
3174 ssize_t result;
3175
3176 if (!qpair)
3177 return VMCI_ERROR_INVALID_ARGS;
3178
3179 qp_lock(qpair);
3180
3181 do {
3182 result = qp_enqueue_locked(qpair->produce_q,
3183 qpair->consume_q,
3184 qpair->produce_q_size,
3185 &msg->msg_iter);
3186
3187 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3188 !qp_wait_for_ready_queue(qpair))
3189 result = VMCI_ERROR_WOULD_BLOCK;
3190
3191 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3192
3193 qp_unlock(qpair);
3194
3195 return result;
3196}
3197EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3198
3199/*
3200 * vmci_qpair_dequev() - Get data from the queue using iov.
3201 * @qpair: Pointer to the queue pair struct.
3202 * @iov: Pointer to buffer for the data
3203 * @iov_size: Length of buffer.
3204 * @buf_type: Buffer type (Unused).
3205 *
3206 * This is the client interface for dequeueing data from the queue.
3207 * This function uses IO vectors to handle the work. Returns number
3208 * of bytes dequeued or < 0 on error.
3209 */
3210ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3211 struct msghdr *msg,
3212 size_t iov_size,
3213 int buf_type)
3214{
3215 ssize_t result;
3216
3217 if (!qpair)
3218 return VMCI_ERROR_INVALID_ARGS;
3219
3220 qp_lock(qpair);
3221
3222 do {
3223 result = qp_dequeue_locked(qpair->produce_q,
3224 qpair->consume_q,
3225 qpair->consume_q_size,
3226 &msg->msg_iter, true);
3227
3228 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3229 !qp_wait_for_ready_queue(qpair))
3230 result = VMCI_ERROR_WOULD_BLOCK;
3231
3232 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3233
3234 qp_unlock(qpair);
3235
3236 return result;
3237}
3238EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3239
3240/*
3241 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3242 * @qpair: Pointer to the queue pair struct.
3243 * @iov: Pointer to buffer for the data
3244 * @iov_size: Length of buffer.
3245 * @buf_type: Buffer type (Unused on Linux).
3246 *
3247 * This is the client interface for peeking into a queue. (I.e.,
3248 * copy data from the queue without updating the head pointer.)
3249 * This function uses IO vectors to handle the work. Returns number
3250 * of bytes peeked or < 0 on error.
3251 */
3252ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3253 struct msghdr *msg,
3254 size_t iov_size,
3255 int buf_type)
3256{
3257 ssize_t result;
3258
3259 if (!qpair)
3260 return VMCI_ERROR_INVALID_ARGS;
3261
3262 qp_lock(qpair);
3263
3264 do {
3265 result = qp_dequeue_locked(qpair->produce_q,
3266 qpair->consume_q,
3267 qpair->consume_q_size,
3268 &msg->msg_iter, false);
3269
3270 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3271 !qp_wait_for_ready_queue(qpair))
3272 result = VMCI_ERROR_WOULD_BLOCK;
3273
3274 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3275
3276 qp_unlock(qpair);
3277 return result;
3278}
3279EXPORT_SYMBOL_GPL(vmci_qpair_peekv);
1/*
2 * VMware VMCI Driver
3 *
4 * Copyright (C) 2012 VMware, Inc. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation version 2 and no later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 */
15
16#include <linux/vmw_vmci_defs.h>
17#include <linux/vmw_vmci_api.h>
18#include <linux/highmem.h>
19#include <linux/kernel.h>
20#include <linux/mm.h>
21#include <linux/module.h>
22#include <linux/mutex.h>
23#include <linux/pagemap.h>
24#include <linux/pci.h>
25#include <linux/sched.h>
26#include <linux/slab.h>
27#include <linux/uio.h>
28#include <linux/wait.h>
29#include <linux/vmalloc.h>
30#include <linux/skbuff.h>
31
32#include "vmci_handle_array.h"
33#include "vmci_queue_pair.h"
34#include "vmci_datagram.h"
35#include "vmci_resource.h"
36#include "vmci_context.h"
37#include "vmci_driver.h"
38#include "vmci_event.h"
39#include "vmci_route.h"
40
41/*
42 * In the following, we will distinguish between two kinds of VMX processes -
43 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
44 * VMCI page files in the VMX and supporting VM to VM communication and the
45 * newer ones that use the guest memory directly. We will in the following
46 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
47 * new-style VMX'en.
48 *
49 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
50 * removed for readability) - see below for more details on the transtions:
51 *
52 * -------------- NEW -------------
53 * | |
54 * \_/ \_/
55 * CREATED_NO_MEM <-----------------> CREATED_MEM
56 * | | |
57 * | o-----------------------o |
58 * | | |
59 * \_/ \_/ \_/
60 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
61 * | | |
62 * | o----------------------o |
63 * | | |
64 * \_/ \_/ \_/
65 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
66 * | |
67 * | |
68 * -------------> gone <-------------
69 *
70 * In more detail. When a VMCI queue pair is first created, it will be in the
71 * VMCIQPB_NEW state. It will then move into one of the following states:
72 *
73 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
74 *
75 * - the created was performed by a host endpoint, in which case there is
76 * no backing memory yet.
77 *
78 * - the create was initiated by an old-style VMX, that uses
79 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
80 * a later point in time. This state can be distinguished from the one
81 * above by the context ID of the creator. A host side is not allowed to
82 * attach until the page store has been set.
83 *
84 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
85 * is created by a VMX using the queue pair device backend that
86 * sets the UVAs of the queue pair immediately and stores the
87 * information for later attachers. At this point, it is ready for
88 * the host side to attach to it.
89 *
90 * Once the queue pair is in one of the created states (with the exception of
91 * the case mentioned for older VMX'en above), it is possible to attach to the
92 * queue pair. Again we have two new states possible:
93 *
94 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
95 * paths:
96 *
97 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
98 * pair, and attaches to a queue pair previously created by the host side.
99 *
100 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
101 * already created by a guest.
102 *
103 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
104 * vmci_qp_broker_set_page_store (see below).
105 *
106 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
107 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
108 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
109 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
110 * will be entered.
111 *
112 * From the attached queue pair, the queue pair can enter the shutdown states
113 * when either side of the queue pair detaches. If the guest side detaches
114 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
115 * the content of the queue pair will no longer be available. If the host
116 * side detaches first, the queue pair will either enter the
117 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
118 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
119 * (e.g., the host detaches while a guest is stunned).
120 *
121 * New-style VMX'en will also unmap guest memory, if the guest is
122 * quiesced, e.g., during a snapshot operation. In that case, the guest
123 * memory will no longer be available, and the queue pair will transition from
124 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
125 * in which case the queue pair will transition from the *_NO_MEM state at that
126 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
127 * since the peer may have either attached or detached in the meantime. The
128 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
129 * *_MEM state, and vice versa.
130 */
131
132/*
133 * VMCIMemcpy{To,From}QueueFunc() prototypes. Functions of these
134 * types are passed around to enqueue and dequeue routines. Note that
135 * often the functions passed are simply wrappers around memcpy
136 * itself.
137 *
138 * Note: In order for the memcpy typedefs to be compatible with the VMKernel,
139 * there's an unused last parameter for the hosted side. In
140 * ESX, that parameter holds a buffer type.
141 */
142typedef int vmci_memcpy_to_queue_func(struct vmci_queue *queue,
143 u64 queue_offset, const void *src,
144 size_t src_offset, size_t size);
145typedef int vmci_memcpy_from_queue_func(void *dest, size_t dest_offset,
146 const struct vmci_queue *queue,
147 u64 queue_offset, size_t size);
148
149/* The Kernel specific component of the struct vmci_queue structure. */
150struct vmci_queue_kern_if {
151 struct mutex __mutex; /* Protects the queue. */
152 struct mutex *mutex; /* Shared by producer and consumer queues. */
153 size_t num_pages; /* Number of pages incl. header. */
154 bool host; /* Host or guest? */
155 union {
156 struct {
157 dma_addr_t *pas;
158 void **vas;
159 } g; /* Used by the guest. */
160 struct {
161 struct page **page;
162 struct page **header_page;
163 } h; /* Used by the host. */
164 } u;
165};
166
167/*
168 * This structure is opaque to the clients.
169 */
170struct vmci_qp {
171 struct vmci_handle handle;
172 struct vmci_queue *produce_q;
173 struct vmci_queue *consume_q;
174 u64 produce_q_size;
175 u64 consume_q_size;
176 u32 peer;
177 u32 flags;
178 u32 priv_flags;
179 bool guest_endpoint;
180 unsigned int blocked;
181 unsigned int generation;
182 wait_queue_head_t event;
183};
184
185enum qp_broker_state {
186 VMCIQPB_NEW,
187 VMCIQPB_CREATED_NO_MEM,
188 VMCIQPB_CREATED_MEM,
189 VMCIQPB_ATTACHED_NO_MEM,
190 VMCIQPB_ATTACHED_MEM,
191 VMCIQPB_SHUTDOWN_NO_MEM,
192 VMCIQPB_SHUTDOWN_MEM,
193 VMCIQPB_GONE
194};
195
196#define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
197 _qpb->state == VMCIQPB_ATTACHED_MEM || \
198 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
199
200/*
201 * In the queue pair broker, we always use the guest point of view for
202 * the produce and consume queue values and references, e.g., the
203 * produce queue size stored is the guests produce queue size. The
204 * host endpoint will need to swap these around. The only exception is
205 * the local queue pairs on the host, in which case the host endpoint
206 * that creates the queue pair will have the right orientation, and
207 * the attaching host endpoint will need to swap.
208 */
209struct qp_entry {
210 struct list_head list_item;
211 struct vmci_handle handle;
212 u32 peer;
213 u32 flags;
214 u64 produce_size;
215 u64 consume_size;
216 u32 ref_count;
217};
218
219struct qp_broker_entry {
220 struct vmci_resource resource;
221 struct qp_entry qp;
222 u32 create_id;
223 u32 attach_id;
224 enum qp_broker_state state;
225 bool require_trusted_attach;
226 bool created_by_trusted;
227 bool vmci_page_files; /* Created by VMX using VMCI page files */
228 struct vmci_queue *produce_q;
229 struct vmci_queue *consume_q;
230 struct vmci_queue_header saved_produce_q;
231 struct vmci_queue_header saved_consume_q;
232 vmci_event_release_cb wakeup_cb;
233 void *client_data;
234 void *local_mem; /* Kernel memory for local queue pair */
235};
236
237struct qp_guest_endpoint {
238 struct vmci_resource resource;
239 struct qp_entry qp;
240 u64 num_ppns;
241 void *produce_q;
242 void *consume_q;
243 struct ppn_set ppn_set;
244};
245
246struct qp_list {
247 struct list_head head;
248 struct mutex mutex; /* Protect queue list. */
249};
250
251static struct qp_list qp_broker_list = {
252 .head = LIST_HEAD_INIT(qp_broker_list.head),
253 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
254};
255
256static struct qp_list qp_guest_endpoints = {
257 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
258 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
259};
260
261#define INVALID_VMCI_GUEST_MEM_ID 0
262#define QPE_NUM_PAGES(_QPE) ((u32) \
263 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
264 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
265
266
267/*
268 * Frees kernel VA space for a given queue and its queue header, and
269 * frees physical data pages.
270 */
271static void qp_free_queue(void *q, u64 size)
272{
273 struct vmci_queue *queue = q;
274
275 if (queue) {
276 u64 i;
277
278 /* Given size does not include header, so add in a page here. */
279 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
280 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
281 queue->kernel_if->u.g.vas[i],
282 queue->kernel_if->u.g.pas[i]);
283 }
284
285 vfree(queue);
286 }
287}
288
289/*
290 * Allocates kernel queue pages of specified size with IOMMU mappings,
291 * plus space for the queue structure/kernel interface and the queue
292 * header.
293 */
294static void *qp_alloc_queue(u64 size, u32 flags)
295{
296 u64 i;
297 struct vmci_queue *queue;
298 size_t pas_size;
299 size_t vas_size;
300 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
301 const u64 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
302
303 if (num_pages >
304 (SIZE_MAX - queue_size) /
305 (sizeof(*queue->kernel_if->u.g.pas) +
306 sizeof(*queue->kernel_if->u.g.vas)))
307 return NULL;
308
309 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
310 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
311 queue_size += pas_size + vas_size;
312
313 queue = vmalloc(queue_size);
314 if (!queue)
315 return NULL;
316
317 queue->q_header = NULL;
318 queue->saved_header = NULL;
319 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
320 queue->kernel_if->mutex = NULL;
321 queue->kernel_if->num_pages = num_pages;
322 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
323 queue->kernel_if->u.g.vas =
324 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
325 queue->kernel_if->host = false;
326
327 for (i = 0; i < num_pages; i++) {
328 queue->kernel_if->u.g.vas[i] =
329 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
330 &queue->kernel_if->u.g.pas[i],
331 GFP_KERNEL);
332 if (!queue->kernel_if->u.g.vas[i]) {
333 /* Size excl. the header. */
334 qp_free_queue(queue, i * PAGE_SIZE);
335 return NULL;
336 }
337 }
338
339 /* Queue header is the first page. */
340 queue->q_header = queue->kernel_if->u.g.vas[0];
341
342 return queue;
343}
344
345/*
346 * Copies from a given buffer or iovector to a VMCI Queue. Uses
347 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
348 * by traversing the offset -> page translation structure for the queue.
349 * Assumes that offset + size does not wrap around in the queue.
350 */
351static int __qp_memcpy_to_queue(struct vmci_queue *queue,
352 u64 queue_offset,
353 const void *src,
354 size_t size,
355 bool is_iovec)
356{
357 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
358 size_t bytes_copied = 0;
359
360 while (bytes_copied < size) {
361 const u64 page_index =
362 (queue_offset + bytes_copied) / PAGE_SIZE;
363 const size_t page_offset =
364 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
365 void *va;
366 size_t to_copy;
367
368 if (kernel_if->host)
369 va = kmap(kernel_if->u.h.page[page_index]);
370 else
371 va = kernel_if->u.g.vas[page_index + 1];
372 /* Skip header. */
373
374 if (size - bytes_copied > PAGE_SIZE - page_offset)
375 /* Enough payload to fill up from this page. */
376 to_copy = PAGE_SIZE - page_offset;
377 else
378 to_copy = size - bytes_copied;
379
380 if (is_iovec) {
381 struct msghdr *msg = (struct msghdr *)src;
382 int err;
383
384 /* The iovec will track bytes_copied internally. */
385 err = memcpy_from_msg((u8 *)va + page_offset,
386 msg, to_copy);
387 if (err != 0) {
388 if (kernel_if->host)
389 kunmap(kernel_if->u.h.page[page_index]);
390 return VMCI_ERROR_INVALID_ARGS;
391 }
392 } else {
393 memcpy((u8 *)va + page_offset,
394 (u8 *)src + bytes_copied, to_copy);
395 }
396
397 bytes_copied += to_copy;
398 if (kernel_if->host)
399 kunmap(kernel_if->u.h.page[page_index]);
400 }
401
402 return VMCI_SUCCESS;
403}
404
405/*
406 * Copies to a given buffer or iovector from a VMCI Queue. Uses
407 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
408 * by traversing the offset -> page translation structure for the queue.
409 * Assumes that offset + size does not wrap around in the queue.
410 */
411static int __qp_memcpy_from_queue(void *dest,
412 const struct vmci_queue *queue,
413 u64 queue_offset,
414 size_t size,
415 bool is_iovec)
416{
417 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
418 size_t bytes_copied = 0;
419
420 while (bytes_copied < size) {
421 const u64 page_index =
422 (queue_offset + bytes_copied) / PAGE_SIZE;
423 const size_t page_offset =
424 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
425 void *va;
426 size_t to_copy;
427
428 if (kernel_if->host)
429 va = kmap(kernel_if->u.h.page[page_index]);
430 else
431 va = kernel_if->u.g.vas[page_index + 1];
432 /* Skip header. */
433
434 if (size - bytes_copied > PAGE_SIZE - page_offset)
435 /* Enough payload to fill up this page. */
436 to_copy = PAGE_SIZE - page_offset;
437 else
438 to_copy = size - bytes_copied;
439
440 if (is_iovec) {
441 struct msghdr *msg = dest;
442 int err;
443
444 /* The iovec will track bytes_copied internally. */
445 err = memcpy_to_msg(msg, (u8 *)va + page_offset,
446 to_copy);
447 if (err != 0) {
448 if (kernel_if->host)
449 kunmap(kernel_if->u.h.page[page_index]);
450 return VMCI_ERROR_INVALID_ARGS;
451 }
452 } else {
453 memcpy((u8 *)dest + bytes_copied,
454 (u8 *)va + page_offset, to_copy);
455 }
456
457 bytes_copied += to_copy;
458 if (kernel_if->host)
459 kunmap(kernel_if->u.h.page[page_index]);
460 }
461
462 return VMCI_SUCCESS;
463}
464
465/*
466 * Allocates two list of PPNs --- one for the pages in the produce queue,
467 * and the other for the pages in the consume queue. Intializes the list
468 * of PPNs with the page frame numbers of the KVA for the two queues (and
469 * the queue headers).
470 */
471static int qp_alloc_ppn_set(void *prod_q,
472 u64 num_produce_pages,
473 void *cons_q,
474 u64 num_consume_pages, struct ppn_set *ppn_set)
475{
476 u32 *produce_ppns;
477 u32 *consume_ppns;
478 struct vmci_queue *produce_q = prod_q;
479 struct vmci_queue *consume_q = cons_q;
480 u64 i;
481
482 if (!produce_q || !num_produce_pages || !consume_q ||
483 !num_consume_pages || !ppn_set)
484 return VMCI_ERROR_INVALID_ARGS;
485
486 if (ppn_set->initialized)
487 return VMCI_ERROR_ALREADY_EXISTS;
488
489 produce_ppns =
490 kmalloc(num_produce_pages * sizeof(*produce_ppns), GFP_KERNEL);
491 if (!produce_ppns)
492 return VMCI_ERROR_NO_MEM;
493
494 consume_ppns =
495 kmalloc(num_consume_pages * sizeof(*consume_ppns), GFP_KERNEL);
496 if (!consume_ppns) {
497 kfree(produce_ppns);
498 return VMCI_ERROR_NO_MEM;
499 }
500
501 for (i = 0; i < num_produce_pages; i++) {
502 unsigned long pfn;
503
504 produce_ppns[i] =
505 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
506 pfn = produce_ppns[i];
507
508 /* Fail allocation if PFN isn't supported by hypervisor. */
509 if (sizeof(pfn) > sizeof(*produce_ppns)
510 && pfn != produce_ppns[i])
511 goto ppn_error;
512 }
513
514 for (i = 0; i < num_consume_pages; i++) {
515 unsigned long pfn;
516
517 consume_ppns[i] =
518 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
519 pfn = consume_ppns[i];
520
521 /* Fail allocation if PFN isn't supported by hypervisor. */
522 if (sizeof(pfn) > sizeof(*consume_ppns)
523 && pfn != consume_ppns[i])
524 goto ppn_error;
525 }
526
527 ppn_set->num_produce_pages = num_produce_pages;
528 ppn_set->num_consume_pages = num_consume_pages;
529 ppn_set->produce_ppns = produce_ppns;
530 ppn_set->consume_ppns = consume_ppns;
531 ppn_set->initialized = true;
532 return VMCI_SUCCESS;
533
534 ppn_error:
535 kfree(produce_ppns);
536 kfree(consume_ppns);
537 return VMCI_ERROR_INVALID_ARGS;
538}
539
540/*
541 * Frees the two list of PPNs for a queue pair.
542 */
543static void qp_free_ppn_set(struct ppn_set *ppn_set)
544{
545 if (ppn_set->initialized) {
546 /* Do not call these functions on NULL inputs. */
547 kfree(ppn_set->produce_ppns);
548 kfree(ppn_set->consume_ppns);
549 }
550 memset(ppn_set, 0, sizeof(*ppn_set));
551}
552
553/*
554 * Populates the list of PPNs in the hypercall structure with the PPNS
555 * of the produce queue and the consume queue.
556 */
557static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
558{
559 memcpy(call_buf, ppn_set->produce_ppns,
560 ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns));
561 memcpy(call_buf +
562 ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns),
563 ppn_set->consume_ppns,
564 ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns));
565
566 return VMCI_SUCCESS;
567}
568
569static int qp_memcpy_to_queue(struct vmci_queue *queue,
570 u64 queue_offset,
571 const void *src, size_t src_offset, size_t size)
572{
573 return __qp_memcpy_to_queue(queue, queue_offset,
574 (u8 *)src + src_offset, size, false);
575}
576
577static int qp_memcpy_from_queue(void *dest,
578 size_t dest_offset,
579 const struct vmci_queue *queue,
580 u64 queue_offset, size_t size)
581{
582 return __qp_memcpy_from_queue((u8 *)dest + dest_offset,
583 queue, queue_offset, size, false);
584}
585
586/*
587 * Copies from a given iovec from a VMCI Queue.
588 */
589static int qp_memcpy_to_queue_iov(struct vmci_queue *queue,
590 u64 queue_offset,
591 const void *msg,
592 size_t src_offset, size_t size)
593{
594
595 /*
596 * We ignore src_offset because src is really a struct iovec * and will
597 * maintain offset internally.
598 */
599 return __qp_memcpy_to_queue(queue, queue_offset, msg, size, true);
600}
601
602/*
603 * Copies to a given iovec from a VMCI Queue.
604 */
605static int qp_memcpy_from_queue_iov(void *dest,
606 size_t dest_offset,
607 const struct vmci_queue *queue,
608 u64 queue_offset, size_t size)
609{
610 /*
611 * We ignore dest_offset because dest is really a struct iovec * and
612 * will maintain offset internally.
613 */
614 return __qp_memcpy_from_queue(dest, queue, queue_offset, size, true);
615}
616
617/*
618 * Allocates kernel VA space of specified size plus space for the queue
619 * and kernel interface. This is different from the guest queue allocator,
620 * because we do not allocate our own queue header/data pages here but
621 * share those of the guest.
622 */
623static struct vmci_queue *qp_host_alloc_queue(u64 size)
624{
625 struct vmci_queue *queue;
626 size_t queue_page_size;
627 const u64 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
628 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
629
630 if (num_pages > (SIZE_MAX - queue_size) /
631 sizeof(*queue->kernel_if->u.h.page))
632 return NULL;
633
634 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
635
636 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
637 if (queue) {
638 queue->q_header = NULL;
639 queue->saved_header = NULL;
640 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
641 queue->kernel_if->host = true;
642 queue->kernel_if->mutex = NULL;
643 queue->kernel_if->num_pages = num_pages;
644 queue->kernel_if->u.h.header_page =
645 (struct page **)((u8 *)queue + queue_size);
646 queue->kernel_if->u.h.page =
647 &queue->kernel_if->u.h.header_page[1];
648 }
649
650 return queue;
651}
652
653/*
654 * Frees kernel memory for a given queue (header plus translation
655 * structure).
656 */
657static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
658{
659 kfree(queue);
660}
661
662/*
663 * Initialize the mutex for the pair of queues. This mutex is used to
664 * protect the q_header and the buffer from changing out from under any
665 * users of either queue. Of course, it's only any good if the mutexes
666 * are actually acquired. Queue structure must lie on non-paged memory
667 * or we cannot guarantee access to the mutex.
668 */
669static void qp_init_queue_mutex(struct vmci_queue *produce_q,
670 struct vmci_queue *consume_q)
671{
672 /*
673 * Only the host queue has shared state - the guest queues do not
674 * need to synchronize access using a queue mutex.
675 */
676
677 if (produce_q->kernel_if->host) {
678 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
679 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
680 mutex_init(produce_q->kernel_if->mutex);
681 }
682}
683
684/*
685 * Cleans up the mutex for the pair of queues.
686 */
687static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
688 struct vmci_queue *consume_q)
689{
690 if (produce_q->kernel_if->host) {
691 produce_q->kernel_if->mutex = NULL;
692 consume_q->kernel_if->mutex = NULL;
693 }
694}
695
696/*
697 * Acquire the mutex for the queue. Note that the produce_q and
698 * the consume_q share a mutex. So, only one of the two need to
699 * be passed in to this routine. Either will work just fine.
700 */
701static void qp_acquire_queue_mutex(struct vmci_queue *queue)
702{
703 if (queue->kernel_if->host)
704 mutex_lock(queue->kernel_if->mutex);
705}
706
707/*
708 * Release the mutex for the queue. Note that the produce_q and
709 * the consume_q share a mutex. So, only one of the two need to
710 * be passed in to this routine. Either will work just fine.
711 */
712static void qp_release_queue_mutex(struct vmci_queue *queue)
713{
714 if (queue->kernel_if->host)
715 mutex_unlock(queue->kernel_if->mutex);
716}
717
718/*
719 * Helper function to release pages in the PageStoreAttachInfo
720 * previously obtained using get_user_pages.
721 */
722static void qp_release_pages(struct page **pages,
723 u64 num_pages, bool dirty)
724{
725 int i;
726
727 for (i = 0; i < num_pages; i++) {
728 if (dirty)
729 set_page_dirty(pages[i]);
730
731 put_page(pages[i]);
732 pages[i] = NULL;
733 }
734}
735
736/*
737 * Lock the user pages referenced by the {produce,consume}Buffer
738 * struct into memory and populate the {produce,consume}Pages
739 * arrays in the attach structure with them.
740 */
741static int qp_host_get_user_memory(u64 produce_uva,
742 u64 consume_uva,
743 struct vmci_queue *produce_q,
744 struct vmci_queue *consume_q)
745{
746 int retval;
747 int err = VMCI_SUCCESS;
748
749 retval = get_user_pages_fast((uintptr_t) produce_uva,
750 produce_q->kernel_if->num_pages, 1,
751 produce_q->kernel_if->u.h.header_page);
752 if (retval < produce_q->kernel_if->num_pages) {
753 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
754 retval);
755 qp_release_pages(produce_q->kernel_if->u.h.header_page,
756 retval, false);
757 err = VMCI_ERROR_NO_MEM;
758 goto out;
759 }
760
761 retval = get_user_pages_fast((uintptr_t) consume_uva,
762 consume_q->kernel_if->num_pages, 1,
763 consume_q->kernel_if->u.h.header_page);
764 if (retval < consume_q->kernel_if->num_pages) {
765 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
766 retval);
767 qp_release_pages(consume_q->kernel_if->u.h.header_page,
768 retval, false);
769 qp_release_pages(produce_q->kernel_if->u.h.header_page,
770 produce_q->kernel_if->num_pages, false);
771 err = VMCI_ERROR_NO_MEM;
772 }
773
774 out:
775 return err;
776}
777
778/*
779 * Registers the specification of the user pages used for backing a queue
780 * pair. Enough information to map in pages is stored in the OS specific
781 * part of the struct vmci_queue structure.
782 */
783static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
784 struct vmci_queue *produce_q,
785 struct vmci_queue *consume_q)
786{
787 u64 produce_uva;
788 u64 consume_uva;
789
790 /*
791 * The new style and the old style mapping only differs in
792 * that we either get a single or two UVAs, so we split the
793 * single UVA range at the appropriate spot.
794 */
795 produce_uva = page_store->pages;
796 consume_uva = page_store->pages +
797 produce_q->kernel_if->num_pages * PAGE_SIZE;
798 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
799 consume_q);
800}
801
802/*
803 * Releases and removes the references to user pages stored in the attach
804 * struct. Pages are released from the page cache and may become
805 * swappable again.
806 */
807static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
808 struct vmci_queue *consume_q)
809{
810 qp_release_pages(produce_q->kernel_if->u.h.header_page,
811 produce_q->kernel_if->num_pages, true);
812 memset(produce_q->kernel_if->u.h.header_page, 0,
813 sizeof(*produce_q->kernel_if->u.h.header_page) *
814 produce_q->kernel_if->num_pages);
815 qp_release_pages(consume_q->kernel_if->u.h.header_page,
816 consume_q->kernel_if->num_pages, true);
817 memset(consume_q->kernel_if->u.h.header_page, 0,
818 sizeof(*consume_q->kernel_if->u.h.header_page) *
819 consume_q->kernel_if->num_pages);
820}
821
822/*
823 * Once qp_host_register_user_memory has been performed on a
824 * queue, the queue pair headers can be mapped into the
825 * kernel. Once mapped, they must be unmapped with
826 * qp_host_unmap_queues prior to calling
827 * qp_host_unregister_user_memory.
828 * Pages are pinned.
829 */
830static int qp_host_map_queues(struct vmci_queue *produce_q,
831 struct vmci_queue *consume_q)
832{
833 int result;
834
835 if (!produce_q->q_header || !consume_q->q_header) {
836 struct page *headers[2];
837
838 if (produce_q->q_header != consume_q->q_header)
839 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
840
841 if (produce_q->kernel_if->u.h.header_page == NULL ||
842 *produce_q->kernel_if->u.h.header_page == NULL)
843 return VMCI_ERROR_UNAVAILABLE;
844
845 headers[0] = *produce_q->kernel_if->u.h.header_page;
846 headers[1] = *consume_q->kernel_if->u.h.header_page;
847
848 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
849 if (produce_q->q_header != NULL) {
850 consume_q->q_header =
851 (struct vmci_queue_header *)((u8 *)
852 produce_q->q_header +
853 PAGE_SIZE);
854 result = VMCI_SUCCESS;
855 } else {
856 pr_warn("vmap failed\n");
857 result = VMCI_ERROR_NO_MEM;
858 }
859 } else {
860 result = VMCI_SUCCESS;
861 }
862
863 return result;
864}
865
866/*
867 * Unmaps previously mapped queue pair headers from the kernel.
868 * Pages are unpinned.
869 */
870static int qp_host_unmap_queues(u32 gid,
871 struct vmci_queue *produce_q,
872 struct vmci_queue *consume_q)
873{
874 if (produce_q->q_header) {
875 if (produce_q->q_header < consume_q->q_header)
876 vunmap(produce_q->q_header);
877 else
878 vunmap(consume_q->q_header);
879
880 produce_q->q_header = NULL;
881 consume_q->q_header = NULL;
882 }
883
884 return VMCI_SUCCESS;
885}
886
887/*
888 * Finds the entry in the list corresponding to a given handle. Assumes
889 * that the list is locked.
890 */
891static struct qp_entry *qp_list_find(struct qp_list *qp_list,
892 struct vmci_handle handle)
893{
894 struct qp_entry *entry;
895
896 if (vmci_handle_is_invalid(handle))
897 return NULL;
898
899 list_for_each_entry(entry, &qp_list->head, list_item) {
900 if (vmci_handle_is_equal(entry->handle, handle))
901 return entry;
902 }
903
904 return NULL;
905}
906
907/*
908 * Finds the entry in the list corresponding to a given handle.
909 */
910static struct qp_guest_endpoint *
911qp_guest_handle_to_entry(struct vmci_handle handle)
912{
913 struct qp_guest_endpoint *entry;
914 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
915
916 entry = qp ? container_of(
917 qp, struct qp_guest_endpoint, qp) : NULL;
918 return entry;
919}
920
921/*
922 * Finds the entry in the list corresponding to a given handle.
923 */
924static struct qp_broker_entry *
925qp_broker_handle_to_entry(struct vmci_handle handle)
926{
927 struct qp_broker_entry *entry;
928 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
929
930 entry = qp ? container_of(
931 qp, struct qp_broker_entry, qp) : NULL;
932 return entry;
933}
934
935/*
936 * Dispatches a queue pair event message directly into the local event
937 * queue.
938 */
939static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
940{
941 u32 context_id = vmci_get_context_id();
942 struct vmci_event_qp ev;
943
944 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
945 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
946 VMCI_CONTEXT_RESOURCE_ID);
947 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
948 ev.msg.event_data.event =
949 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
950 ev.payload.peer_id = context_id;
951 ev.payload.handle = handle;
952
953 return vmci_event_dispatch(&ev.msg.hdr);
954}
955
956/*
957 * Allocates and initializes a qp_guest_endpoint structure.
958 * Allocates a queue_pair rid (and handle) iff the given entry has
959 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
960 * are reserved handles. Assumes that the QP list mutex is held
961 * by the caller.
962 */
963static struct qp_guest_endpoint *
964qp_guest_endpoint_create(struct vmci_handle handle,
965 u32 peer,
966 u32 flags,
967 u64 produce_size,
968 u64 consume_size,
969 void *produce_q,
970 void *consume_q)
971{
972 int result;
973 struct qp_guest_endpoint *entry;
974 /* One page each for the queue headers. */
975 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
976 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
977
978 if (vmci_handle_is_invalid(handle)) {
979 u32 context_id = vmci_get_context_id();
980
981 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
982 }
983
984 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
985 if (entry) {
986 entry->qp.peer = peer;
987 entry->qp.flags = flags;
988 entry->qp.produce_size = produce_size;
989 entry->qp.consume_size = consume_size;
990 entry->qp.ref_count = 0;
991 entry->num_ppns = num_ppns;
992 entry->produce_q = produce_q;
993 entry->consume_q = consume_q;
994 INIT_LIST_HEAD(&entry->qp.list_item);
995
996 /* Add resource obj */
997 result = vmci_resource_add(&entry->resource,
998 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
999 handle);
1000 entry->qp.handle = vmci_resource_handle(&entry->resource);
1001 if ((result != VMCI_SUCCESS) ||
1002 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
1003 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1004 handle.context, handle.resource, result);
1005 kfree(entry);
1006 entry = NULL;
1007 }
1008 }
1009 return entry;
1010}
1011
1012/*
1013 * Frees a qp_guest_endpoint structure.
1014 */
1015static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
1016{
1017 qp_free_ppn_set(&entry->ppn_set);
1018 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
1019 qp_free_queue(entry->produce_q, entry->qp.produce_size);
1020 qp_free_queue(entry->consume_q, entry->qp.consume_size);
1021 /* Unlink from resource hash table and free callback */
1022 vmci_resource_remove(&entry->resource);
1023
1024 kfree(entry);
1025}
1026
1027/*
1028 * Helper to make a queue_pairAlloc hypercall when the driver is
1029 * supporting a guest device.
1030 */
1031static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
1032{
1033 struct vmci_qp_alloc_msg *alloc_msg;
1034 size_t msg_size;
1035 int result;
1036
1037 if (!entry || entry->num_ppns <= 2)
1038 return VMCI_ERROR_INVALID_ARGS;
1039
1040 msg_size = sizeof(*alloc_msg) +
1041 (size_t) entry->num_ppns * sizeof(u32);
1042 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
1043 if (!alloc_msg)
1044 return VMCI_ERROR_NO_MEM;
1045
1046 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1047 VMCI_QUEUEPAIR_ALLOC);
1048 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
1049 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
1050 alloc_msg->handle = entry->qp.handle;
1051 alloc_msg->peer = entry->qp.peer;
1052 alloc_msg->flags = entry->qp.flags;
1053 alloc_msg->produce_size = entry->qp.produce_size;
1054 alloc_msg->consume_size = entry->qp.consume_size;
1055 alloc_msg->num_ppns = entry->num_ppns;
1056
1057 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
1058 &entry->ppn_set);
1059 if (result == VMCI_SUCCESS)
1060 result = vmci_send_datagram(&alloc_msg->hdr);
1061
1062 kfree(alloc_msg);
1063
1064 return result;
1065}
1066
1067/*
1068 * Helper to make a queue_pairDetach hypercall when the driver is
1069 * supporting a guest device.
1070 */
1071static int qp_detatch_hypercall(struct vmci_handle handle)
1072{
1073 struct vmci_qp_detach_msg detach_msg;
1074
1075 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1076 VMCI_QUEUEPAIR_DETACH);
1077 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
1078 detach_msg.hdr.payload_size = sizeof(handle);
1079 detach_msg.handle = handle;
1080
1081 return vmci_send_datagram(&detach_msg.hdr);
1082}
1083
1084/*
1085 * Adds the given entry to the list. Assumes that the list is locked.
1086 */
1087static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
1088{
1089 if (entry)
1090 list_add(&entry->list_item, &qp_list->head);
1091}
1092
1093/*
1094 * Removes the given entry from the list. Assumes that the list is locked.
1095 */
1096static void qp_list_remove_entry(struct qp_list *qp_list,
1097 struct qp_entry *entry)
1098{
1099 if (entry)
1100 list_del(&entry->list_item);
1101}
1102
1103/*
1104 * Helper for VMCI queue_pair detach interface. Frees the physical
1105 * pages for the queue pair.
1106 */
1107static int qp_detatch_guest_work(struct vmci_handle handle)
1108{
1109 int result;
1110 struct qp_guest_endpoint *entry;
1111 u32 ref_count = ~0; /* To avoid compiler warning below */
1112
1113 mutex_lock(&qp_guest_endpoints.mutex);
1114
1115 entry = qp_guest_handle_to_entry(handle);
1116 if (!entry) {
1117 mutex_unlock(&qp_guest_endpoints.mutex);
1118 return VMCI_ERROR_NOT_FOUND;
1119 }
1120
1121 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1122 result = VMCI_SUCCESS;
1123
1124 if (entry->qp.ref_count > 1) {
1125 result = qp_notify_peer_local(false, handle);
1126 /*
1127 * We can fail to notify a local queuepair
1128 * because we can't allocate. We still want
1129 * to release the entry if that happens, so
1130 * don't bail out yet.
1131 */
1132 }
1133 } else {
1134 result = qp_detatch_hypercall(handle);
1135 if (result < VMCI_SUCCESS) {
1136 /*
1137 * We failed to notify a non-local queuepair.
1138 * That other queuepair might still be
1139 * accessing the shared memory, so don't
1140 * release the entry yet. It will get cleaned
1141 * up by VMCIqueue_pair_Exit() if necessary
1142 * (assuming we are going away, otherwise why
1143 * did this fail?).
1144 */
1145
1146 mutex_unlock(&qp_guest_endpoints.mutex);
1147 return result;
1148 }
1149 }
1150
1151 /*
1152 * If we get here then we either failed to notify a local queuepair, or
1153 * we succeeded in all cases. Release the entry if required.
1154 */
1155
1156 entry->qp.ref_count--;
1157 if (entry->qp.ref_count == 0)
1158 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1159
1160 /* If we didn't remove the entry, this could change once we unlock. */
1161 if (entry)
1162 ref_count = entry->qp.ref_count;
1163
1164 mutex_unlock(&qp_guest_endpoints.mutex);
1165
1166 if (ref_count == 0)
1167 qp_guest_endpoint_destroy(entry);
1168
1169 return result;
1170}
1171
1172/*
1173 * This functions handles the actual allocation of a VMCI queue
1174 * pair guest endpoint. Allocates physical pages for the queue
1175 * pair. It makes OS dependent calls through generic wrappers.
1176 */
1177static int qp_alloc_guest_work(struct vmci_handle *handle,
1178 struct vmci_queue **produce_q,
1179 u64 produce_size,
1180 struct vmci_queue **consume_q,
1181 u64 consume_size,
1182 u32 peer,
1183 u32 flags,
1184 u32 priv_flags)
1185{
1186 const u64 num_produce_pages =
1187 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1188 const u64 num_consume_pages =
1189 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1190 void *my_produce_q = NULL;
1191 void *my_consume_q = NULL;
1192 int result;
1193 struct qp_guest_endpoint *queue_pair_entry = NULL;
1194
1195 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1196 return VMCI_ERROR_NO_ACCESS;
1197
1198 mutex_lock(&qp_guest_endpoints.mutex);
1199
1200 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1201 if (queue_pair_entry) {
1202 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1203 /* Local attach case. */
1204 if (queue_pair_entry->qp.ref_count > 1) {
1205 pr_devel("Error attempting to attach more than once\n");
1206 result = VMCI_ERROR_UNAVAILABLE;
1207 goto error_keep_entry;
1208 }
1209
1210 if (queue_pair_entry->qp.produce_size != consume_size ||
1211 queue_pair_entry->qp.consume_size !=
1212 produce_size ||
1213 queue_pair_entry->qp.flags !=
1214 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1215 pr_devel("Error mismatched queue pair in local attach\n");
1216 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1217 goto error_keep_entry;
1218 }
1219
1220 /*
1221 * Do a local attach. We swap the consume and
1222 * produce queues for the attacher and deliver
1223 * an attach event.
1224 */
1225 result = qp_notify_peer_local(true, *handle);
1226 if (result < VMCI_SUCCESS)
1227 goto error_keep_entry;
1228
1229 my_produce_q = queue_pair_entry->consume_q;
1230 my_consume_q = queue_pair_entry->produce_q;
1231 goto out;
1232 }
1233
1234 result = VMCI_ERROR_ALREADY_EXISTS;
1235 goto error_keep_entry;
1236 }
1237
1238 my_produce_q = qp_alloc_queue(produce_size, flags);
1239 if (!my_produce_q) {
1240 pr_warn("Error allocating pages for produce queue\n");
1241 result = VMCI_ERROR_NO_MEM;
1242 goto error;
1243 }
1244
1245 my_consume_q = qp_alloc_queue(consume_size, flags);
1246 if (!my_consume_q) {
1247 pr_warn("Error allocating pages for consume queue\n");
1248 result = VMCI_ERROR_NO_MEM;
1249 goto error;
1250 }
1251
1252 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1253 produce_size, consume_size,
1254 my_produce_q, my_consume_q);
1255 if (!queue_pair_entry) {
1256 pr_warn("Error allocating memory in %s\n", __func__);
1257 result = VMCI_ERROR_NO_MEM;
1258 goto error;
1259 }
1260
1261 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1262 num_consume_pages,
1263 &queue_pair_entry->ppn_set);
1264 if (result < VMCI_SUCCESS) {
1265 pr_warn("qp_alloc_ppn_set failed\n");
1266 goto error;
1267 }
1268
1269 /*
1270 * It's only necessary to notify the host if this queue pair will be
1271 * attached to from another context.
1272 */
1273 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1274 /* Local create case. */
1275 u32 context_id = vmci_get_context_id();
1276
1277 /*
1278 * Enforce similar checks on local queue pairs as we
1279 * do for regular ones. The handle's context must
1280 * match the creator or attacher context id (here they
1281 * are both the current context id) and the
1282 * attach-only flag cannot exist during create. We
1283 * also ensure specified peer is this context or an
1284 * invalid one.
1285 */
1286 if (queue_pair_entry->qp.handle.context != context_id ||
1287 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1288 queue_pair_entry->qp.peer != context_id)) {
1289 result = VMCI_ERROR_NO_ACCESS;
1290 goto error;
1291 }
1292
1293 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1294 result = VMCI_ERROR_NOT_FOUND;
1295 goto error;
1296 }
1297 } else {
1298 result = qp_alloc_hypercall(queue_pair_entry);
1299 if (result < VMCI_SUCCESS) {
1300 pr_warn("qp_alloc_hypercall result = %d\n", result);
1301 goto error;
1302 }
1303 }
1304
1305 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1306 (struct vmci_queue *)my_consume_q);
1307
1308 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1309
1310 out:
1311 queue_pair_entry->qp.ref_count++;
1312 *handle = queue_pair_entry->qp.handle;
1313 *produce_q = (struct vmci_queue *)my_produce_q;
1314 *consume_q = (struct vmci_queue *)my_consume_q;
1315
1316 /*
1317 * We should initialize the queue pair header pages on a local
1318 * queue pair create. For non-local queue pairs, the
1319 * hypervisor initializes the header pages in the create step.
1320 */
1321 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1322 queue_pair_entry->qp.ref_count == 1) {
1323 vmci_q_header_init((*produce_q)->q_header, *handle);
1324 vmci_q_header_init((*consume_q)->q_header, *handle);
1325 }
1326
1327 mutex_unlock(&qp_guest_endpoints.mutex);
1328
1329 return VMCI_SUCCESS;
1330
1331 error:
1332 mutex_unlock(&qp_guest_endpoints.mutex);
1333 if (queue_pair_entry) {
1334 /* The queues will be freed inside the destroy routine. */
1335 qp_guest_endpoint_destroy(queue_pair_entry);
1336 } else {
1337 qp_free_queue(my_produce_q, produce_size);
1338 qp_free_queue(my_consume_q, consume_size);
1339 }
1340 return result;
1341
1342 error_keep_entry:
1343 /* This path should only be used when an existing entry was found. */
1344 mutex_unlock(&qp_guest_endpoints.mutex);
1345 return result;
1346}
1347
1348/*
1349 * The first endpoint issuing a queue pair allocation will create the state
1350 * of the queue pair in the queue pair broker.
1351 *
1352 * If the creator is a guest, it will associate a VMX virtual address range
1353 * with the queue pair as specified by the page_store. For compatibility with
1354 * older VMX'en, that would use a separate step to set the VMX virtual
1355 * address range, the virtual address range can be registered later using
1356 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1357 * used.
1358 *
1359 * If the creator is the host, a page_store of NULL should be used as well,
1360 * since the host is not able to supply a page store for the queue pair.
1361 *
1362 * For older VMX and host callers, the queue pair will be created in the
1363 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1364 * created in VMCOQPB_CREATED_MEM state.
1365 */
1366static int qp_broker_create(struct vmci_handle handle,
1367 u32 peer,
1368 u32 flags,
1369 u32 priv_flags,
1370 u64 produce_size,
1371 u64 consume_size,
1372 struct vmci_qp_page_store *page_store,
1373 struct vmci_ctx *context,
1374 vmci_event_release_cb wakeup_cb,
1375 void *client_data, struct qp_broker_entry **ent)
1376{
1377 struct qp_broker_entry *entry = NULL;
1378 const u32 context_id = vmci_ctx_get_id(context);
1379 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1380 int result;
1381 u64 guest_produce_size;
1382 u64 guest_consume_size;
1383
1384 /* Do not create if the caller asked not to. */
1385 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1386 return VMCI_ERROR_NOT_FOUND;
1387
1388 /*
1389 * Creator's context ID should match handle's context ID or the creator
1390 * must allow the context in handle's context ID as the "peer".
1391 */
1392 if (handle.context != context_id && handle.context != peer)
1393 return VMCI_ERROR_NO_ACCESS;
1394
1395 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1396 return VMCI_ERROR_DST_UNREACHABLE;
1397
1398 /*
1399 * Creator's context ID for local queue pairs should match the
1400 * peer, if a peer is specified.
1401 */
1402 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1403 return VMCI_ERROR_NO_ACCESS;
1404
1405 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1406 if (!entry)
1407 return VMCI_ERROR_NO_MEM;
1408
1409 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1410 /*
1411 * The queue pair broker entry stores values from the guest
1412 * point of view, so a creating host side endpoint should swap
1413 * produce and consume values -- unless it is a local queue
1414 * pair, in which case no swapping is necessary, since the local
1415 * attacher will swap queues.
1416 */
1417
1418 guest_produce_size = consume_size;
1419 guest_consume_size = produce_size;
1420 } else {
1421 guest_produce_size = produce_size;
1422 guest_consume_size = consume_size;
1423 }
1424
1425 entry->qp.handle = handle;
1426 entry->qp.peer = peer;
1427 entry->qp.flags = flags;
1428 entry->qp.produce_size = guest_produce_size;
1429 entry->qp.consume_size = guest_consume_size;
1430 entry->qp.ref_count = 1;
1431 entry->create_id = context_id;
1432 entry->attach_id = VMCI_INVALID_ID;
1433 entry->state = VMCIQPB_NEW;
1434 entry->require_trusted_attach =
1435 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1436 entry->created_by_trusted =
1437 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1438 entry->vmci_page_files = false;
1439 entry->wakeup_cb = wakeup_cb;
1440 entry->client_data = client_data;
1441 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1442 if (entry->produce_q == NULL) {
1443 result = VMCI_ERROR_NO_MEM;
1444 goto error;
1445 }
1446 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1447 if (entry->consume_q == NULL) {
1448 result = VMCI_ERROR_NO_MEM;
1449 goto error;
1450 }
1451
1452 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1453
1454 INIT_LIST_HEAD(&entry->qp.list_item);
1455
1456 if (is_local) {
1457 u8 *tmp;
1458
1459 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1460 PAGE_SIZE, GFP_KERNEL);
1461 if (entry->local_mem == NULL) {
1462 result = VMCI_ERROR_NO_MEM;
1463 goto error;
1464 }
1465 entry->state = VMCIQPB_CREATED_MEM;
1466 entry->produce_q->q_header = entry->local_mem;
1467 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1468 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1469 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1470 } else if (page_store) {
1471 /*
1472 * The VMX already initialized the queue pair headers, so no
1473 * need for the kernel side to do that.
1474 */
1475 result = qp_host_register_user_memory(page_store,
1476 entry->produce_q,
1477 entry->consume_q);
1478 if (result < VMCI_SUCCESS)
1479 goto error;
1480
1481 entry->state = VMCIQPB_CREATED_MEM;
1482 } else {
1483 /*
1484 * A create without a page_store may be either a host
1485 * side create (in which case we are waiting for the
1486 * guest side to supply the memory) or an old style
1487 * queue pair create (in which case we will expect a
1488 * set page store call as the next step).
1489 */
1490 entry->state = VMCIQPB_CREATED_NO_MEM;
1491 }
1492
1493 qp_list_add_entry(&qp_broker_list, &entry->qp);
1494 if (ent != NULL)
1495 *ent = entry;
1496
1497 /* Add to resource obj */
1498 result = vmci_resource_add(&entry->resource,
1499 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1500 handle);
1501 if (result != VMCI_SUCCESS) {
1502 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1503 handle.context, handle.resource, result);
1504 goto error;
1505 }
1506
1507 entry->qp.handle = vmci_resource_handle(&entry->resource);
1508 if (is_local) {
1509 vmci_q_header_init(entry->produce_q->q_header,
1510 entry->qp.handle);
1511 vmci_q_header_init(entry->consume_q->q_header,
1512 entry->qp.handle);
1513 }
1514
1515 vmci_ctx_qp_create(context, entry->qp.handle);
1516
1517 return VMCI_SUCCESS;
1518
1519 error:
1520 if (entry != NULL) {
1521 qp_host_free_queue(entry->produce_q, guest_produce_size);
1522 qp_host_free_queue(entry->consume_q, guest_consume_size);
1523 kfree(entry);
1524 }
1525
1526 return result;
1527}
1528
1529/*
1530 * Enqueues an event datagram to notify the peer VM attached to
1531 * the given queue pair handle about attach/detach event by the
1532 * given VM. Returns Payload size of datagram enqueued on
1533 * success, error code otherwise.
1534 */
1535static int qp_notify_peer(bool attach,
1536 struct vmci_handle handle,
1537 u32 my_id,
1538 u32 peer_id)
1539{
1540 int rv;
1541 struct vmci_event_qp ev;
1542
1543 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1544 peer_id == VMCI_INVALID_ID)
1545 return VMCI_ERROR_INVALID_ARGS;
1546
1547 /*
1548 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1549 * number of pending events from the hypervisor to a given VM
1550 * otherwise a rogue VM could do an arbitrary number of attach
1551 * and detach operations causing memory pressure in the host
1552 * kernel.
1553 */
1554
1555 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1556 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1557 VMCI_CONTEXT_RESOURCE_ID);
1558 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1559 ev.msg.event_data.event = attach ?
1560 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1561 ev.payload.handle = handle;
1562 ev.payload.peer_id = my_id;
1563
1564 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1565 &ev.msg.hdr, false);
1566 if (rv < VMCI_SUCCESS)
1567 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1568 attach ? "ATTACH" : "DETACH", peer_id);
1569
1570 return rv;
1571}
1572
1573/*
1574 * The second endpoint issuing a queue pair allocation will attach to
1575 * the queue pair registered with the queue pair broker.
1576 *
1577 * If the attacher is a guest, it will associate a VMX virtual address
1578 * range with the queue pair as specified by the page_store. At this
1579 * point, the already attach host endpoint may start using the queue
1580 * pair, and an attach event is sent to it. For compatibility with
1581 * older VMX'en, that used a separate step to set the VMX virtual
1582 * address range, the virtual address range can be registered later
1583 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1584 * NULL should be used, and the attach event will be generated once
1585 * the actual page store has been set.
1586 *
1587 * If the attacher is the host, a page_store of NULL should be used as
1588 * well, since the page store information is already set by the guest.
1589 *
1590 * For new VMX and host callers, the queue pair will be moved to the
1591 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1592 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1593 */
1594static int qp_broker_attach(struct qp_broker_entry *entry,
1595 u32 peer,
1596 u32 flags,
1597 u32 priv_flags,
1598 u64 produce_size,
1599 u64 consume_size,
1600 struct vmci_qp_page_store *page_store,
1601 struct vmci_ctx *context,
1602 vmci_event_release_cb wakeup_cb,
1603 void *client_data,
1604 struct qp_broker_entry **ent)
1605{
1606 const u32 context_id = vmci_ctx_get_id(context);
1607 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1608 int result;
1609
1610 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1611 entry->state != VMCIQPB_CREATED_MEM)
1612 return VMCI_ERROR_UNAVAILABLE;
1613
1614 if (is_local) {
1615 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1616 context_id != entry->create_id) {
1617 return VMCI_ERROR_INVALID_ARGS;
1618 }
1619 } else if (context_id == entry->create_id ||
1620 context_id == entry->attach_id) {
1621 return VMCI_ERROR_ALREADY_EXISTS;
1622 }
1623
1624 if (VMCI_CONTEXT_IS_VM(context_id) &&
1625 VMCI_CONTEXT_IS_VM(entry->create_id))
1626 return VMCI_ERROR_DST_UNREACHABLE;
1627
1628 /*
1629 * If we are attaching from a restricted context then the queuepair
1630 * must have been created by a trusted endpoint.
1631 */
1632 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1633 !entry->created_by_trusted)
1634 return VMCI_ERROR_NO_ACCESS;
1635
1636 /*
1637 * If we are attaching to a queuepair that was created by a restricted
1638 * context then we must be trusted.
1639 */
1640 if (entry->require_trusted_attach &&
1641 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1642 return VMCI_ERROR_NO_ACCESS;
1643
1644 /*
1645 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1646 * control check is not performed.
1647 */
1648 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1649 return VMCI_ERROR_NO_ACCESS;
1650
1651 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1652 /*
1653 * Do not attach if the caller doesn't support Host Queue Pairs
1654 * and a host created this queue pair.
1655 */
1656
1657 if (!vmci_ctx_supports_host_qp(context))
1658 return VMCI_ERROR_INVALID_RESOURCE;
1659
1660 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1661 struct vmci_ctx *create_context;
1662 bool supports_host_qp;
1663
1664 /*
1665 * Do not attach a host to a user created queue pair if that
1666 * user doesn't support host queue pair end points.
1667 */
1668
1669 create_context = vmci_ctx_get(entry->create_id);
1670 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1671 vmci_ctx_put(create_context);
1672
1673 if (!supports_host_qp)
1674 return VMCI_ERROR_INVALID_RESOURCE;
1675 }
1676
1677 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1678 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1679
1680 if (context_id != VMCI_HOST_CONTEXT_ID) {
1681 /*
1682 * The queue pair broker entry stores values from the guest
1683 * point of view, so an attaching guest should match the values
1684 * stored in the entry.
1685 */
1686
1687 if (entry->qp.produce_size != produce_size ||
1688 entry->qp.consume_size != consume_size) {
1689 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1690 }
1691 } else if (entry->qp.produce_size != consume_size ||
1692 entry->qp.consume_size != produce_size) {
1693 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1694 }
1695
1696 if (context_id != VMCI_HOST_CONTEXT_ID) {
1697 /*
1698 * If a guest attached to a queue pair, it will supply
1699 * the backing memory. If this is a pre NOVMVM vmx,
1700 * the backing memory will be supplied by calling
1701 * vmci_qp_broker_set_page_store() following the
1702 * return of the vmci_qp_broker_alloc() call. If it is
1703 * a vmx of version NOVMVM or later, the page store
1704 * must be supplied as part of the
1705 * vmci_qp_broker_alloc call. Under all circumstances
1706 * must the initially created queue pair not have any
1707 * memory associated with it already.
1708 */
1709
1710 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1711 return VMCI_ERROR_INVALID_ARGS;
1712
1713 if (page_store != NULL) {
1714 /*
1715 * Patch up host state to point to guest
1716 * supplied memory. The VMX already
1717 * initialized the queue pair headers, so no
1718 * need for the kernel side to do that.
1719 */
1720
1721 result = qp_host_register_user_memory(page_store,
1722 entry->produce_q,
1723 entry->consume_q);
1724 if (result < VMCI_SUCCESS)
1725 return result;
1726
1727 entry->state = VMCIQPB_ATTACHED_MEM;
1728 } else {
1729 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1730 }
1731 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1732 /*
1733 * The host side is attempting to attach to a queue
1734 * pair that doesn't have any memory associated with
1735 * it. This must be a pre NOVMVM vmx that hasn't set
1736 * the page store information yet, or a quiesced VM.
1737 */
1738
1739 return VMCI_ERROR_UNAVAILABLE;
1740 } else {
1741 /* The host side has successfully attached to a queue pair. */
1742 entry->state = VMCIQPB_ATTACHED_MEM;
1743 }
1744
1745 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1746 result =
1747 qp_notify_peer(true, entry->qp.handle, context_id,
1748 entry->create_id);
1749 if (result < VMCI_SUCCESS)
1750 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1751 entry->create_id, entry->qp.handle.context,
1752 entry->qp.handle.resource);
1753 }
1754
1755 entry->attach_id = context_id;
1756 entry->qp.ref_count++;
1757 if (wakeup_cb) {
1758 entry->wakeup_cb = wakeup_cb;
1759 entry->client_data = client_data;
1760 }
1761
1762 /*
1763 * When attaching to local queue pairs, the context already has
1764 * an entry tracking the queue pair, so don't add another one.
1765 */
1766 if (!is_local)
1767 vmci_ctx_qp_create(context, entry->qp.handle);
1768
1769 if (ent != NULL)
1770 *ent = entry;
1771
1772 return VMCI_SUCCESS;
1773}
1774
1775/*
1776 * queue_pair_Alloc for use when setting up queue pair endpoints
1777 * on the host.
1778 */
1779static int qp_broker_alloc(struct vmci_handle handle,
1780 u32 peer,
1781 u32 flags,
1782 u32 priv_flags,
1783 u64 produce_size,
1784 u64 consume_size,
1785 struct vmci_qp_page_store *page_store,
1786 struct vmci_ctx *context,
1787 vmci_event_release_cb wakeup_cb,
1788 void *client_data,
1789 struct qp_broker_entry **ent,
1790 bool *swap)
1791{
1792 const u32 context_id = vmci_ctx_get_id(context);
1793 bool create;
1794 struct qp_broker_entry *entry = NULL;
1795 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1796 int result;
1797
1798 if (vmci_handle_is_invalid(handle) ||
1799 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1800 !(produce_size || consume_size) ||
1801 !context || context_id == VMCI_INVALID_ID ||
1802 handle.context == VMCI_INVALID_ID) {
1803 return VMCI_ERROR_INVALID_ARGS;
1804 }
1805
1806 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1807 return VMCI_ERROR_INVALID_ARGS;
1808
1809 /*
1810 * In the initial argument check, we ensure that non-vmkernel hosts
1811 * are not allowed to create local queue pairs.
1812 */
1813
1814 mutex_lock(&qp_broker_list.mutex);
1815
1816 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1817 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1818 context_id, handle.context, handle.resource);
1819 mutex_unlock(&qp_broker_list.mutex);
1820 return VMCI_ERROR_ALREADY_EXISTS;
1821 }
1822
1823 if (handle.resource != VMCI_INVALID_ID)
1824 entry = qp_broker_handle_to_entry(handle);
1825
1826 if (!entry) {
1827 create = true;
1828 result =
1829 qp_broker_create(handle, peer, flags, priv_flags,
1830 produce_size, consume_size, page_store,
1831 context, wakeup_cb, client_data, ent);
1832 } else {
1833 create = false;
1834 result =
1835 qp_broker_attach(entry, peer, flags, priv_flags,
1836 produce_size, consume_size, page_store,
1837 context, wakeup_cb, client_data, ent);
1838 }
1839
1840 mutex_unlock(&qp_broker_list.mutex);
1841
1842 if (swap)
1843 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1844 !(create && is_local);
1845
1846 return result;
1847}
1848
1849/*
1850 * This function implements the kernel API for allocating a queue
1851 * pair.
1852 */
1853static int qp_alloc_host_work(struct vmci_handle *handle,
1854 struct vmci_queue **produce_q,
1855 u64 produce_size,
1856 struct vmci_queue **consume_q,
1857 u64 consume_size,
1858 u32 peer,
1859 u32 flags,
1860 u32 priv_flags,
1861 vmci_event_release_cb wakeup_cb,
1862 void *client_data)
1863{
1864 struct vmci_handle new_handle;
1865 struct vmci_ctx *context;
1866 struct qp_broker_entry *entry;
1867 int result;
1868 bool swap;
1869
1870 if (vmci_handle_is_invalid(*handle)) {
1871 new_handle = vmci_make_handle(
1872 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1873 } else
1874 new_handle = *handle;
1875
1876 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1877 entry = NULL;
1878 result =
1879 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1880 produce_size, consume_size, NULL, context,
1881 wakeup_cb, client_data, &entry, &swap);
1882 if (result == VMCI_SUCCESS) {
1883 if (swap) {
1884 /*
1885 * If this is a local queue pair, the attacher
1886 * will swap around produce and consume
1887 * queues.
1888 */
1889
1890 *produce_q = entry->consume_q;
1891 *consume_q = entry->produce_q;
1892 } else {
1893 *produce_q = entry->produce_q;
1894 *consume_q = entry->consume_q;
1895 }
1896
1897 *handle = vmci_resource_handle(&entry->resource);
1898 } else {
1899 *handle = VMCI_INVALID_HANDLE;
1900 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1901 result);
1902 }
1903 vmci_ctx_put(context);
1904 return result;
1905}
1906
1907/*
1908 * Allocates a VMCI queue_pair. Only checks validity of input
1909 * arguments. The real work is done in the host or guest
1910 * specific function.
1911 */
1912int vmci_qp_alloc(struct vmci_handle *handle,
1913 struct vmci_queue **produce_q,
1914 u64 produce_size,
1915 struct vmci_queue **consume_q,
1916 u64 consume_size,
1917 u32 peer,
1918 u32 flags,
1919 u32 priv_flags,
1920 bool guest_endpoint,
1921 vmci_event_release_cb wakeup_cb,
1922 void *client_data)
1923{
1924 if (!handle || !produce_q || !consume_q ||
1925 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1926 return VMCI_ERROR_INVALID_ARGS;
1927
1928 if (guest_endpoint) {
1929 return qp_alloc_guest_work(handle, produce_q,
1930 produce_size, consume_q,
1931 consume_size, peer,
1932 flags, priv_flags);
1933 } else {
1934 return qp_alloc_host_work(handle, produce_q,
1935 produce_size, consume_q,
1936 consume_size, peer, flags,
1937 priv_flags, wakeup_cb, client_data);
1938 }
1939}
1940
1941/*
1942 * This function implements the host kernel API for detaching from
1943 * a queue pair.
1944 */
1945static int qp_detatch_host_work(struct vmci_handle handle)
1946{
1947 int result;
1948 struct vmci_ctx *context;
1949
1950 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1951
1952 result = vmci_qp_broker_detach(handle, context);
1953
1954 vmci_ctx_put(context);
1955 return result;
1956}
1957
1958/*
1959 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1960 * Real work is done in the host or guest specific function.
1961 */
1962static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1963{
1964 if (vmci_handle_is_invalid(handle))
1965 return VMCI_ERROR_INVALID_ARGS;
1966
1967 if (guest_endpoint)
1968 return qp_detatch_guest_work(handle);
1969 else
1970 return qp_detatch_host_work(handle);
1971}
1972
1973/*
1974 * Returns the entry from the head of the list. Assumes that the list is
1975 * locked.
1976 */
1977static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1978{
1979 if (!list_empty(&qp_list->head)) {
1980 struct qp_entry *entry =
1981 list_first_entry(&qp_list->head, struct qp_entry,
1982 list_item);
1983 return entry;
1984 }
1985
1986 return NULL;
1987}
1988
1989void vmci_qp_broker_exit(void)
1990{
1991 struct qp_entry *entry;
1992 struct qp_broker_entry *be;
1993
1994 mutex_lock(&qp_broker_list.mutex);
1995
1996 while ((entry = qp_list_get_head(&qp_broker_list))) {
1997 be = (struct qp_broker_entry *)entry;
1998
1999 qp_list_remove_entry(&qp_broker_list, entry);
2000 kfree(be);
2001 }
2002
2003 mutex_unlock(&qp_broker_list.mutex);
2004}
2005
2006/*
2007 * Requests that a queue pair be allocated with the VMCI queue
2008 * pair broker. Allocates a queue pair entry if one does not
2009 * exist. Attaches to one if it exists, and retrieves the page
2010 * files backing that queue_pair. Assumes that the queue pair
2011 * broker lock is held.
2012 */
2013int vmci_qp_broker_alloc(struct vmci_handle handle,
2014 u32 peer,
2015 u32 flags,
2016 u32 priv_flags,
2017 u64 produce_size,
2018 u64 consume_size,
2019 struct vmci_qp_page_store *page_store,
2020 struct vmci_ctx *context)
2021{
2022 return qp_broker_alloc(handle, peer, flags, priv_flags,
2023 produce_size, consume_size,
2024 page_store, context, NULL, NULL, NULL, NULL);
2025}
2026
2027/*
2028 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
2029 * step to add the UVAs of the VMX mapping of the queue pair. This function
2030 * provides backwards compatibility with such VMX'en, and takes care of
2031 * registering the page store for a queue pair previously allocated by the
2032 * VMX during create or attach. This function will move the queue pair state
2033 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
2034 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
2035 * attached state with memory, the queue pair is ready to be used by the
2036 * host peer, and an attached event will be generated.
2037 *
2038 * Assumes that the queue pair broker lock is held.
2039 *
2040 * This function is only used by the hosted platform, since there is no
2041 * issue with backwards compatibility for vmkernel.
2042 */
2043int vmci_qp_broker_set_page_store(struct vmci_handle handle,
2044 u64 produce_uva,
2045 u64 consume_uva,
2046 struct vmci_ctx *context)
2047{
2048 struct qp_broker_entry *entry;
2049 int result;
2050 const u32 context_id = vmci_ctx_get_id(context);
2051
2052 if (vmci_handle_is_invalid(handle) || !context ||
2053 context_id == VMCI_INVALID_ID)
2054 return VMCI_ERROR_INVALID_ARGS;
2055
2056 /*
2057 * We only support guest to host queue pairs, so the VMX must
2058 * supply UVAs for the mapped page files.
2059 */
2060
2061 if (produce_uva == 0 || consume_uva == 0)
2062 return VMCI_ERROR_INVALID_ARGS;
2063
2064 mutex_lock(&qp_broker_list.mutex);
2065
2066 if (!vmci_ctx_qp_exists(context, handle)) {
2067 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2068 context_id, handle.context, handle.resource);
2069 result = VMCI_ERROR_NOT_FOUND;
2070 goto out;
2071 }
2072
2073 entry = qp_broker_handle_to_entry(handle);
2074 if (!entry) {
2075 result = VMCI_ERROR_NOT_FOUND;
2076 goto out;
2077 }
2078
2079 /*
2080 * If I'm the owner then I can set the page store.
2081 *
2082 * Or, if a host created the queue_pair and I'm the attached peer
2083 * then I can set the page store.
2084 */
2085 if (entry->create_id != context_id &&
2086 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
2087 entry->attach_id != context_id)) {
2088 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
2089 goto out;
2090 }
2091
2092 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2093 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2094 result = VMCI_ERROR_UNAVAILABLE;
2095 goto out;
2096 }
2097
2098 result = qp_host_get_user_memory(produce_uva, consume_uva,
2099 entry->produce_q, entry->consume_q);
2100 if (result < VMCI_SUCCESS)
2101 goto out;
2102
2103 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2104 if (result < VMCI_SUCCESS) {
2105 qp_host_unregister_user_memory(entry->produce_q,
2106 entry->consume_q);
2107 goto out;
2108 }
2109
2110 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2111 entry->state = VMCIQPB_CREATED_MEM;
2112 else
2113 entry->state = VMCIQPB_ATTACHED_MEM;
2114
2115 entry->vmci_page_files = true;
2116
2117 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2118 result =
2119 qp_notify_peer(true, handle, context_id, entry->create_id);
2120 if (result < VMCI_SUCCESS) {
2121 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2122 entry->create_id, entry->qp.handle.context,
2123 entry->qp.handle.resource);
2124 }
2125 }
2126
2127 result = VMCI_SUCCESS;
2128 out:
2129 mutex_unlock(&qp_broker_list.mutex);
2130 return result;
2131}
2132
2133/*
2134 * Resets saved queue headers for the given QP broker
2135 * entry. Should be used when guest memory becomes available
2136 * again, or the guest detaches.
2137 */
2138static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2139{
2140 entry->produce_q->saved_header = NULL;
2141 entry->consume_q->saved_header = NULL;
2142}
2143
2144/*
2145 * The main entry point for detaching from a queue pair registered with the
2146 * queue pair broker. If more than one endpoint is attached to the queue
2147 * pair, the first endpoint will mainly decrement a reference count and
2148 * generate a notification to its peer. The last endpoint will clean up
2149 * the queue pair state registered with the broker.
2150 *
2151 * When a guest endpoint detaches, it will unmap and unregister the guest
2152 * memory backing the queue pair. If the host is still attached, it will
2153 * no longer be able to access the queue pair content.
2154 *
2155 * If the queue pair is already in a state where there is no memory
2156 * registered for the queue pair (any *_NO_MEM state), it will transition to
2157 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2158 * endpoint is the first of two endpoints to detach. If the host endpoint is
2159 * the first out of two to detach, the queue pair will move to the
2160 * VMCIQPB_SHUTDOWN_MEM state.
2161 */
2162int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2163{
2164 struct qp_broker_entry *entry;
2165 const u32 context_id = vmci_ctx_get_id(context);
2166 u32 peer_id;
2167 bool is_local = false;
2168 int result;
2169
2170 if (vmci_handle_is_invalid(handle) || !context ||
2171 context_id == VMCI_INVALID_ID) {
2172 return VMCI_ERROR_INVALID_ARGS;
2173 }
2174
2175 mutex_lock(&qp_broker_list.mutex);
2176
2177 if (!vmci_ctx_qp_exists(context, handle)) {
2178 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2179 context_id, handle.context, handle.resource);
2180 result = VMCI_ERROR_NOT_FOUND;
2181 goto out;
2182 }
2183
2184 entry = qp_broker_handle_to_entry(handle);
2185 if (!entry) {
2186 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2187 context_id, handle.context, handle.resource);
2188 result = VMCI_ERROR_NOT_FOUND;
2189 goto out;
2190 }
2191
2192 if (context_id != entry->create_id && context_id != entry->attach_id) {
2193 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2194 goto out;
2195 }
2196
2197 if (context_id == entry->create_id) {
2198 peer_id = entry->attach_id;
2199 entry->create_id = VMCI_INVALID_ID;
2200 } else {
2201 peer_id = entry->create_id;
2202 entry->attach_id = VMCI_INVALID_ID;
2203 }
2204 entry->qp.ref_count--;
2205
2206 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2207
2208 if (context_id != VMCI_HOST_CONTEXT_ID) {
2209 bool headers_mapped;
2210
2211 /*
2212 * Pre NOVMVM vmx'en may detach from a queue pair
2213 * before setting the page store, and in that case
2214 * there is no user memory to detach from. Also, more
2215 * recent VMX'en may detach from a queue pair in the
2216 * quiesced state.
2217 */
2218
2219 qp_acquire_queue_mutex(entry->produce_q);
2220 headers_mapped = entry->produce_q->q_header ||
2221 entry->consume_q->q_header;
2222 if (QPBROKERSTATE_HAS_MEM(entry)) {
2223 result =
2224 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2225 entry->produce_q,
2226 entry->consume_q);
2227 if (result < VMCI_SUCCESS)
2228 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2229 handle.context, handle.resource,
2230 result);
2231
2232 if (entry->vmci_page_files)
2233 qp_host_unregister_user_memory(entry->produce_q,
2234 entry->
2235 consume_q);
2236 else
2237 qp_host_unregister_user_memory(entry->produce_q,
2238 entry->
2239 consume_q);
2240
2241 }
2242
2243 if (!headers_mapped)
2244 qp_reset_saved_headers(entry);
2245
2246 qp_release_queue_mutex(entry->produce_q);
2247
2248 if (!headers_mapped && entry->wakeup_cb)
2249 entry->wakeup_cb(entry->client_data);
2250
2251 } else {
2252 if (entry->wakeup_cb) {
2253 entry->wakeup_cb = NULL;
2254 entry->client_data = NULL;
2255 }
2256 }
2257
2258 if (entry->qp.ref_count == 0) {
2259 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2260
2261 if (is_local)
2262 kfree(entry->local_mem);
2263
2264 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2265 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2266 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2267 /* Unlink from resource hash table and free callback */
2268 vmci_resource_remove(&entry->resource);
2269
2270 kfree(entry);
2271
2272 vmci_ctx_qp_destroy(context, handle);
2273 } else {
2274 qp_notify_peer(false, handle, context_id, peer_id);
2275 if (context_id == VMCI_HOST_CONTEXT_ID &&
2276 QPBROKERSTATE_HAS_MEM(entry)) {
2277 entry->state = VMCIQPB_SHUTDOWN_MEM;
2278 } else {
2279 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2280 }
2281
2282 if (!is_local)
2283 vmci_ctx_qp_destroy(context, handle);
2284
2285 }
2286 result = VMCI_SUCCESS;
2287 out:
2288 mutex_unlock(&qp_broker_list.mutex);
2289 return result;
2290}
2291
2292/*
2293 * Establishes the necessary mappings for a queue pair given a
2294 * reference to the queue pair guest memory. This is usually
2295 * called when a guest is unquiesced and the VMX is allowed to
2296 * map guest memory once again.
2297 */
2298int vmci_qp_broker_map(struct vmci_handle handle,
2299 struct vmci_ctx *context,
2300 u64 guest_mem)
2301{
2302 struct qp_broker_entry *entry;
2303 const u32 context_id = vmci_ctx_get_id(context);
2304 bool is_local = false;
2305 int result;
2306
2307 if (vmci_handle_is_invalid(handle) || !context ||
2308 context_id == VMCI_INVALID_ID)
2309 return VMCI_ERROR_INVALID_ARGS;
2310
2311 mutex_lock(&qp_broker_list.mutex);
2312
2313 if (!vmci_ctx_qp_exists(context, handle)) {
2314 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2315 context_id, handle.context, handle.resource);
2316 result = VMCI_ERROR_NOT_FOUND;
2317 goto out;
2318 }
2319
2320 entry = qp_broker_handle_to_entry(handle);
2321 if (!entry) {
2322 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2323 context_id, handle.context, handle.resource);
2324 result = VMCI_ERROR_NOT_FOUND;
2325 goto out;
2326 }
2327
2328 if (context_id != entry->create_id && context_id != entry->attach_id) {
2329 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2330 goto out;
2331 }
2332
2333 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2334 result = VMCI_SUCCESS;
2335
2336 if (context_id != VMCI_HOST_CONTEXT_ID) {
2337 struct vmci_qp_page_store page_store;
2338
2339 page_store.pages = guest_mem;
2340 page_store.len = QPE_NUM_PAGES(entry->qp);
2341
2342 qp_acquire_queue_mutex(entry->produce_q);
2343 qp_reset_saved_headers(entry);
2344 result =
2345 qp_host_register_user_memory(&page_store,
2346 entry->produce_q,
2347 entry->consume_q);
2348 qp_release_queue_mutex(entry->produce_q);
2349 if (result == VMCI_SUCCESS) {
2350 /* Move state from *_NO_MEM to *_MEM */
2351
2352 entry->state++;
2353
2354 if (entry->wakeup_cb)
2355 entry->wakeup_cb(entry->client_data);
2356 }
2357 }
2358
2359 out:
2360 mutex_unlock(&qp_broker_list.mutex);
2361 return result;
2362}
2363
2364/*
2365 * Saves a snapshot of the queue headers for the given QP broker
2366 * entry. Should be used when guest memory is unmapped.
2367 * Results:
2368 * VMCI_SUCCESS on success, appropriate error code if guest memory
2369 * can't be accessed..
2370 */
2371static int qp_save_headers(struct qp_broker_entry *entry)
2372{
2373 int result;
2374
2375 if (entry->produce_q->saved_header != NULL &&
2376 entry->consume_q->saved_header != NULL) {
2377 /*
2378 * If the headers have already been saved, we don't need to do
2379 * it again, and we don't want to map in the headers
2380 * unnecessarily.
2381 */
2382
2383 return VMCI_SUCCESS;
2384 }
2385
2386 if (NULL == entry->produce_q->q_header ||
2387 NULL == entry->consume_q->q_header) {
2388 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2389 if (result < VMCI_SUCCESS)
2390 return result;
2391 }
2392
2393 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2394 sizeof(entry->saved_produce_q));
2395 entry->produce_q->saved_header = &entry->saved_produce_q;
2396 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2397 sizeof(entry->saved_consume_q));
2398 entry->consume_q->saved_header = &entry->saved_consume_q;
2399
2400 return VMCI_SUCCESS;
2401}
2402
2403/*
2404 * Removes all references to the guest memory of a given queue pair, and
2405 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2406 * called when a VM is being quiesced where access to guest memory should
2407 * avoided.
2408 */
2409int vmci_qp_broker_unmap(struct vmci_handle handle,
2410 struct vmci_ctx *context,
2411 u32 gid)
2412{
2413 struct qp_broker_entry *entry;
2414 const u32 context_id = vmci_ctx_get_id(context);
2415 bool is_local = false;
2416 int result;
2417
2418 if (vmci_handle_is_invalid(handle) || !context ||
2419 context_id == VMCI_INVALID_ID)
2420 return VMCI_ERROR_INVALID_ARGS;
2421
2422 mutex_lock(&qp_broker_list.mutex);
2423
2424 if (!vmci_ctx_qp_exists(context, handle)) {
2425 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2426 context_id, handle.context, handle.resource);
2427 result = VMCI_ERROR_NOT_FOUND;
2428 goto out;
2429 }
2430
2431 entry = qp_broker_handle_to_entry(handle);
2432 if (!entry) {
2433 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2434 context_id, handle.context, handle.resource);
2435 result = VMCI_ERROR_NOT_FOUND;
2436 goto out;
2437 }
2438
2439 if (context_id != entry->create_id && context_id != entry->attach_id) {
2440 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2441 goto out;
2442 }
2443
2444 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2445
2446 if (context_id != VMCI_HOST_CONTEXT_ID) {
2447 qp_acquire_queue_mutex(entry->produce_q);
2448 result = qp_save_headers(entry);
2449 if (result < VMCI_SUCCESS)
2450 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2451 handle.context, handle.resource, result);
2452
2453 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2454
2455 /*
2456 * On hosted, when we unmap queue pairs, the VMX will also
2457 * unmap the guest memory, so we invalidate the previously
2458 * registered memory. If the queue pair is mapped again at a
2459 * later point in time, we will need to reregister the user
2460 * memory with a possibly new user VA.
2461 */
2462 qp_host_unregister_user_memory(entry->produce_q,
2463 entry->consume_q);
2464
2465 /*
2466 * Move state from *_MEM to *_NO_MEM.
2467 */
2468 entry->state--;
2469
2470 qp_release_queue_mutex(entry->produce_q);
2471 }
2472
2473 result = VMCI_SUCCESS;
2474
2475 out:
2476 mutex_unlock(&qp_broker_list.mutex);
2477 return result;
2478}
2479
2480/*
2481 * Destroys all guest queue pair endpoints. If active guest queue
2482 * pairs still exist, hypercalls to attempt detach from these
2483 * queue pairs will be made. Any failure to detach is silently
2484 * ignored.
2485 */
2486void vmci_qp_guest_endpoints_exit(void)
2487{
2488 struct qp_entry *entry;
2489 struct qp_guest_endpoint *ep;
2490
2491 mutex_lock(&qp_guest_endpoints.mutex);
2492
2493 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2494 ep = (struct qp_guest_endpoint *)entry;
2495
2496 /* Don't make a hypercall for local queue_pairs. */
2497 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2498 qp_detatch_hypercall(entry->handle);
2499
2500 /* We cannot fail the exit, so let's reset ref_count. */
2501 entry->ref_count = 0;
2502 qp_list_remove_entry(&qp_guest_endpoints, entry);
2503
2504 qp_guest_endpoint_destroy(ep);
2505 }
2506
2507 mutex_unlock(&qp_guest_endpoints.mutex);
2508}
2509
2510/*
2511 * Helper routine that will lock the queue pair before subsequent
2512 * operations.
2513 * Note: Non-blocking on the host side is currently only implemented in ESX.
2514 * Since non-blocking isn't yet implemented on the host personality we
2515 * have no reason to acquire a spin lock. So to avoid the use of an
2516 * unnecessary lock only acquire the mutex if we can block.
2517 */
2518static void qp_lock(const struct vmci_qp *qpair)
2519{
2520 qp_acquire_queue_mutex(qpair->produce_q);
2521}
2522
2523/*
2524 * Helper routine that unlocks the queue pair after calling
2525 * qp_lock.
2526 */
2527static void qp_unlock(const struct vmci_qp *qpair)
2528{
2529 qp_release_queue_mutex(qpair->produce_q);
2530}
2531
2532/*
2533 * The queue headers may not be mapped at all times. If a queue is
2534 * currently not mapped, it will be attempted to do so.
2535 */
2536static int qp_map_queue_headers(struct vmci_queue *produce_q,
2537 struct vmci_queue *consume_q)
2538{
2539 int result;
2540
2541 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2542 result = qp_host_map_queues(produce_q, consume_q);
2543 if (result < VMCI_SUCCESS)
2544 return (produce_q->saved_header &&
2545 consume_q->saved_header) ?
2546 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2547 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2548 }
2549
2550 return VMCI_SUCCESS;
2551}
2552
2553/*
2554 * Helper routine that will retrieve the produce and consume
2555 * headers of a given queue pair. If the guest memory of the
2556 * queue pair is currently not available, the saved queue headers
2557 * will be returned, if these are available.
2558 */
2559static int qp_get_queue_headers(const struct vmci_qp *qpair,
2560 struct vmci_queue_header **produce_q_header,
2561 struct vmci_queue_header **consume_q_header)
2562{
2563 int result;
2564
2565 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2566 if (result == VMCI_SUCCESS) {
2567 *produce_q_header = qpair->produce_q->q_header;
2568 *consume_q_header = qpair->consume_q->q_header;
2569 } else if (qpair->produce_q->saved_header &&
2570 qpair->consume_q->saved_header) {
2571 *produce_q_header = qpair->produce_q->saved_header;
2572 *consume_q_header = qpair->consume_q->saved_header;
2573 result = VMCI_SUCCESS;
2574 }
2575
2576 return result;
2577}
2578
2579/*
2580 * Callback from VMCI queue pair broker indicating that a queue
2581 * pair that was previously not ready, now either is ready or
2582 * gone forever.
2583 */
2584static int qp_wakeup_cb(void *client_data)
2585{
2586 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2587
2588 qp_lock(qpair);
2589 while (qpair->blocked > 0) {
2590 qpair->blocked--;
2591 qpair->generation++;
2592 wake_up(&qpair->event);
2593 }
2594 qp_unlock(qpair);
2595
2596 return VMCI_SUCCESS;
2597}
2598
2599/*
2600 * Makes the calling thread wait for the queue pair to become
2601 * ready for host side access. Returns true when thread is
2602 * woken up after queue pair state change, false otherwise.
2603 */
2604static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2605{
2606 unsigned int generation;
2607
2608 qpair->blocked++;
2609 generation = qpair->generation;
2610 qp_unlock(qpair);
2611 wait_event(qpair->event, generation != qpair->generation);
2612 qp_lock(qpair);
2613
2614 return true;
2615}
2616
2617/*
2618 * Enqueues a given buffer to the produce queue using the provided
2619 * function. As many bytes as possible (space available in the queue)
2620 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2621 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2622 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2623 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2624 * an error occured when accessing the buffer,
2625 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2626 * available. Otherwise, the number of bytes written to the queue is
2627 * returned. Updates the tail pointer of the produce queue.
2628 */
2629static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2630 struct vmci_queue *consume_q,
2631 const u64 produce_q_size,
2632 const void *buf,
2633 size_t buf_size,
2634 vmci_memcpy_to_queue_func memcpy_to_queue)
2635{
2636 s64 free_space;
2637 u64 tail;
2638 size_t written;
2639 ssize_t result;
2640
2641 result = qp_map_queue_headers(produce_q, consume_q);
2642 if (unlikely(result != VMCI_SUCCESS))
2643 return result;
2644
2645 free_space = vmci_q_header_free_space(produce_q->q_header,
2646 consume_q->q_header,
2647 produce_q_size);
2648 if (free_space == 0)
2649 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2650
2651 if (free_space < VMCI_SUCCESS)
2652 return (ssize_t) free_space;
2653
2654 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2655 tail = vmci_q_header_producer_tail(produce_q->q_header);
2656 if (likely(tail + written < produce_q_size)) {
2657 result = memcpy_to_queue(produce_q, tail, buf, 0, written);
2658 } else {
2659 /* Tail pointer wraps around. */
2660
2661 const size_t tmp = (size_t) (produce_q_size - tail);
2662
2663 result = memcpy_to_queue(produce_q, tail, buf, 0, tmp);
2664 if (result >= VMCI_SUCCESS)
2665 result = memcpy_to_queue(produce_q, 0, buf, tmp,
2666 written - tmp);
2667 }
2668
2669 if (result < VMCI_SUCCESS)
2670 return result;
2671
2672 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2673 produce_q_size);
2674 return written;
2675}
2676
2677/*
2678 * Dequeues data (if available) from the given consume queue. Writes data
2679 * to the user provided buffer using the provided function.
2680 * Assumes the queue->mutex has been acquired.
2681 * Results:
2682 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2683 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2684 * (as defined by the queue size).
2685 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2686 * Otherwise the number of bytes dequeued is returned.
2687 * Side effects:
2688 * Updates the head pointer of the consume queue.
2689 */
2690static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2691 struct vmci_queue *consume_q,
2692 const u64 consume_q_size,
2693 void *buf,
2694 size_t buf_size,
2695 vmci_memcpy_from_queue_func memcpy_from_queue,
2696 bool update_consumer)
2697{
2698 s64 buf_ready;
2699 u64 head;
2700 size_t read;
2701 ssize_t result;
2702
2703 result = qp_map_queue_headers(produce_q, consume_q);
2704 if (unlikely(result != VMCI_SUCCESS))
2705 return result;
2706
2707 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2708 produce_q->q_header,
2709 consume_q_size);
2710 if (buf_ready == 0)
2711 return VMCI_ERROR_QUEUEPAIR_NODATA;
2712
2713 if (buf_ready < VMCI_SUCCESS)
2714 return (ssize_t) buf_ready;
2715
2716 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2717 head = vmci_q_header_consumer_head(produce_q->q_header);
2718 if (likely(head + read < consume_q_size)) {
2719 result = memcpy_from_queue(buf, 0, consume_q, head, read);
2720 } else {
2721 /* Head pointer wraps around. */
2722
2723 const size_t tmp = (size_t) (consume_q_size - head);
2724
2725 result = memcpy_from_queue(buf, 0, consume_q, head, tmp);
2726 if (result >= VMCI_SUCCESS)
2727 result = memcpy_from_queue(buf, tmp, consume_q, 0,
2728 read - tmp);
2729
2730 }
2731
2732 if (result < VMCI_SUCCESS)
2733 return result;
2734
2735 if (update_consumer)
2736 vmci_q_header_add_consumer_head(produce_q->q_header,
2737 read, consume_q_size);
2738
2739 return read;
2740}
2741
2742/*
2743 * vmci_qpair_alloc() - Allocates a queue pair.
2744 * @qpair: Pointer for the new vmci_qp struct.
2745 * @handle: Handle to track the resource.
2746 * @produce_qsize: Desired size of the producer queue.
2747 * @consume_qsize: Desired size of the consumer queue.
2748 * @peer: ContextID of the peer.
2749 * @flags: VMCI flags.
2750 * @priv_flags: VMCI priviledge flags.
2751 *
2752 * This is the client interface for allocating the memory for a
2753 * vmci_qp structure and then attaching to the underlying
2754 * queue. If an error occurs allocating the memory for the
2755 * vmci_qp structure no attempt is made to attach. If an
2756 * error occurs attaching, then the structure is freed.
2757 */
2758int vmci_qpair_alloc(struct vmci_qp **qpair,
2759 struct vmci_handle *handle,
2760 u64 produce_qsize,
2761 u64 consume_qsize,
2762 u32 peer,
2763 u32 flags,
2764 u32 priv_flags)
2765{
2766 struct vmci_qp *my_qpair;
2767 int retval;
2768 struct vmci_handle src = VMCI_INVALID_HANDLE;
2769 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2770 enum vmci_route route;
2771 vmci_event_release_cb wakeup_cb;
2772 void *client_data;
2773
2774 /*
2775 * Restrict the size of a queuepair. The device already
2776 * enforces a limit on the total amount of memory that can be
2777 * allocated to queuepairs for a guest. However, we try to
2778 * allocate this memory before we make the queuepair
2779 * allocation hypercall. On Linux, we allocate each page
2780 * separately, which means rather than fail, the guest will
2781 * thrash while it tries to allocate, and will become
2782 * increasingly unresponsive to the point where it appears to
2783 * be hung. So we place a limit on the size of an individual
2784 * queuepair here, and leave the device to enforce the
2785 * restriction on total queuepair memory. (Note that this
2786 * doesn't prevent all cases; a user with only this much
2787 * physical memory could still get into trouble.) The error
2788 * used by the device is NO_RESOURCES, so use that here too.
2789 */
2790
2791 if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) ||
2792 produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY)
2793 return VMCI_ERROR_NO_RESOURCES;
2794
2795 retval = vmci_route(&src, &dst, false, &route);
2796 if (retval < VMCI_SUCCESS)
2797 route = vmci_guest_code_active() ?
2798 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2799
2800 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2801 pr_devel("NONBLOCK OR PINNED set");
2802 return VMCI_ERROR_INVALID_ARGS;
2803 }
2804
2805 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2806 if (!my_qpair)
2807 return VMCI_ERROR_NO_MEM;
2808
2809 my_qpair->produce_q_size = produce_qsize;
2810 my_qpair->consume_q_size = consume_qsize;
2811 my_qpair->peer = peer;
2812 my_qpair->flags = flags;
2813 my_qpair->priv_flags = priv_flags;
2814
2815 wakeup_cb = NULL;
2816 client_data = NULL;
2817
2818 if (VMCI_ROUTE_AS_HOST == route) {
2819 my_qpair->guest_endpoint = false;
2820 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2821 my_qpair->blocked = 0;
2822 my_qpair->generation = 0;
2823 init_waitqueue_head(&my_qpair->event);
2824 wakeup_cb = qp_wakeup_cb;
2825 client_data = (void *)my_qpair;
2826 }
2827 } else {
2828 my_qpair->guest_endpoint = true;
2829 }
2830
2831 retval = vmci_qp_alloc(handle,
2832 &my_qpair->produce_q,
2833 my_qpair->produce_q_size,
2834 &my_qpair->consume_q,
2835 my_qpair->consume_q_size,
2836 my_qpair->peer,
2837 my_qpair->flags,
2838 my_qpair->priv_flags,
2839 my_qpair->guest_endpoint,
2840 wakeup_cb, client_data);
2841
2842 if (retval < VMCI_SUCCESS) {
2843 kfree(my_qpair);
2844 return retval;
2845 }
2846
2847 *qpair = my_qpair;
2848 my_qpair->handle = *handle;
2849
2850 return retval;
2851}
2852EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2853
2854/*
2855 * vmci_qpair_detach() - Detatches the client from a queue pair.
2856 * @qpair: Reference of a pointer to the qpair struct.
2857 *
2858 * This is the client interface for detaching from a VMCIQPair.
2859 * Note that this routine will free the memory allocated for the
2860 * vmci_qp structure too.
2861 */
2862int vmci_qpair_detach(struct vmci_qp **qpair)
2863{
2864 int result;
2865 struct vmci_qp *old_qpair;
2866
2867 if (!qpair || !(*qpair))
2868 return VMCI_ERROR_INVALID_ARGS;
2869
2870 old_qpair = *qpair;
2871 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2872
2873 /*
2874 * The guest can fail to detach for a number of reasons, and
2875 * if it does so, it will cleanup the entry (if there is one).
2876 * The host can fail too, but it won't cleanup the entry
2877 * immediately, it will do that later when the context is
2878 * freed. Either way, we need to release the qpair struct
2879 * here; there isn't much the caller can do, and we don't want
2880 * to leak.
2881 */
2882
2883 memset(old_qpair, 0, sizeof(*old_qpair));
2884 old_qpair->handle = VMCI_INVALID_HANDLE;
2885 old_qpair->peer = VMCI_INVALID_ID;
2886 kfree(old_qpair);
2887 *qpair = NULL;
2888
2889 return result;
2890}
2891EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2892
2893/*
2894 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2895 * @qpair: Pointer to the queue pair struct.
2896 * @producer_tail: Reference used for storing producer tail index.
2897 * @consumer_head: Reference used for storing the consumer head index.
2898 *
2899 * This is the client interface for getting the current indexes of the
2900 * QPair from the point of the view of the caller as the producer.
2901 */
2902int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2903 u64 *producer_tail,
2904 u64 *consumer_head)
2905{
2906 struct vmci_queue_header *produce_q_header;
2907 struct vmci_queue_header *consume_q_header;
2908 int result;
2909
2910 if (!qpair)
2911 return VMCI_ERROR_INVALID_ARGS;
2912
2913 qp_lock(qpair);
2914 result =
2915 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2916 if (result == VMCI_SUCCESS)
2917 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2918 producer_tail, consumer_head);
2919 qp_unlock(qpair);
2920
2921 if (result == VMCI_SUCCESS &&
2922 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2923 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2924 return VMCI_ERROR_INVALID_SIZE;
2925
2926 return result;
2927}
2928EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2929
2930/*
2931 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the comsumer.
2932 * @qpair: Pointer to the queue pair struct.
2933 * @consumer_tail: Reference used for storing consumer tail index.
2934 * @producer_head: Reference used for storing the producer head index.
2935 *
2936 * This is the client interface for getting the current indexes of the
2937 * QPair from the point of the view of the caller as the consumer.
2938 */
2939int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2940 u64 *consumer_tail,
2941 u64 *producer_head)
2942{
2943 struct vmci_queue_header *produce_q_header;
2944 struct vmci_queue_header *consume_q_header;
2945 int result;
2946
2947 if (!qpair)
2948 return VMCI_ERROR_INVALID_ARGS;
2949
2950 qp_lock(qpair);
2951 result =
2952 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2953 if (result == VMCI_SUCCESS)
2954 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2955 consumer_tail, producer_head);
2956 qp_unlock(qpair);
2957
2958 if (result == VMCI_SUCCESS &&
2959 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2960 (producer_head && *producer_head >= qpair->consume_q_size)))
2961 return VMCI_ERROR_INVALID_SIZE;
2962
2963 return result;
2964}
2965EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2966
2967/*
2968 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2969 * @qpair: Pointer to the queue pair struct.
2970 *
2971 * This is the client interface for getting the amount of free
2972 * space in the QPair from the point of the view of the caller as
2973 * the producer which is the common case. Returns < 0 if err, else
2974 * available bytes into which data can be enqueued if > 0.
2975 */
2976s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2977{
2978 struct vmci_queue_header *produce_q_header;
2979 struct vmci_queue_header *consume_q_header;
2980 s64 result;
2981
2982 if (!qpair)
2983 return VMCI_ERROR_INVALID_ARGS;
2984
2985 qp_lock(qpair);
2986 result =
2987 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2988 if (result == VMCI_SUCCESS)
2989 result = vmci_q_header_free_space(produce_q_header,
2990 consume_q_header,
2991 qpair->produce_q_size);
2992 else
2993 result = 0;
2994
2995 qp_unlock(qpair);
2996
2997 return result;
2998}
2999EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
3000
3001/*
3002 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
3003 * @qpair: Pointer to the queue pair struct.
3004 *
3005 * This is the client interface for getting the amount of free
3006 * space in the QPair from the point of the view of the caller as
3007 * the consumer which is not the common case. Returns < 0 if err, else
3008 * available bytes into which data can be enqueued if > 0.
3009 */
3010s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
3011{
3012 struct vmci_queue_header *produce_q_header;
3013 struct vmci_queue_header *consume_q_header;
3014 s64 result;
3015
3016 if (!qpair)
3017 return VMCI_ERROR_INVALID_ARGS;
3018
3019 qp_lock(qpair);
3020 result =
3021 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3022 if (result == VMCI_SUCCESS)
3023 result = vmci_q_header_free_space(consume_q_header,
3024 produce_q_header,
3025 qpair->consume_q_size);
3026 else
3027 result = 0;
3028
3029 qp_unlock(qpair);
3030
3031 return result;
3032}
3033EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
3034
3035/*
3036 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
3037 * producer queue.
3038 * @qpair: Pointer to the queue pair struct.
3039 *
3040 * This is the client interface for getting the amount of
3041 * enqueued data in the QPair from the point of the view of the
3042 * caller as the producer which is not the common case. Returns < 0 if err,
3043 * else available bytes that may be read.
3044 */
3045s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
3046{
3047 struct vmci_queue_header *produce_q_header;
3048 struct vmci_queue_header *consume_q_header;
3049 s64 result;
3050
3051 if (!qpair)
3052 return VMCI_ERROR_INVALID_ARGS;
3053
3054 qp_lock(qpair);
3055 result =
3056 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3057 if (result == VMCI_SUCCESS)
3058 result = vmci_q_header_buf_ready(produce_q_header,
3059 consume_q_header,
3060 qpair->produce_q_size);
3061 else
3062 result = 0;
3063
3064 qp_unlock(qpair);
3065
3066 return result;
3067}
3068EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
3069
3070/*
3071 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
3072 * consumer queue.
3073 * @qpair: Pointer to the queue pair struct.
3074 *
3075 * This is the client interface for getting the amount of
3076 * enqueued data in the QPair from the point of the view of the
3077 * caller as the consumer which is the normal case. Returns < 0 if err,
3078 * else available bytes that may be read.
3079 */
3080s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
3081{
3082 struct vmci_queue_header *produce_q_header;
3083 struct vmci_queue_header *consume_q_header;
3084 s64 result;
3085
3086 if (!qpair)
3087 return VMCI_ERROR_INVALID_ARGS;
3088
3089 qp_lock(qpair);
3090 result =
3091 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3092 if (result == VMCI_SUCCESS)
3093 result = vmci_q_header_buf_ready(consume_q_header,
3094 produce_q_header,
3095 qpair->consume_q_size);
3096 else
3097 result = 0;
3098
3099 qp_unlock(qpair);
3100
3101 return result;
3102}
3103EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
3104
3105/*
3106 * vmci_qpair_enqueue() - Throw data on the queue.
3107 * @qpair: Pointer to the queue pair struct.
3108 * @buf: Pointer to buffer containing data
3109 * @buf_size: Length of buffer.
3110 * @buf_type: Buffer type (Unused).
3111 *
3112 * This is the client interface for enqueueing data into the queue.
3113 * Returns number of bytes enqueued or < 0 on error.
3114 */
3115ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3116 const void *buf,
3117 size_t buf_size,
3118 int buf_type)
3119{
3120 ssize_t result;
3121
3122 if (!qpair || !buf)
3123 return VMCI_ERROR_INVALID_ARGS;
3124
3125 qp_lock(qpair);
3126
3127 do {
3128 result = qp_enqueue_locked(qpair->produce_q,
3129 qpair->consume_q,
3130 qpair->produce_q_size,
3131 buf, buf_size,
3132 qp_memcpy_to_queue);
3133
3134 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3135 !qp_wait_for_ready_queue(qpair))
3136 result = VMCI_ERROR_WOULD_BLOCK;
3137
3138 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3139
3140 qp_unlock(qpair);
3141
3142 return result;
3143}
3144EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3145
3146/*
3147 * vmci_qpair_dequeue() - Get data from the queue.
3148 * @qpair: Pointer to the queue pair struct.
3149 * @buf: Pointer to buffer for the data
3150 * @buf_size: Length of buffer.
3151 * @buf_type: Buffer type (Unused).
3152 *
3153 * This is the client interface for dequeueing data from the queue.
3154 * Returns number of bytes dequeued or < 0 on error.
3155 */
3156ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3157 void *buf,
3158 size_t buf_size,
3159 int buf_type)
3160{
3161 ssize_t result;
3162
3163 if (!qpair || !buf)
3164 return VMCI_ERROR_INVALID_ARGS;
3165
3166 qp_lock(qpair);
3167
3168 do {
3169 result = qp_dequeue_locked(qpair->produce_q,
3170 qpair->consume_q,
3171 qpair->consume_q_size,
3172 buf, buf_size,
3173 qp_memcpy_from_queue, true);
3174
3175 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3176 !qp_wait_for_ready_queue(qpair))
3177 result = VMCI_ERROR_WOULD_BLOCK;
3178
3179 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3180
3181 qp_unlock(qpair);
3182
3183 return result;
3184}
3185EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3186
3187/*
3188 * vmci_qpair_peek() - Peek at the data in the queue.
3189 * @qpair: Pointer to the queue pair struct.
3190 * @buf: Pointer to buffer for the data
3191 * @buf_size: Length of buffer.
3192 * @buf_type: Buffer type (Unused on Linux).
3193 *
3194 * This is the client interface for peeking into a queue. (I.e.,
3195 * copy data from the queue without updating the head pointer.)
3196 * Returns number of bytes dequeued or < 0 on error.
3197 */
3198ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3199 void *buf,
3200 size_t buf_size,
3201 int buf_type)
3202{
3203 ssize_t result;
3204
3205 if (!qpair || !buf)
3206 return VMCI_ERROR_INVALID_ARGS;
3207
3208 qp_lock(qpair);
3209
3210 do {
3211 result = qp_dequeue_locked(qpair->produce_q,
3212 qpair->consume_q,
3213 qpair->consume_q_size,
3214 buf, buf_size,
3215 qp_memcpy_from_queue, false);
3216
3217 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3218 !qp_wait_for_ready_queue(qpair))
3219 result = VMCI_ERROR_WOULD_BLOCK;
3220
3221 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3222
3223 qp_unlock(qpair);
3224
3225 return result;
3226}
3227EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3228
3229/*
3230 * vmci_qpair_enquev() - Throw data on the queue using iov.
3231 * @qpair: Pointer to the queue pair struct.
3232 * @iov: Pointer to buffer containing data
3233 * @iov_size: Length of buffer.
3234 * @buf_type: Buffer type (Unused).
3235 *
3236 * This is the client interface for enqueueing data into the queue.
3237 * This function uses IO vectors to handle the work. Returns number
3238 * of bytes enqueued or < 0 on error.
3239 */
3240ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3241 struct msghdr *msg,
3242 size_t iov_size,
3243 int buf_type)
3244{
3245 ssize_t result;
3246
3247 if (!qpair)
3248 return VMCI_ERROR_INVALID_ARGS;
3249
3250 qp_lock(qpair);
3251
3252 do {
3253 result = qp_enqueue_locked(qpair->produce_q,
3254 qpair->consume_q,
3255 qpair->produce_q_size,
3256 msg, iov_size,
3257 qp_memcpy_to_queue_iov);
3258
3259 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3260 !qp_wait_for_ready_queue(qpair))
3261 result = VMCI_ERROR_WOULD_BLOCK;
3262
3263 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3264
3265 qp_unlock(qpair);
3266
3267 return result;
3268}
3269EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3270
3271/*
3272 * vmci_qpair_dequev() - Get data from the queue using iov.
3273 * @qpair: Pointer to the queue pair struct.
3274 * @iov: Pointer to buffer for the data
3275 * @iov_size: Length of buffer.
3276 * @buf_type: Buffer type (Unused).
3277 *
3278 * This is the client interface for dequeueing data from the queue.
3279 * This function uses IO vectors to handle the work. Returns number
3280 * of bytes dequeued or < 0 on error.
3281 */
3282ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3283 struct msghdr *msg,
3284 size_t iov_size,
3285 int buf_type)
3286{
3287 ssize_t result;
3288
3289 if (!qpair)
3290 return VMCI_ERROR_INVALID_ARGS;
3291
3292 qp_lock(qpair);
3293
3294 do {
3295 result = qp_dequeue_locked(qpair->produce_q,
3296 qpair->consume_q,
3297 qpair->consume_q_size,
3298 msg, iov_size,
3299 qp_memcpy_from_queue_iov,
3300 true);
3301
3302 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3303 !qp_wait_for_ready_queue(qpair))
3304 result = VMCI_ERROR_WOULD_BLOCK;
3305
3306 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3307
3308 qp_unlock(qpair);
3309
3310 return result;
3311}
3312EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3313
3314/*
3315 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3316 * @qpair: Pointer to the queue pair struct.
3317 * @iov: Pointer to buffer for the data
3318 * @iov_size: Length of buffer.
3319 * @buf_type: Buffer type (Unused on Linux).
3320 *
3321 * This is the client interface for peeking into a queue. (I.e.,
3322 * copy data from the queue without updating the head pointer.)
3323 * This function uses IO vectors to handle the work. Returns number
3324 * of bytes peeked or < 0 on error.
3325 */
3326ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3327 struct msghdr *msg,
3328 size_t iov_size,
3329 int buf_type)
3330{
3331 ssize_t result;
3332
3333 if (!qpair)
3334 return VMCI_ERROR_INVALID_ARGS;
3335
3336 qp_lock(qpair);
3337
3338 do {
3339 result = qp_dequeue_locked(qpair->produce_q,
3340 qpair->consume_q,
3341 qpair->consume_q_size,
3342 msg, iov_size,
3343 qp_memcpy_from_queue_iov,
3344 false);
3345
3346 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3347 !qp_wait_for_ready_queue(qpair))
3348 result = VMCI_ERROR_WOULD_BLOCK;
3349
3350 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3351
3352 qp_unlock(qpair);
3353 return result;
3354}
3355EXPORT_SYMBOL_GPL(vmci_qpair_peekv);