Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VMware VMCI Driver
4 *
5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 */
7
8#include <linux/vmw_vmci_defs.h>
9#include <linux/vmw_vmci_api.h>
10#include <linux/highmem.h>
11#include <linux/kernel.h>
12#include <linux/mm.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/pagemap.h>
16#include <linux/pci.h>
17#include <linux/sched.h>
18#include <linux/slab.h>
19#include <linux/uio.h>
20#include <linux/wait.h>
21#include <linux/vmalloc.h>
22#include <linux/skbuff.h>
23
24#include "vmci_handle_array.h"
25#include "vmci_queue_pair.h"
26#include "vmci_datagram.h"
27#include "vmci_resource.h"
28#include "vmci_context.h"
29#include "vmci_driver.h"
30#include "vmci_event.h"
31#include "vmci_route.h"
32
33/*
34 * In the following, we will distinguish between two kinds of VMX processes -
35 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
36 * VMCI page files in the VMX and supporting VM to VM communication and the
37 * newer ones that use the guest memory directly. We will in the following
38 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
39 * new-style VMX'en.
40 *
41 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
42 * removed for readability) - see below for more details on the transtions:
43 *
44 * -------------- NEW -------------
45 * | |
46 * \_/ \_/
47 * CREATED_NO_MEM <-----------------> CREATED_MEM
48 * | | |
49 * | o-----------------------o |
50 * | | |
51 * \_/ \_/ \_/
52 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
53 * | | |
54 * | o----------------------o |
55 * | | |
56 * \_/ \_/ \_/
57 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
58 * | |
59 * | |
60 * -------------> gone <-------------
61 *
62 * In more detail. When a VMCI queue pair is first created, it will be in the
63 * VMCIQPB_NEW state. It will then move into one of the following states:
64 *
65 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
66 *
67 * - the created was performed by a host endpoint, in which case there is
68 * no backing memory yet.
69 *
70 * - the create was initiated by an old-style VMX, that uses
71 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
72 * a later point in time. This state can be distinguished from the one
73 * above by the context ID of the creator. A host side is not allowed to
74 * attach until the page store has been set.
75 *
76 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
77 * is created by a VMX using the queue pair device backend that
78 * sets the UVAs of the queue pair immediately and stores the
79 * information for later attachers. At this point, it is ready for
80 * the host side to attach to it.
81 *
82 * Once the queue pair is in one of the created states (with the exception of
83 * the case mentioned for older VMX'en above), it is possible to attach to the
84 * queue pair. Again we have two new states possible:
85 *
86 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
87 * paths:
88 *
89 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
90 * pair, and attaches to a queue pair previously created by the host side.
91 *
92 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
93 * already created by a guest.
94 *
95 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
96 * vmci_qp_broker_set_page_store (see below).
97 *
98 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
99 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
100 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
101 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
102 * will be entered.
103 *
104 * From the attached queue pair, the queue pair can enter the shutdown states
105 * when either side of the queue pair detaches. If the guest side detaches
106 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
107 * the content of the queue pair will no longer be available. If the host
108 * side detaches first, the queue pair will either enter the
109 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
110 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
111 * (e.g., the host detaches while a guest is stunned).
112 *
113 * New-style VMX'en will also unmap guest memory, if the guest is
114 * quiesced, e.g., during a snapshot operation. In that case, the guest
115 * memory will no longer be available, and the queue pair will transition from
116 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
117 * in which case the queue pair will transition from the *_NO_MEM state at that
118 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
119 * since the peer may have either attached or detached in the meantime. The
120 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
121 * *_MEM state, and vice versa.
122 */
123
124/* The Kernel specific component of the struct vmci_queue structure. */
125struct vmci_queue_kern_if {
126 struct mutex __mutex; /* Protects the queue. */
127 struct mutex *mutex; /* Shared by producer and consumer queues. */
128 size_t num_pages; /* Number of pages incl. header. */
129 bool host; /* Host or guest? */
130 union {
131 struct {
132 dma_addr_t *pas;
133 void **vas;
134 } g; /* Used by the guest. */
135 struct {
136 struct page **page;
137 struct page **header_page;
138 } h; /* Used by the host. */
139 } u;
140};
141
142/*
143 * This structure is opaque to the clients.
144 */
145struct vmci_qp {
146 struct vmci_handle handle;
147 struct vmci_queue *produce_q;
148 struct vmci_queue *consume_q;
149 u64 produce_q_size;
150 u64 consume_q_size;
151 u32 peer;
152 u32 flags;
153 u32 priv_flags;
154 bool guest_endpoint;
155 unsigned int blocked;
156 unsigned int generation;
157 wait_queue_head_t event;
158};
159
160enum qp_broker_state {
161 VMCIQPB_NEW,
162 VMCIQPB_CREATED_NO_MEM,
163 VMCIQPB_CREATED_MEM,
164 VMCIQPB_ATTACHED_NO_MEM,
165 VMCIQPB_ATTACHED_MEM,
166 VMCIQPB_SHUTDOWN_NO_MEM,
167 VMCIQPB_SHUTDOWN_MEM,
168 VMCIQPB_GONE
169};
170
171#define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
172 _qpb->state == VMCIQPB_ATTACHED_MEM || \
173 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
174
175/*
176 * In the queue pair broker, we always use the guest point of view for
177 * the produce and consume queue values and references, e.g., the
178 * produce queue size stored is the guests produce queue size. The
179 * host endpoint will need to swap these around. The only exception is
180 * the local queue pairs on the host, in which case the host endpoint
181 * that creates the queue pair will have the right orientation, and
182 * the attaching host endpoint will need to swap.
183 */
184struct qp_entry {
185 struct list_head list_item;
186 struct vmci_handle handle;
187 u32 peer;
188 u32 flags;
189 u64 produce_size;
190 u64 consume_size;
191 u32 ref_count;
192};
193
194struct qp_broker_entry {
195 struct vmci_resource resource;
196 struct qp_entry qp;
197 u32 create_id;
198 u32 attach_id;
199 enum qp_broker_state state;
200 bool require_trusted_attach;
201 bool created_by_trusted;
202 bool vmci_page_files; /* Created by VMX using VMCI page files */
203 struct vmci_queue *produce_q;
204 struct vmci_queue *consume_q;
205 struct vmci_queue_header saved_produce_q;
206 struct vmci_queue_header saved_consume_q;
207 vmci_event_release_cb wakeup_cb;
208 void *client_data;
209 void *local_mem; /* Kernel memory for local queue pair */
210};
211
212struct qp_guest_endpoint {
213 struct vmci_resource resource;
214 struct qp_entry qp;
215 u64 num_ppns;
216 void *produce_q;
217 void *consume_q;
218 struct ppn_set ppn_set;
219};
220
221struct qp_list {
222 struct list_head head;
223 struct mutex mutex; /* Protect queue list. */
224};
225
226static struct qp_list qp_broker_list = {
227 .head = LIST_HEAD_INIT(qp_broker_list.head),
228 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
229};
230
231static struct qp_list qp_guest_endpoints = {
232 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
233 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
234};
235
236#define INVALID_VMCI_GUEST_MEM_ID 0
237#define QPE_NUM_PAGES(_QPE) ((u32) \
238 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
239 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
240
241
242/*
243 * Frees kernel VA space for a given queue and its queue header, and
244 * frees physical data pages.
245 */
246static void qp_free_queue(void *q, u64 size)
247{
248 struct vmci_queue *queue = q;
249
250 if (queue) {
251 u64 i;
252
253 /* Given size does not include header, so add in a page here. */
254 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
255 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
256 queue->kernel_if->u.g.vas[i],
257 queue->kernel_if->u.g.pas[i]);
258 }
259
260 vfree(queue);
261 }
262}
263
264/*
265 * Allocates kernel queue pages of specified size with IOMMU mappings,
266 * plus space for the queue structure/kernel interface and the queue
267 * header.
268 */
269static void *qp_alloc_queue(u64 size, u32 flags)
270{
271 u64 i;
272 struct vmci_queue *queue;
273 size_t pas_size;
274 size_t vas_size;
275 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
276 u64 num_pages;
277
278 if (size > SIZE_MAX - PAGE_SIZE)
279 return NULL;
280 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
281 if (num_pages >
282 (SIZE_MAX - queue_size) /
283 (sizeof(*queue->kernel_if->u.g.pas) +
284 sizeof(*queue->kernel_if->u.g.vas)))
285 return NULL;
286
287 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
288 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
289 queue_size += pas_size + vas_size;
290
291 queue = vmalloc(queue_size);
292 if (!queue)
293 return NULL;
294
295 queue->q_header = NULL;
296 queue->saved_header = NULL;
297 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
298 queue->kernel_if->mutex = NULL;
299 queue->kernel_if->num_pages = num_pages;
300 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
301 queue->kernel_if->u.g.vas =
302 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
303 queue->kernel_if->host = false;
304
305 for (i = 0; i < num_pages; i++) {
306 queue->kernel_if->u.g.vas[i] =
307 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
308 &queue->kernel_if->u.g.pas[i],
309 GFP_KERNEL);
310 if (!queue->kernel_if->u.g.vas[i]) {
311 /* Size excl. the header. */
312 qp_free_queue(queue, i * PAGE_SIZE);
313 return NULL;
314 }
315 }
316
317 /* Queue header is the first page. */
318 queue->q_header = queue->kernel_if->u.g.vas[0];
319
320 return queue;
321}
322
323/*
324 * Copies from a given buffer or iovector to a VMCI Queue. Uses
325 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
326 * by traversing the offset -> page translation structure for the queue.
327 * Assumes that offset + size does not wrap around in the queue.
328 */
329static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
330 u64 queue_offset,
331 struct iov_iter *from,
332 size_t size)
333{
334 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
335 size_t bytes_copied = 0;
336
337 while (bytes_copied < size) {
338 const u64 page_index =
339 (queue_offset + bytes_copied) / PAGE_SIZE;
340 const size_t page_offset =
341 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
342 void *va;
343 size_t to_copy;
344
345 if (kernel_if->host)
346 va = kmap(kernel_if->u.h.page[page_index]);
347 else
348 va = kernel_if->u.g.vas[page_index + 1];
349 /* Skip header. */
350
351 if (size - bytes_copied > PAGE_SIZE - page_offset)
352 /* Enough payload to fill up from this page. */
353 to_copy = PAGE_SIZE - page_offset;
354 else
355 to_copy = size - bytes_copied;
356
357 if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
358 from)) {
359 if (kernel_if->host)
360 kunmap(kernel_if->u.h.page[page_index]);
361 return VMCI_ERROR_INVALID_ARGS;
362 }
363 bytes_copied += to_copy;
364 if (kernel_if->host)
365 kunmap(kernel_if->u.h.page[page_index]);
366 }
367
368 return VMCI_SUCCESS;
369}
370
371/*
372 * Copies to a given buffer or iovector from a VMCI Queue. Uses
373 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
374 * by traversing the offset -> page translation structure for the queue.
375 * Assumes that offset + size does not wrap around in the queue.
376 */
377static int qp_memcpy_from_queue_iter(struct iov_iter *to,
378 const struct vmci_queue *queue,
379 u64 queue_offset, size_t size)
380{
381 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
382 size_t bytes_copied = 0;
383
384 while (bytes_copied < size) {
385 const u64 page_index =
386 (queue_offset + bytes_copied) / PAGE_SIZE;
387 const size_t page_offset =
388 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
389 void *va;
390 size_t to_copy;
391 int err;
392
393 if (kernel_if->host)
394 va = kmap(kernel_if->u.h.page[page_index]);
395 else
396 va = kernel_if->u.g.vas[page_index + 1];
397 /* Skip header. */
398
399 if (size - bytes_copied > PAGE_SIZE - page_offset)
400 /* Enough payload to fill up this page. */
401 to_copy = PAGE_SIZE - page_offset;
402 else
403 to_copy = size - bytes_copied;
404
405 err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
406 if (err != to_copy) {
407 if (kernel_if->host)
408 kunmap(kernel_if->u.h.page[page_index]);
409 return VMCI_ERROR_INVALID_ARGS;
410 }
411 bytes_copied += to_copy;
412 if (kernel_if->host)
413 kunmap(kernel_if->u.h.page[page_index]);
414 }
415
416 return VMCI_SUCCESS;
417}
418
419/*
420 * Allocates two list of PPNs --- one for the pages in the produce queue,
421 * and the other for the pages in the consume queue. Intializes the list
422 * of PPNs with the page frame numbers of the KVA for the two queues (and
423 * the queue headers).
424 */
425static int qp_alloc_ppn_set(void *prod_q,
426 u64 num_produce_pages,
427 void *cons_q,
428 u64 num_consume_pages, struct ppn_set *ppn_set)
429{
430 u64 *produce_ppns;
431 u64 *consume_ppns;
432 struct vmci_queue *produce_q = prod_q;
433 struct vmci_queue *consume_q = cons_q;
434 u64 i;
435
436 if (!produce_q || !num_produce_pages || !consume_q ||
437 !num_consume_pages || !ppn_set)
438 return VMCI_ERROR_INVALID_ARGS;
439
440 if (ppn_set->initialized)
441 return VMCI_ERROR_ALREADY_EXISTS;
442
443 produce_ppns =
444 kmalloc_array(num_produce_pages, sizeof(*produce_ppns),
445 GFP_KERNEL);
446 if (!produce_ppns)
447 return VMCI_ERROR_NO_MEM;
448
449 consume_ppns =
450 kmalloc_array(num_consume_pages, sizeof(*consume_ppns),
451 GFP_KERNEL);
452 if (!consume_ppns) {
453 kfree(produce_ppns);
454 return VMCI_ERROR_NO_MEM;
455 }
456
457 for (i = 0; i < num_produce_pages; i++)
458 produce_ppns[i] =
459 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
460
461 for (i = 0; i < num_consume_pages; i++)
462 consume_ppns[i] =
463 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
464
465 ppn_set->num_produce_pages = num_produce_pages;
466 ppn_set->num_consume_pages = num_consume_pages;
467 ppn_set->produce_ppns = produce_ppns;
468 ppn_set->consume_ppns = consume_ppns;
469 ppn_set->initialized = true;
470 return VMCI_SUCCESS;
471}
472
473/*
474 * Frees the two list of PPNs for a queue pair.
475 */
476static void qp_free_ppn_set(struct ppn_set *ppn_set)
477{
478 if (ppn_set->initialized) {
479 /* Do not call these functions on NULL inputs. */
480 kfree(ppn_set->produce_ppns);
481 kfree(ppn_set->consume_ppns);
482 }
483 memset(ppn_set, 0, sizeof(*ppn_set));
484}
485
486/*
487 * Populates the list of PPNs in the hypercall structure with the PPNS
488 * of the produce queue and the consume queue.
489 */
490static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
491{
492 if (vmci_use_ppn64()) {
493 memcpy(call_buf, ppn_set->produce_ppns,
494 ppn_set->num_produce_pages *
495 sizeof(*ppn_set->produce_ppns));
496 memcpy(call_buf +
497 ppn_set->num_produce_pages *
498 sizeof(*ppn_set->produce_ppns),
499 ppn_set->consume_ppns,
500 ppn_set->num_consume_pages *
501 sizeof(*ppn_set->consume_ppns));
502 } else {
503 int i;
504 u32 *ppns = (u32 *) call_buf;
505
506 for (i = 0; i < ppn_set->num_produce_pages; i++)
507 ppns[i] = (u32) ppn_set->produce_ppns[i];
508
509 ppns = &ppns[ppn_set->num_produce_pages];
510
511 for (i = 0; i < ppn_set->num_consume_pages; i++)
512 ppns[i] = (u32) ppn_set->consume_ppns[i];
513 }
514
515 return VMCI_SUCCESS;
516}
517
518/*
519 * Allocates kernel VA space of specified size plus space for the queue
520 * and kernel interface. This is different from the guest queue allocator,
521 * because we do not allocate our own queue header/data pages here but
522 * share those of the guest.
523 */
524static struct vmci_queue *qp_host_alloc_queue(u64 size)
525{
526 struct vmci_queue *queue;
527 size_t queue_page_size;
528 u64 num_pages;
529 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
530
531 if (size > SIZE_MAX - PAGE_SIZE)
532 return NULL;
533 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
534 if (num_pages > (SIZE_MAX - queue_size) /
535 sizeof(*queue->kernel_if->u.h.page))
536 return NULL;
537
538 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
539
540 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
541 if (queue) {
542 queue->q_header = NULL;
543 queue->saved_header = NULL;
544 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
545 queue->kernel_if->host = true;
546 queue->kernel_if->mutex = NULL;
547 queue->kernel_if->num_pages = num_pages;
548 queue->kernel_if->u.h.header_page =
549 (struct page **)((u8 *)queue + queue_size);
550 queue->kernel_if->u.h.page =
551 &queue->kernel_if->u.h.header_page[1];
552 }
553
554 return queue;
555}
556
557/*
558 * Frees kernel memory for a given queue (header plus translation
559 * structure).
560 */
561static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
562{
563 kfree(queue);
564}
565
566/*
567 * Initialize the mutex for the pair of queues. This mutex is used to
568 * protect the q_header and the buffer from changing out from under any
569 * users of either queue. Of course, it's only any good if the mutexes
570 * are actually acquired. Queue structure must lie on non-paged memory
571 * or we cannot guarantee access to the mutex.
572 */
573static void qp_init_queue_mutex(struct vmci_queue *produce_q,
574 struct vmci_queue *consume_q)
575{
576 /*
577 * Only the host queue has shared state - the guest queues do not
578 * need to synchronize access using a queue mutex.
579 */
580
581 if (produce_q->kernel_if->host) {
582 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
583 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
584 mutex_init(produce_q->kernel_if->mutex);
585 }
586}
587
588/*
589 * Cleans up the mutex for the pair of queues.
590 */
591static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
592 struct vmci_queue *consume_q)
593{
594 if (produce_q->kernel_if->host) {
595 produce_q->kernel_if->mutex = NULL;
596 consume_q->kernel_if->mutex = NULL;
597 }
598}
599
600/*
601 * Acquire the mutex for the queue. Note that the produce_q and
602 * the consume_q share a mutex. So, only one of the two need to
603 * be passed in to this routine. Either will work just fine.
604 */
605static void qp_acquire_queue_mutex(struct vmci_queue *queue)
606{
607 if (queue->kernel_if->host)
608 mutex_lock(queue->kernel_if->mutex);
609}
610
611/*
612 * Release the mutex for the queue. Note that the produce_q and
613 * the consume_q share a mutex. So, only one of the two need to
614 * be passed in to this routine. Either will work just fine.
615 */
616static void qp_release_queue_mutex(struct vmci_queue *queue)
617{
618 if (queue->kernel_if->host)
619 mutex_unlock(queue->kernel_if->mutex);
620}
621
622/*
623 * Helper function to release pages in the PageStoreAttachInfo
624 * previously obtained using get_user_pages.
625 */
626static void qp_release_pages(struct page **pages,
627 u64 num_pages, bool dirty)
628{
629 int i;
630
631 for (i = 0; i < num_pages; i++) {
632 if (dirty)
633 set_page_dirty(pages[i]);
634
635 put_page(pages[i]);
636 pages[i] = NULL;
637 }
638}
639
640/*
641 * Lock the user pages referenced by the {produce,consume}Buffer
642 * struct into memory and populate the {produce,consume}Pages
643 * arrays in the attach structure with them.
644 */
645static int qp_host_get_user_memory(u64 produce_uva,
646 u64 consume_uva,
647 struct vmci_queue *produce_q,
648 struct vmci_queue *consume_q)
649{
650 int retval;
651 int err = VMCI_SUCCESS;
652
653 retval = get_user_pages_fast((uintptr_t) produce_uva,
654 produce_q->kernel_if->num_pages,
655 FOLL_WRITE,
656 produce_q->kernel_if->u.h.header_page);
657 if (retval < (int)produce_q->kernel_if->num_pages) {
658 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
659 retval);
660 qp_release_pages(produce_q->kernel_if->u.h.header_page,
661 retval, false);
662 err = VMCI_ERROR_NO_MEM;
663 goto out;
664 }
665
666 retval = get_user_pages_fast((uintptr_t) consume_uva,
667 consume_q->kernel_if->num_pages,
668 FOLL_WRITE,
669 consume_q->kernel_if->u.h.header_page);
670 if (retval < (int)consume_q->kernel_if->num_pages) {
671 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
672 retval);
673 qp_release_pages(consume_q->kernel_if->u.h.header_page,
674 retval, false);
675 qp_release_pages(produce_q->kernel_if->u.h.header_page,
676 produce_q->kernel_if->num_pages, false);
677 err = VMCI_ERROR_NO_MEM;
678 }
679
680 out:
681 return err;
682}
683
684/*
685 * Registers the specification of the user pages used for backing a queue
686 * pair. Enough information to map in pages is stored in the OS specific
687 * part of the struct vmci_queue structure.
688 */
689static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
690 struct vmci_queue *produce_q,
691 struct vmci_queue *consume_q)
692{
693 u64 produce_uva;
694 u64 consume_uva;
695
696 /*
697 * The new style and the old style mapping only differs in
698 * that we either get a single or two UVAs, so we split the
699 * single UVA range at the appropriate spot.
700 */
701 produce_uva = page_store->pages;
702 consume_uva = page_store->pages +
703 produce_q->kernel_if->num_pages * PAGE_SIZE;
704 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
705 consume_q);
706}
707
708/*
709 * Releases and removes the references to user pages stored in the attach
710 * struct. Pages are released from the page cache and may become
711 * swappable again.
712 */
713static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
714 struct vmci_queue *consume_q)
715{
716 qp_release_pages(produce_q->kernel_if->u.h.header_page,
717 produce_q->kernel_if->num_pages, true);
718 memset(produce_q->kernel_if->u.h.header_page, 0,
719 sizeof(*produce_q->kernel_if->u.h.header_page) *
720 produce_q->kernel_if->num_pages);
721 qp_release_pages(consume_q->kernel_if->u.h.header_page,
722 consume_q->kernel_if->num_pages, true);
723 memset(consume_q->kernel_if->u.h.header_page, 0,
724 sizeof(*consume_q->kernel_if->u.h.header_page) *
725 consume_q->kernel_if->num_pages);
726}
727
728/*
729 * Once qp_host_register_user_memory has been performed on a
730 * queue, the queue pair headers can be mapped into the
731 * kernel. Once mapped, they must be unmapped with
732 * qp_host_unmap_queues prior to calling
733 * qp_host_unregister_user_memory.
734 * Pages are pinned.
735 */
736static int qp_host_map_queues(struct vmci_queue *produce_q,
737 struct vmci_queue *consume_q)
738{
739 int result;
740
741 if (!produce_q->q_header || !consume_q->q_header) {
742 struct page *headers[2];
743
744 if (produce_q->q_header != consume_q->q_header)
745 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
746
747 if (produce_q->kernel_if->u.h.header_page == NULL ||
748 *produce_q->kernel_if->u.h.header_page == NULL)
749 return VMCI_ERROR_UNAVAILABLE;
750
751 headers[0] = *produce_q->kernel_if->u.h.header_page;
752 headers[1] = *consume_q->kernel_if->u.h.header_page;
753
754 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
755 if (produce_q->q_header != NULL) {
756 consume_q->q_header =
757 (struct vmci_queue_header *)((u8 *)
758 produce_q->q_header +
759 PAGE_SIZE);
760 result = VMCI_SUCCESS;
761 } else {
762 pr_warn("vmap failed\n");
763 result = VMCI_ERROR_NO_MEM;
764 }
765 } else {
766 result = VMCI_SUCCESS;
767 }
768
769 return result;
770}
771
772/*
773 * Unmaps previously mapped queue pair headers from the kernel.
774 * Pages are unpinned.
775 */
776static int qp_host_unmap_queues(u32 gid,
777 struct vmci_queue *produce_q,
778 struct vmci_queue *consume_q)
779{
780 if (produce_q->q_header) {
781 if (produce_q->q_header < consume_q->q_header)
782 vunmap(produce_q->q_header);
783 else
784 vunmap(consume_q->q_header);
785
786 produce_q->q_header = NULL;
787 consume_q->q_header = NULL;
788 }
789
790 return VMCI_SUCCESS;
791}
792
793/*
794 * Finds the entry in the list corresponding to a given handle. Assumes
795 * that the list is locked.
796 */
797static struct qp_entry *qp_list_find(struct qp_list *qp_list,
798 struct vmci_handle handle)
799{
800 struct qp_entry *entry;
801
802 if (vmci_handle_is_invalid(handle))
803 return NULL;
804
805 list_for_each_entry(entry, &qp_list->head, list_item) {
806 if (vmci_handle_is_equal(entry->handle, handle))
807 return entry;
808 }
809
810 return NULL;
811}
812
813/*
814 * Finds the entry in the list corresponding to a given handle.
815 */
816static struct qp_guest_endpoint *
817qp_guest_handle_to_entry(struct vmci_handle handle)
818{
819 struct qp_guest_endpoint *entry;
820 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
821
822 entry = qp ? container_of(
823 qp, struct qp_guest_endpoint, qp) : NULL;
824 return entry;
825}
826
827/*
828 * Finds the entry in the list corresponding to a given handle.
829 */
830static struct qp_broker_entry *
831qp_broker_handle_to_entry(struct vmci_handle handle)
832{
833 struct qp_broker_entry *entry;
834 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
835
836 entry = qp ? container_of(
837 qp, struct qp_broker_entry, qp) : NULL;
838 return entry;
839}
840
841/*
842 * Dispatches a queue pair event message directly into the local event
843 * queue.
844 */
845static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
846{
847 u32 context_id = vmci_get_context_id();
848 struct vmci_event_qp ev;
849
850 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
851 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
852 VMCI_CONTEXT_RESOURCE_ID);
853 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
854 ev.msg.event_data.event =
855 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
856 ev.payload.peer_id = context_id;
857 ev.payload.handle = handle;
858
859 return vmci_event_dispatch(&ev.msg.hdr);
860}
861
862/*
863 * Allocates and initializes a qp_guest_endpoint structure.
864 * Allocates a queue_pair rid (and handle) iff the given entry has
865 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
866 * are reserved handles. Assumes that the QP list mutex is held
867 * by the caller.
868 */
869static struct qp_guest_endpoint *
870qp_guest_endpoint_create(struct vmci_handle handle,
871 u32 peer,
872 u32 flags,
873 u64 produce_size,
874 u64 consume_size,
875 void *produce_q,
876 void *consume_q)
877{
878 int result;
879 struct qp_guest_endpoint *entry;
880 /* One page each for the queue headers. */
881 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
882 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
883
884 if (vmci_handle_is_invalid(handle)) {
885 u32 context_id = vmci_get_context_id();
886
887 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
888 }
889
890 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
891 if (entry) {
892 entry->qp.peer = peer;
893 entry->qp.flags = flags;
894 entry->qp.produce_size = produce_size;
895 entry->qp.consume_size = consume_size;
896 entry->qp.ref_count = 0;
897 entry->num_ppns = num_ppns;
898 entry->produce_q = produce_q;
899 entry->consume_q = consume_q;
900 INIT_LIST_HEAD(&entry->qp.list_item);
901
902 /* Add resource obj */
903 result = vmci_resource_add(&entry->resource,
904 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
905 handle);
906 entry->qp.handle = vmci_resource_handle(&entry->resource);
907 if ((result != VMCI_SUCCESS) ||
908 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
909 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
910 handle.context, handle.resource, result);
911 kfree(entry);
912 entry = NULL;
913 }
914 }
915 return entry;
916}
917
918/*
919 * Frees a qp_guest_endpoint structure.
920 */
921static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
922{
923 qp_free_ppn_set(&entry->ppn_set);
924 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
925 qp_free_queue(entry->produce_q, entry->qp.produce_size);
926 qp_free_queue(entry->consume_q, entry->qp.consume_size);
927 /* Unlink from resource hash table and free callback */
928 vmci_resource_remove(&entry->resource);
929
930 kfree(entry);
931}
932
933/*
934 * Helper to make a queue_pairAlloc hypercall when the driver is
935 * supporting a guest device.
936 */
937static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
938{
939 struct vmci_qp_alloc_msg *alloc_msg;
940 size_t msg_size;
941 size_t ppn_size;
942 int result;
943
944 if (!entry || entry->num_ppns <= 2)
945 return VMCI_ERROR_INVALID_ARGS;
946
947 ppn_size = vmci_use_ppn64() ? sizeof(u64) : sizeof(u32);
948 msg_size = sizeof(*alloc_msg) +
949 (size_t) entry->num_ppns * ppn_size;
950 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
951 if (!alloc_msg)
952 return VMCI_ERROR_NO_MEM;
953
954 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
955 VMCI_QUEUEPAIR_ALLOC);
956 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
957 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
958 alloc_msg->handle = entry->qp.handle;
959 alloc_msg->peer = entry->qp.peer;
960 alloc_msg->flags = entry->qp.flags;
961 alloc_msg->produce_size = entry->qp.produce_size;
962 alloc_msg->consume_size = entry->qp.consume_size;
963 alloc_msg->num_ppns = entry->num_ppns;
964
965 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
966 &entry->ppn_set);
967 if (result == VMCI_SUCCESS)
968 result = vmci_send_datagram(&alloc_msg->hdr);
969
970 kfree(alloc_msg);
971
972 return result;
973}
974
975/*
976 * Helper to make a queue_pairDetach hypercall when the driver is
977 * supporting a guest device.
978 */
979static int qp_detatch_hypercall(struct vmci_handle handle)
980{
981 struct vmci_qp_detach_msg detach_msg;
982
983 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
984 VMCI_QUEUEPAIR_DETACH);
985 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
986 detach_msg.hdr.payload_size = sizeof(handle);
987 detach_msg.handle = handle;
988
989 return vmci_send_datagram(&detach_msg.hdr);
990}
991
992/*
993 * Adds the given entry to the list. Assumes that the list is locked.
994 */
995static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
996{
997 if (entry)
998 list_add(&entry->list_item, &qp_list->head);
999}
1000
1001/*
1002 * Removes the given entry from the list. Assumes that the list is locked.
1003 */
1004static void qp_list_remove_entry(struct qp_list *qp_list,
1005 struct qp_entry *entry)
1006{
1007 if (entry)
1008 list_del(&entry->list_item);
1009}
1010
1011/*
1012 * Helper for VMCI queue_pair detach interface. Frees the physical
1013 * pages for the queue pair.
1014 */
1015static int qp_detatch_guest_work(struct vmci_handle handle)
1016{
1017 int result;
1018 struct qp_guest_endpoint *entry;
1019 u32 ref_count = ~0; /* To avoid compiler warning below */
1020
1021 mutex_lock(&qp_guest_endpoints.mutex);
1022
1023 entry = qp_guest_handle_to_entry(handle);
1024 if (!entry) {
1025 mutex_unlock(&qp_guest_endpoints.mutex);
1026 return VMCI_ERROR_NOT_FOUND;
1027 }
1028
1029 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1030 result = VMCI_SUCCESS;
1031
1032 if (entry->qp.ref_count > 1) {
1033 result = qp_notify_peer_local(false, handle);
1034 /*
1035 * We can fail to notify a local queuepair
1036 * because we can't allocate. We still want
1037 * to release the entry if that happens, so
1038 * don't bail out yet.
1039 */
1040 }
1041 } else {
1042 result = qp_detatch_hypercall(handle);
1043 if (result < VMCI_SUCCESS) {
1044 /*
1045 * We failed to notify a non-local queuepair.
1046 * That other queuepair might still be
1047 * accessing the shared memory, so don't
1048 * release the entry yet. It will get cleaned
1049 * up by VMCIqueue_pair_Exit() if necessary
1050 * (assuming we are going away, otherwise why
1051 * did this fail?).
1052 */
1053
1054 mutex_unlock(&qp_guest_endpoints.mutex);
1055 return result;
1056 }
1057 }
1058
1059 /*
1060 * If we get here then we either failed to notify a local queuepair, or
1061 * we succeeded in all cases. Release the entry if required.
1062 */
1063
1064 entry->qp.ref_count--;
1065 if (entry->qp.ref_count == 0)
1066 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1067
1068 /* If we didn't remove the entry, this could change once we unlock. */
1069 if (entry)
1070 ref_count = entry->qp.ref_count;
1071
1072 mutex_unlock(&qp_guest_endpoints.mutex);
1073
1074 if (ref_count == 0)
1075 qp_guest_endpoint_destroy(entry);
1076
1077 return result;
1078}
1079
1080/*
1081 * This functions handles the actual allocation of a VMCI queue
1082 * pair guest endpoint. Allocates physical pages for the queue
1083 * pair. It makes OS dependent calls through generic wrappers.
1084 */
1085static int qp_alloc_guest_work(struct vmci_handle *handle,
1086 struct vmci_queue **produce_q,
1087 u64 produce_size,
1088 struct vmci_queue **consume_q,
1089 u64 consume_size,
1090 u32 peer,
1091 u32 flags,
1092 u32 priv_flags)
1093{
1094 const u64 num_produce_pages =
1095 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1096 const u64 num_consume_pages =
1097 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1098 void *my_produce_q = NULL;
1099 void *my_consume_q = NULL;
1100 int result;
1101 struct qp_guest_endpoint *queue_pair_entry = NULL;
1102
1103 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1104 return VMCI_ERROR_NO_ACCESS;
1105
1106 mutex_lock(&qp_guest_endpoints.mutex);
1107
1108 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1109 if (queue_pair_entry) {
1110 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1111 /* Local attach case. */
1112 if (queue_pair_entry->qp.ref_count > 1) {
1113 pr_devel("Error attempting to attach more than once\n");
1114 result = VMCI_ERROR_UNAVAILABLE;
1115 goto error_keep_entry;
1116 }
1117
1118 if (queue_pair_entry->qp.produce_size != consume_size ||
1119 queue_pair_entry->qp.consume_size !=
1120 produce_size ||
1121 queue_pair_entry->qp.flags !=
1122 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1123 pr_devel("Error mismatched queue pair in local attach\n");
1124 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1125 goto error_keep_entry;
1126 }
1127
1128 /*
1129 * Do a local attach. We swap the consume and
1130 * produce queues for the attacher and deliver
1131 * an attach event.
1132 */
1133 result = qp_notify_peer_local(true, *handle);
1134 if (result < VMCI_SUCCESS)
1135 goto error_keep_entry;
1136
1137 my_produce_q = queue_pair_entry->consume_q;
1138 my_consume_q = queue_pair_entry->produce_q;
1139 goto out;
1140 }
1141
1142 result = VMCI_ERROR_ALREADY_EXISTS;
1143 goto error_keep_entry;
1144 }
1145
1146 my_produce_q = qp_alloc_queue(produce_size, flags);
1147 if (!my_produce_q) {
1148 pr_warn("Error allocating pages for produce queue\n");
1149 result = VMCI_ERROR_NO_MEM;
1150 goto error;
1151 }
1152
1153 my_consume_q = qp_alloc_queue(consume_size, flags);
1154 if (!my_consume_q) {
1155 pr_warn("Error allocating pages for consume queue\n");
1156 result = VMCI_ERROR_NO_MEM;
1157 goto error;
1158 }
1159
1160 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1161 produce_size, consume_size,
1162 my_produce_q, my_consume_q);
1163 if (!queue_pair_entry) {
1164 pr_warn("Error allocating memory in %s\n", __func__);
1165 result = VMCI_ERROR_NO_MEM;
1166 goto error;
1167 }
1168
1169 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1170 num_consume_pages,
1171 &queue_pair_entry->ppn_set);
1172 if (result < VMCI_SUCCESS) {
1173 pr_warn("qp_alloc_ppn_set failed\n");
1174 goto error;
1175 }
1176
1177 /*
1178 * It's only necessary to notify the host if this queue pair will be
1179 * attached to from another context.
1180 */
1181 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1182 /* Local create case. */
1183 u32 context_id = vmci_get_context_id();
1184
1185 /*
1186 * Enforce similar checks on local queue pairs as we
1187 * do for regular ones. The handle's context must
1188 * match the creator or attacher context id (here they
1189 * are both the current context id) and the
1190 * attach-only flag cannot exist during create. We
1191 * also ensure specified peer is this context or an
1192 * invalid one.
1193 */
1194 if (queue_pair_entry->qp.handle.context != context_id ||
1195 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1196 queue_pair_entry->qp.peer != context_id)) {
1197 result = VMCI_ERROR_NO_ACCESS;
1198 goto error;
1199 }
1200
1201 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1202 result = VMCI_ERROR_NOT_FOUND;
1203 goto error;
1204 }
1205 } else {
1206 result = qp_alloc_hypercall(queue_pair_entry);
1207 if (result < VMCI_SUCCESS) {
1208 pr_warn("qp_alloc_hypercall result = %d\n", result);
1209 goto error;
1210 }
1211 }
1212
1213 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1214 (struct vmci_queue *)my_consume_q);
1215
1216 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1217
1218 out:
1219 queue_pair_entry->qp.ref_count++;
1220 *handle = queue_pair_entry->qp.handle;
1221 *produce_q = (struct vmci_queue *)my_produce_q;
1222 *consume_q = (struct vmci_queue *)my_consume_q;
1223
1224 /*
1225 * We should initialize the queue pair header pages on a local
1226 * queue pair create. For non-local queue pairs, the
1227 * hypervisor initializes the header pages in the create step.
1228 */
1229 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1230 queue_pair_entry->qp.ref_count == 1) {
1231 vmci_q_header_init((*produce_q)->q_header, *handle);
1232 vmci_q_header_init((*consume_q)->q_header, *handle);
1233 }
1234
1235 mutex_unlock(&qp_guest_endpoints.mutex);
1236
1237 return VMCI_SUCCESS;
1238
1239 error:
1240 mutex_unlock(&qp_guest_endpoints.mutex);
1241 if (queue_pair_entry) {
1242 /* The queues will be freed inside the destroy routine. */
1243 qp_guest_endpoint_destroy(queue_pair_entry);
1244 } else {
1245 qp_free_queue(my_produce_q, produce_size);
1246 qp_free_queue(my_consume_q, consume_size);
1247 }
1248 return result;
1249
1250 error_keep_entry:
1251 /* This path should only be used when an existing entry was found. */
1252 mutex_unlock(&qp_guest_endpoints.mutex);
1253 return result;
1254}
1255
1256/*
1257 * The first endpoint issuing a queue pair allocation will create the state
1258 * of the queue pair in the queue pair broker.
1259 *
1260 * If the creator is a guest, it will associate a VMX virtual address range
1261 * with the queue pair as specified by the page_store. For compatibility with
1262 * older VMX'en, that would use a separate step to set the VMX virtual
1263 * address range, the virtual address range can be registered later using
1264 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1265 * used.
1266 *
1267 * If the creator is the host, a page_store of NULL should be used as well,
1268 * since the host is not able to supply a page store for the queue pair.
1269 *
1270 * For older VMX and host callers, the queue pair will be created in the
1271 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1272 * created in VMCOQPB_CREATED_MEM state.
1273 */
1274static int qp_broker_create(struct vmci_handle handle,
1275 u32 peer,
1276 u32 flags,
1277 u32 priv_flags,
1278 u64 produce_size,
1279 u64 consume_size,
1280 struct vmci_qp_page_store *page_store,
1281 struct vmci_ctx *context,
1282 vmci_event_release_cb wakeup_cb,
1283 void *client_data, struct qp_broker_entry **ent)
1284{
1285 struct qp_broker_entry *entry = NULL;
1286 const u32 context_id = vmci_ctx_get_id(context);
1287 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1288 int result;
1289 u64 guest_produce_size;
1290 u64 guest_consume_size;
1291
1292 /* Do not create if the caller asked not to. */
1293 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1294 return VMCI_ERROR_NOT_FOUND;
1295
1296 /*
1297 * Creator's context ID should match handle's context ID or the creator
1298 * must allow the context in handle's context ID as the "peer".
1299 */
1300 if (handle.context != context_id && handle.context != peer)
1301 return VMCI_ERROR_NO_ACCESS;
1302
1303 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1304 return VMCI_ERROR_DST_UNREACHABLE;
1305
1306 /*
1307 * Creator's context ID for local queue pairs should match the
1308 * peer, if a peer is specified.
1309 */
1310 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1311 return VMCI_ERROR_NO_ACCESS;
1312
1313 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1314 if (!entry)
1315 return VMCI_ERROR_NO_MEM;
1316
1317 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1318 /*
1319 * The queue pair broker entry stores values from the guest
1320 * point of view, so a creating host side endpoint should swap
1321 * produce and consume values -- unless it is a local queue
1322 * pair, in which case no swapping is necessary, since the local
1323 * attacher will swap queues.
1324 */
1325
1326 guest_produce_size = consume_size;
1327 guest_consume_size = produce_size;
1328 } else {
1329 guest_produce_size = produce_size;
1330 guest_consume_size = consume_size;
1331 }
1332
1333 entry->qp.handle = handle;
1334 entry->qp.peer = peer;
1335 entry->qp.flags = flags;
1336 entry->qp.produce_size = guest_produce_size;
1337 entry->qp.consume_size = guest_consume_size;
1338 entry->qp.ref_count = 1;
1339 entry->create_id = context_id;
1340 entry->attach_id = VMCI_INVALID_ID;
1341 entry->state = VMCIQPB_NEW;
1342 entry->require_trusted_attach =
1343 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1344 entry->created_by_trusted =
1345 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1346 entry->vmci_page_files = false;
1347 entry->wakeup_cb = wakeup_cb;
1348 entry->client_data = client_data;
1349 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1350 if (entry->produce_q == NULL) {
1351 result = VMCI_ERROR_NO_MEM;
1352 goto error;
1353 }
1354 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1355 if (entry->consume_q == NULL) {
1356 result = VMCI_ERROR_NO_MEM;
1357 goto error;
1358 }
1359
1360 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1361
1362 INIT_LIST_HEAD(&entry->qp.list_item);
1363
1364 if (is_local) {
1365 u8 *tmp;
1366
1367 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1368 PAGE_SIZE, GFP_KERNEL);
1369 if (entry->local_mem == NULL) {
1370 result = VMCI_ERROR_NO_MEM;
1371 goto error;
1372 }
1373 entry->state = VMCIQPB_CREATED_MEM;
1374 entry->produce_q->q_header = entry->local_mem;
1375 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1376 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1377 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1378 } else if (page_store) {
1379 /*
1380 * The VMX already initialized the queue pair headers, so no
1381 * need for the kernel side to do that.
1382 */
1383 result = qp_host_register_user_memory(page_store,
1384 entry->produce_q,
1385 entry->consume_q);
1386 if (result < VMCI_SUCCESS)
1387 goto error;
1388
1389 entry->state = VMCIQPB_CREATED_MEM;
1390 } else {
1391 /*
1392 * A create without a page_store may be either a host
1393 * side create (in which case we are waiting for the
1394 * guest side to supply the memory) or an old style
1395 * queue pair create (in which case we will expect a
1396 * set page store call as the next step).
1397 */
1398 entry->state = VMCIQPB_CREATED_NO_MEM;
1399 }
1400
1401 qp_list_add_entry(&qp_broker_list, &entry->qp);
1402 if (ent != NULL)
1403 *ent = entry;
1404
1405 /* Add to resource obj */
1406 result = vmci_resource_add(&entry->resource,
1407 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1408 handle);
1409 if (result != VMCI_SUCCESS) {
1410 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1411 handle.context, handle.resource, result);
1412 goto error;
1413 }
1414
1415 entry->qp.handle = vmci_resource_handle(&entry->resource);
1416 if (is_local) {
1417 vmci_q_header_init(entry->produce_q->q_header,
1418 entry->qp.handle);
1419 vmci_q_header_init(entry->consume_q->q_header,
1420 entry->qp.handle);
1421 }
1422
1423 vmci_ctx_qp_create(context, entry->qp.handle);
1424
1425 return VMCI_SUCCESS;
1426
1427 error:
1428 if (entry != NULL) {
1429 qp_host_free_queue(entry->produce_q, guest_produce_size);
1430 qp_host_free_queue(entry->consume_q, guest_consume_size);
1431 kfree(entry);
1432 }
1433
1434 return result;
1435}
1436
1437/*
1438 * Enqueues an event datagram to notify the peer VM attached to
1439 * the given queue pair handle about attach/detach event by the
1440 * given VM. Returns Payload size of datagram enqueued on
1441 * success, error code otherwise.
1442 */
1443static int qp_notify_peer(bool attach,
1444 struct vmci_handle handle,
1445 u32 my_id,
1446 u32 peer_id)
1447{
1448 int rv;
1449 struct vmci_event_qp ev;
1450
1451 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1452 peer_id == VMCI_INVALID_ID)
1453 return VMCI_ERROR_INVALID_ARGS;
1454
1455 /*
1456 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1457 * number of pending events from the hypervisor to a given VM
1458 * otherwise a rogue VM could do an arbitrary number of attach
1459 * and detach operations causing memory pressure in the host
1460 * kernel.
1461 */
1462
1463 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1464 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1465 VMCI_CONTEXT_RESOURCE_ID);
1466 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1467 ev.msg.event_data.event = attach ?
1468 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1469 ev.payload.handle = handle;
1470 ev.payload.peer_id = my_id;
1471
1472 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1473 &ev.msg.hdr, false);
1474 if (rv < VMCI_SUCCESS)
1475 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1476 attach ? "ATTACH" : "DETACH", peer_id);
1477
1478 return rv;
1479}
1480
1481/*
1482 * The second endpoint issuing a queue pair allocation will attach to
1483 * the queue pair registered with the queue pair broker.
1484 *
1485 * If the attacher is a guest, it will associate a VMX virtual address
1486 * range with the queue pair as specified by the page_store. At this
1487 * point, the already attach host endpoint may start using the queue
1488 * pair, and an attach event is sent to it. For compatibility with
1489 * older VMX'en, that used a separate step to set the VMX virtual
1490 * address range, the virtual address range can be registered later
1491 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1492 * NULL should be used, and the attach event will be generated once
1493 * the actual page store has been set.
1494 *
1495 * If the attacher is the host, a page_store of NULL should be used as
1496 * well, since the page store information is already set by the guest.
1497 *
1498 * For new VMX and host callers, the queue pair will be moved to the
1499 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1500 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1501 */
1502static int qp_broker_attach(struct qp_broker_entry *entry,
1503 u32 peer,
1504 u32 flags,
1505 u32 priv_flags,
1506 u64 produce_size,
1507 u64 consume_size,
1508 struct vmci_qp_page_store *page_store,
1509 struct vmci_ctx *context,
1510 vmci_event_release_cb wakeup_cb,
1511 void *client_data,
1512 struct qp_broker_entry **ent)
1513{
1514 const u32 context_id = vmci_ctx_get_id(context);
1515 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1516 int result;
1517
1518 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1519 entry->state != VMCIQPB_CREATED_MEM)
1520 return VMCI_ERROR_UNAVAILABLE;
1521
1522 if (is_local) {
1523 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1524 context_id != entry->create_id) {
1525 return VMCI_ERROR_INVALID_ARGS;
1526 }
1527 } else if (context_id == entry->create_id ||
1528 context_id == entry->attach_id) {
1529 return VMCI_ERROR_ALREADY_EXISTS;
1530 }
1531
1532 if (VMCI_CONTEXT_IS_VM(context_id) &&
1533 VMCI_CONTEXT_IS_VM(entry->create_id))
1534 return VMCI_ERROR_DST_UNREACHABLE;
1535
1536 /*
1537 * If we are attaching from a restricted context then the queuepair
1538 * must have been created by a trusted endpoint.
1539 */
1540 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1541 !entry->created_by_trusted)
1542 return VMCI_ERROR_NO_ACCESS;
1543
1544 /*
1545 * If we are attaching to a queuepair that was created by a restricted
1546 * context then we must be trusted.
1547 */
1548 if (entry->require_trusted_attach &&
1549 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1550 return VMCI_ERROR_NO_ACCESS;
1551
1552 /*
1553 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1554 * control check is not performed.
1555 */
1556 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1557 return VMCI_ERROR_NO_ACCESS;
1558
1559 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1560 /*
1561 * Do not attach if the caller doesn't support Host Queue Pairs
1562 * and a host created this queue pair.
1563 */
1564
1565 if (!vmci_ctx_supports_host_qp(context))
1566 return VMCI_ERROR_INVALID_RESOURCE;
1567
1568 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1569 struct vmci_ctx *create_context;
1570 bool supports_host_qp;
1571
1572 /*
1573 * Do not attach a host to a user created queue pair if that
1574 * user doesn't support host queue pair end points.
1575 */
1576
1577 create_context = vmci_ctx_get(entry->create_id);
1578 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1579 vmci_ctx_put(create_context);
1580
1581 if (!supports_host_qp)
1582 return VMCI_ERROR_INVALID_RESOURCE;
1583 }
1584
1585 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1586 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1587
1588 if (context_id != VMCI_HOST_CONTEXT_ID) {
1589 /*
1590 * The queue pair broker entry stores values from the guest
1591 * point of view, so an attaching guest should match the values
1592 * stored in the entry.
1593 */
1594
1595 if (entry->qp.produce_size != produce_size ||
1596 entry->qp.consume_size != consume_size) {
1597 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1598 }
1599 } else if (entry->qp.produce_size != consume_size ||
1600 entry->qp.consume_size != produce_size) {
1601 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1602 }
1603
1604 if (context_id != VMCI_HOST_CONTEXT_ID) {
1605 /*
1606 * If a guest attached to a queue pair, it will supply
1607 * the backing memory. If this is a pre NOVMVM vmx,
1608 * the backing memory will be supplied by calling
1609 * vmci_qp_broker_set_page_store() following the
1610 * return of the vmci_qp_broker_alloc() call. If it is
1611 * a vmx of version NOVMVM or later, the page store
1612 * must be supplied as part of the
1613 * vmci_qp_broker_alloc call. Under all circumstances
1614 * must the initially created queue pair not have any
1615 * memory associated with it already.
1616 */
1617
1618 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1619 return VMCI_ERROR_INVALID_ARGS;
1620
1621 if (page_store != NULL) {
1622 /*
1623 * Patch up host state to point to guest
1624 * supplied memory. The VMX already
1625 * initialized the queue pair headers, so no
1626 * need for the kernel side to do that.
1627 */
1628
1629 result = qp_host_register_user_memory(page_store,
1630 entry->produce_q,
1631 entry->consume_q);
1632 if (result < VMCI_SUCCESS)
1633 return result;
1634
1635 entry->state = VMCIQPB_ATTACHED_MEM;
1636 } else {
1637 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1638 }
1639 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1640 /*
1641 * The host side is attempting to attach to a queue
1642 * pair that doesn't have any memory associated with
1643 * it. This must be a pre NOVMVM vmx that hasn't set
1644 * the page store information yet, or a quiesced VM.
1645 */
1646
1647 return VMCI_ERROR_UNAVAILABLE;
1648 } else {
1649 /* The host side has successfully attached to a queue pair. */
1650 entry->state = VMCIQPB_ATTACHED_MEM;
1651 }
1652
1653 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1654 result =
1655 qp_notify_peer(true, entry->qp.handle, context_id,
1656 entry->create_id);
1657 if (result < VMCI_SUCCESS)
1658 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1659 entry->create_id, entry->qp.handle.context,
1660 entry->qp.handle.resource);
1661 }
1662
1663 entry->attach_id = context_id;
1664 entry->qp.ref_count++;
1665 if (wakeup_cb) {
1666 entry->wakeup_cb = wakeup_cb;
1667 entry->client_data = client_data;
1668 }
1669
1670 /*
1671 * When attaching to local queue pairs, the context already has
1672 * an entry tracking the queue pair, so don't add another one.
1673 */
1674 if (!is_local)
1675 vmci_ctx_qp_create(context, entry->qp.handle);
1676
1677 if (ent != NULL)
1678 *ent = entry;
1679
1680 return VMCI_SUCCESS;
1681}
1682
1683/*
1684 * queue_pair_Alloc for use when setting up queue pair endpoints
1685 * on the host.
1686 */
1687static int qp_broker_alloc(struct vmci_handle handle,
1688 u32 peer,
1689 u32 flags,
1690 u32 priv_flags,
1691 u64 produce_size,
1692 u64 consume_size,
1693 struct vmci_qp_page_store *page_store,
1694 struct vmci_ctx *context,
1695 vmci_event_release_cb wakeup_cb,
1696 void *client_data,
1697 struct qp_broker_entry **ent,
1698 bool *swap)
1699{
1700 const u32 context_id = vmci_ctx_get_id(context);
1701 bool create;
1702 struct qp_broker_entry *entry = NULL;
1703 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1704 int result;
1705
1706 if (vmci_handle_is_invalid(handle) ||
1707 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1708 !(produce_size || consume_size) ||
1709 !context || context_id == VMCI_INVALID_ID ||
1710 handle.context == VMCI_INVALID_ID) {
1711 return VMCI_ERROR_INVALID_ARGS;
1712 }
1713
1714 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1715 return VMCI_ERROR_INVALID_ARGS;
1716
1717 /*
1718 * In the initial argument check, we ensure that non-vmkernel hosts
1719 * are not allowed to create local queue pairs.
1720 */
1721
1722 mutex_lock(&qp_broker_list.mutex);
1723
1724 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1725 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1726 context_id, handle.context, handle.resource);
1727 mutex_unlock(&qp_broker_list.mutex);
1728 return VMCI_ERROR_ALREADY_EXISTS;
1729 }
1730
1731 if (handle.resource != VMCI_INVALID_ID)
1732 entry = qp_broker_handle_to_entry(handle);
1733
1734 if (!entry) {
1735 create = true;
1736 result =
1737 qp_broker_create(handle, peer, flags, priv_flags,
1738 produce_size, consume_size, page_store,
1739 context, wakeup_cb, client_data, ent);
1740 } else {
1741 create = false;
1742 result =
1743 qp_broker_attach(entry, peer, flags, priv_flags,
1744 produce_size, consume_size, page_store,
1745 context, wakeup_cb, client_data, ent);
1746 }
1747
1748 mutex_unlock(&qp_broker_list.mutex);
1749
1750 if (swap)
1751 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1752 !(create && is_local);
1753
1754 return result;
1755}
1756
1757/*
1758 * This function implements the kernel API for allocating a queue
1759 * pair.
1760 */
1761static int qp_alloc_host_work(struct vmci_handle *handle,
1762 struct vmci_queue **produce_q,
1763 u64 produce_size,
1764 struct vmci_queue **consume_q,
1765 u64 consume_size,
1766 u32 peer,
1767 u32 flags,
1768 u32 priv_flags,
1769 vmci_event_release_cb wakeup_cb,
1770 void *client_data)
1771{
1772 struct vmci_handle new_handle;
1773 struct vmci_ctx *context;
1774 struct qp_broker_entry *entry;
1775 int result;
1776 bool swap;
1777
1778 if (vmci_handle_is_invalid(*handle)) {
1779 new_handle = vmci_make_handle(
1780 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1781 } else
1782 new_handle = *handle;
1783
1784 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1785 entry = NULL;
1786 result =
1787 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1788 produce_size, consume_size, NULL, context,
1789 wakeup_cb, client_data, &entry, &swap);
1790 if (result == VMCI_SUCCESS) {
1791 if (swap) {
1792 /*
1793 * If this is a local queue pair, the attacher
1794 * will swap around produce and consume
1795 * queues.
1796 */
1797
1798 *produce_q = entry->consume_q;
1799 *consume_q = entry->produce_q;
1800 } else {
1801 *produce_q = entry->produce_q;
1802 *consume_q = entry->consume_q;
1803 }
1804
1805 *handle = vmci_resource_handle(&entry->resource);
1806 } else {
1807 *handle = VMCI_INVALID_HANDLE;
1808 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1809 result);
1810 }
1811 vmci_ctx_put(context);
1812 return result;
1813}
1814
1815/*
1816 * Allocates a VMCI queue_pair. Only checks validity of input
1817 * arguments. The real work is done in the host or guest
1818 * specific function.
1819 */
1820int vmci_qp_alloc(struct vmci_handle *handle,
1821 struct vmci_queue **produce_q,
1822 u64 produce_size,
1823 struct vmci_queue **consume_q,
1824 u64 consume_size,
1825 u32 peer,
1826 u32 flags,
1827 u32 priv_flags,
1828 bool guest_endpoint,
1829 vmci_event_release_cb wakeup_cb,
1830 void *client_data)
1831{
1832 if (!handle || !produce_q || !consume_q ||
1833 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1834 return VMCI_ERROR_INVALID_ARGS;
1835
1836 if (guest_endpoint) {
1837 return qp_alloc_guest_work(handle, produce_q,
1838 produce_size, consume_q,
1839 consume_size, peer,
1840 flags, priv_flags);
1841 } else {
1842 return qp_alloc_host_work(handle, produce_q,
1843 produce_size, consume_q,
1844 consume_size, peer, flags,
1845 priv_flags, wakeup_cb, client_data);
1846 }
1847}
1848
1849/*
1850 * This function implements the host kernel API for detaching from
1851 * a queue pair.
1852 */
1853static int qp_detatch_host_work(struct vmci_handle handle)
1854{
1855 int result;
1856 struct vmci_ctx *context;
1857
1858 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1859
1860 result = vmci_qp_broker_detach(handle, context);
1861
1862 vmci_ctx_put(context);
1863 return result;
1864}
1865
1866/*
1867 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1868 * Real work is done in the host or guest specific function.
1869 */
1870static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1871{
1872 if (vmci_handle_is_invalid(handle))
1873 return VMCI_ERROR_INVALID_ARGS;
1874
1875 if (guest_endpoint)
1876 return qp_detatch_guest_work(handle);
1877 else
1878 return qp_detatch_host_work(handle);
1879}
1880
1881/*
1882 * Returns the entry from the head of the list. Assumes that the list is
1883 * locked.
1884 */
1885static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1886{
1887 if (!list_empty(&qp_list->head)) {
1888 struct qp_entry *entry =
1889 list_first_entry(&qp_list->head, struct qp_entry,
1890 list_item);
1891 return entry;
1892 }
1893
1894 return NULL;
1895}
1896
1897void vmci_qp_broker_exit(void)
1898{
1899 struct qp_entry *entry;
1900 struct qp_broker_entry *be;
1901
1902 mutex_lock(&qp_broker_list.mutex);
1903
1904 while ((entry = qp_list_get_head(&qp_broker_list))) {
1905 be = (struct qp_broker_entry *)entry;
1906
1907 qp_list_remove_entry(&qp_broker_list, entry);
1908 kfree(be);
1909 }
1910
1911 mutex_unlock(&qp_broker_list.mutex);
1912}
1913
1914/*
1915 * Requests that a queue pair be allocated with the VMCI queue
1916 * pair broker. Allocates a queue pair entry if one does not
1917 * exist. Attaches to one if it exists, and retrieves the page
1918 * files backing that queue_pair. Assumes that the queue pair
1919 * broker lock is held.
1920 */
1921int vmci_qp_broker_alloc(struct vmci_handle handle,
1922 u32 peer,
1923 u32 flags,
1924 u32 priv_flags,
1925 u64 produce_size,
1926 u64 consume_size,
1927 struct vmci_qp_page_store *page_store,
1928 struct vmci_ctx *context)
1929{
1930 return qp_broker_alloc(handle, peer, flags, priv_flags,
1931 produce_size, consume_size,
1932 page_store, context, NULL, NULL, NULL, NULL);
1933}
1934
1935/*
1936 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
1937 * step to add the UVAs of the VMX mapping of the queue pair. This function
1938 * provides backwards compatibility with such VMX'en, and takes care of
1939 * registering the page store for a queue pair previously allocated by the
1940 * VMX during create or attach. This function will move the queue pair state
1941 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
1942 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
1943 * attached state with memory, the queue pair is ready to be used by the
1944 * host peer, and an attached event will be generated.
1945 *
1946 * Assumes that the queue pair broker lock is held.
1947 *
1948 * This function is only used by the hosted platform, since there is no
1949 * issue with backwards compatibility for vmkernel.
1950 */
1951int vmci_qp_broker_set_page_store(struct vmci_handle handle,
1952 u64 produce_uva,
1953 u64 consume_uva,
1954 struct vmci_ctx *context)
1955{
1956 struct qp_broker_entry *entry;
1957 int result;
1958 const u32 context_id = vmci_ctx_get_id(context);
1959
1960 if (vmci_handle_is_invalid(handle) || !context ||
1961 context_id == VMCI_INVALID_ID)
1962 return VMCI_ERROR_INVALID_ARGS;
1963
1964 /*
1965 * We only support guest to host queue pairs, so the VMX must
1966 * supply UVAs for the mapped page files.
1967 */
1968
1969 if (produce_uva == 0 || consume_uva == 0)
1970 return VMCI_ERROR_INVALID_ARGS;
1971
1972 mutex_lock(&qp_broker_list.mutex);
1973
1974 if (!vmci_ctx_qp_exists(context, handle)) {
1975 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
1976 context_id, handle.context, handle.resource);
1977 result = VMCI_ERROR_NOT_FOUND;
1978 goto out;
1979 }
1980
1981 entry = qp_broker_handle_to_entry(handle);
1982 if (!entry) {
1983 result = VMCI_ERROR_NOT_FOUND;
1984 goto out;
1985 }
1986
1987 /*
1988 * If I'm the owner then I can set the page store.
1989 *
1990 * Or, if a host created the queue_pair and I'm the attached peer
1991 * then I can set the page store.
1992 */
1993 if (entry->create_id != context_id &&
1994 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
1995 entry->attach_id != context_id)) {
1996 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
1997 goto out;
1998 }
1999
2000 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2001 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2002 result = VMCI_ERROR_UNAVAILABLE;
2003 goto out;
2004 }
2005
2006 result = qp_host_get_user_memory(produce_uva, consume_uva,
2007 entry->produce_q, entry->consume_q);
2008 if (result < VMCI_SUCCESS)
2009 goto out;
2010
2011 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2012 if (result < VMCI_SUCCESS) {
2013 qp_host_unregister_user_memory(entry->produce_q,
2014 entry->consume_q);
2015 goto out;
2016 }
2017
2018 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2019 entry->state = VMCIQPB_CREATED_MEM;
2020 else
2021 entry->state = VMCIQPB_ATTACHED_MEM;
2022
2023 entry->vmci_page_files = true;
2024
2025 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2026 result =
2027 qp_notify_peer(true, handle, context_id, entry->create_id);
2028 if (result < VMCI_SUCCESS) {
2029 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2030 entry->create_id, entry->qp.handle.context,
2031 entry->qp.handle.resource);
2032 }
2033 }
2034
2035 result = VMCI_SUCCESS;
2036 out:
2037 mutex_unlock(&qp_broker_list.mutex);
2038 return result;
2039}
2040
2041/*
2042 * Resets saved queue headers for the given QP broker
2043 * entry. Should be used when guest memory becomes available
2044 * again, or the guest detaches.
2045 */
2046static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2047{
2048 entry->produce_q->saved_header = NULL;
2049 entry->consume_q->saved_header = NULL;
2050}
2051
2052/*
2053 * The main entry point for detaching from a queue pair registered with the
2054 * queue pair broker. If more than one endpoint is attached to the queue
2055 * pair, the first endpoint will mainly decrement a reference count and
2056 * generate a notification to its peer. The last endpoint will clean up
2057 * the queue pair state registered with the broker.
2058 *
2059 * When a guest endpoint detaches, it will unmap and unregister the guest
2060 * memory backing the queue pair. If the host is still attached, it will
2061 * no longer be able to access the queue pair content.
2062 *
2063 * If the queue pair is already in a state where there is no memory
2064 * registered for the queue pair (any *_NO_MEM state), it will transition to
2065 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2066 * endpoint is the first of two endpoints to detach. If the host endpoint is
2067 * the first out of two to detach, the queue pair will move to the
2068 * VMCIQPB_SHUTDOWN_MEM state.
2069 */
2070int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2071{
2072 struct qp_broker_entry *entry;
2073 const u32 context_id = vmci_ctx_get_id(context);
2074 u32 peer_id;
2075 bool is_local = false;
2076 int result;
2077
2078 if (vmci_handle_is_invalid(handle) || !context ||
2079 context_id == VMCI_INVALID_ID) {
2080 return VMCI_ERROR_INVALID_ARGS;
2081 }
2082
2083 mutex_lock(&qp_broker_list.mutex);
2084
2085 if (!vmci_ctx_qp_exists(context, handle)) {
2086 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2087 context_id, handle.context, handle.resource);
2088 result = VMCI_ERROR_NOT_FOUND;
2089 goto out;
2090 }
2091
2092 entry = qp_broker_handle_to_entry(handle);
2093 if (!entry) {
2094 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2095 context_id, handle.context, handle.resource);
2096 result = VMCI_ERROR_NOT_FOUND;
2097 goto out;
2098 }
2099
2100 if (context_id != entry->create_id && context_id != entry->attach_id) {
2101 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2102 goto out;
2103 }
2104
2105 if (context_id == entry->create_id) {
2106 peer_id = entry->attach_id;
2107 entry->create_id = VMCI_INVALID_ID;
2108 } else {
2109 peer_id = entry->create_id;
2110 entry->attach_id = VMCI_INVALID_ID;
2111 }
2112 entry->qp.ref_count--;
2113
2114 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2115
2116 if (context_id != VMCI_HOST_CONTEXT_ID) {
2117 bool headers_mapped;
2118
2119 /*
2120 * Pre NOVMVM vmx'en may detach from a queue pair
2121 * before setting the page store, and in that case
2122 * there is no user memory to detach from. Also, more
2123 * recent VMX'en may detach from a queue pair in the
2124 * quiesced state.
2125 */
2126
2127 qp_acquire_queue_mutex(entry->produce_q);
2128 headers_mapped = entry->produce_q->q_header ||
2129 entry->consume_q->q_header;
2130 if (QPBROKERSTATE_HAS_MEM(entry)) {
2131 result =
2132 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2133 entry->produce_q,
2134 entry->consume_q);
2135 if (result < VMCI_SUCCESS)
2136 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2137 handle.context, handle.resource,
2138 result);
2139
2140 qp_host_unregister_user_memory(entry->produce_q,
2141 entry->consume_q);
2142
2143 }
2144
2145 if (!headers_mapped)
2146 qp_reset_saved_headers(entry);
2147
2148 qp_release_queue_mutex(entry->produce_q);
2149
2150 if (!headers_mapped && entry->wakeup_cb)
2151 entry->wakeup_cb(entry->client_data);
2152
2153 } else {
2154 if (entry->wakeup_cb) {
2155 entry->wakeup_cb = NULL;
2156 entry->client_data = NULL;
2157 }
2158 }
2159
2160 if (entry->qp.ref_count == 0) {
2161 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2162
2163 if (is_local)
2164 kfree(entry->local_mem);
2165
2166 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2167 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2168 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2169 /* Unlink from resource hash table and free callback */
2170 vmci_resource_remove(&entry->resource);
2171
2172 kfree(entry);
2173
2174 vmci_ctx_qp_destroy(context, handle);
2175 } else {
2176 qp_notify_peer(false, handle, context_id, peer_id);
2177 if (context_id == VMCI_HOST_CONTEXT_ID &&
2178 QPBROKERSTATE_HAS_MEM(entry)) {
2179 entry->state = VMCIQPB_SHUTDOWN_MEM;
2180 } else {
2181 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2182 }
2183
2184 if (!is_local)
2185 vmci_ctx_qp_destroy(context, handle);
2186
2187 }
2188 result = VMCI_SUCCESS;
2189 out:
2190 mutex_unlock(&qp_broker_list.mutex);
2191 return result;
2192}
2193
2194/*
2195 * Establishes the necessary mappings for a queue pair given a
2196 * reference to the queue pair guest memory. This is usually
2197 * called when a guest is unquiesced and the VMX is allowed to
2198 * map guest memory once again.
2199 */
2200int vmci_qp_broker_map(struct vmci_handle handle,
2201 struct vmci_ctx *context,
2202 u64 guest_mem)
2203{
2204 struct qp_broker_entry *entry;
2205 const u32 context_id = vmci_ctx_get_id(context);
2206 int result;
2207
2208 if (vmci_handle_is_invalid(handle) || !context ||
2209 context_id == VMCI_INVALID_ID)
2210 return VMCI_ERROR_INVALID_ARGS;
2211
2212 mutex_lock(&qp_broker_list.mutex);
2213
2214 if (!vmci_ctx_qp_exists(context, handle)) {
2215 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2216 context_id, handle.context, handle.resource);
2217 result = VMCI_ERROR_NOT_FOUND;
2218 goto out;
2219 }
2220
2221 entry = qp_broker_handle_to_entry(handle);
2222 if (!entry) {
2223 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2224 context_id, handle.context, handle.resource);
2225 result = VMCI_ERROR_NOT_FOUND;
2226 goto out;
2227 }
2228
2229 if (context_id != entry->create_id && context_id != entry->attach_id) {
2230 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2231 goto out;
2232 }
2233
2234 result = VMCI_SUCCESS;
2235
2236 if (context_id != VMCI_HOST_CONTEXT_ID) {
2237 struct vmci_qp_page_store page_store;
2238
2239 page_store.pages = guest_mem;
2240 page_store.len = QPE_NUM_PAGES(entry->qp);
2241
2242 qp_acquire_queue_mutex(entry->produce_q);
2243 qp_reset_saved_headers(entry);
2244 result =
2245 qp_host_register_user_memory(&page_store,
2246 entry->produce_q,
2247 entry->consume_q);
2248 qp_release_queue_mutex(entry->produce_q);
2249 if (result == VMCI_SUCCESS) {
2250 /* Move state from *_NO_MEM to *_MEM */
2251
2252 entry->state++;
2253
2254 if (entry->wakeup_cb)
2255 entry->wakeup_cb(entry->client_data);
2256 }
2257 }
2258
2259 out:
2260 mutex_unlock(&qp_broker_list.mutex);
2261 return result;
2262}
2263
2264/*
2265 * Saves a snapshot of the queue headers for the given QP broker
2266 * entry. Should be used when guest memory is unmapped.
2267 * Results:
2268 * VMCI_SUCCESS on success, appropriate error code if guest memory
2269 * can't be accessed..
2270 */
2271static int qp_save_headers(struct qp_broker_entry *entry)
2272{
2273 int result;
2274
2275 if (entry->produce_q->saved_header != NULL &&
2276 entry->consume_q->saved_header != NULL) {
2277 /*
2278 * If the headers have already been saved, we don't need to do
2279 * it again, and we don't want to map in the headers
2280 * unnecessarily.
2281 */
2282
2283 return VMCI_SUCCESS;
2284 }
2285
2286 if (NULL == entry->produce_q->q_header ||
2287 NULL == entry->consume_q->q_header) {
2288 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2289 if (result < VMCI_SUCCESS)
2290 return result;
2291 }
2292
2293 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2294 sizeof(entry->saved_produce_q));
2295 entry->produce_q->saved_header = &entry->saved_produce_q;
2296 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2297 sizeof(entry->saved_consume_q));
2298 entry->consume_q->saved_header = &entry->saved_consume_q;
2299
2300 return VMCI_SUCCESS;
2301}
2302
2303/*
2304 * Removes all references to the guest memory of a given queue pair, and
2305 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2306 * called when a VM is being quiesced where access to guest memory should
2307 * avoided.
2308 */
2309int vmci_qp_broker_unmap(struct vmci_handle handle,
2310 struct vmci_ctx *context,
2311 u32 gid)
2312{
2313 struct qp_broker_entry *entry;
2314 const u32 context_id = vmci_ctx_get_id(context);
2315 int result;
2316
2317 if (vmci_handle_is_invalid(handle) || !context ||
2318 context_id == VMCI_INVALID_ID)
2319 return VMCI_ERROR_INVALID_ARGS;
2320
2321 mutex_lock(&qp_broker_list.mutex);
2322
2323 if (!vmci_ctx_qp_exists(context, handle)) {
2324 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2325 context_id, handle.context, handle.resource);
2326 result = VMCI_ERROR_NOT_FOUND;
2327 goto out;
2328 }
2329
2330 entry = qp_broker_handle_to_entry(handle);
2331 if (!entry) {
2332 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2333 context_id, handle.context, handle.resource);
2334 result = VMCI_ERROR_NOT_FOUND;
2335 goto out;
2336 }
2337
2338 if (context_id != entry->create_id && context_id != entry->attach_id) {
2339 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2340 goto out;
2341 }
2342
2343 if (context_id != VMCI_HOST_CONTEXT_ID) {
2344 qp_acquire_queue_mutex(entry->produce_q);
2345 result = qp_save_headers(entry);
2346 if (result < VMCI_SUCCESS)
2347 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2348 handle.context, handle.resource, result);
2349
2350 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2351
2352 /*
2353 * On hosted, when we unmap queue pairs, the VMX will also
2354 * unmap the guest memory, so we invalidate the previously
2355 * registered memory. If the queue pair is mapped again at a
2356 * later point in time, we will need to reregister the user
2357 * memory with a possibly new user VA.
2358 */
2359 qp_host_unregister_user_memory(entry->produce_q,
2360 entry->consume_q);
2361
2362 /*
2363 * Move state from *_MEM to *_NO_MEM.
2364 */
2365 entry->state--;
2366
2367 qp_release_queue_mutex(entry->produce_q);
2368 }
2369
2370 result = VMCI_SUCCESS;
2371
2372 out:
2373 mutex_unlock(&qp_broker_list.mutex);
2374 return result;
2375}
2376
2377/*
2378 * Destroys all guest queue pair endpoints. If active guest queue
2379 * pairs still exist, hypercalls to attempt detach from these
2380 * queue pairs will be made. Any failure to detach is silently
2381 * ignored.
2382 */
2383void vmci_qp_guest_endpoints_exit(void)
2384{
2385 struct qp_entry *entry;
2386 struct qp_guest_endpoint *ep;
2387
2388 mutex_lock(&qp_guest_endpoints.mutex);
2389
2390 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2391 ep = (struct qp_guest_endpoint *)entry;
2392
2393 /* Don't make a hypercall for local queue_pairs. */
2394 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2395 qp_detatch_hypercall(entry->handle);
2396
2397 /* We cannot fail the exit, so let's reset ref_count. */
2398 entry->ref_count = 0;
2399 qp_list_remove_entry(&qp_guest_endpoints, entry);
2400
2401 qp_guest_endpoint_destroy(ep);
2402 }
2403
2404 mutex_unlock(&qp_guest_endpoints.mutex);
2405}
2406
2407/*
2408 * Helper routine that will lock the queue pair before subsequent
2409 * operations.
2410 * Note: Non-blocking on the host side is currently only implemented in ESX.
2411 * Since non-blocking isn't yet implemented on the host personality we
2412 * have no reason to acquire a spin lock. So to avoid the use of an
2413 * unnecessary lock only acquire the mutex if we can block.
2414 */
2415static void qp_lock(const struct vmci_qp *qpair)
2416{
2417 qp_acquire_queue_mutex(qpair->produce_q);
2418}
2419
2420/*
2421 * Helper routine that unlocks the queue pair after calling
2422 * qp_lock.
2423 */
2424static void qp_unlock(const struct vmci_qp *qpair)
2425{
2426 qp_release_queue_mutex(qpair->produce_q);
2427}
2428
2429/*
2430 * The queue headers may not be mapped at all times. If a queue is
2431 * currently not mapped, it will be attempted to do so.
2432 */
2433static int qp_map_queue_headers(struct vmci_queue *produce_q,
2434 struct vmci_queue *consume_q)
2435{
2436 int result;
2437
2438 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2439 result = qp_host_map_queues(produce_q, consume_q);
2440 if (result < VMCI_SUCCESS)
2441 return (produce_q->saved_header &&
2442 consume_q->saved_header) ?
2443 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2444 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2445 }
2446
2447 return VMCI_SUCCESS;
2448}
2449
2450/*
2451 * Helper routine that will retrieve the produce and consume
2452 * headers of a given queue pair. If the guest memory of the
2453 * queue pair is currently not available, the saved queue headers
2454 * will be returned, if these are available.
2455 */
2456static int qp_get_queue_headers(const struct vmci_qp *qpair,
2457 struct vmci_queue_header **produce_q_header,
2458 struct vmci_queue_header **consume_q_header)
2459{
2460 int result;
2461
2462 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2463 if (result == VMCI_SUCCESS) {
2464 *produce_q_header = qpair->produce_q->q_header;
2465 *consume_q_header = qpair->consume_q->q_header;
2466 } else if (qpair->produce_q->saved_header &&
2467 qpair->consume_q->saved_header) {
2468 *produce_q_header = qpair->produce_q->saved_header;
2469 *consume_q_header = qpair->consume_q->saved_header;
2470 result = VMCI_SUCCESS;
2471 }
2472
2473 return result;
2474}
2475
2476/*
2477 * Callback from VMCI queue pair broker indicating that a queue
2478 * pair that was previously not ready, now either is ready or
2479 * gone forever.
2480 */
2481static int qp_wakeup_cb(void *client_data)
2482{
2483 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2484
2485 qp_lock(qpair);
2486 while (qpair->blocked > 0) {
2487 qpair->blocked--;
2488 qpair->generation++;
2489 wake_up(&qpair->event);
2490 }
2491 qp_unlock(qpair);
2492
2493 return VMCI_SUCCESS;
2494}
2495
2496/*
2497 * Makes the calling thread wait for the queue pair to become
2498 * ready for host side access. Returns true when thread is
2499 * woken up after queue pair state change, false otherwise.
2500 */
2501static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2502{
2503 unsigned int generation;
2504
2505 qpair->blocked++;
2506 generation = qpair->generation;
2507 qp_unlock(qpair);
2508 wait_event(qpair->event, generation != qpair->generation);
2509 qp_lock(qpair);
2510
2511 return true;
2512}
2513
2514/*
2515 * Enqueues a given buffer to the produce queue using the provided
2516 * function. As many bytes as possible (space available in the queue)
2517 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2518 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2519 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2520 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2521 * an error occured when accessing the buffer,
2522 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2523 * available. Otherwise, the number of bytes written to the queue is
2524 * returned. Updates the tail pointer of the produce queue.
2525 */
2526static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2527 struct vmci_queue *consume_q,
2528 const u64 produce_q_size,
2529 struct iov_iter *from)
2530{
2531 s64 free_space;
2532 u64 tail;
2533 size_t buf_size = iov_iter_count(from);
2534 size_t written;
2535 ssize_t result;
2536
2537 result = qp_map_queue_headers(produce_q, consume_q);
2538 if (unlikely(result != VMCI_SUCCESS))
2539 return result;
2540
2541 free_space = vmci_q_header_free_space(produce_q->q_header,
2542 consume_q->q_header,
2543 produce_q_size);
2544 if (free_space == 0)
2545 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2546
2547 if (free_space < VMCI_SUCCESS)
2548 return (ssize_t) free_space;
2549
2550 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2551 tail = vmci_q_header_producer_tail(produce_q->q_header);
2552 if (likely(tail + written < produce_q_size)) {
2553 result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2554 } else {
2555 /* Tail pointer wraps around. */
2556
2557 const size_t tmp = (size_t) (produce_q_size - tail);
2558
2559 result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2560 if (result >= VMCI_SUCCESS)
2561 result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2562 written - tmp);
2563 }
2564
2565 if (result < VMCI_SUCCESS)
2566 return result;
2567
2568 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2569 produce_q_size);
2570 return written;
2571}
2572
2573/*
2574 * Dequeues data (if available) from the given consume queue. Writes data
2575 * to the user provided buffer using the provided function.
2576 * Assumes the queue->mutex has been acquired.
2577 * Results:
2578 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2579 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2580 * (as defined by the queue size).
2581 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2582 * Otherwise the number of bytes dequeued is returned.
2583 * Side effects:
2584 * Updates the head pointer of the consume queue.
2585 */
2586static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2587 struct vmci_queue *consume_q,
2588 const u64 consume_q_size,
2589 struct iov_iter *to,
2590 bool update_consumer)
2591{
2592 size_t buf_size = iov_iter_count(to);
2593 s64 buf_ready;
2594 u64 head;
2595 size_t read;
2596 ssize_t result;
2597
2598 result = qp_map_queue_headers(produce_q, consume_q);
2599 if (unlikely(result != VMCI_SUCCESS))
2600 return result;
2601
2602 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2603 produce_q->q_header,
2604 consume_q_size);
2605 if (buf_ready == 0)
2606 return VMCI_ERROR_QUEUEPAIR_NODATA;
2607
2608 if (buf_ready < VMCI_SUCCESS)
2609 return (ssize_t) buf_ready;
2610
2611 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2612 head = vmci_q_header_consumer_head(produce_q->q_header);
2613 if (likely(head + read < consume_q_size)) {
2614 result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
2615 } else {
2616 /* Head pointer wraps around. */
2617
2618 const size_t tmp = (size_t) (consume_q_size - head);
2619
2620 result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
2621 if (result >= VMCI_SUCCESS)
2622 result = qp_memcpy_from_queue_iter(to, consume_q, 0,
2623 read - tmp);
2624
2625 }
2626
2627 if (result < VMCI_SUCCESS)
2628 return result;
2629
2630 if (update_consumer)
2631 vmci_q_header_add_consumer_head(produce_q->q_header,
2632 read, consume_q_size);
2633
2634 return read;
2635}
2636
2637/*
2638 * vmci_qpair_alloc() - Allocates a queue pair.
2639 * @qpair: Pointer for the new vmci_qp struct.
2640 * @handle: Handle to track the resource.
2641 * @produce_qsize: Desired size of the producer queue.
2642 * @consume_qsize: Desired size of the consumer queue.
2643 * @peer: ContextID of the peer.
2644 * @flags: VMCI flags.
2645 * @priv_flags: VMCI priviledge flags.
2646 *
2647 * This is the client interface for allocating the memory for a
2648 * vmci_qp structure and then attaching to the underlying
2649 * queue. If an error occurs allocating the memory for the
2650 * vmci_qp structure no attempt is made to attach. If an
2651 * error occurs attaching, then the structure is freed.
2652 */
2653int vmci_qpair_alloc(struct vmci_qp **qpair,
2654 struct vmci_handle *handle,
2655 u64 produce_qsize,
2656 u64 consume_qsize,
2657 u32 peer,
2658 u32 flags,
2659 u32 priv_flags)
2660{
2661 struct vmci_qp *my_qpair;
2662 int retval;
2663 struct vmci_handle src = VMCI_INVALID_HANDLE;
2664 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2665 enum vmci_route route;
2666 vmci_event_release_cb wakeup_cb;
2667 void *client_data;
2668
2669 /*
2670 * Restrict the size of a queuepair. The device already
2671 * enforces a limit on the total amount of memory that can be
2672 * allocated to queuepairs for a guest. However, we try to
2673 * allocate this memory before we make the queuepair
2674 * allocation hypercall. On Linux, we allocate each page
2675 * separately, which means rather than fail, the guest will
2676 * thrash while it tries to allocate, and will become
2677 * increasingly unresponsive to the point where it appears to
2678 * be hung. So we place a limit on the size of an individual
2679 * queuepair here, and leave the device to enforce the
2680 * restriction on total queuepair memory. (Note that this
2681 * doesn't prevent all cases; a user with only this much
2682 * physical memory could still get into trouble.) The error
2683 * used by the device is NO_RESOURCES, so use that here too.
2684 */
2685
2686 if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) ||
2687 produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY)
2688 return VMCI_ERROR_NO_RESOURCES;
2689
2690 retval = vmci_route(&src, &dst, false, &route);
2691 if (retval < VMCI_SUCCESS)
2692 route = vmci_guest_code_active() ?
2693 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2694
2695 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2696 pr_devel("NONBLOCK OR PINNED set");
2697 return VMCI_ERROR_INVALID_ARGS;
2698 }
2699
2700 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2701 if (!my_qpair)
2702 return VMCI_ERROR_NO_MEM;
2703
2704 my_qpair->produce_q_size = produce_qsize;
2705 my_qpair->consume_q_size = consume_qsize;
2706 my_qpair->peer = peer;
2707 my_qpair->flags = flags;
2708 my_qpair->priv_flags = priv_flags;
2709
2710 wakeup_cb = NULL;
2711 client_data = NULL;
2712
2713 if (VMCI_ROUTE_AS_HOST == route) {
2714 my_qpair->guest_endpoint = false;
2715 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2716 my_qpair->blocked = 0;
2717 my_qpair->generation = 0;
2718 init_waitqueue_head(&my_qpair->event);
2719 wakeup_cb = qp_wakeup_cb;
2720 client_data = (void *)my_qpair;
2721 }
2722 } else {
2723 my_qpair->guest_endpoint = true;
2724 }
2725
2726 retval = vmci_qp_alloc(handle,
2727 &my_qpair->produce_q,
2728 my_qpair->produce_q_size,
2729 &my_qpair->consume_q,
2730 my_qpair->consume_q_size,
2731 my_qpair->peer,
2732 my_qpair->flags,
2733 my_qpair->priv_flags,
2734 my_qpair->guest_endpoint,
2735 wakeup_cb, client_data);
2736
2737 if (retval < VMCI_SUCCESS) {
2738 kfree(my_qpair);
2739 return retval;
2740 }
2741
2742 *qpair = my_qpair;
2743 my_qpair->handle = *handle;
2744
2745 return retval;
2746}
2747EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2748
2749/*
2750 * vmci_qpair_detach() - Detatches the client from a queue pair.
2751 * @qpair: Reference of a pointer to the qpair struct.
2752 *
2753 * This is the client interface for detaching from a VMCIQPair.
2754 * Note that this routine will free the memory allocated for the
2755 * vmci_qp structure too.
2756 */
2757int vmci_qpair_detach(struct vmci_qp **qpair)
2758{
2759 int result;
2760 struct vmci_qp *old_qpair;
2761
2762 if (!qpair || !(*qpair))
2763 return VMCI_ERROR_INVALID_ARGS;
2764
2765 old_qpair = *qpair;
2766 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2767
2768 /*
2769 * The guest can fail to detach for a number of reasons, and
2770 * if it does so, it will cleanup the entry (if there is one).
2771 * The host can fail too, but it won't cleanup the entry
2772 * immediately, it will do that later when the context is
2773 * freed. Either way, we need to release the qpair struct
2774 * here; there isn't much the caller can do, and we don't want
2775 * to leak.
2776 */
2777
2778 memset(old_qpair, 0, sizeof(*old_qpair));
2779 old_qpair->handle = VMCI_INVALID_HANDLE;
2780 old_qpair->peer = VMCI_INVALID_ID;
2781 kfree(old_qpair);
2782 *qpair = NULL;
2783
2784 return result;
2785}
2786EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2787
2788/*
2789 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2790 * @qpair: Pointer to the queue pair struct.
2791 * @producer_tail: Reference used for storing producer tail index.
2792 * @consumer_head: Reference used for storing the consumer head index.
2793 *
2794 * This is the client interface for getting the current indexes of the
2795 * QPair from the point of the view of the caller as the producer.
2796 */
2797int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2798 u64 *producer_tail,
2799 u64 *consumer_head)
2800{
2801 struct vmci_queue_header *produce_q_header;
2802 struct vmci_queue_header *consume_q_header;
2803 int result;
2804
2805 if (!qpair)
2806 return VMCI_ERROR_INVALID_ARGS;
2807
2808 qp_lock(qpair);
2809 result =
2810 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2811 if (result == VMCI_SUCCESS)
2812 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2813 producer_tail, consumer_head);
2814 qp_unlock(qpair);
2815
2816 if (result == VMCI_SUCCESS &&
2817 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2818 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2819 return VMCI_ERROR_INVALID_SIZE;
2820
2821 return result;
2822}
2823EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2824
2825/*
2826 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
2827 * @qpair: Pointer to the queue pair struct.
2828 * @consumer_tail: Reference used for storing consumer tail index.
2829 * @producer_head: Reference used for storing the producer head index.
2830 *
2831 * This is the client interface for getting the current indexes of the
2832 * QPair from the point of the view of the caller as the consumer.
2833 */
2834int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2835 u64 *consumer_tail,
2836 u64 *producer_head)
2837{
2838 struct vmci_queue_header *produce_q_header;
2839 struct vmci_queue_header *consume_q_header;
2840 int result;
2841
2842 if (!qpair)
2843 return VMCI_ERROR_INVALID_ARGS;
2844
2845 qp_lock(qpair);
2846 result =
2847 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2848 if (result == VMCI_SUCCESS)
2849 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2850 consumer_tail, producer_head);
2851 qp_unlock(qpair);
2852
2853 if (result == VMCI_SUCCESS &&
2854 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2855 (producer_head && *producer_head >= qpair->consume_q_size)))
2856 return VMCI_ERROR_INVALID_SIZE;
2857
2858 return result;
2859}
2860EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2861
2862/*
2863 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2864 * @qpair: Pointer to the queue pair struct.
2865 *
2866 * This is the client interface for getting the amount of free
2867 * space in the QPair from the point of the view of the caller as
2868 * the producer which is the common case. Returns < 0 if err, else
2869 * available bytes into which data can be enqueued if > 0.
2870 */
2871s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2872{
2873 struct vmci_queue_header *produce_q_header;
2874 struct vmci_queue_header *consume_q_header;
2875 s64 result;
2876
2877 if (!qpair)
2878 return VMCI_ERROR_INVALID_ARGS;
2879
2880 qp_lock(qpair);
2881 result =
2882 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2883 if (result == VMCI_SUCCESS)
2884 result = vmci_q_header_free_space(produce_q_header,
2885 consume_q_header,
2886 qpair->produce_q_size);
2887 else
2888 result = 0;
2889
2890 qp_unlock(qpair);
2891
2892 return result;
2893}
2894EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
2895
2896/*
2897 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
2898 * @qpair: Pointer to the queue pair struct.
2899 *
2900 * This is the client interface for getting the amount of free
2901 * space in the QPair from the point of the view of the caller as
2902 * the consumer which is not the common case. Returns < 0 if err, else
2903 * available bytes into which data can be enqueued if > 0.
2904 */
2905s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
2906{
2907 struct vmci_queue_header *produce_q_header;
2908 struct vmci_queue_header *consume_q_header;
2909 s64 result;
2910
2911 if (!qpair)
2912 return VMCI_ERROR_INVALID_ARGS;
2913
2914 qp_lock(qpair);
2915 result =
2916 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2917 if (result == VMCI_SUCCESS)
2918 result = vmci_q_header_free_space(consume_q_header,
2919 produce_q_header,
2920 qpair->consume_q_size);
2921 else
2922 result = 0;
2923
2924 qp_unlock(qpair);
2925
2926 return result;
2927}
2928EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
2929
2930/*
2931 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
2932 * producer queue.
2933 * @qpair: Pointer to the queue pair struct.
2934 *
2935 * This is the client interface for getting the amount of
2936 * enqueued data in the QPair from the point of the view of the
2937 * caller as the producer which is not the common case. Returns < 0 if err,
2938 * else available bytes that may be read.
2939 */
2940s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
2941{
2942 struct vmci_queue_header *produce_q_header;
2943 struct vmci_queue_header *consume_q_header;
2944 s64 result;
2945
2946 if (!qpair)
2947 return VMCI_ERROR_INVALID_ARGS;
2948
2949 qp_lock(qpair);
2950 result =
2951 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2952 if (result == VMCI_SUCCESS)
2953 result = vmci_q_header_buf_ready(produce_q_header,
2954 consume_q_header,
2955 qpair->produce_q_size);
2956 else
2957 result = 0;
2958
2959 qp_unlock(qpair);
2960
2961 return result;
2962}
2963EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
2964
2965/*
2966 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
2967 * consumer queue.
2968 * @qpair: Pointer to the queue pair struct.
2969 *
2970 * This is the client interface for getting the amount of
2971 * enqueued data in the QPair from the point of the view of the
2972 * caller as the consumer which is the normal case. Returns < 0 if err,
2973 * else available bytes that may be read.
2974 */
2975s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
2976{
2977 struct vmci_queue_header *produce_q_header;
2978 struct vmci_queue_header *consume_q_header;
2979 s64 result;
2980
2981 if (!qpair)
2982 return VMCI_ERROR_INVALID_ARGS;
2983
2984 qp_lock(qpair);
2985 result =
2986 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2987 if (result == VMCI_SUCCESS)
2988 result = vmci_q_header_buf_ready(consume_q_header,
2989 produce_q_header,
2990 qpair->consume_q_size);
2991 else
2992 result = 0;
2993
2994 qp_unlock(qpair);
2995
2996 return result;
2997}
2998EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
2999
3000/*
3001 * vmci_qpair_enqueue() - Throw data on the queue.
3002 * @qpair: Pointer to the queue pair struct.
3003 * @buf: Pointer to buffer containing data
3004 * @buf_size: Length of buffer.
3005 * @buf_type: Buffer type (Unused).
3006 *
3007 * This is the client interface for enqueueing data into the queue.
3008 * Returns number of bytes enqueued or < 0 on error.
3009 */
3010ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3011 const void *buf,
3012 size_t buf_size,
3013 int buf_type)
3014{
3015 ssize_t result;
3016 struct iov_iter from;
3017 struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
3018
3019 if (!qpair || !buf)
3020 return VMCI_ERROR_INVALID_ARGS;
3021
3022 iov_iter_kvec(&from, WRITE, &v, 1, buf_size);
3023
3024 qp_lock(qpair);
3025
3026 do {
3027 result = qp_enqueue_locked(qpair->produce_q,
3028 qpair->consume_q,
3029 qpair->produce_q_size,
3030 &from);
3031
3032 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3033 !qp_wait_for_ready_queue(qpair))
3034 result = VMCI_ERROR_WOULD_BLOCK;
3035
3036 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3037
3038 qp_unlock(qpair);
3039
3040 return result;
3041}
3042EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3043
3044/*
3045 * vmci_qpair_dequeue() - Get data from the queue.
3046 * @qpair: Pointer to the queue pair struct.
3047 * @buf: Pointer to buffer for the data
3048 * @buf_size: Length of buffer.
3049 * @buf_type: Buffer type (Unused).
3050 *
3051 * This is the client interface for dequeueing data from the queue.
3052 * Returns number of bytes dequeued or < 0 on error.
3053 */
3054ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3055 void *buf,
3056 size_t buf_size,
3057 int buf_type)
3058{
3059 ssize_t result;
3060 struct iov_iter to;
3061 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3062
3063 if (!qpair || !buf)
3064 return VMCI_ERROR_INVALID_ARGS;
3065
3066 iov_iter_kvec(&to, READ, &v, 1, buf_size);
3067
3068 qp_lock(qpair);
3069
3070 do {
3071 result = qp_dequeue_locked(qpair->produce_q,
3072 qpair->consume_q,
3073 qpair->consume_q_size,
3074 &to, true);
3075
3076 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3077 !qp_wait_for_ready_queue(qpair))
3078 result = VMCI_ERROR_WOULD_BLOCK;
3079
3080 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3081
3082 qp_unlock(qpair);
3083
3084 return result;
3085}
3086EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3087
3088/*
3089 * vmci_qpair_peek() - Peek at the data in the queue.
3090 * @qpair: Pointer to the queue pair struct.
3091 * @buf: Pointer to buffer for the data
3092 * @buf_size: Length of buffer.
3093 * @buf_type: Buffer type (Unused on Linux).
3094 *
3095 * This is the client interface for peeking into a queue. (I.e.,
3096 * copy data from the queue without updating the head pointer.)
3097 * Returns number of bytes dequeued or < 0 on error.
3098 */
3099ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3100 void *buf,
3101 size_t buf_size,
3102 int buf_type)
3103{
3104 struct iov_iter to;
3105 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3106 ssize_t result;
3107
3108 if (!qpair || !buf)
3109 return VMCI_ERROR_INVALID_ARGS;
3110
3111 iov_iter_kvec(&to, READ, &v, 1, buf_size);
3112
3113 qp_lock(qpair);
3114
3115 do {
3116 result = qp_dequeue_locked(qpair->produce_q,
3117 qpair->consume_q,
3118 qpair->consume_q_size,
3119 &to, false);
3120
3121 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3122 !qp_wait_for_ready_queue(qpair))
3123 result = VMCI_ERROR_WOULD_BLOCK;
3124
3125 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3126
3127 qp_unlock(qpair);
3128
3129 return result;
3130}
3131EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3132
3133/*
3134 * vmci_qpair_enquev() - Throw data on the queue using iov.
3135 * @qpair: Pointer to the queue pair struct.
3136 * @iov: Pointer to buffer containing data
3137 * @iov_size: Length of buffer.
3138 * @buf_type: Buffer type (Unused).
3139 *
3140 * This is the client interface for enqueueing data into the queue.
3141 * This function uses IO vectors to handle the work. Returns number
3142 * of bytes enqueued or < 0 on error.
3143 */
3144ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3145 struct msghdr *msg,
3146 size_t iov_size,
3147 int buf_type)
3148{
3149 ssize_t result;
3150
3151 if (!qpair)
3152 return VMCI_ERROR_INVALID_ARGS;
3153
3154 qp_lock(qpair);
3155
3156 do {
3157 result = qp_enqueue_locked(qpair->produce_q,
3158 qpair->consume_q,
3159 qpair->produce_q_size,
3160 &msg->msg_iter);
3161
3162 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3163 !qp_wait_for_ready_queue(qpair))
3164 result = VMCI_ERROR_WOULD_BLOCK;
3165
3166 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3167
3168 qp_unlock(qpair);
3169
3170 return result;
3171}
3172EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3173
3174/*
3175 * vmci_qpair_dequev() - Get data from the queue using iov.
3176 * @qpair: Pointer to the queue pair struct.
3177 * @iov: Pointer to buffer for the data
3178 * @iov_size: Length of buffer.
3179 * @buf_type: Buffer type (Unused).
3180 *
3181 * This is the client interface for dequeueing data from the queue.
3182 * This function uses IO vectors to handle the work. Returns number
3183 * of bytes dequeued or < 0 on error.
3184 */
3185ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3186 struct msghdr *msg,
3187 size_t iov_size,
3188 int buf_type)
3189{
3190 ssize_t result;
3191
3192 if (!qpair)
3193 return VMCI_ERROR_INVALID_ARGS;
3194
3195 qp_lock(qpair);
3196
3197 do {
3198 result = qp_dequeue_locked(qpair->produce_q,
3199 qpair->consume_q,
3200 qpair->consume_q_size,
3201 &msg->msg_iter, true);
3202
3203 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3204 !qp_wait_for_ready_queue(qpair))
3205 result = VMCI_ERROR_WOULD_BLOCK;
3206
3207 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3208
3209 qp_unlock(qpair);
3210
3211 return result;
3212}
3213EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3214
3215/*
3216 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3217 * @qpair: Pointer to the queue pair struct.
3218 * @iov: Pointer to buffer for the data
3219 * @iov_size: Length of buffer.
3220 * @buf_type: Buffer type (Unused on Linux).
3221 *
3222 * This is the client interface for peeking into a queue. (I.e.,
3223 * copy data from the queue without updating the head pointer.)
3224 * This function uses IO vectors to handle the work. Returns number
3225 * of bytes peeked or < 0 on error.
3226 */
3227ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3228 struct msghdr *msg,
3229 size_t iov_size,
3230 int buf_type)
3231{
3232 ssize_t result;
3233
3234 if (!qpair)
3235 return VMCI_ERROR_INVALID_ARGS;
3236
3237 qp_lock(qpair);
3238
3239 do {
3240 result = qp_dequeue_locked(qpair->produce_q,
3241 qpair->consume_q,
3242 qpair->consume_q_size,
3243 &msg->msg_iter, false);
3244
3245 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3246 !qp_wait_for_ready_queue(qpair))
3247 result = VMCI_ERROR_WOULD_BLOCK;
3248
3249 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3250
3251 qp_unlock(qpair);
3252 return result;
3253}
3254EXPORT_SYMBOL_GPL(vmci_qpair_peekv);