Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VMware VMCI Driver
4 *
5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 */
7
8#include <linux/vmw_vmci_defs.h>
9#include <linux/vmw_vmci_api.h>
10#include <linux/completion.h>
11#include <linux/hash.h>
12#include <linux/kernel.h>
13#include <linux/list.h>
14#include <linux/module.h>
15#include <linux/sched.h>
16#include <linux/slab.h>
17
18#include "vmci_datagram.h"
19#include "vmci_doorbell.h"
20#include "vmci_resource.h"
21#include "vmci_driver.h"
22#include "vmci_route.h"
23
24
25#define VMCI_DOORBELL_INDEX_BITS 6
26#define VMCI_DOORBELL_INDEX_TABLE_SIZE (1 << VMCI_DOORBELL_INDEX_BITS)
27#define VMCI_DOORBELL_HASH(_idx) hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
28
29/*
30 * DoorbellEntry describes the a doorbell notification handle allocated by the
31 * host.
32 */
33struct dbell_entry {
34 struct vmci_resource resource;
35 struct hlist_node node;
36 struct work_struct work;
37 vmci_callback notify_cb;
38 void *client_data;
39 u32 idx;
40 u32 priv_flags;
41 bool run_delayed;
42 atomic_t active; /* Only used by guest personality */
43};
44
45/* The VMCI index table keeps track of currently registered doorbells. */
46struct dbell_index_table {
47 spinlock_t lock; /* Index table lock */
48 struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
49};
50
51static struct dbell_index_table vmci_doorbell_it = {
52 .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
53};
54
55/*
56 * The max_notify_idx is one larger than the currently known bitmap index in
57 * use, and is used to determine how much of the bitmap needs to be scanned.
58 */
59static u32 max_notify_idx;
60
61/*
62 * The notify_idx_count is used for determining whether there are free entries
63 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
64 */
65static u32 notify_idx_count;
66
67/*
68 * The last_notify_idx_reserved is used to track the last index handed out - in
69 * the case where multiple handles share a notification index, we hand out
70 * indexes round robin based on last_notify_idx_reserved.
71 */
72static u32 last_notify_idx_reserved;
73
74/* This is a one entry cache used to by the index allocation. */
75static u32 last_notify_idx_released = PAGE_SIZE;
76
77
78/*
79 * Utility function that retrieves the privilege flags associated
80 * with a given doorbell handle. For guest endpoints, the
81 * privileges are determined by the context ID, but for host
82 * endpoints privileges are associated with the complete
83 * handle. Hypervisor endpoints are not yet supported.
84 */
85int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
86{
87 if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
88 return VMCI_ERROR_INVALID_ARGS;
89
90 if (handle.context == VMCI_HOST_CONTEXT_ID) {
91 struct dbell_entry *entry;
92 struct vmci_resource *resource;
93
94 resource = vmci_resource_by_handle(handle,
95 VMCI_RESOURCE_TYPE_DOORBELL);
96 if (!resource)
97 return VMCI_ERROR_NOT_FOUND;
98
99 entry = container_of(resource, struct dbell_entry, resource);
100 *priv_flags = entry->priv_flags;
101 vmci_resource_put(resource);
102 } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
103 /*
104 * Hypervisor endpoints for notifications are not
105 * supported (yet).
106 */
107 return VMCI_ERROR_INVALID_ARGS;
108 } else {
109 *priv_flags = vmci_context_get_priv_flags(handle.context);
110 }
111
112 return VMCI_SUCCESS;
113}
114
115/*
116 * Find doorbell entry by bitmap index.
117 */
118static struct dbell_entry *dbell_index_table_find(u32 idx)
119{
120 u32 bucket = VMCI_DOORBELL_HASH(idx);
121 struct dbell_entry *dbell;
122
123 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
124 node) {
125 if (idx == dbell->idx)
126 return dbell;
127 }
128
129 return NULL;
130}
131
132/*
133 * Add the given entry to the index table. This willi take a reference to the
134 * entry's resource so that the entry is not deleted before it is removed from
135 * the * table.
136 */
137static void dbell_index_table_add(struct dbell_entry *entry)
138{
139 u32 bucket;
140 u32 new_notify_idx;
141
142 vmci_resource_get(&entry->resource);
143
144 spin_lock_bh(&vmci_doorbell_it.lock);
145
146 /*
147 * Below we try to allocate an index in the notification
148 * bitmap with "not too much" sharing between resources. If we
149 * use less that the full bitmap, we either add to the end if
150 * there are no unused flags within the currently used area,
151 * or we search for unused ones. If we use the full bitmap, we
152 * allocate the index round robin.
153 */
154 if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
155 if (last_notify_idx_released < max_notify_idx &&
156 !dbell_index_table_find(last_notify_idx_released)) {
157 new_notify_idx = last_notify_idx_released;
158 last_notify_idx_released = PAGE_SIZE;
159 } else {
160 bool reused = false;
161 new_notify_idx = last_notify_idx_reserved;
162 if (notify_idx_count + 1 < max_notify_idx) {
163 do {
164 if (!dbell_index_table_find
165 (new_notify_idx)) {
166 reused = true;
167 break;
168 }
169 new_notify_idx = (new_notify_idx + 1) %
170 max_notify_idx;
171 } while (new_notify_idx !=
172 last_notify_idx_released);
173 }
174 if (!reused) {
175 new_notify_idx = max_notify_idx;
176 max_notify_idx++;
177 }
178 }
179 } else {
180 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
181 }
182
183 last_notify_idx_reserved = new_notify_idx;
184 notify_idx_count++;
185
186 entry->idx = new_notify_idx;
187 bucket = VMCI_DOORBELL_HASH(entry->idx);
188 hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
189
190 spin_unlock_bh(&vmci_doorbell_it.lock);
191}
192
193/*
194 * Remove the given entry from the index table. This will release() the
195 * entry's resource.
196 */
197static void dbell_index_table_remove(struct dbell_entry *entry)
198{
199 spin_lock_bh(&vmci_doorbell_it.lock);
200
201 hlist_del_init(&entry->node);
202
203 notify_idx_count--;
204 if (entry->idx == max_notify_idx - 1) {
205 /*
206 * If we delete an entry with the maximum known
207 * notification index, we take the opportunity to
208 * prune the current max. As there might be other
209 * unused indices immediately below, we lower the
210 * maximum until we hit an index in use.
211 */
212 while (max_notify_idx > 0 &&
213 !dbell_index_table_find(max_notify_idx - 1))
214 max_notify_idx--;
215 }
216
217 last_notify_idx_released = entry->idx;
218
219 spin_unlock_bh(&vmci_doorbell_it.lock);
220
221 vmci_resource_put(&entry->resource);
222}
223
224/*
225 * Creates a link between the given doorbell handle and the given
226 * index in the bitmap in the device backend. A notification state
227 * is created in hypervisor.
228 */
229static int dbell_link(struct vmci_handle handle, u32 notify_idx)
230{
231 struct vmci_doorbell_link_msg link_msg;
232
233 link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
234 VMCI_DOORBELL_LINK);
235 link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
236 link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
237 link_msg.handle = handle;
238 link_msg.notify_idx = notify_idx;
239
240 return vmci_send_datagram(&link_msg.hdr);
241}
242
243/*
244 * Unlinks the given doorbell handle from an index in the bitmap in
245 * the device backend. The notification state is destroyed in hypervisor.
246 */
247static int dbell_unlink(struct vmci_handle handle)
248{
249 struct vmci_doorbell_unlink_msg unlink_msg;
250
251 unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
252 VMCI_DOORBELL_UNLINK);
253 unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
254 unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
255 unlink_msg.handle = handle;
256
257 return vmci_send_datagram(&unlink_msg.hdr);
258}
259
260/*
261 * Notify another guest or the host. We send a datagram down to the
262 * host via the hypervisor with the notification info.
263 */
264static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
265{
266 struct vmci_doorbell_notify_msg notify_msg;
267
268 notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
269 VMCI_DOORBELL_NOTIFY);
270 notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
271 notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
272 notify_msg.handle = handle;
273
274 return vmci_send_datagram(¬ify_msg.hdr);
275}
276
277/*
278 * Calls the specified callback in a delayed context.
279 */
280static void dbell_delayed_dispatch(struct work_struct *work)
281{
282 struct dbell_entry *entry = container_of(work,
283 struct dbell_entry, work);
284
285 entry->notify_cb(entry->client_data);
286 vmci_resource_put(&entry->resource);
287}
288
289/*
290 * Dispatches a doorbell notification to the host context.
291 */
292int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
293{
294 struct dbell_entry *entry;
295 struct vmci_resource *resource;
296
297 if (vmci_handle_is_invalid(handle)) {
298 pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
299 handle.context, handle.resource);
300 return VMCI_ERROR_INVALID_ARGS;
301 }
302
303 resource = vmci_resource_by_handle(handle,
304 VMCI_RESOURCE_TYPE_DOORBELL);
305 if (!resource) {
306 pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
307 handle.context, handle.resource);
308 return VMCI_ERROR_NOT_FOUND;
309 }
310
311 entry = container_of(resource, struct dbell_entry, resource);
312 if (entry->run_delayed) {
313 if (!schedule_work(&entry->work))
314 vmci_resource_put(resource);
315 } else {
316 entry->notify_cb(entry->client_data);
317 vmci_resource_put(resource);
318 }
319
320 return VMCI_SUCCESS;
321}
322
323/*
324 * Register the notification bitmap with the host.
325 */
326bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)
327{
328 int result;
329 struct vmci_notify_bm_set_msg bitmap_set_msg = { };
330
331 bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
332 VMCI_SET_NOTIFY_BITMAP);
333 bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
334 bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
335 VMCI_DG_HEADERSIZE;
336 if (vmci_use_ppn64())
337 bitmap_set_msg.bitmap_ppn64 = bitmap_ppn;
338 else
339 bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn;
340
341 result = vmci_send_datagram(&bitmap_set_msg.hdr);
342 if (result != VMCI_SUCCESS) {
343 pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n",
344 bitmap_ppn, result);
345 return false;
346 }
347 return true;
348}
349
350/*
351 * Executes or schedules the handlers for a given notify index.
352 */
353static void dbell_fire_entries(u32 notify_idx)
354{
355 u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
356 struct dbell_entry *dbell;
357
358 spin_lock_bh(&vmci_doorbell_it.lock);
359
360 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
361 if (dbell->idx == notify_idx &&
362 atomic_read(&dbell->active) == 1) {
363 if (dbell->run_delayed) {
364 vmci_resource_get(&dbell->resource);
365 if (!schedule_work(&dbell->work))
366 vmci_resource_put(&dbell->resource);
367 } else {
368 dbell->notify_cb(dbell->client_data);
369 }
370 }
371 }
372
373 spin_unlock_bh(&vmci_doorbell_it.lock);
374}
375
376/*
377 * Scans the notification bitmap, collects pending notifications,
378 * resets the bitmap and invokes appropriate callbacks.
379 */
380void vmci_dbell_scan_notification_entries(u8 *bitmap)
381{
382 u32 idx;
383
384 for (idx = 0; idx < max_notify_idx; idx++) {
385 if (bitmap[idx] & 0x1) {
386 bitmap[idx] &= ~1;
387 dbell_fire_entries(idx);
388 }
389 }
390}
391
392/*
393 * vmci_doorbell_create() - Creates a doorbell
394 * @handle: A handle used to track the resource. Can be invalid.
395 * @flags: Flag that determines context of callback.
396 * @priv_flags: Privileges flags.
397 * @notify_cb: The callback to be ivoked when the doorbell fires.
398 * @client_data: A parameter to be passed to the callback.
399 *
400 * Creates a doorbell with the given callback. If the handle is
401 * VMCI_INVALID_HANDLE, a free handle will be assigned, if
402 * possible. The callback can be run immediately (potentially with
403 * locks held - the default) or delayed (in a kernel thread) by
404 * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
405 * is selected, a given callback may not be run if the kernel is
406 * unable to allocate memory for the delayed execution (highly
407 * unlikely).
408 */
409int vmci_doorbell_create(struct vmci_handle *handle,
410 u32 flags,
411 u32 priv_flags,
412 vmci_callback notify_cb, void *client_data)
413{
414 struct dbell_entry *entry;
415 struct vmci_handle new_handle;
416 int result;
417
418 if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
419 priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
420 return VMCI_ERROR_INVALID_ARGS;
421
422 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
423 if (entry == NULL) {
424 pr_warn("Failed allocating memory for datagram entry\n");
425 return VMCI_ERROR_NO_MEM;
426 }
427
428 if (vmci_handle_is_invalid(*handle)) {
429 u32 context_id = vmci_get_context_id();
430
431 if (context_id == VMCI_INVALID_ID) {
432 pr_warn("Failed to get context ID\n");
433 result = VMCI_ERROR_NO_RESOURCES;
434 goto free_mem;
435 }
436
437 /* Let resource code allocate a free ID for us */
438 new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
439 } else {
440 bool valid_context = false;
441
442 /*
443 * Validate the handle. We must do both of the checks below
444 * because we can be acting as both a host and a guest at the
445 * same time. We always allow the host context ID, since the
446 * host functionality is in practice always there with the
447 * unified driver.
448 */
449 if (handle->context == VMCI_HOST_CONTEXT_ID ||
450 (vmci_guest_code_active() &&
451 vmci_get_context_id() == handle->context)) {
452 valid_context = true;
453 }
454
455 if (!valid_context || handle->resource == VMCI_INVALID_ID) {
456 pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
457 handle->context, handle->resource);
458 result = VMCI_ERROR_INVALID_ARGS;
459 goto free_mem;
460 }
461
462 new_handle = *handle;
463 }
464
465 entry->idx = 0;
466 INIT_HLIST_NODE(&entry->node);
467 entry->priv_flags = priv_flags;
468 INIT_WORK(&entry->work, dbell_delayed_dispatch);
469 entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
470 entry->notify_cb = notify_cb;
471 entry->client_data = client_data;
472 atomic_set(&entry->active, 0);
473
474 result = vmci_resource_add(&entry->resource,
475 VMCI_RESOURCE_TYPE_DOORBELL,
476 new_handle);
477 if (result != VMCI_SUCCESS) {
478 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
479 new_handle.context, new_handle.resource, result);
480 goto free_mem;
481 }
482
483 new_handle = vmci_resource_handle(&entry->resource);
484 if (vmci_guest_code_active()) {
485 dbell_index_table_add(entry);
486 result = dbell_link(new_handle, entry->idx);
487 if (VMCI_SUCCESS != result)
488 goto destroy_resource;
489
490 atomic_set(&entry->active, 1);
491 }
492
493 *handle = new_handle;
494
495 return result;
496
497 destroy_resource:
498 dbell_index_table_remove(entry);
499 vmci_resource_remove(&entry->resource);
500 free_mem:
501 kfree(entry);
502 return result;
503}
504EXPORT_SYMBOL_GPL(vmci_doorbell_create);
505
506/*
507 * vmci_doorbell_destroy() - Destroy a doorbell.
508 * @handle: The handle tracking the resource.
509 *
510 * Destroys a doorbell previously created with vmcii_doorbell_create. This
511 * operation may block waiting for a callback to finish.
512 */
513int vmci_doorbell_destroy(struct vmci_handle handle)
514{
515 struct dbell_entry *entry;
516 struct vmci_resource *resource;
517
518 if (vmci_handle_is_invalid(handle))
519 return VMCI_ERROR_INVALID_ARGS;
520
521 resource = vmci_resource_by_handle(handle,
522 VMCI_RESOURCE_TYPE_DOORBELL);
523 if (!resource) {
524 pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
525 handle.context, handle.resource);
526 return VMCI_ERROR_NOT_FOUND;
527 }
528
529 entry = container_of(resource, struct dbell_entry, resource);
530
531 if (!hlist_unhashed(&entry->node)) {
532 int result;
533
534 dbell_index_table_remove(entry);
535
536 result = dbell_unlink(handle);
537 if (VMCI_SUCCESS != result) {
538
539 /*
540 * The only reason this should fail would be
541 * an inconsistency between guest and
542 * hypervisor state, where the guest believes
543 * it has an active registration whereas the
544 * hypervisor doesn't. One case where this may
545 * happen is if a doorbell is unregistered
546 * following a hibernation at a time where the
547 * doorbell state hasn't been restored on the
548 * hypervisor side yet. Since the handle has
549 * now been removed in the guest, we just
550 * print a warning and return success.
551 */
552 pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
553 handle.context, handle.resource, result);
554 }
555 }
556
557 /*
558 * Now remove the resource from the table. It might still be in use
559 * after this, in a callback or still on the delayed work queue.
560 */
561 vmci_resource_put(&entry->resource);
562 vmci_resource_remove(&entry->resource);
563
564 kfree(entry);
565
566 return VMCI_SUCCESS;
567}
568EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
569
570/*
571 * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
572 * @dst: The handlle identifying the doorbell resource
573 * @priv_flags: Priviledge flags.
574 *
575 * Generates a notification on the doorbell identified by the
576 * handle. For host side generation of notifications, the caller
577 * can specify what the privilege of the calling side is.
578 */
579int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
580{
581 int retval;
582 enum vmci_route route;
583 struct vmci_handle src;
584
585 if (vmci_handle_is_invalid(dst) ||
586 (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
587 return VMCI_ERROR_INVALID_ARGS;
588
589 src = VMCI_INVALID_HANDLE;
590 retval = vmci_route(&src, &dst, false, &route);
591 if (retval < VMCI_SUCCESS)
592 return retval;
593
594 if (VMCI_ROUTE_AS_HOST == route)
595 return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
596 dst, priv_flags);
597
598 if (VMCI_ROUTE_AS_GUEST == route)
599 return dbell_notify_as_guest(dst, priv_flags);
600
601 pr_warn("Unknown route (%d) for doorbell\n", route);
602 return VMCI_ERROR_DST_UNREACHABLE;
603}
604EXPORT_SYMBOL_GPL(vmci_doorbell_notify);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VMware VMCI Driver
4 *
5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
6 */
7
8#include <linux/vmw_vmci_defs.h>
9#include <linux/vmw_vmci_api.h>
10#include <linux/completion.h>
11#include <linux/hash.h>
12#include <linux/kernel.h>
13#include <linux/list.h>
14#include <linux/module.h>
15#include <linux/sched.h>
16#include <linux/slab.h>
17
18#include "vmci_datagram.h"
19#include "vmci_doorbell.h"
20#include "vmci_resource.h"
21#include "vmci_driver.h"
22#include "vmci_route.h"
23
24
25#define VMCI_DOORBELL_INDEX_BITS 6
26#define VMCI_DOORBELL_INDEX_TABLE_SIZE (1 << VMCI_DOORBELL_INDEX_BITS)
27#define VMCI_DOORBELL_HASH(_idx) hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
28
29/*
30 * DoorbellEntry describes the a doorbell notification handle allocated by the
31 * host.
32 */
33struct dbell_entry {
34 struct vmci_resource resource;
35 struct hlist_node node;
36 struct work_struct work;
37 vmci_callback notify_cb;
38 void *client_data;
39 u32 idx;
40 u32 priv_flags;
41 bool run_delayed;
42 atomic_t active; /* Only used by guest personality */
43};
44
45/* The VMCI index table keeps track of currently registered doorbells. */
46struct dbell_index_table {
47 spinlock_t lock; /* Index table lock */
48 struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
49};
50
51static struct dbell_index_table vmci_doorbell_it = {
52 .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
53};
54
55/*
56 * The max_notify_idx is one larger than the currently known bitmap index in
57 * use, and is used to determine how much of the bitmap needs to be scanned.
58 */
59static u32 max_notify_idx;
60
61/*
62 * The notify_idx_count is used for determining whether there are free entries
63 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
64 */
65static u32 notify_idx_count;
66
67/*
68 * The last_notify_idx_reserved is used to track the last index handed out - in
69 * the case where multiple handles share a notification index, we hand out
70 * indexes round robin based on last_notify_idx_reserved.
71 */
72static u32 last_notify_idx_reserved;
73
74/* This is a one entry cache used to by the index allocation. */
75static u32 last_notify_idx_released = PAGE_SIZE;
76
77
78/*
79 * Utility function that retrieves the privilege flags associated
80 * with a given doorbell handle. For guest endpoints, the
81 * privileges are determined by the context ID, but for host
82 * endpoints privileges are associated with the complete
83 * handle. Hypervisor endpoints are not yet supported.
84 */
85int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
86{
87 if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
88 return VMCI_ERROR_INVALID_ARGS;
89
90 if (handle.context == VMCI_HOST_CONTEXT_ID) {
91 struct dbell_entry *entry;
92 struct vmci_resource *resource;
93
94 resource = vmci_resource_by_handle(handle,
95 VMCI_RESOURCE_TYPE_DOORBELL);
96 if (!resource)
97 return VMCI_ERROR_NOT_FOUND;
98
99 entry = container_of(resource, struct dbell_entry, resource);
100 *priv_flags = entry->priv_flags;
101 vmci_resource_put(resource);
102 } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
103 /*
104 * Hypervisor endpoints for notifications are not
105 * supported (yet).
106 */
107 return VMCI_ERROR_INVALID_ARGS;
108 } else {
109 *priv_flags = vmci_context_get_priv_flags(handle.context);
110 }
111
112 return VMCI_SUCCESS;
113}
114
115/*
116 * Find doorbell entry by bitmap index.
117 */
118static struct dbell_entry *dbell_index_table_find(u32 idx)
119{
120 u32 bucket = VMCI_DOORBELL_HASH(idx);
121 struct dbell_entry *dbell;
122
123 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
124 node) {
125 if (idx == dbell->idx)
126 return dbell;
127 }
128
129 return NULL;
130}
131
132/*
133 * Add the given entry to the index table. This willi take a reference to the
134 * entry's resource so that the entry is not deleted before it is removed from
135 * the * table.
136 */
137static void dbell_index_table_add(struct dbell_entry *entry)
138{
139 u32 bucket;
140 u32 new_notify_idx;
141
142 vmci_resource_get(&entry->resource);
143
144 spin_lock_bh(&vmci_doorbell_it.lock);
145
146 /*
147 * Below we try to allocate an index in the notification
148 * bitmap with "not too much" sharing between resources. If we
149 * use less that the full bitmap, we either add to the end if
150 * there are no unused flags within the currently used area,
151 * or we search for unused ones. If we use the full bitmap, we
152 * allocate the index round robin.
153 */
154 if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
155 if (last_notify_idx_released < max_notify_idx &&
156 !dbell_index_table_find(last_notify_idx_released)) {
157 new_notify_idx = last_notify_idx_released;
158 last_notify_idx_released = PAGE_SIZE;
159 } else {
160 bool reused = false;
161 new_notify_idx = last_notify_idx_reserved;
162 if (notify_idx_count + 1 < max_notify_idx) {
163 do {
164 if (!dbell_index_table_find
165 (new_notify_idx)) {
166 reused = true;
167 break;
168 }
169 new_notify_idx = (new_notify_idx + 1) %
170 max_notify_idx;
171 } while (new_notify_idx !=
172 last_notify_idx_released);
173 }
174 if (!reused) {
175 new_notify_idx = max_notify_idx;
176 max_notify_idx++;
177 }
178 }
179 } else {
180 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
181 }
182
183 last_notify_idx_reserved = new_notify_idx;
184 notify_idx_count++;
185
186 entry->idx = new_notify_idx;
187 bucket = VMCI_DOORBELL_HASH(entry->idx);
188 hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
189
190 spin_unlock_bh(&vmci_doorbell_it.lock);
191}
192
193/*
194 * Remove the given entry from the index table. This will release() the
195 * entry's resource.
196 */
197static void dbell_index_table_remove(struct dbell_entry *entry)
198{
199 spin_lock_bh(&vmci_doorbell_it.lock);
200
201 hlist_del_init(&entry->node);
202
203 notify_idx_count--;
204 if (entry->idx == max_notify_idx - 1) {
205 /*
206 * If we delete an entry with the maximum known
207 * notification index, we take the opportunity to
208 * prune the current max. As there might be other
209 * unused indices immediately below, we lower the
210 * maximum until we hit an index in use.
211 */
212 while (max_notify_idx > 0 &&
213 !dbell_index_table_find(max_notify_idx - 1))
214 max_notify_idx--;
215 }
216
217 last_notify_idx_released = entry->idx;
218
219 spin_unlock_bh(&vmci_doorbell_it.lock);
220
221 vmci_resource_put(&entry->resource);
222}
223
224/*
225 * Creates a link between the given doorbell handle and the given
226 * index in the bitmap in the device backend. A notification state
227 * is created in hypervisor.
228 */
229static int dbell_link(struct vmci_handle handle, u32 notify_idx)
230{
231 struct vmci_doorbell_link_msg link_msg;
232
233 link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
234 VMCI_DOORBELL_LINK);
235 link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
236 link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
237 link_msg.handle = handle;
238 link_msg.notify_idx = notify_idx;
239
240 return vmci_send_datagram(&link_msg.hdr);
241}
242
243/*
244 * Unlinks the given doorbell handle from an index in the bitmap in
245 * the device backend. The notification state is destroyed in hypervisor.
246 */
247static int dbell_unlink(struct vmci_handle handle)
248{
249 struct vmci_doorbell_unlink_msg unlink_msg;
250
251 unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
252 VMCI_DOORBELL_UNLINK);
253 unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
254 unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
255 unlink_msg.handle = handle;
256
257 return vmci_send_datagram(&unlink_msg.hdr);
258}
259
260/*
261 * Notify another guest or the host. We send a datagram down to the
262 * host via the hypervisor with the notification info.
263 */
264static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
265{
266 struct vmci_doorbell_notify_msg notify_msg;
267
268 notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
269 VMCI_DOORBELL_NOTIFY);
270 notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
271 notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
272 notify_msg.handle = handle;
273
274 return vmci_send_datagram(¬ify_msg.hdr);
275}
276
277/*
278 * Calls the specified callback in a delayed context.
279 */
280static void dbell_delayed_dispatch(struct work_struct *work)
281{
282 struct dbell_entry *entry = container_of(work,
283 struct dbell_entry, work);
284
285 entry->notify_cb(entry->client_data);
286 vmci_resource_put(&entry->resource);
287}
288
289/*
290 * Dispatches a doorbell notification to the host context.
291 */
292int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
293{
294 struct dbell_entry *entry;
295 struct vmci_resource *resource;
296
297 if (vmci_handle_is_invalid(handle)) {
298 pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
299 handle.context, handle.resource);
300 return VMCI_ERROR_INVALID_ARGS;
301 }
302
303 resource = vmci_resource_by_handle(handle,
304 VMCI_RESOURCE_TYPE_DOORBELL);
305 if (!resource) {
306 pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
307 handle.context, handle.resource);
308 return VMCI_ERROR_NOT_FOUND;
309 }
310
311 entry = container_of(resource, struct dbell_entry, resource);
312 if (entry->run_delayed) {
313 if (!schedule_work(&entry->work))
314 vmci_resource_put(resource);
315 } else {
316 entry->notify_cb(entry->client_data);
317 vmci_resource_put(resource);
318 }
319
320 return VMCI_SUCCESS;
321}
322
323/*
324 * Register the notification bitmap with the host.
325 */
326bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)
327{
328 int result;
329 struct vmci_notify_bm_set_msg bitmap_set_msg = { };
330
331 bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
332 VMCI_SET_NOTIFY_BITMAP);
333 bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
334 bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
335 VMCI_DG_HEADERSIZE;
336 if (vmci_use_ppn64())
337 bitmap_set_msg.bitmap_ppn64 = bitmap_ppn;
338 else
339 bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn;
340
341 result = vmci_send_datagram(&bitmap_set_msg.hdr);
342 if (result != VMCI_SUCCESS) {
343 pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n",
344 bitmap_ppn, result);
345 return false;
346 }
347 return true;
348}
349
350/*
351 * Executes or schedules the handlers for a given notify index.
352 */
353static void dbell_fire_entries(u32 notify_idx)
354{
355 u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
356 struct dbell_entry *dbell;
357
358 spin_lock_bh(&vmci_doorbell_it.lock);
359
360 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
361 if (dbell->idx == notify_idx &&
362 atomic_read(&dbell->active) == 1) {
363 if (dbell->run_delayed) {
364 vmci_resource_get(&dbell->resource);
365 if (!schedule_work(&dbell->work))
366 vmci_resource_put(&dbell->resource);
367 } else {
368 dbell->notify_cb(dbell->client_data);
369 }
370 }
371 }
372
373 spin_unlock_bh(&vmci_doorbell_it.lock);
374}
375
376/*
377 * Scans the notification bitmap, collects pending notifications,
378 * resets the bitmap and invokes appropriate callbacks.
379 */
380void vmci_dbell_scan_notification_entries(u8 *bitmap)
381{
382 u32 idx;
383
384 for (idx = 0; idx < max_notify_idx; idx++) {
385 if (bitmap[idx] & 0x1) {
386 bitmap[idx] &= ~1;
387 dbell_fire_entries(idx);
388 }
389 }
390}
391
392/*
393 * vmci_doorbell_create() - Creates a doorbell
394 * @handle: A handle used to track the resource. Can be invalid.
395 * @flags: Flag that determines context of callback.
396 * @priv_flags: Privileges flags.
397 * @notify_cb: The callback to be ivoked when the doorbell fires.
398 * @client_data: A parameter to be passed to the callback.
399 *
400 * Creates a doorbell with the given callback. If the handle is
401 * VMCI_INVALID_HANDLE, a free handle will be assigned, if
402 * possible. The callback can be run immediately (potentially with
403 * locks held - the default) or delayed (in a kernel thread) by
404 * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
405 * is selected, a given callback may not be run if the kernel is
406 * unable to allocate memory for the delayed execution (highly
407 * unlikely).
408 */
409int vmci_doorbell_create(struct vmci_handle *handle,
410 u32 flags,
411 u32 priv_flags,
412 vmci_callback notify_cb, void *client_data)
413{
414 struct dbell_entry *entry;
415 struct vmci_handle new_handle;
416 int result;
417
418 if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
419 priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
420 return VMCI_ERROR_INVALID_ARGS;
421
422 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
423 if (entry == NULL) {
424 pr_warn("Failed allocating memory for datagram entry\n");
425 return VMCI_ERROR_NO_MEM;
426 }
427
428 if (vmci_handle_is_invalid(*handle)) {
429 u32 context_id = vmci_get_context_id();
430
431 if (context_id == VMCI_INVALID_ID) {
432 pr_warn("Failed to get context ID\n");
433 result = VMCI_ERROR_NO_RESOURCES;
434 goto free_mem;
435 }
436
437 /* Let resource code allocate a free ID for us */
438 new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
439 } else {
440 bool valid_context = false;
441
442 /*
443 * Validate the handle. We must do both of the checks below
444 * because we can be acting as both a host and a guest at the
445 * same time. We always allow the host context ID, since the
446 * host functionality is in practice always there with the
447 * unified driver.
448 */
449 if (handle->context == VMCI_HOST_CONTEXT_ID ||
450 (vmci_guest_code_active() &&
451 vmci_get_context_id() == handle->context)) {
452 valid_context = true;
453 }
454
455 if (!valid_context || handle->resource == VMCI_INVALID_ID) {
456 pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
457 handle->context, handle->resource);
458 result = VMCI_ERROR_INVALID_ARGS;
459 goto free_mem;
460 }
461
462 new_handle = *handle;
463 }
464
465 entry->idx = 0;
466 INIT_HLIST_NODE(&entry->node);
467 entry->priv_flags = priv_flags;
468 INIT_WORK(&entry->work, dbell_delayed_dispatch);
469 entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
470 entry->notify_cb = notify_cb;
471 entry->client_data = client_data;
472 atomic_set(&entry->active, 0);
473
474 result = vmci_resource_add(&entry->resource,
475 VMCI_RESOURCE_TYPE_DOORBELL,
476 new_handle);
477 if (result != VMCI_SUCCESS) {
478 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
479 new_handle.context, new_handle.resource, result);
480 goto free_mem;
481 }
482
483 new_handle = vmci_resource_handle(&entry->resource);
484 if (vmci_guest_code_active()) {
485 dbell_index_table_add(entry);
486 result = dbell_link(new_handle, entry->idx);
487 if (VMCI_SUCCESS != result)
488 goto destroy_resource;
489
490 atomic_set(&entry->active, 1);
491 }
492
493 *handle = new_handle;
494
495 return result;
496
497 destroy_resource:
498 dbell_index_table_remove(entry);
499 vmci_resource_remove(&entry->resource);
500 free_mem:
501 kfree(entry);
502 return result;
503}
504EXPORT_SYMBOL_GPL(vmci_doorbell_create);
505
506/*
507 * vmci_doorbell_destroy() - Destroy a doorbell.
508 * @handle: The handle tracking the resource.
509 *
510 * Destroys a doorbell previously created with vmcii_doorbell_create. This
511 * operation may block waiting for a callback to finish.
512 */
513int vmci_doorbell_destroy(struct vmci_handle handle)
514{
515 struct dbell_entry *entry;
516 struct vmci_resource *resource;
517
518 if (vmci_handle_is_invalid(handle))
519 return VMCI_ERROR_INVALID_ARGS;
520
521 resource = vmci_resource_by_handle(handle,
522 VMCI_RESOURCE_TYPE_DOORBELL);
523 if (!resource) {
524 pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
525 handle.context, handle.resource);
526 return VMCI_ERROR_NOT_FOUND;
527 }
528
529 entry = container_of(resource, struct dbell_entry, resource);
530
531 if (!hlist_unhashed(&entry->node)) {
532 int result;
533
534 dbell_index_table_remove(entry);
535
536 result = dbell_unlink(handle);
537 if (VMCI_SUCCESS != result) {
538
539 /*
540 * The only reason this should fail would be
541 * an inconsistency between guest and
542 * hypervisor state, where the guest believes
543 * it has an active registration whereas the
544 * hypervisor doesn't. One case where this may
545 * happen is if a doorbell is unregistered
546 * following a hibernation at a time where the
547 * doorbell state hasn't been restored on the
548 * hypervisor side yet. Since the handle has
549 * now been removed in the guest, we just
550 * print a warning and return success.
551 */
552 pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
553 handle.context, handle.resource, result);
554 }
555 }
556
557 /*
558 * Now remove the resource from the table. It might still be in use
559 * after this, in a callback or still on the delayed work queue.
560 */
561 vmci_resource_put(&entry->resource);
562 vmci_resource_remove(&entry->resource);
563
564 kfree(entry);
565
566 return VMCI_SUCCESS;
567}
568EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
569
570/*
571 * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
572 * @dst: The handlle identifying the doorbell resource
573 * @priv_flags: Priviledge flags.
574 *
575 * Generates a notification on the doorbell identified by the
576 * handle. For host side generation of notifications, the caller
577 * can specify what the privilege of the calling side is.
578 */
579int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
580{
581 int retval;
582 enum vmci_route route;
583 struct vmci_handle src;
584
585 if (vmci_handle_is_invalid(dst) ||
586 (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
587 return VMCI_ERROR_INVALID_ARGS;
588
589 src = VMCI_INVALID_HANDLE;
590 retval = vmci_route(&src, &dst, false, &route);
591 if (retval < VMCI_SUCCESS)
592 return retval;
593
594 if (VMCI_ROUTE_AS_HOST == route)
595 return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
596 dst, priv_flags);
597
598 if (VMCI_ROUTE_AS_GUEST == route)
599 return dbell_notify_as_guest(dst, priv_flags);
600
601 pr_warn("Unknown route (%d) for doorbell\n", route);
602 return VMCI_ERROR_DST_UNREACHABLE;
603}
604EXPORT_SYMBOL_GPL(vmci_doorbell_notify);