Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/sched.h>
14#include <linux/wait.h>
15#include <linux/mm.h>
16#include <linux/slab.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/completion.h>
20#include <linux/delay.h>
21#include <linux/cpu.h>
22#include <linux/hyperv.h>
23#include <asm/mshyperv.h>
24#include <linux/sched/isolation.h>
25
26#include "hyperv_vmbus.h"
27
28static void init_vp_index(struct vmbus_channel *channel);
29
30const struct vmbus_device vmbus_devs[] = {
31 /* IDE */
32 { .dev_type = HV_IDE,
33 HV_IDE_GUID,
34 .perf_device = true,
35 .allowed_in_isolated = false,
36 },
37
38 /* SCSI */
39 { .dev_type = HV_SCSI,
40 HV_SCSI_GUID,
41 .perf_device = true,
42 .allowed_in_isolated = true,
43 },
44
45 /* Fibre Channel */
46 { .dev_type = HV_FC,
47 HV_SYNTHFC_GUID,
48 .perf_device = true,
49 .allowed_in_isolated = false,
50 },
51
52 /* Synthetic NIC */
53 { .dev_type = HV_NIC,
54 HV_NIC_GUID,
55 .perf_device = true,
56 .allowed_in_isolated = true,
57 },
58
59 /* Network Direct */
60 { .dev_type = HV_ND,
61 HV_ND_GUID,
62 .perf_device = true,
63 .allowed_in_isolated = false,
64 },
65
66 /* PCIE */
67 { .dev_type = HV_PCIE,
68 HV_PCIE_GUID,
69 .perf_device = false,
70 .allowed_in_isolated = true,
71 },
72
73 /* Synthetic Frame Buffer */
74 { .dev_type = HV_FB,
75 HV_SYNTHVID_GUID,
76 .perf_device = false,
77 .allowed_in_isolated = false,
78 },
79
80 /* Synthetic Keyboard */
81 { .dev_type = HV_KBD,
82 HV_KBD_GUID,
83 .perf_device = false,
84 .allowed_in_isolated = false,
85 },
86
87 /* Synthetic MOUSE */
88 { .dev_type = HV_MOUSE,
89 HV_MOUSE_GUID,
90 .perf_device = false,
91 .allowed_in_isolated = false,
92 },
93
94 /* KVP */
95 { .dev_type = HV_KVP,
96 HV_KVP_GUID,
97 .perf_device = false,
98 .allowed_in_isolated = false,
99 },
100
101 /* Time Synch */
102 { .dev_type = HV_TS,
103 HV_TS_GUID,
104 .perf_device = false,
105 .allowed_in_isolated = true,
106 },
107
108 /* Heartbeat */
109 { .dev_type = HV_HB,
110 HV_HEART_BEAT_GUID,
111 .perf_device = false,
112 .allowed_in_isolated = true,
113 },
114
115 /* Shutdown */
116 { .dev_type = HV_SHUTDOWN,
117 HV_SHUTDOWN_GUID,
118 .perf_device = false,
119 .allowed_in_isolated = true,
120 },
121
122 /* File copy */
123 { .dev_type = HV_FCOPY,
124 HV_FCOPY_GUID,
125 .perf_device = false,
126 .allowed_in_isolated = false,
127 },
128
129 /* Backup */
130 { .dev_type = HV_BACKUP,
131 HV_VSS_GUID,
132 .perf_device = false,
133 .allowed_in_isolated = false,
134 },
135
136 /* Dynamic Memory */
137 { .dev_type = HV_DM,
138 HV_DM_GUID,
139 .perf_device = false,
140 .allowed_in_isolated = false,
141 },
142
143 /* Unknown GUID */
144 { .dev_type = HV_UNKNOWN,
145 .perf_device = false,
146 .allowed_in_isolated = false,
147 },
148};
149
150static const struct {
151 guid_t guid;
152} vmbus_unsupported_devs[] = {
153 { HV_AVMA1_GUID },
154 { HV_AVMA2_GUID },
155 { HV_RDV_GUID },
156 { HV_IMC_GUID },
157};
158
159/*
160 * The rescinded channel may be blocked waiting for a response from the host;
161 * take care of that.
162 */
163static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
164{
165 struct vmbus_channel_msginfo *msginfo;
166 unsigned long flags;
167
168
169 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
170 channel->rescind = true;
171 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
172 msglistentry) {
173
174 if (msginfo->waiting_channel == channel) {
175 complete(&msginfo->waitevent);
176 break;
177 }
178 }
179 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
180}
181
182static bool is_unsupported_vmbus_devs(const guid_t *guid)
183{
184 int i;
185
186 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
187 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))
188 return true;
189 return false;
190}
191
192static u16 hv_get_dev_type(const struct vmbus_channel *channel)
193{
194 const guid_t *guid = &channel->offermsg.offer.if_type;
195 u16 i;
196
197 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
198 return HV_UNKNOWN;
199
200 for (i = HV_IDE; i < HV_UNKNOWN; i++) {
201 if (guid_equal(guid, &vmbus_devs[i].guid))
202 return i;
203 }
204 pr_info("Unknown GUID: %pUl\n", guid);
205 return i;
206}
207
208/**
209 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
210 * @icmsghdrp: Pointer to msg header structure
211 * @buf: Raw buffer channel data
212 * @buflen: Length of the raw buffer channel data.
213 * @fw_version: The framework versions we can support.
214 * @fw_vercnt: The size of @fw_version.
215 * @srv_version: The service versions we can support.
216 * @srv_vercnt: The size of @srv_version.
217 * @nego_fw_version: The selected framework version.
218 * @nego_srv_version: The selected service version.
219 *
220 * Note: Versions are given in decreasing order.
221 *
222 * Set up and fill in default negotiate response message.
223 * Mainly used by Hyper-V drivers.
224 */
225bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf,
226 u32 buflen, const int *fw_version, int fw_vercnt,
227 const int *srv_version, int srv_vercnt,
228 int *nego_fw_version, int *nego_srv_version)
229{
230 int icframe_major, icframe_minor;
231 int icmsg_major, icmsg_minor;
232 int fw_major, fw_minor;
233 int srv_major, srv_minor;
234 int i, j;
235 bool found_match = false;
236 struct icmsg_negotiate *negop;
237
238 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */
239 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) {
240 pr_err_ratelimited("Invalid icmsg negotiate\n");
241 return false;
242 }
243
244 icmsghdrp->icmsgsize = 0x10;
245 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR];
246
247 icframe_major = negop->icframe_vercnt;
248 icframe_minor = 0;
249
250 icmsg_major = negop->icmsg_vercnt;
251 icmsg_minor = 0;
252
253 /* Validate negop packet */
254 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
255 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
256 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) {
257 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",
258 icframe_major, icmsg_major);
259 goto fw_error;
260 }
261
262 /*
263 * Select the framework version number we will
264 * support.
265 */
266
267 for (i = 0; i < fw_vercnt; i++) {
268 fw_major = (fw_version[i] >> 16);
269 fw_minor = (fw_version[i] & 0xFFFF);
270
271 for (j = 0; j < negop->icframe_vercnt; j++) {
272 if ((negop->icversion_data[j].major == fw_major) &&
273 (negop->icversion_data[j].minor == fw_minor)) {
274 icframe_major = negop->icversion_data[j].major;
275 icframe_minor = negop->icversion_data[j].minor;
276 found_match = true;
277 break;
278 }
279 }
280
281 if (found_match)
282 break;
283 }
284
285 if (!found_match)
286 goto fw_error;
287
288 found_match = false;
289
290 for (i = 0; i < srv_vercnt; i++) {
291 srv_major = (srv_version[i] >> 16);
292 srv_minor = (srv_version[i] & 0xFFFF);
293
294 for (j = negop->icframe_vercnt;
295 (j < negop->icframe_vercnt + negop->icmsg_vercnt);
296 j++) {
297
298 if ((negop->icversion_data[j].major == srv_major) &&
299 (negop->icversion_data[j].minor == srv_minor)) {
300
301 icmsg_major = negop->icversion_data[j].major;
302 icmsg_minor = negop->icversion_data[j].minor;
303 found_match = true;
304 break;
305 }
306 }
307
308 if (found_match)
309 break;
310 }
311
312 /*
313 * Respond with the framework and service
314 * version numbers we can support.
315 */
316
317fw_error:
318 if (!found_match) {
319 negop->icframe_vercnt = 0;
320 negop->icmsg_vercnt = 0;
321 } else {
322 negop->icframe_vercnt = 1;
323 negop->icmsg_vercnt = 1;
324 }
325
326 if (nego_fw_version)
327 *nego_fw_version = (icframe_major << 16) | icframe_minor;
328
329 if (nego_srv_version)
330 *nego_srv_version = (icmsg_major << 16) | icmsg_minor;
331
332 negop->icversion_data[0].major = icframe_major;
333 negop->icversion_data[0].minor = icframe_minor;
334 negop->icversion_data[1].major = icmsg_major;
335 negop->icversion_data[1].minor = icmsg_minor;
336 return found_match;
337}
338EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
339
340/*
341 * alloc_channel - Allocate and initialize a vmbus channel object
342 */
343static struct vmbus_channel *alloc_channel(void)
344{
345 struct vmbus_channel *channel;
346
347 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
348 if (!channel)
349 return NULL;
350
351 spin_lock_init(&channel->sched_lock);
352 init_completion(&channel->rescind_event);
353
354 INIT_LIST_HEAD(&channel->sc_list);
355
356 tasklet_init(&channel->callback_event,
357 vmbus_on_event, (unsigned long)channel);
358
359 hv_ringbuffer_pre_init(channel);
360
361 return channel;
362}
363
364/*
365 * free_channel - Release the resources used by the vmbus channel object
366 */
367static void free_channel(struct vmbus_channel *channel)
368{
369 tasklet_kill(&channel->callback_event);
370 vmbus_remove_channel_attr_group(channel);
371
372 kobject_put(&channel->kobj);
373}
374
375void vmbus_channel_map_relid(struct vmbus_channel *channel)
376{
377 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
378 return;
379 /*
380 * The mapping of the channel's relid is visible from the CPUs that
381 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will
382 * execute:
383 *
384 * (a) In the "normal (i.e., not resuming from hibernation)" path,
385 * the full barrier in virt_store_mb() guarantees that the store
386 * is propagated to all CPUs before the add_channel_work work
387 * is queued. In turn, add_channel_work is queued before the
388 * channel's ring buffer is allocated/initialized and the
389 * OPENCHANNEL message for the channel is sent in vmbus_open().
390 * Hyper-V won't start sending the interrupts for the channel
391 * before the OPENCHANNEL message is acked. The memory barrier
392 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures
393 * that vmbus_chan_sched() must find the channel's relid in
394 * recv_int_page before retrieving the channel pointer from the
395 * array of channels.
396 *
397 * (b) In the "resuming from hibernation" path, the virt_store_mb()
398 * guarantees that the store is propagated to all CPUs before
399 * the VMBus connection is marked as ready for the resume event
400 * (cf. check_ready_for_resume_event()). The interrupt handler
401 * of the VMBus driver and vmbus_chan_sched() can not run before
402 * vmbus_bus_resume() has completed execution (cf. resume_noirq).
403 */
404 virt_store_mb(
405 vmbus_connection.channels[channel->offermsg.child_relid],
406 channel);
407}
408
409void vmbus_channel_unmap_relid(struct vmbus_channel *channel)
410{
411 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
412 return;
413 WRITE_ONCE(
414 vmbus_connection.channels[channel->offermsg.child_relid],
415 NULL);
416}
417
418static void vmbus_release_relid(u32 relid)
419{
420 struct vmbus_channel_relid_released msg;
421 int ret;
422
423 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
424 msg.child_relid = relid;
425 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
426 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
427 true);
428
429 trace_vmbus_release_relid(&msg, ret);
430}
431
432void hv_process_channel_removal(struct vmbus_channel *channel)
433{
434 lockdep_assert_held(&vmbus_connection.channel_mutex);
435 BUG_ON(!channel->rescind);
436
437 /*
438 * hv_process_channel_removal() could find INVALID_RELID only for
439 * hv_sock channels. See the inline comments in vmbus_onoffer().
440 */
441 WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&
442 !is_hvsock_channel(channel));
443
444 /*
445 * Upon suspend, an in-use hv_sock channel is removed from the array of
446 * channels and the relid is invalidated. After hibernation, when the
447 * user-space application destroys the channel, it's unnecessary and
448 * unsafe to remove the channel from the array of channels. See also
449 * the inline comments before the call of vmbus_release_relid() below.
450 */
451 if (channel->offermsg.child_relid != INVALID_RELID)
452 vmbus_channel_unmap_relid(channel);
453
454 if (channel->primary_channel == NULL)
455 list_del(&channel->listentry);
456 else
457 list_del(&channel->sc_list);
458
459 /*
460 * If this is a "perf" channel, updates the hv_numa_map[] masks so that
461 * init_vp_index() can (re-)use the CPU.
462 */
463 if (hv_is_perf_channel(channel))
464 hv_clear_allocated_cpu(channel->target_cpu);
465
466 /*
467 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and
468 * the relid is invalidated; after hibernation, when the user-space app
469 * destroys the channel, the relid is INVALID_RELID, and in this case
470 * it's unnecessary and unsafe to release the old relid, since the same
471 * relid can refer to a completely different channel now.
472 */
473 if (channel->offermsg.child_relid != INVALID_RELID)
474 vmbus_release_relid(channel->offermsg.child_relid);
475
476 free_channel(channel);
477}
478
479void vmbus_free_channels(void)
480{
481 struct vmbus_channel *channel, *tmp;
482
483 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
484 listentry) {
485 /* hv_process_channel_removal() needs this */
486 channel->rescind = true;
487
488 vmbus_device_unregister(channel->device_obj);
489 }
490}
491
492/* Note: the function can run concurrently for primary/sub channels. */
493static void vmbus_add_channel_work(struct work_struct *work)
494{
495 struct vmbus_channel *newchannel =
496 container_of(work, struct vmbus_channel, add_channel_work);
497 struct vmbus_channel *primary_channel = newchannel->primary_channel;
498 int ret;
499
500 /*
501 * This state is used to indicate a successful open
502 * so that when we do close the channel normally, we
503 * can cleanup properly.
504 */
505 newchannel->state = CHANNEL_OPEN_STATE;
506
507 if (primary_channel != NULL) {
508 /* newchannel is a sub-channel. */
509 struct hv_device *dev = primary_channel->device_obj;
510
511 if (vmbus_add_channel_kobj(dev, newchannel))
512 goto err_deq_chan;
513
514 if (primary_channel->sc_creation_callback != NULL)
515 primary_channel->sc_creation_callback(newchannel);
516
517 newchannel->probe_done = true;
518 return;
519 }
520
521 /*
522 * Start the process of binding the primary channel to the driver
523 */
524 newchannel->device_obj = vmbus_device_create(
525 &newchannel->offermsg.offer.if_type,
526 &newchannel->offermsg.offer.if_instance,
527 newchannel);
528 if (!newchannel->device_obj)
529 goto err_deq_chan;
530
531 newchannel->device_obj->device_id = newchannel->device_id;
532 /*
533 * Add the new device to the bus. This will kick off device-driver
534 * binding which eventually invokes the device driver's AddDevice()
535 * method.
536 *
537 * If vmbus_device_register() fails, the 'device_obj' is freed in
538 * vmbus_device_release() as called by device_unregister() in the
539 * error path of vmbus_device_register(). In the outside error
540 * path, there's no need to free it.
541 */
542 ret = vmbus_device_register(newchannel->device_obj);
543
544 if (ret != 0) {
545 pr_err("unable to add child device object (relid %d)\n",
546 newchannel->offermsg.child_relid);
547 goto err_deq_chan;
548 }
549
550 newchannel->probe_done = true;
551 return;
552
553err_deq_chan:
554 mutex_lock(&vmbus_connection.channel_mutex);
555
556 /*
557 * We need to set the flag, otherwise
558 * vmbus_onoffer_rescind() can be blocked.
559 */
560 newchannel->probe_done = true;
561
562 if (primary_channel == NULL)
563 list_del(&newchannel->listentry);
564 else
565 list_del(&newchannel->sc_list);
566
567 /* vmbus_process_offer() has mapped the channel. */
568 vmbus_channel_unmap_relid(newchannel);
569
570 mutex_unlock(&vmbus_connection.channel_mutex);
571
572 vmbus_release_relid(newchannel->offermsg.child_relid);
573
574 free_channel(newchannel);
575}
576
577/*
578 * vmbus_process_offer - Process the offer by creating a channel/device
579 * associated with this offer
580 */
581static void vmbus_process_offer(struct vmbus_channel *newchannel)
582{
583 struct vmbus_channel *channel;
584 struct workqueue_struct *wq;
585 bool fnew = true;
586
587 /*
588 * Synchronize vmbus_process_offer() and CPU hotplugging:
589 *
590 * CPU1 CPU2
591 *
592 * [vmbus_process_offer()] [Hot removal of the CPU]
593 *
594 * CPU_READ_LOCK CPUS_WRITE_LOCK
595 * LOAD cpu_online_mask SEARCH chn_list
596 * STORE target_cpu LOAD target_cpu
597 * INSERT chn_list STORE cpu_online_mask
598 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK
599 *
600 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE &&
601 * CPU2's SEARCH from *not* seeing CPU1's INSERT
602 *
603 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT &&
604 * CPU2's LOAD from *not* seing CPU1's STORE
605 */
606 cpus_read_lock();
607
608 /*
609 * Serializes the modifications of the chn_list list as well as
610 * the accesses to next_numa_node_id in init_vp_index().
611 */
612 mutex_lock(&vmbus_connection.channel_mutex);
613
614 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
615 if (guid_equal(&channel->offermsg.offer.if_type,
616 &newchannel->offermsg.offer.if_type) &&
617 guid_equal(&channel->offermsg.offer.if_instance,
618 &newchannel->offermsg.offer.if_instance)) {
619 fnew = false;
620 newchannel->primary_channel = channel;
621 break;
622 }
623 }
624
625 init_vp_index(newchannel);
626
627 /* Remember the channels that should be cleaned up upon suspend. */
628 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))
629 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);
630
631 /*
632 * Now that we have acquired the channel_mutex,
633 * we can release the potentially racing rescind thread.
634 */
635 atomic_dec(&vmbus_connection.offer_in_progress);
636
637 if (fnew) {
638 list_add_tail(&newchannel->listentry,
639 &vmbus_connection.chn_list);
640 } else {
641 /*
642 * Check to see if this is a valid sub-channel.
643 */
644 if (newchannel->offermsg.offer.sub_channel_index == 0) {
645 mutex_unlock(&vmbus_connection.channel_mutex);
646 cpus_read_unlock();
647 /*
648 * Don't call free_channel(), because newchannel->kobj
649 * is not initialized yet.
650 */
651 kfree(newchannel);
652 WARN_ON_ONCE(1);
653 return;
654 }
655 /*
656 * Process the sub-channel.
657 */
658 list_add_tail(&newchannel->sc_list, &channel->sc_list);
659 }
660
661 vmbus_channel_map_relid(newchannel);
662
663 mutex_unlock(&vmbus_connection.channel_mutex);
664 cpus_read_unlock();
665
666 /*
667 * vmbus_process_offer() mustn't call channel->sc_creation_callback()
668 * directly for sub-channels, because sc_creation_callback() ->
669 * vmbus_open() may never get the host's response to the
670 * OPEN_CHANNEL message (the host may rescind a channel at any time,
671 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()
672 * may not wake up the vmbus_open() as it's blocked due to a non-zero
673 * vmbus_connection.offer_in_progress, and finally we have a deadlock.
674 *
675 * The above is also true for primary channels, if the related device
676 * drivers use sync probing mode by default.
677 *
678 * And, usually the handling of primary channels and sub-channels can
679 * depend on each other, so we should offload them to different
680 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,
681 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->
682 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock
683 * and waits for all the sub-channels to appear, but the latter
684 * can't get the rtnl_lock and this blocks the handling of
685 * sub-channels.
686 */
687 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);
688 wq = fnew ? vmbus_connection.handle_primary_chan_wq :
689 vmbus_connection.handle_sub_chan_wq;
690 queue_work(wq, &newchannel->add_channel_work);
691}
692
693/*
694 * Check if CPUs used by other channels of the same device.
695 * It should only be called by init_vp_index().
696 */
697static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn)
698{
699 struct vmbus_channel *primary = chn->primary_channel;
700 struct vmbus_channel *sc;
701
702 lockdep_assert_held(&vmbus_connection.channel_mutex);
703
704 if (!primary)
705 return false;
706
707 if (primary->target_cpu == cpu)
708 return true;
709
710 list_for_each_entry(sc, &primary->sc_list, sc_list)
711 if (sc != chn && sc->target_cpu == cpu)
712 return true;
713
714 return false;
715}
716
717/*
718 * We use this state to statically distribute the channel interrupt load.
719 */
720static int next_numa_node_id;
721
722/*
723 * We can statically distribute the incoming channel interrupt load
724 * by binding a channel to VCPU.
725 *
726 * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.
727 * Performance critical channels will be distributed evenly among all
728 * the available NUMA nodes. Once the node is assigned, we will assign
729 * the CPU based on a simple round robin scheme.
730 */
731static void init_vp_index(struct vmbus_channel *channel)
732{
733 bool perf_chn = hv_is_perf_channel(channel);
734 u32 i, ncpu = num_online_cpus();
735 cpumask_var_t available_mask;
736 struct cpumask *allocated_mask;
737 const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
738 u32 target_cpu;
739 int numa_node;
740
741 if (!perf_chn ||
742 !alloc_cpumask_var(&available_mask, GFP_KERNEL) ||
743 cpumask_empty(hk_mask)) {
744 /*
745 * If the channel is not a performance critical
746 * channel, bind it to VMBUS_CONNECT_CPU.
747 * In case alloc_cpumask_var() fails, bind it to
748 * VMBUS_CONNECT_CPU.
749 * If all the cpus are isolated, bind it to
750 * VMBUS_CONNECT_CPU.
751 */
752 channel->target_cpu = VMBUS_CONNECT_CPU;
753 if (perf_chn)
754 hv_set_allocated_cpu(VMBUS_CONNECT_CPU);
755 return;
756 }
757
758 for (i = 1; i <= ncpu + 1; i++) {
759 while (true) {
760 numa_node = next_numa_node_id++;
761 if (numa_node == nr_node_ids) {
762 next_numa_node_id = 0;
763 continue;
764 }
765 if (cpumask_empty(cpumask_of_node(numa_node)))
766 continue;
767 break;
768 }
769 allocated_mask = &hv_context.hv_numa_map[numa_node];
770
771retry:
772 cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node));
773 cpumask_and(available_mask, available_mask, hk_mask);
774
775 if (cpumask_empty(available_mask)) {
776 /*
777 * We have cycled through all the CPUs in the node;
778 * reset the allocated map.
779 */
780 cpumask_clear(allocated_mask);
781 goto retry;
782 }
783
784 target_cpu = cpumask_first(available_mask);
785 cpumask_set_cpu(target_cpu, allocated_mask);
786
787 if (channel->offermsg.offer.sub_channel_index >= ncpu ||
788 i > ncpu || !hv_cpuself_used(target_cpu, channel))
789 break;
790 }
791
792 channel->target_cpu = target_cpu;
793
794 free_cpumask_var(available_mask);
795}
796
797#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */
798#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */
799#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS)
800#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */
801#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS)
802
803static void vmbus_wait_for_unload(void)
804{
805 int cpu;
806 void *page_addr;
807 struct hv_message *msg;
808 struct vmbus_channel_message_header *hdr;
809 u32 message_type, i;
810
811 /*
812 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
813 * used for initial contact or to CPU0 depending on host version. When
814 * we're crashing on a different CPU let's hope that IRQ handler on
815 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
816 * functional and vmbus_unload_response() will complete
817 * vmbus_connection.unload_event. If not, the last thing we can do is
818 * read message pages for all CPUs directly.
819 *
820 * Wait up to 100 seconds since an Azure host must writeback any dirty
821 * data in its disk cache before the VMbus UNLOAD request will
822 * complete. This flushing has been empirically observed to take up
823 * to 50 seconds in cases with a lot of dirty data, so allow additional
824 * leeway and for inaccuracies in mdelay(). But eventually time out so
825 * that the panic path can't get hung forever in case the response
826 * message isn't seen.
827 */
828 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) {
829 if (completion_done(&vmbus_connection.unload_event))
830 goto completed;
831
832 for_each_present_cpu(cpu) {
833 struct hv_per_cpu_context *hv_cpu
834 = per_cpu_ptr(hv_context.cpu_context, cpu);
835
836 /*
837 * In a CoCo VM the synic_message_page is not allocated
838 * in hv_synic_alloc(). Instead it is set/cleared in
839 * hv_synic_enable_regs() and hv_synic_disable_regs()
840 * such that it is set only when the CPU is online. If
841 * not all present CPUs are online, the message page
842 * might be NULL, so skip such CPUs.
843 */
844 page_addr = hv_cpu->synic_message_page;
845 if (!page_addr)
846 continue;
847
848 msg = (struct hv_message *)page_addr
849 + VMBUS_MESSAGE_SINT;
850
851 message_type = READ_ONCE(msg->header.message_type);
852 if (message_type == HVMSG_NONE)
853 continue;
854
855 hdr = (struct vmbus_channel_message_header *)
856 msg->u.payload;
857
858 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
859 complete(&vmbus_connection.unload_event);
860
861 vmbus_signal_eom(msg, message_type);
862 }
863
864 /*
865 * Give a notice periodically so someone watching the
866 * serial output won't think it is completely hung.
867 */
868 if (!(i % UNLOAD_MSG_LOOPS))
869 pr_notice("Waiting for VMBus UNLOAD to complete\n");
870
871 mdelay(UNLOAD_DELAY_UNIT_MS);
872 }
873 pr_err("Continuing even though VMBus UNLOAD did not complete\n");
874
875completed:
876 /*
877 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
878 * maybe-pending messages on all CPUs to be able to receive new
879 * messages after we reconnect.
880 */
881 for_each_present_cpu(cpu) {
882 struct hv_per_cpu_context *hv_cpu
883 = per_cpu_ptr(hv_context.cpu_context, cpu);
884
885 page_addr = hv_cpu->synic_message_page;
886 if (!page_addr)
887 continue;
888
889 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
890 msg->header.message_type = HVMSG_NONE;
891 }
892}
893
894/*
895 * vmbus_unload_response - Handler for the unload response.
896 */
897static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
898{
899 /*
900 * This is a global event; just wakeup the waiting thread.
901 * Once we successfully unload, we can cleanup the monitor state.
902 *
903 * NB. A malicious or compromised Hyper-V could send a spurious
904 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call
905 * of the complete() below. Make sure that unload_event has been
906 * initialized by the time this complete() is executed.
907 */
908 complete(&vmbus_connection.unload_event);
909}
910
911void vmbus_initiate_unload(bool crash)
912{
913 struct vmbus_channel_message_header hdr;
914
915 if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED)
916 return;
917
918 /* Pre-Win2012R2 hosts don't support reconnect */
919 if (vmbus_proto_version < VERSION_WIN8_1)
920 return;
921
922 reinit_completion(&vmbus_connection.unload_event);
923 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
924 hdr.msgtype = CHANNELMSG_UNLOAD;
925 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
926 !crash);
927
928 /*
929 * vmbus_initiate_unload() is also called on crash and the crash can be
930 * happening in an interrupt context, where scheduling is impossible.
931 */
932 if (!crash)
933 wait_for_completion(&vmbus_connection.unload_event);
934 else
935 vmbus_wait_for_unload();
936}
937
938static void check_ready_for_resume_event(void)
939{
940 /*
941 * If all the old primary channels have been fixed up, then it's safe
942 * to resume.
943 */
944 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))
945 complete(&vmbus_connection.ready_for_resume_event);
946}
947
948static void vmbus_setup_channel_state(struct vmbus_channel *channel,
949 struct vmbus_channel_offer_channel *offer)
950{
951 /*
952 * Setup state for signalling the host.
953 */
954 channel->sig_event = VMBUS_EVENT_CONNECTION_ID;
955
956 channel->is_dedicated_interrupt =
957 (offer->is_dedicated_interrupt != 0);
958 channel->sig_event = offer->connection_id;
959
960 memcpy(&channel->offermsg, offer,
961 sizeof(struct vmbus_channel_offer_channel));
962 channel->monitor_grp = (u8)offer->monitorid / 32;
963 channel->monitor_bit = (u8)offer->monitorid % 32;
964 channel->device_id = hv_get_dev_type(channel);
965}
966
967/*
968 * find_primary_channel_by_offer - Get the channel object given the new offer.
969 * This is only used in the resume path of hibernation.
970 */
971static struct vmbus_channel *
972find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)
973{
974 struct vmbus_channel *channel = NULL, *iter;
975 const guid_t *inst1, *inst2;
976
977 /* Ignore sub-channel offers. */
978 if (offer->offer.sub_channel_index != 0)
979 return NULL;
980
981 mutex_lock(&vmbus_connection.channel_mutex);
982
983 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {
984 inst1 = &iter->offermsg.offer.if_instance;
985 inst2 = &offer->offer.if_instance;
986
987 if (guid_equal(inst1, inst2)) {
988 channel = iter;
989 break;
990 }
991 }
992
993 mutex_unlock(&vmbus_connection.channel_mutex);
994
995 return channel;
996}
997
998static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer)
999{
1000 const guid_t *guid = &offer->offer.if_type;
1001 u16 i;
1002
1003 if (!hv_is_isolation_supported())
1004 return true;
1005
1006 if (is_hvsock_offer(offer))
1007 return true;
1008
1009 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) {
1010 if (guid_equal(guid, &vmbus_devs[i].guid))
1011 return vmbus_devs[i].allowed_in_isolated;
1012 }
1013 return false;
1014}
1015
1016/*
1017 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1018 *
1019 */
1020static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
1021{
1022 struct vmbus_channel_offer_channel *offer;
1023 struct vmbus_channel *oldchannel, *newchannel;
1024 size_t offer_sz;
1025
1026 offer = (struct vmbus_channel_offer_channel *)hdr;
1027
1028 trace_vmbus_onoffer(offer);
1029
1030 if (!vmbus_is_valid_offer(offer)) {
1031 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n",
1032 offer->child_relid);
1033 atomic_dec(&vmbus_connection.offer_in_progress);
1034 return;
1035 }
1036
1037 oldchannel = find_primary_channel_by_offer(offer);
1038
1039 if (oldchannel != NULL) {
1040 /*
1041 * We're resuming from hibernation: all the sub-channel and
1042 * hv_sock channels we had before the hibernation should have
1043 * been cleaned up, and now we must be seeing a re-offered
1044 * primary channel that we had before the hibernation.
1045 */
1046
1047 /*
1048 * { Initially: channel relid = INVALID_RELID,
1049 * channels[valid_relid] = NULL }
1050 *
1051 * CPU1 CPU2
1052 *
1053 * [vmbus_onoffer()] [vmbus_device_release()]
1054 *
1055 * LOCK channel_mutex LOCK channel_mutex
1056 * STORE channel relid = valid_relid LOAD r1 = channel relid
1057 * MAP_RELID channel if (r1 != INVALID_RELID)
1058 * UNLOCK channel_mutex UNMAP_RELID channel
1059 * UNLOCK channel_mutex
1060 *
1061 * Forbids: r1 == valid_relid &&
1062 * channels[valid_relid] == channel
1063 *
1064 * Note. r1 can be INVALID_RELID only for an hv_sock channel.
1065 * None of the hv_sock channels which were present before the
1066 * suspend are re-offered upon the resume. See the WARN_ON()
1067 * in hv_process_channel_removal().
1068 */
1069 mutex_lock(&vmbus_connection.channel_mutex);
1070
1071 atomic_dec(&vmbus_connection.offer_in_progress);
1072
1073 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);
1074 /* Fix up the relid. */
1075 oldchannel->offermsg.child_relid = offer->child_relid;
1076
1077 offer_sz = sizeof(*offer);
1078 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {
1079 /*
1080 * This is not an error, since the host can also change
1081 * the other field(s) of the offer, e.g. on WS RS5
1082 * (Build 17763), the offer->connection_id of the
1083 * Mellanox VF vmbus device can change when the host
1084 * reoffers the device upon resume.
1085 */
1086 pr_debug("vmbus offer changed: relid=%d\n",
1087 offer->child_relid);
1088
1089 print_hex_dump_debug("Old vmbus offer: ",
1090 DUMP_PREFIX_OFFSET, 16, 4,
1091 &oldchannel->offermsg, offer_sz,
1092 false);
1093 print_hex_dump_debug("New vmbus offer: ",
1094 DUMP_PREFIX_OFFSET, 16, 4,
1095 offer, offer_sz, false);
1096
1097 /* Fix up the old channel. */
1098 vmbus_setup_channel_state(oldchannel, offer);
1099 }
1100
1101 /* Add the channel back to the array of channels. */
1102 vmbus_channel_map_relid(oldchannel);
1103 check_ready_for_resume_event();
1104
1105 mutex_unlock(&vmbus_connection.channel_mutex);
1106 return;
1107 }
1108
1109 /* Allocate the channel object and save this offer. */
1110 newchannel = alloc_channel();
1111 if (!newchannel) {
1112 vmbus_release_relid(offer->child_relid);
1113 atomic_dec(&vmbus_connection.offer_in_progress);
1114 pr_err("Unable to allocate channel object\n");
1115 return;
1116 }
1117
1118 vmbus_setup_channel_state(newchannel, offer);
1119
1120 vmbus_process_offer(newchannel);
1121}
1122
1123static void check_ready_for_suspend_event(void)
1124{
1125 /*
1126 * If all the sub-channels or hv_sock channels have been cleaned up,
1127 * then it's safe to suspend.
1128 */
1129 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))
1130 complete(&vmbus_connection.ready_for_suspend_event);
1131}
1132
1133/*
1134 * vmbus_onoffer_rescind - Rescind offer handler.
1135 *
1136 * We queue a work item to process this offer synchronously
1137 */
1138static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
1139{
1140 struct vmbus_channel_rescind_offer *rescind;
1141 struct vmbus_channel *channel;
1142 struct device *dev;
1143 bool clean_up_chan_for_suspend;
1144
1145 rescind = (struct vmbus_channel_rescind_offer *)hdr;
1146
1147 trace_vmbus_onoffer_rescind(rescind);
1148
1149 /*
1150 * The offer msg and the corresponding rescind msg
1151 * from the host are guranteed to be ordered -
1152 * offer comes in first and then the rescind.
1153 * Since we process these events in work elements,
1154 * and with preemption, we may end up processing
1155 * the events out of order. We rely on the synchronization
1156 * provided by offer_in_progress and by channel_mutex for
1157 * ordering these events:
1158 *
1159 * { Initially: offer_in_progress = 1 }
1160 *
1161 * CPU1 CPU2
1162 *
1163 * [vmbus_onoffer()] [vmbus_onoffer_rescind()]
1164 *
1165 * LOCK channel_mutex WAIT_ON offer_in_progress == 0
1166 * DECREMENT offer_in_progress LOCK channel_mutex
1167 * STORE channels[] LOAD channels[]
1168 * UNLOCK channel_mutex UNLOCK channel_mutex
1169 *
1170 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE
1171 */
1172
1173 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {
1174 /*
1175 * We wait here until any channel offer is currently
1176 * being processed.
1177 */
1178 msleep(1);
1179 }
1180
1181 mutex_lock(&vmbus_connection.channel_mutex);
1182 channel = relid2channel(rescind->child_relid);
1183 if (channel != NULL) {
1184 /*
1185 * Guarantee that no other instance of vmbus_onoffer_rescind()
1186 * has got a reference to the channel object. Synchronize on
1187 * &vmbus_connection.channel_mutex.
1188 */
1189 if (channel->rescind_ref) {
1190 mutex_unlock(&vmbus_connection.channel_mutex);
1191 return;
1192 }
1193 channel->rescind_ref = true;
1194 }
1195 mutex_unlock(&vmbus_connection.channel_mutex);
1196
1197 if (channel == NULL) {
1198 /*
1199 * We failed in processing the offer message;
1200 * we would have cleaned up the relid in that
1201 * failure path.
1202 */
1203 return;
1204 }
1205
1206 clean_up_chan_for_suspend = is_hvsock_channel(channel) ||
1207 is_sub_channel(channel);
1208 /*
1209 * Before setting channel->rescind in vmbus_rescind_cleanup(), we
1210 * should make sure the channel callback is not running any more.
1211 */
1212 vmbus_reset_channel_cb(channel);
1213
1214 /*
1215 * Now wait for offer handling to complete.
1216 */
1217 vmbus_rescind_cleanup(channel);
1218 while (READ_ONCE(channel->probe_done) == false) {
1219 /*
1220 * We wait here until any channel offer is currently
1221 * being processed.
1222 */
1223 msleep(1);
1224 }
1225
1226 /*
1227 * At this point, the rescind handling can proceed safely.
1228 */
1229
1230 if (channel->device_obj) {
1231 if (channel->chn_rescind_callback) {
1232 channel->chn_rescind_callback(channel);
1233
1234 if (clean_up_chan_for_suspend)
1235 check_ready_for_suspend_event();
1236
1237 return;
1238 }
1239 /*
1240 * We will have to unregister this device from the
1241 * driver core.
1242 */
1243 dev = get_device(&channel->device_obj->device);
1244 if (dev) {
1245 vmbus_device_unregister(channel->device_obj);
1246 put_device(dev);
1247 }
1248 } else if (channel->primary_channel != NULL) {
1249 /*
1250 * Sub-channel is being rescinded. Following is the channel
1251 * close sequence when initiated from the driveri (refer to
1252 * vmbus_close() for details):
1253 * 1. Close all sub-channels first
1254 * 2. Then close the primary channel.
1255 */
1256 mutex_lock(&vmbus_connection.channel_mutex);
1257 if (channel->state == CHANNEL_OPEN_STATE) {
1258 /*
1259 * The channel is currently not open;
1260 * it is safe for us to cleanup the channel.
1261 */
1262 hv_process_channel_removal(channel);
1263 } else {
1264 complete(&channel->rescind_event);
1265 }
1266 mutex_unlock(&vmbus_connection.channel_mutex);
1267 }
1268
1269 /* The "channel" may have been freed. Do not access it any longer. */
1270
1271 if (clean_up_chan_for_suspend)
1272 check_ready_for_suspend_event();
1273}
1274
1275void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
1276{
1277 BUG_ON(!is_hvsock_channel(channel));
1278
1279 /* We always get a rescind msg when a connection is closed. */
1280 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))
1281 msleep(1);
1282
1283 vmbus_device_unregister(channel->device_obj);
1284}
1285EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
1286
1287
1288/*
1289 * vmbus_onoffers_delivered -
1290 * This is invoked when all offers have been delivered.
1291 *
1292 * Nothing to do here.
1293 */
1294static void vmbus_onoffers_delivered(
1295 struct vmbus_channel_message_header *hdr)
1296{
1297}
1298
1299/*
1300 * vmbus_onopen_result - Open result handler.
1301 *
1302 * This is invoked when we received a response to our channel open request.
1303 * Find the matching request, copy the response and signal the requesting
1304 * thread.
1305 */
1306static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
1307{
1308 struct vmbus_channel_open_result *result;
1309 struct vmbus_channel_msginfo *msginfo;
1310 struct vmbus_channel_message_header *requestheader;
1311 struct vmbus_channel_open_channel *openmsg;
1312 unsigned long flags;
1313
1314 result = (struct vmbus_channel_open_result *)hdr;
1315
1316 trace_vmbus_onopen_result(result);
1317
1318 /*
1319 * Find the open msg, copy the result and signal/unblock the wait event
1320 */
1321 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1322
1323 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1324 msglistentry) {
1325 requestheader =
1326 (struct vmbus_channel_message_header *)msginfo->msg;
1327
1328 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
1329 openmsg =
1330 (struct vmbus_channel_open_channel *)msginfo->msg;
1331 if (openmsg->child_relid == result->child_relid &&
1332 openmsg->openid == result->openid) {
1333 memcpy(&msginfo->response.open_result,
1334 result,
1335 sizeof(
1336 struct vmbus_channel_open_result));
1337 complete(&msginfo->waitevent);
1338 break;
1339 }
1340 }
1341 }
1342 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1343}
1344
1345/*
1346 * vmbus_ongpadl_created - GPADL created handler.
1347 *
1348 * This is invoked when we received a response to our gpadl create request.
1349 * Find the matching request, copy the response and signal the requesting
1350 * thread.
1351 */
1352static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
1353{
1354 struct vmbus_channel_gpadl_created *gpadlcreated;
1355 struct vmbus_channel_msginfo *msginfo;
1356 struct vmbus_channel_message_header *requestheader;
1357 struct vmbus_channel_gpadl_header *gpadlheader;
1358 unsigned long flags;
1359
1360 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
1361
1362 trace_vmbus_ongpadl_created(gpadlcreated);
1363
1364 /*
1365 * Find the establish msg, copy the result and signal/unblock the wait
1366 * event
1367 */
1368 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1369
1370 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1371 msglistentry) {
1372 requestheader =
1373 (struct vmbus_channel_message_header *)msginfo->msg;
1374
1375 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
1376 gpadlheader =
1377 (struct vmbus_channel_gpadl_header *)requestheader;
1378
1379 if ((gpadlcreated->child_relid ==
1380 gpadlheader->child_relid) &&
1381 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
1382 memcpy(&msginfo->response.gpadl_created,
1383 gpadlcreated,
1384 sizeof(
1385 struct vmbus_channel_gpadl_created));
1386 complete(&msginfo->waitevent);
1387 break;
1388 }
1389 }
1390 }
1391 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1392}
1393
1394/*
1395 * vmbus_onmodifychannel_response - Modify Channel response handler.
1396 *
1397 * This is invoked when we received a response to our channel modify request.
1398 * Find the matching request, copy the response and signal the requesting thread.
1399 */
1400static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr)
1401{
1402 struct vmbus_channel_modifychannel_response *response;
1403 struct vmbus_channel_msginfo *msginfo;
1404 unsigned long flags;
1405
1406 response = (struct vmbus_channel_modifychannel_response *)hdr;
1407
1408 trace_vmbus_onmodifychannel_response(response);
1409
1410 /*
1411 * Find the modify msg, copy the response and signal/unblock the wait event.
1412 */
1413 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1414
1415 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) {
1416 struct vmbus_channel_message_header *responseheader =
1417 (struct vmbus_channel_message_header *)msginfo->msg;
1418
1419 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {
1420 struct vmbus_channel_modifychannel *modifymsg;
1421
1422 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;
1423 if (modifymsg->child_relid == response->child_relid) {
1424 memcpy(&msginfo->response.modify_response, response,
1425 sizeof(*response));
1426 complete(&msginfo->waitevent);
1427 break;
1428 }
1429 }
1430 }
1431 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1432}
1433
1434/*
1435 * vmbus_ongpadl_torndown - GPADL torndown handler.
1436 *
1437 * This is invoked when we received a response to our gpadl teardown request.
1438 * Find the matching request, copy the response and signal the requesting
1439 * thread.
1440 */
1441static void vmbus_ongpadl_torndown(
1442 struct vmbus_channel_message_header *hdr)
1443{
1444 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1445 struct vmbus_channel_msginfo *msginfo;
1446 struct vmbus_channel_message_header *requestheader;
1447 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1448 unsigned long flags;
1449
1450 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1451
1452 trace_vmbus_ongpadl_torndown(gpadl_torndown);
1453
1454 /*
1455 * Find the open msg, copy the result and signal/unblock the wait event
1456 */
1457 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1458
1459 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1460 msglistentry) {
1461 requestheader =
1462 (struct vmbus_channel_message_header *)msginfo->msg;
1463
1464 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1465 gpadl_teardown =
1466 (struct vmbus_channel_gpadl_teardown *)requestheader;
1467
1468 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1469 memcpy(&msginfo->response.gpadl_torndown,
1470 gpadl_torndown,
1471 sizeof(
1472 struct vmbus_channel_gpadl_torndown));
1473 complete(&msginfo->waitevent);
1474 break;
1475 }
1476 }
1477 }
1478 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1479}
1480
1481/*
1482 * vmbus_onversion_response - Version response handler
1483 *
1484 * This is invoked when we received a response to our initiate contact request.
1485 * Find the matching request, copy the response and signal the requesting
1486 * thread.
1487 */
1488static void vmbus_onversion_response(
1489 struct vmbus_channel_message_header *hdr)
1490{
1491 struct vmbus_channel_msginfo *msginfo;
1492 struct vmbus_channel_message_header *requestheader;
1493 struct vmbus_channel_version_response *version_response;
1494 unsigned long flags;
1495
1496 version_response = (struct vmbus_channel_version_response *)hdr;
1497
1498 trace_vmbus_onversion_response(version_response);
1499
1500 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1501
1502 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1503 msglistentry) {
1504 requestheader =
1505 (struct vmbus_channel_message_header *)msginfo->msg;
1506
1507 if (requestheader->msgtype ==
1508 CHANNELMSG_INITIATE_CONTACT) {
1509 memcpy(&msginfo->response.version_response,
1510 version_response,
1511 sizeof(struct vmbus_channel_version_response));
1512 complete(&msginfo->waitevent);
1513 }
1514 }
1515 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1516}
1517
1518/* Channel message dispatch table */
1519const struct vmbus_channel_message_table_entry
1520channel_message_table[CHANNELMSG_COUNT] = {
1521 { CHANNELMSG_INVALID, 0, NULL, 0},
1522 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer,
1523 sizeof(struct vmbus_channel_offer_channel)},
1524 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind,
1525 sizeof(struct vmbus_channel_rescind_offer) },
1526 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0},
1527 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0},
1528 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0},
1529 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result,
1530 sizeof(struct vmbus_channel_open_result)},
1531 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0},
1532 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0},
1533 { CHANNELMSG_GPADL_BODY, 0, NULL, 0},
1534 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created,
1535 sizeof(struct vmbus_channel_gpadl_created)},
1536 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0},
1537 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown,
1538 sizeof(struct vmbus_channel_gpadl_torndown) },
1539 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0},
1540 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0},
1541 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response,
1542 sizeof(struct vmbus_channel_version_response)},
1543 { CHANNELMSG_UNLOAD, 0, NULL, 0},
1544 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0},
1545 { CHANNELMSG_18, 0, NULL, 0},
1546 { CHANNELMSG_19, 0, NULL, 0},
1547 { CHANNELMSG_20, 0, NULL, 0},
1548 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0},
1549 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0},
1550 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0},
1551 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response,
1552 sizeof(struct vmbus_channel_modifychannel_response)},
1553};
1554
1555/*
1556 * vmbus_onmessage - Handler for channel protocol messages.
1557 *
1558 * This is invoked in the vmbus worker thread context.
1559 */
1560void vmbus_onmessage(struct vmbus_channel_message_header *hdr)
1561{
1562 trace_vmbus_on_message(hdr);
1563
1564 /*
1565 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go
1566 * out of bound and the message_handler pointer can not be NULL.
1567 */
1568 channel_message_table[hdr->msgtype].message_handler(hdr);
1569}
1570
1571/*
1572 * vmbus_request_offers - Send a request to get all our pending offers.
1573 */
1574int vmbus_request_offers(void)
1575{
1576 struct vmbus_channel_message_header *msg;
1577 struct vmbus_channel_msginfo *msginfo;
1578 int ret;
1579
1580 msginfo = kzalloc(sizeof(*msginfo) +
1581 sizeof(struct vmbus_channel_message_header),
1582 GFP_KERNEL);
1583 if (!msginfo)
1584 return -ENOMEM;
1585
1586 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1587
1588 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1589
1590 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1591 true);
1592
1593 trace_vmbus_request_offers(ret);
1594
1595 if (ret != 0) {
1596 pr_err("Unable to request offers - %d\n", ret);
1597
1598 goto cleanup;
1599 }
1600
1601cleanup:
1602 kfree(msginfo);
1603
1604 return ret;
1605}
1606
1607void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1608 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1609{
1610 primary_channel->sc_creation_callback = sc_cr_cb;
1611}
1612EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1613
1614void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1615 void (*chn_rescind_cb)(struct vmbus_channel *))
1616{
1617 channel->chn_rescind_callback = chn_rescind_cb;
1618}
1619EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);