Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/sched.h>
14#include <linux/wait.h>
15#include <linux/mm.h>
16#include <linux/slab.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/completion.h>
20#include <linux/delay.h>
21#include <linux/cpu.h>
22#include <linux/hyperv.h>
23#include <asm/mshyperv.h>
24#include <linux/sched/isolation.h>
25
26#include "hyperv_vmbus.h"
27
28static void init_vp_index(struct vmbus_channel *channel);
29
30const struct vmbus_device vmbus_devs[] = {
31 /* IDE */
32 { .dev_type = HV_IDE,
33 HV_IDE_GUID,
34 .perf_device = true,
35 .allowed_in_isolated = false,
36 },
37
38 /* SCSI */
39 { .dev_type = HV_SCSI,
40 HV_SCSI_GUID,
41 .perf_device = true,
42 .allowed_in_isolated = true,
43 },
44
45 /* Fibre Channel */
46 { .dev_type = HV_FC,
47 HV_SYNTHFC_GUID,
48 .perf_device = true,
49 .allowed_in_isolated = false,
50 },
51
52 /* Synthetic NIC */
53 { .dev_type = HV_NIC,
54 HV_NIC_GUID,
55 .perf_device = true,
56 .allowed_in_isolated = true,
57 },
58
59 /* Network Direct */
60 { .dev_type = HV_ND,
61 HV_ND_GUID,
62 .perf_device = true,
63 .allowed_in_isolated = false,
64 },
65
66 /* PCIE */
67 { .dev_type = HV_PCIE,
68 HV_PCIE_GUID,
69 .perf_device = false,
70 .allowed_in_isolated = false,
71 },
72
73 /* Synthetic Frame Buffer */
74 { .dev_type = HV_FB,
75 HV_SYNTHVID_GUID,
76 .perf_device = false,
77 .allowed_in_isolated = false,
78 },
79
80 /* Synthetic Keyboard */
81 { .dev_type = HV_KBD,
82 HV_KBD_GUID,
83 .perf_device = false,
84 .allowed_in_isolated = false,
85 },
86
87 /* Synthetic MOUSE */
88 { .dev_type = HV_MOUSE,
89 HV_MOUSE_GUID,
90 .perf_device = false,
91 .allowed_in_isolated = false,
92 },
93
94 /* KVP */
95 { .dev_type = HV_KVP,
96 HV_KVP_GUID,
97 .perf_device = false,
98 .allowed_in_isolated = false,
99 },
100
101 /* Time Synch */
102 { .dev_type = HV_TS,
103 HV_TS_GUID,
104 .perf_device = false,
105 .allowed_in_isolated = true,
106 },
107
108 /* Heartbeat */
109 { .dev_type = HV_HB,
110 HV_HEART_BEAT_GUID,
111 .perf_device = false,
112 .allowed_in_isolated = true,
113 },
114
115 /* Shutdown */
116 { .dev_type = HV_SHUTDOWN,
117 HV_SHUTDOWN_GUID,
118 .perf_device = false,
119 .allowed_in_isolated = true,
120 },
121
122 /* File copy */
123 { .dev_type = HV_FCOPY,
124 HV_FCOPY_GUID,
125 .perf_device = false,
126 .allowed_in_isolated = false,
127 },
128
129 /* Backup */
130 { .dev_type = HV_BACKUP,
131 HV_VSS_GUID,
132 .perf_device = false,
133 .allowed_in_isolated = false,
134 },
135
136 /* Dynamic Memory */
137 { .dev_type = HV_DM,
138 HV_DM_GUID,
139 .perf_device = false,
140 .allowed_in_isolated = false,
141 },
142
143 /* Unknown GUID */
144 { .dev_type = HV_UNKNOWN,
145 .perf_device = false,
146 .allowed_in_isolated = false,
147 },
148};
149
150static const struct {
151 guid_t guid;
152} vmbus_unsupported_devs[] = {
153 { HV_AVMA1_GUID },
154 { HV_AVMA2_GUID },
155 { HV_RDV_GUID },
156 { HV_IMC_GUID },
157};
158
159/*
160 * The rescinded channel may be blocked waiting for a response from the host;
161 * take care of that.
162 */
163static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
164{
165 struct vmbus_channel_msginfo *msginfo;
166 unsigned long flags;
167
168
169 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
170 channel->rescind = true;
171 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
172 msglistentry) {
173
174 if (msginfo->waiting_channel == channel) {
175 complete(&msginfo->waitevent);
176 break;
177 }
178 }
179 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
180}
181
182static bool is_unsupported_vmbus_devs(const guid_t *guid)
183{
184 int i;
185
186 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
187 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))
188 return true;
189 return false;
190}
191
192static u16 hv_get_dev_type(const struct vmbus_channel *channel)
193{
194 const guid_t *guid = &channel->offermsg.offer.if_type;
195 u16 i;
196
197 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
198 return HV_UNKNOWN;
199
200 for (i = HV_IDE; i < HV_UNKNOWN; i++) {
201 if (guid_equal(guid, &vmbus_devs[i].guid))
202 return i;
203 }
204 pr_info("Unknown GUID: %pUl\n", guid);
205 return i;
206}
207
208/**
209 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
210 * @icmsghdrp: Pointer to msg header structure
211 * @buf: Raw buffer channel data
212 * @buflen: Length of the raw buffer channel data.
213 * @fw_version: The framework versions we can support.
214 * @fw_vercnt: The size of @fw_version.
215 * @srv_version: The service versions we can support.
216 * @srv_vercnt: The size of @srv_version.
217 * @nego_fw_version: The selected framework version.
218 * @nego_srv_version: The selected service version.
219 *
220 * Note: Versions are given in decreasing order.
221 *
222 * Set up and fill in default negotiate response message.
223 * Mainly used by Hyper-V drivers.
224 */
225bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf,
226 u32 buflen, const int *fw_version, int fw_vercnt,
227 const int *srv_version, int srv_vercnt,
228 int *nego_fw_version, int *nego_srv_version)
229{
230 int icframe_major, icframe_minor;
231 int icmsg_major, icmsg_minor;
232 int fw_major, fw_minor;
233 int srv_major, srv_minor;
234 int i, j;
235 bool found_match = false;
236 struct icmsg_negotiate *negop;
237
238 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */
239 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) {
240 pr_err_ratelimited("Invalid icmsg negotiate\n");
241 return false;
242 }
243
244 icmsghdrp->icmsgsize = 0x10;
245 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR];
246
247 icframe_major = negop->icframe_vercnt;
248 icframe_minor = 0;
249
250 icmsg_major = negop->icmsg_vercnt;
251 icmsg_minor = 0;
252
253 /* Validate negop packet */
254 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
255 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
256 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) {
257 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",
258 icframe_major, icmsg_major);
259 goto fw_error;
260 }
261
262 /*
263 * Select the framework version number we will
264 * support.
265 */
266
267 for (i = 0; i < fw_vercnt; i++) {
268 fw_major = (fw_version[i] >> 16);
269 fw_minor = (fw_version[i] & 0xFFFF);
270
271 for (j = 0; j < negop->icframe_vercnt; j++) {
272 if ((negop->icversion_data[j].major == fw_major) &&
273 (negop->icversion_data[j].minor == fw_minor)) {
274 icframe_major = negop->icversion_data[j].major;
275 icframe_minor = negop->icversion_data[j].minor;
276 found_match = true;
277 break;
278 }
279 }
280
281 if (found_match)
282 break;
283 }
284
285 if (!found_match)
286 goto fw_error;
287
288 found_match = false;
289
290 for (i = 0; i < srv_vercnt; i++) {
291 srv_major = (srv_version[i] >> 16);
292 srv_minor = (srv_version[i] & 0xFFFF);
293
294 for (j = negop->icframe_vercnt;
295 (j < negop->icframe_vercnt + negop->icmsg_vercnt);
296 j++) {
297
298 if ((negop->icversion_data[j].major == srv_major) &&
299 (negop->icversion_data[j].minor == srv_minor)) {
300
301 icmsg_major = negop->icversion_data[j].major;
302 icmsg_minor = negop->icversion_data[j].minor;
303 found_match = true;
304 break;
305 }
306 }
307
308 if (found_match)
309 break;
310 }
311
312 /*
313 * Respond with the framework and service
314 * version numbers we can support.
315 */
316
317fw_error:
318 if (!found_match) {
319 negop->icframe_vercnt = 0;
320 negop->icmsg_vercnt = 0;
321 } else {
322 negop->icframe_vercnt = 1;
323 negop->icmsg_vercnt = 1;
324 }
325
326 if (nego_fw_version)
327 *nego_fw_version = (icframe_major << 16) | icframe_minor;
328
329 if (nego_srv_version)
330 *nego_srv_version = (icmsg_major << 16) | icmsg_minor;
331
332 negop->icversion_data[0].major = icframe_major;
333 negop->icversion_data[0].minor = icframe_minor;
334 negop->icversion_data[1].major = icmsg_major;
335 negop->icversion_data[1].minor = icmsg_minor;
336 return found_match;
337}
338EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
339
340/*
341 * alloc_channel - Allocate and initialize a vmbus channel object
342 */
343static struct vmbus_channel *alloc_channel(void)
344{
345 struct vmbus_channel *channel;
346
347 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
348 if (!channel)
349 return NULL;
350
351 spin_lock_init(&channel->sched_lock);
352 init_completion(&channel->rescind_event);
353
354 INIT_LIST_HEAD(&channel->sc_list);
355
356 tasklet_init(&channel->callback_event,
357 vmbus_on_event, (unsigned long)channel);
358
359 hv_ringbuffer_pre_init(channel);
360
361 return channel;
362}
363
364/*
365 * free_channel - Release the resources used by the vmbus channel object
366 */
367static void free_channel(struct vmbus_channel *channel)
368{
369 tasklet_kill(&channel->callback_event);
370 vmbus_remove_channel_attr_group(channel);
371
372 kobject_put(&channel->kobj);
373}
374
375void vmbus_channel_map_relid(struct vmbus_channel *channel)
376{
377 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
378 return;
379 /*
380 * The mapping of the channel's relid is visible from the CPUs that
381 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will
382 * execute:
383 *
384 * (a) In the "normal (i.e., not resuming from hibernation)" path,
385 * the full barrier in virt_store_mb() guarantees that the store
386 * is propagated to all CPUs before the add_channel_work work
387 * is queued. In turn, add_channel_work is queued before the
388 * channel's ring buffer is allocated/initialized and the
389 * OPENCHANNEL message for the channel is sent in vmbus_open().
390 * Hyper-V won't start sending the interrupts for the channel
391 * before the OPENCHANNEL message is acked. The memory barrier
392 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures
393 * that vmbus_chan_sched() must find the channel's relid in
394 * recv_int_page before retrieving the channel pointer from the
395 * array of channels.
396 *
397 * (b) In the "resuming from hibernation" path, the virt_store_mb()
398 * guarantees that the store is propagated to all CPUs before
399 * the VMBus connection is marked as ready for the resume event
400 * (cf. check_ready_for_resume_event()). The interrupt handler
401 * of the VMBus driver and vmbus_chan_sched() can not run before
402 * vmbus_bus_resume() has completed execution (cf. resume_noirq).
403 */
404 virt_store_mb(
405 vmbus_connection.channels[channel->offermsg.child_relid],
406 channel);
407}
408
409void vmbus_channel_unmap_relid(struct vmbus_channel *channel)
410{
411 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
412 return;
413 WRITE_ONCE(
414 vmbus_connection.channels[channel->offermsg.child_relid],
415 NULL);
416}
417
418static void vmbus_release_relid(u32 relid)
419{
420 struct vmbus_channel_relid_released msg;
421 int ret;
422
423 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
424 msg.child_relid = relid;
425 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
426 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
427 true);
428
429 trace_vmbus_release_relid(&msg, ret);
430}
431
432void hv_process_channel_removal(struct vmbus_channel *channel)
433{
434 lockdep_assert_held(&vmbus_connection.channel_mutex);
435 BUG_ON(!channel->rescind);
436
437 /*
438 * hv_process_channel_removal() could find INVALID_RELID only for
439 * hv_sock channels. See the inline comments in vmbus_onoffer().
440 */
441 WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&
442 !is_hvsock_channel(channel));
443
444 /*
445 * Upon suspend, an in-use hv_sock channel is removed from the array of
446 * channels and the relid is invalidated. After hibernation, when the
447 * user-space application destroys the channel, it's unnecessary and
448 * unsafe to remove the channel from the array of channels. See also
449 * the inline comments before the call of vmbus_release_relid() below.
450 */
451 if (channel->offermsg.child_relid != INVALID_RELID)
452 vmbus_channel_unmap_relid(channel);
453
454 if (channel->primary_channel == NULL)
455 list_del(&channel->listentry);
456 else
457 list_del(&channel->sc_list);
458
459 /*
460 * If this is a "perf" channel, updates the hv_numa_map[] masks so that
461 * init_vp_index() can (re-)use the CPU.
462 */
463 if (hv_is_perf_channel(channel))
464 hv_clear_allocated_cpu(channel->target_cpu);
465
466 /*
467 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and
468 * the relid is invalidated; after hibernation, when the user-space app
469 * destroys the channel, the relid is INVALID_RELID, and in this case
470 * it's unnecessary and unsafe to release the old relid, since the same
471 * relid can refer to a completely different channel now.
472 */
473 if (channel->offermsg.child_relid != INVALID_RELID)
474 vmbus_release_relid(channel->offermsg.child_relid);
475
476 free_channel(channel);
477}
478
479void vmbus_free_channels(void)
480{
481 struct vmbus_channel *channel, *tmp;
482
483 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
484 listentry) {
485 /* hv_process_channel_removal() needs this */
486 channel->rescind = true;
487
488 vmbus_device_unregister(channel->device_obj);
489 }
490}
491
492/* Note: the function can run concurrently for primary/sub channels. */
493static void vmbus_add_channel_work(struct work_struct *work)
494{
495 struct vmbus_channel *newchannel =
496 container_of(work, struct vmbus_channel, add_channel_work);
497 struct vmbus_channel *primary_channel = newchannel->primary_channel;
498 int ret;
499
500 /*
501 * This state is used to indicate a successful open
502 * so that when we do close the channel normally, we
503 * can cleanup properly.
504 */
505 newchannel->state = CHANNEL_OPEN_STATE;
506
507 if (primary_channel != NULL) {
508 /* newchannel is a sub-channel. */
509 struct hv_device *dev = primary_channel->device_obj;
510
511 if (vmbus_add_channel_kobj(dev, newchannel))
512 goto err_deq_chan;
513
514 if (primary_channel->sc_creation_callback != NULL)
515 primary_channel->sc_creation_callback(newchannel);
516
517 newchannel->probe_done = true;
518 return;
519 }
520
521 /*
522 * Start the process of binding the primary channel to the driver
523 */
524 newchannel->device_obj = vmbus_device_create(
525 &newchannel->offermsg.offer.if_type,
526 &newchannel->offermsg.offer.if_instance,
527 newchannel);
528 if (!newchannel->device_obj)
529 goto err_deq_chan;
530
531 newchannel->device_obj->device_id = newchannel->device_id;
532 /*
533 * Add the new device to the bus. This will kick off device-driver
534 * binding which eventually invokes the device driver's AddDevice()
535 * method.
536 *
537 * If vmbus_device_register() fails, the 'device_obj' is freed in
538 * vmbus_device_release() as called by device_unregister() in the
539 * error path of vmbus_device_register(). In the outside error
540 * path, there's no need to free it.
541 */
542 ret = vmbus_device_register(newchannel->device_obj);
543
544 if (ret != 0) {
545 pr_err("unable to add child device object (relid %d)\n",
546 newchannel->offermsg.child_relid);
547 goto err_deq_chan;
548 }
549
550 newchannel->probe_done = true;
551 return;
552
553err_deq_chan:
554 mutex_lock(&vmbus_connection.channel_mutex);
555
556 /*
557 * We need to set the flag, otherwise
558 * vmbus_onoffer_rescind() can be blocked.
559 */
560 newchannel->probe_done = true;
561
562 if (primary_channel == NULL)
563 list_del(&newchannel->listentry);
564 else
565 list_del(&newchannel->sc_list);
566
567 /* vmbus_process_offer() has mapped the channel. */
568 vmbus_channel_unmap_relid(newchannel);
569
570 mutex_unlock(&vmbus_connection.channel_mutex);
571
572 vmbus_release_relid(newchannel->offermsg.child_relid);
573
574 free_channel(newchannel);
575}
576
577/*
578 * vmbus_process_offer - Process the offer by creating a channel/device
579 * associated with this offer
580 */
581static void vmbus_process_offer(struct vmbus_channel *newchannel)
582{
583 struct vmbus_channel *channel;
584 struct workqueue_struct *wq;
585 bool fnew = true;
586
587 /*
588 * Synchronize vmbus_process_offer() and CPU hotplugging:
589 *
590 * CPU1 CPU2
591 *
592 * [vmbus_process_offer()] [Hot removal of the CPU]
593 *
594 * CPU_READ_LOCK CPUS_WRITE_LOCK
595 * LOAD cpu_online_mask SEARCH chn_list
596 * STORE target_cpu LOAD target_cpu
597 * INSERT chn_list STORE cpu_online_mask
598 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK
599 *
600 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE &&
601 * CPU2's SEARCH from *not* seeing CPU1's INSERT
602 *
603 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT &&
604 * CPU2's LOAD from *not* seing CPU1's STORE
605 */
606 cpus_read_lock();
607
608 /*
609 * Serializes the modifications of the chn_list list as well as
610 * the accesses to next_numa_node_id in init_vp_index().
611 */
612 mutex_lock(&vmbus_connection.channel_mutex);
613
614 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
615 if (guid_equal(&channel->offermsg.offer.if_type,
616 &newchannel->offermsg.offer.if_type) &&
617 guid_equal(&channel->offermsg.offer.if_instance,
618 &newchannel->offermsg.offer.if_instance)) {
619 fnew = false;
620 newchannel->primary_channel = channel;
621 break;
622 }
623 }
624
625 init_vp_index(newchannel);
626
627 /* Remember the channels that should be cleaned up upon suspend. */
628 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))
629 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);
630
631 /*
632 * Now that we have acquired the channel_mutex,
633 * we can release the potentially racing rescind thread.
634 */
635 atomic_dec(&vmbus_connection.offer_in_progress);
636
637 if (fnew) {
638 list_add_tail(&newchannel->listentry,
639 &vmbus_connection.chn_list);
640 } else {
641 /*
642 * Check to see if this is a valid sub-channel.
643 */
644 if (newchannel->offermsg.offer.sub_channel_index == 0) {
645 mutex_unlock(&vmbus_connection.channel_mutex);
646 cpus_read_unlock();
647 /*
648 * Don't call free_channel(), because newchannel->kobj
649 * is not initialized yet.
650 */
651 kfree(newchannel);
652 WARN_ON_ONCE(1);
653 return;
654 }
655 /*
656 * Process the sub-channel.
657 */
658 list_add_tail(&newchannel->sc_list, &channel->sc_list);
659 }
660
661 vmbus_channel_map_relid(newchannel);
662
663 mutex_unlock(&vmbus_connection.channel_mutex);
664 cpus_read_unlock();
665
666 /*
667 * vmbus_process_offer() mustn't call channel->sc_creation_callback()
668 * directly for sub-channels, because sc_creation_callback() ->
669 * vmbus_open() may never get the host's response to the
670 * OPEN_CHANNEL message (the host may rescind a channel at any time,
671 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()
672 * may not wake up the vmbus_open() as it's blocked due to a non-zero
673 * vmbus_connection.offer_in_progress, and finally we have a deadlock.
674 *
675 * The above is also true for primary channels, if the related device
676 * drivers use sync probing mode by default.
677 *
678 * And, usually the handling of primary channels and sub-channels can
679 * depend on each other, so we should offload them to different
680 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,
681 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->
682 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock
683 * and waits for all the sub-channels to appear, but the latter
684 * can't get the rtnl_lock and this blocks the handling of
685 * sub-channels.
686 */
687 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);
688 wq = fnew ? vmbus_connection.handle_primary_chan_wq :
689 vmbus_connection.handle_sub_chan_wq;
690 queue_work(wq, &newchannel->add_channel_work);
691}
692
693/*
694 * Check if CPUs used by other channels of the same device.
695 * It should only be called by init_vp_index().
696 */
697static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn)
698{
699 struct vmbus_channel *primary = chn->primary_channel;
700 struct vmbus_channel *sc;
701
702 lockdep_assert_held(&vmbus_connection.channel_mutex);
703
704 if (!primary)
705 return false;
706
707 if (primary->target_cpu == cpu)
708 return true;
709
710 list_for_each_entry(sc, &primary->sc_list, sc_list)
711 if (sc != chn && sc->target_cpu == cpu)
712 return true;
713
714 return false;
715}
716
717/*
718 * We use this state to statically distribute the channel interrupt load.
719 */
720static int next_numa_node_id;
721
722/*
723 * We can statically distribute the incoming channel interrupt load
724 * by binding a channel to VCPU.
725 *
726 * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.
727 * Performance critical channels will be distributed evenly among all
728 * the available NUMA nodes. Once the node is assigned, we will assign
729 * the CPU based on a simple round robin scheme.
730 */
731static void init_vp_index(struct vmbus_channel *channel)
732{
733 bool perf_chn = hv_is_perf_channel(channel);
734 u32 i, ncpu = num_online_cpus();
735 cpumask_var_t available_mask;
736 struct cpumask *allocated_mask;
737 const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
738 u32 target_cpu;
739 int numa_node;
740
741 if (!perf_chn ||
742 !alloc_cpumask_var(&available_mask, GFP_KERNEL) ||
743 cpumask_empty(hk_mask)) {
744 /*
745 * If the channel is not a performance critical
746 * channel, bind it to VMBUS_CONNECT_CPU.
747 * In case alloc_cpumask_var() fails, bind it to
748 * VMBUS_CONNECT_CPU.
749 * If all the cpus are isolated, bind it to
750 * VMBUS_CONNECT_CPU.
751 */
752 channel->target_cpu = VMBUS_CONNECT_CPU;
753 if (perf_chn)
754 hv_set_allocated_cpu(VMBUS_CONNECT_CPU);
755 return;
756 }
757
758 for (i = 1; i <= ncpu + 1; i++) {
759 while (true) {
760 numa_node = next_numa_node_id++;
761 if (numa_node == nr_node_ids) {
762 next_numa_node_id = 0;
763 continue;
764 }
765 if (cpumask_empty(cpumask_of_node(numa_node)))
766 continue;
767 break;
768 }
769 allocated_mask = &hv_context.hv_numa_map[numa_node];
770
771retry:
772 cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node));
773 cpumask_and(available_mask, available_mask, hk_mask);
774
775 if (cpumask_empty(available_mask)) {
776 /*
777 * We have cycled through all the CPUs in the node;
778 * reset the allocated map.
779 */
780 cpumask_clear(allocated_mask);
781 goto retry;
782 }
783
784 target_cpu = cpumask_first(available_mask);
785 cpumask_set_cpu(target_cpu, allocated_mask);
786
787 if (channel->offermsg.offer.sub_channel_index >= ncpu ||
788 i > ncpu || !hv_cpuself_used(target_cpu, channel))
789 break;
790 }
791
792 channel->target_cpu = target_cpu;
793
794 free_cpumask_var(available_mask);
795}
796
797#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */
798#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */
799#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS)
800#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */
801#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS)
802
803static void vmbus_wait_for_unload(void)
804{
805 int cpu;
806 void *page_addr;
807 struct hv_message *msg;
808 struct vmbus_channel_message_header *hdr;
809 u32 message_type, i;
810
811 /*
812 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
813 * used for initial contact or to CPU0 depending on host version. When
814 * we're crashing on a different CPU let's hope that IRQ handler on
815 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
816 * functional and vmbus_unload_response() will complete
817 * vmbus_connection.unload_event. If not, the last thing we can do is
818 * read message pages for all CPUs directly.
819 *
820 * Wait up to 100 seconds since an Azure host must writeback any dirty
821 * data in its disk cache before the VMbus UNLOAD request will
822 * complete. This flushing has been empirically observed to take up
823 * to 50 seconds in cases with a lot of dirty data, so allow additional
824 * leeway and for inaccuracies in mdelay(). But eventually time out so
825 * that the panic path can't get hung forever in case the response
826 * message isn't seen.
827 */
828 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) {
829 if (completion_done(&vmbus_connection.unload_event))
830 goto completed;
831
832 for_each_online_cpu(cpu) {
833 struct hv_per_cpu_context *hv_cpu
834 = per_cpu_ptr(hv_context.cpu_context, cpu);
835
836 page_addr = hv_cpu->synic_message_page;
837 msg = (struct hv_message *)page_addr
838 + VMBUS_MESSAGE_SINT;
839
840 message_type = READ_ONCE(msg->header.message_type);
841 if (message_type == HVMSG_NONE)
842 continue;
843
844 hdr = (struct vmbus_channel_message_header *)
845 msg->u.payload;
846
847 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
848 complete(&vmbus_connection.unload_event);
849
850 vmbus_signal_eom(msg, message_type);
851 }
852
853 /*
854 * Give a notice periodically so someone watching the
855 * serial output won't think it is completely hung.
856 */
857 if (!(i % UNLOAD_MSG_LOOPS))
858 pr_notice("Waiting for VMBus UNLOAD to complete\n");
859
860 mdelay(UNLOAD_DELAY_UNIT_MS);
861 }
862 pr_err("Continuing even though VMBus UNLOAD did not complete\n");
863
864completed:
865 /*
866 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
867 * maybe-pending messages on all CPUs to be able to receive new
868 * messages after we reconnect.
869 */
870 for_each_online_cpu(cpu) {
871 struct hv_per_cpu_context *hv_cpu
872 = per_cpu_ptr(hv_context.cpu_context, cpu);
873
874 page_addr = hv_cpu->synic_message_page;
875 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
876 msg->header.message_type = HVMSG_NONE;
877 }
878}
879
880/*
881 * vmbus_unload_response - Handler for the unload response.
882 */
883static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
884{
885 /*
886 * This is a global event; just wakeup the waiting thread.
887 * Once we successfully unload, we can cleanup the monitor state.
888 *
889 * NB. A malicious or compromised Hyper-V could send a spurious
890 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call
891 * of the complete() below. Make sure that unload_event has been
892 * initialized by the time this complete() is executed.
893 */
894 complete(&vmbus_connection.unload_event);
895}
896
897void vmbus_initiate_unload(bool crash)
898{
899 struct vmbus_channel_message_header hdr;
900
901 if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED)
902 return;
903
904 /* Pre-Win2012R2 hosts don't support reconnect */
905 if (vmbus_proto_version < VERSION_WIN8_1)
906 return;
907
908 reinit_completion(&vmbus_connection.unload_event);
909 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
910 hdr.msgtype = CHANNELMSG_UNLOAD;
911 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
912 !crash);
913
914 /*
915 * vmbus_initiate_unload() is also called on crash and the crash can be
916 * happening in an interrupt context, where scheduling is impossible.
917 */
918 if (!crash)
919 wait_for_completion(&vmbus_connection.unload_event);
920 else
921 vmbus_wait_for_unload();
922}
923
924static void check_ready_for_resume_event(void)
925{
926 /*
927 * If all the old primary channels have been fixed up, then it's safe
928 * to resume.
929 */
930 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))
931 complete(&vmbus_connection.ready_for_resume_event);
932}
933
934static void vmbus_setup_channel_state(struct vmbus_channel *channel,
935 struct vmbus_channel_offer_channel *offer)
936{
937 /*
938 * Setup state for signalling the host.
939 */
940 channel->sig_event = VMBUS_EVENT_CONNECTION_ID;
941
942 channel->is_dedicated_interrupt =
943 (offer->is_dedicated_interrupt != 0);
944 channel->sig_event = offer->connection_id;
945
946 memcpy(&channel->offermsg, offer,
947 sizeof(struct vmbus_channel_offer_channel));
948 channel->monitor_grp = (u8)offer->monitorid / 32;
949 channel->monitor_bit = (u8)offer->monitorid % 32;
950 channel->device_id = hv_get_dev_type(channel);
951}
952
953/*
954 * find_primary_channel_by_offer - Get the channel object given the new offer.
955 * This is only used in the resume path of hibernation.
956 */
957static struct vmbus_channel *
958find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)
959{
960 struct vmbus_channel *channel = NULL, *iter;
961 const guid_t *inst1, *inst2;
962
963 /* Ignore sub-channel offers. */
964 if (offer->offer.sub_channel_index != 0)
965 return NULL;
966
967 mutex_lock(&vmbus_connection.channel_mutex);
968
969 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {
970 inst1 = &iter->offermsg.offer.if_instance;
971 inst2 = &offer->offer.if_instance;
972
973 if (guid_equal(inst1, inst2)) {
974 channel = iter;
975 break;
976 }
977 }
978
979 mutex_unlock(&vmbus_connection.channel_mutex);
980
981 return channel;
982}
983
984static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer)
985{
986 const guid_t *guid = &offer->offer.if_type;
987 u16 i;
988
989 if (!hv_is_isolation_supported())
990 return true;
991
992 if (is_hvsock_offer(offer))
993 return true;
994
995 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) {
996 if (guid_equal(guid, &vmbus_devs[i].guid))
997 return vmbus_devs[i].allowed_in_isolated;
998 }
999 return false;
1000}
1001
1002/*
1003 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1004 *
1005 */
1006static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
1007{
1008 struct vmbus_channel_offer_channel *offer;
1009 struct vmbus_channel *oldchannel, *newchannel;
1010 size_t offer_sz;
1011
1012 offer = (struct vmbus_channel_offer_channel *)hdr;
1013
1014 trace_vmbus_onoffer(offer);
1015
1016 if (!vmbus_is_valid_offer(offer)) {
1017 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n",
1018 offer->child_relid);
1019 atomic_dec(&vmbus_connection.offer_in_progress);
1020 return;
1021 }
1022
1023 oldchannel = find_primary_channel_by_offer(offer);
1024
1025 if (oldchannel != NULL) {
1026 /*
1027 * We're resuming from hibernation: all the sub-channel and
1028 * hv_sock channels we had before the hibernation should have
1029 * been cleaned up, and now we must be seeing a re-offered
1030 * primary channel that we had before the hibernation.
1031 */
1032
1033 /*
1034 * { Initially: channel relid = INVALID_RELID,
1035 * channels[valid_relid] = NULL }
1036 *
1037 * CPU1 CPU2
1038 *
1039 * [vmbus_onoffer()] [vmbus_device_release()]
1040 *
1041 * LOCK channel_mutex LOCK channel_mutex
1042 * STORE channel relid = valid_relid LOAD r1 = channel relid
1043 * MAP_RELID channel if (r1 != INVALID_RELID)
1044 * UNLOCK channel_mutex UNMAP_RELID channel
1045 * UNLOCK channel_mutex
1046 *
1047 * Forbids: r1 == valid_relid &&
1048 * channels[valid_relid] == channel
1049 *
1050 * Note. r1 can be INVALID_RELID only for an hv_sock channel.
1051 * None of the hv_sock channels which were present before the
1052 * suspend are re-offered upon the resume. See the WARN_ON()
1053 * in hv_process_channel_removal().
1054 */
1055 mutex_lock(&vmbus_connection.channel_mutex);
1056
1057 atomic_dec(&vmbus_connection.offer_in_progress);
1058
1059 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);
1060 /* Fix up the relid. */
1061 oldchannel->offermsg.child_relid = offer->child_relid;
1062
1063 offer_sz = sizeof(*offer);
1064 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {
1065 /*
1066 * This is not an error, since the host can also change
1067 * the other field(s) of the offer, e.g. on WS RS5
1068 * (Build 17763), the offer->connection_id of the
1069 * Mellanox VF vmbus device can change when the host
1070 * reoffers the device upon resume.
1071 */
1072 pr_debug("vmbus offer changed: relid=%d\n",
1073 offer->child_relid);
1074
1075 print_hex_dump_debug("Old vmbus offer: ",
1076 DUMP_PREFIX_OFFSET, 16, 4,
1077 &oldchannel->offermsg, offer_sz,
1078 false);
1079 print_hex_dump_debug("New vmbus offer: ",
1080 DUMP_PREFIX_OFFSET, 16, 4,
1081 offer, offer_sz, false);
1082
1083 /* Fix up the old channel. */
1084 vmbus_setup_channel_state(oldchannel, offer);
1085 }
1086
1087 /* Add the channel back to the array of channels. */
1088 vmbus_channel_map_relid(oldchannel);
1089 check_ready_for_resume_event();
1090
1091 mutex_unlock(&vmbus_connection.channel_mutex);
1092 return;
1093 }
1094
1095 /* Allocate the channel object and save this offer. */
1096 newchannel = alloc_channel();
1097 if (!newchannel) {
1098 vmbus_release_relid(offer->child_relid);
1099 atomic_dec(&vmbus_connection.offer_in_progress);
1100 pr_err("Unable to allocate channel object\n");
1101 return;
1102 }
1103
1104 vmbus_setup_channel_state(newchannel, offer);
1105
1106 vmbus_process_offer(newchannel);
1107}
1108
1109static void check_ready_for_suspend_event(void)
1110{
1111 /*
1112 * If all the sub-channels or hv_sock channels have been cleaned up,
1113 * then it's safe to suspend.
1114 */
1115 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))
1116 complete(&vmbus_connection.ready_for_suspend_event);
1117}
1118
1119/*
1120 * vmbus_onoffer_rescind - Rescind offer handler.
1121 *
1122 * We queue a work item to process this offer synchronously
1123 */
1124static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
1125{
1126 struct vmbus_channel_rescind_offer *rescind;
1127 struct vmbus_channel *channel;
1128 struct device *dev;
1129 bool clean_up_chan_for_suspend;
1130
1131 rescind = (struct vmbus_channel_rescind_offer *)hdr;
1132
1133 trace_vmbus_onoffer_rescind(rescind);
1134
1135 /*
1136 * The offer msg and the corresponding rescind msg
1137 * from the host are guranteed to be ordered -
1138 * offer comes in first and then the rescind.
1139 * Since we process these events in work elements,
1140 * and with preemption, we may end up processing
1141 * the events out of order. We rely on the synchronization
1142 * provided by offer_in_progress and by channel_mutex for
1143 * ordering these events:
1144 *
1145 * { Initially: offer_in_progress = 1 }
1146 *
1147 * CPU1 CPU2
1148 *
1149 * [vmbus_onoffer()] [vmbus_onoffer_rescind()]
1150 *
1151 * LOCK channel_mutex WAIT_ON offer_in_progress == 0
1152 * DECREMENT offer_in_progress LOCK channel_mutex
1153 * STORE channels[] LOAD channels[]
1154 * UNLOCK channel_mutex UNLOCK channel_mutex
1155 *
1156 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE
1157 */
1158
1159 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {
1160 /*
1161 * We wait here until any channel offer is currently
1162 * being processed.
1163 */
1164 msleep(1);
1165 }
1166
1167 mutex_lock(&vmbus_connection.channel_mutex);
1168 channel = relid2channel(rescind->child_relid);
1169 if (channel != NULL) {
1170 /*
1171 * Guarantee that no other instance of vmbus_onoffer_rescind()
1172 * has got a reference to the channel object. Synchronize on
1173 * &vmbus_connection.channel_mutex.
1174 */
1175 if (channel->rescind_ref) {
1176 mutex_unlock(&vmbus_connection.channel_mutex);
1177 return;
1178 }
1179 channel->rescind_ref = true;
1180 }
1181 mutex_unlock(&vmbus_connection.channel_mutex);
1182
1183 if (channel == NULL) {
1184 /*
1185 * We failed in processing the offer message;
1186 * we would have cleaned up the relid in that
1187 * failure path.
1188 */
1189 return;
1190 }
1191
1192 clean_up_chan_for_suspend = is_hvsock_channel(channel) ||
1193 is_sub_channel(channel);
1194 /*
1195 * Before setting channel->rescind in vmbus_rescind_cleanup(), we
1196 * should make sure the channel callback is not running any more.
1197 */
1198 vmbus_reset_channel_cb(channel);
1199
1200 /*
1201 * Now wait for offer handling to complete.
1202 */
1203 vmbus_rescind_cleanup(channel);
1204 while (READ_ONCE(channel->probe_done) == false) {
1205 /*
1206 * We wait here until any channel offer is currently
1207 * being processed.
1208 */
1209 msleep(1);
1210 }
1211
1212 /*
1213 * At this point, the rescind handling can proceed safely.
1214 */
1215
1216 if (channel->device_obj) {
1217 if (channel->chn_rescind_callback) {
1218 channel->chn_rescind_callback(channel);
1219
1220 if (clean_up_chan_for_suspend)
1221 check_ready_for_suspend_event();
1222
1223 return;
1224 }
1225 /*
1226 * We will have to unregister this device from the
1227 * driver core.
1228 */
1229 dev = get_device(&channel->device_obj->device);
1230 if (dev) {
1231 vmbus_device_unregister(channel->device_obj);
1232 put_device(dev);
1233 }
1234 } else if (channel->primary_channel != NULL) {
1235 /*
1236 * Sub-channel is being rescinded. Following is the channel
1237 * close sequence when initiated from the driveri (refer to
1238 * vmbus_close() for details):
1239 * 1. Close all sub-channels first
1240 * 2. Then close the primary channel.
1241 */
1242 mutex_lock(&vmbus_connection.channel_mutex);
1243 if (channel->state == CHANNEL_OPEN_STATE) {
1244 /*
1245 * The channel is currently not open;
1246 * it is safe for us to cleanup the channel.
1247 */
1248 hv_process_channel_removal(channel);
1249 } else {
1250 complete(&channel->rescind_event);
1251 }
1252 mutex_unlock(&vmbus_connection.channel_mutex);
1253 }
1254
1255 /* The "channel" may have been freed. Do not access it any longer. */
1256
1257 if (clean_up_chan_for_suspend)
1258 check_ready_for_suspend_event();
1259}
1260
1261void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
1262{
1263 BUG_ON(!is_hvsock_channel(channel));
1264
1265 /* We always get a rescind msg when a connection is closed. */
1266 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))
1267 msleep(1);
1268
1269 vmbus_device_unregister(channel->device_obj);
1270}
1271EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
1272
1273
1274/*
1275 * vmbus_onoffers_delivered -
1276 * This is invoked when all offers have been delivered.
1277 *
1278 * Nothing to do here.
1279 */
1280static void vmbus_onoffers_delivered(
1281 struct vmbus_channel_message_header *hdr)
1282{
1283}
1284
1285/*
1286 * vmbus_onopen_result - Open result handler.
1287 *
1288 * This is invoked when we received a response to our channel open request.
1289 * Find the matching request, copy the response and signal the requesting
1290 * thread.
1291 */
1292static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
1293{
1294 struct vmbus_channel_open_result *result;
1295 struct vmbus_channel_msginfo *msginfo;
1296 struct vmbus_channel_message_header *requestheader;
1297 struct vmbus_channel_open_channel *openmsg;
1298 unsigned long flags;
1299
1300 result = (struct vmbus_channel_open_result *)hdr;
1301
1302 trace_vmbus_onopen_result(result);
1303
1304 /*
1305 * Find the open msg, copy the result and signal/unblock the wait event
1306 */
1307 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1308
1309 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1310 msglistentry) {
1311 requestheader =
1312 (struct vmbus_channel_message_header *)msginfo->msg;
1313
1314 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
1315 openmsg =
1316 (struct vmbus_channel_open_channel *)msginfo->msg;
1317 if (openmsg->child_relid == result->child_relid &&
1318 openmsg->openid == result->openid) {
1319 memcpy(&msginfo->response.open_result,
1320 result,
1321 sizeof(
1322 struct vmbus_channel_open_result));
1323 complete(&msginfo->waitevent);
1324 break;
1325 }
1326 }
1327 }
1328 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1329}
1330
1331/*
1332 * vmbus_ongpadl_created - GPADL created handler.
1333 *
1334 * This is invoked when we received a response to our gpadl create request.
1335 * Find the matching request, copy the response and signal the requesting
1336 * thread.
1337 */
1338static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
1339{
1340 struct vmbus_channel_gpadl_created *gpadlcreated;
1341 struct vmbus_channel_msginfo *msginfo;
1342 struct vmbus_channel_message_header *requestheader;
1343 struct vmbus_channel_gpadl_header *gpadlheader;
1344 unsigned long flags;
1345
1346 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
1347
1348 trace_vmbus_ongpadl_created(gpadlcreated);
1349
1350 /*
1351 * Find the establish msg, copy the result and signal/unblock the wait
1352 * event
1353 */
1354 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1355
1356 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1357 msglistentry) {
1358 requestheader =
1359 (struct vmbus_channel_message_header *)msginfo->msg;
1360
1361 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
1362 gpadlheader =
1363 (struct vmbus_channel_gpadl_header *)requestheader;
1364
1365 if ((gpadlcreated->child_relid ==
1366 gpadlheader->child_relid) &&
1367 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
1368 memcpy(&msginfo->response.gpadl_created,
1369 gpadlcreated,
1370 sizeof(
1371 struct vmbus_channel_gpadl_created));
1372 complete(&msginfo->waitevent);
1373 break;
1374 }
1375 }
1376 }
1377 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1378}
1379
1380/*
1381 * vmbus_onmodifychannel_response - Modify Channel response handler.
1382 *
1383 * This is invoked when we received a response to our channel modify request.
1384 * Find the matching request, copy the response and signal the requesting thread.
1385 */
1386static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr)
1387{
1388 struct vmbus_channel_modifychannel_response *response;
1389 struct vmbus_channel_msginfo *msginfo;
1390 unsigned long flags;
1391
1392 response = (struct vmbus_channel_modifychannel_response *)hdr;
1393
1394 trace_vmbus_onmodifychannel_response(response);
1395
1396 /*
1397 * Find the modify msg, copy the response and signal/unblock the wait event.
1398 */
1399 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1400
1401 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) {
1402 struct vmbus_channel_message_header *responseheader =
1403 (struct vmbus_channel_message_header *)msginfo->msg;
1404
1405 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {
1406 struct vmbus_channel_modifychannel *modifymsg;
1407
1408 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;
1409 if (modifymsg->child_relid == response->child_relid) {
1410 memcpy(&msginfo->response.modify_response, response,
1411 sizeof(*response));
1412 complete(&msginfo->waitevent);
1413 break;
1414 }
1415 }
1416 }
1417 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1418}
1419
1420/*
1421 * vmbus_ongpadl_torndown - GPADL torndown handler.
1422 *
1423 * This is invoked when we received a response to our gpadl teardown request.
1424 * Find the matching request, copy the response and signal the requesting
1425 * thread.
1426 */
1427static void vmbus_ongpadl_torndown(
1428 struct vmbus_channel_message_header *hdr)
1429{
1430 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1431 struct vmbus_channel_msginfo *msginfo;
1432 struct vmbus_channel_message_header *requestheader;
1433 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1434 unsigned long flags;
1435
1436 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1437
1438 trace_vmbus_ongpadl_torndown(gpadl_torndown);
1439
1440 /*
1441 * Find the open msg, copy the result and signal/unblock the wait event
1442 */
1443 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1444
1445 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1446 msglistentry) {
1447 requestheader =
1448 (struct vmbus_channel_message_header *)msginfo->msg;
1449
1450 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1451 gpadl_teardown =
1452 (struct vmbus_channel_gpadl_teardown *)requestheader;
1453
1454 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1455 memcpy(&msginfo->response.gpadl_torndown,
1456 gpadl_torndown,
1457 sizeof(
1458 struct vmbus_channel_gpadl_torndown));
1459 complete(&msginfo->waitevent);
1460 break;
1461 }
1462 }
1463 }
1464 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1465}
1466
1467/*
1468 * vmbus_onversion_response - Version response handler
1469 *
1470 * This is invoked when we received a response to our initiate contact request.
1471 * Find the matching request, copy the response and signal the requesting
1472 * thread.
1473 */
1474static void vmbus_onversion_response(
1475 struct vmbus_channel_message_header *hdr)
1476{
1477 struct vmbus_channel_msginfo *msginfo;
1478 struct vmbus_channel_message_header *requestheader;
1479 struct vmbus_channel_version_response *version_response;
1480 unsigned long flags;
1481
1482 version_response = (struct vmbus_channel_version_response *)hdr;
1483
1484 trace_vmbus_onversion_response(version_response);
1485
1486 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1487
1488 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1489 msglistentry) {
1490 requestheader =
1491 (struct vmbus_channel_message_header *)msginfo->msg;
1492
1493 if (requestheader->msgtype ==
1494 CHANNELMSG_INITIATE_CONTACT) {
1495 memcpy(&msginfo->response.version_response,
1496 version_response,
1497 sizeof(struct vmbus_channel_version_response));
1498 complete(&msginfo->waitevent);
1499 }
1500 }
1501 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1502}
1503
1504/* Channel message dispatch table */
1505const struct vmbus_channel_message_table_entry
1506channel_message_table[CHANNELMSG_COUNT] = {
1507 { CHANNELMSG_INVALID, 0, NULL, 0},
1508 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer,
1509 sizeof(struct vmbus_channel_offer_channel)},
1510 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind,
1511 sizeof(struct vmbus_channel_rescind_offer) },
1512 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0},
1513 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0},
1514 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0},
1515 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result,
1516 sizeof(struct vmbus_channel_open_result)},
1517 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0},
1518 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0},
1519 { CHANNELMSG_GPADL_BODY, 0, NULL, 0},
1520 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created,
1521 sizeof(struct vmbus_channel_gpadl_created)},
1522 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0},
1523 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown,
1524 sizeof(struct vmbus_channel_gpadl_torndown) },
1525 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0},
1526 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0},
1527 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response,
1528 sizeof(struct vmbus_channel_version_response)},
1529 { CHANNELMSG_UNLOAD, 0, NULL, 0},
1530 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0},
1531 { CHANNELMSG_18, 0, NULL, 0},
1532 { CHANNELMSG_19, 0, NULL, 0},
1533 { CHANNELMSG_20, 0, NULL, 0},
1534 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0},
1535 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0},
1536 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0},
1537 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response,
1538 sizeof(struct vmbus_channel_modifychannel_response)},
1539};
1540
1541/*
1542 * vmbus_onmessage - Handler for channel protocol messages.
1543 *
1544 * This is invoked in the vmbus worker thread context.
1545 */
1546void vmbus_onmessage(struct vmbus_channel_message_header *hdr)
1547{
1548 trace_vmbus_on_message(hdr);
1549
1550 /*
1551 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go
1552 * out of bound and the message_handler pointer can not be NULL.
1553 */
1554 channel_message_table[hdr->msgtype].message_handler(hdr);
1555}
1556
1557/*
1558 * vmbus_request_offers - Send a request to get all our pending offers.
1559 */
1560int vmbus_request_offers(void)
1561{
1562 struct vmbus_channel_message_header *msg;
1563 struct vmbus_channel_msginfo *msginfo;
1564 int ret;
1565
1566 msginfo = kzalloc(sizeof(*msginfo) +
1567 sizeof(struct vmbus_channel_message_header),
1568 GFP_KERNEL);
1569 if (!msginfo)
1570 return -ENOMEM;
1571
1572 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1573
1574 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1575
1576 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1577 true);
1578
1579 trace_vmbus_request_offers(ret);
1580
1581 if (ret != 0) {
1582 pr_err("Unable to request offers - %d\n", ret);
1583
1584 goto cleanup;
1585 }
1586
1587cleanup:
1588 kfree(msginfo);
1589
1590 return ret;
1591}
1592
1593void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1594 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1595{
1596 primary_channel->sc_creation_callback = sc_cr_cb;
1597}
1598EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1599
1600void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1601 void (*chn_rescind_cb)(struct vmbus_channel *))
1602{
1603 channel->chn_rescind_callback = chn_rescind_cb;
1604}
1605EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);
1/*
2 * Copyright (c) 2009, Microsoft Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Authors:
18 * Haiyang Zhang <haiyangz@microsoft.com>
19 * Hank Janssen <hjanssen@microsoft.com>
20 */
21#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
22
23#include <linux/kernel.h>
24#include <linux/interrupt.h>
25#include <linux/sched.h>
26#include <linux/wait.h>
27#include <linux/mm.h>
28#include <linux/slab.h>
29#include <linux/list.h>
30#include <linux/module.h>
31#include <linux/completion.h>
32#include <linux/delay.h>
33#include <linux/hyperv.h>
34
35#include "hyperv_vmbus.h"
36
37static void init_vp_index(struct vmbus_channel *channel, u16 dev_type);
38
39static const struct vmbus_device vmbus_devs[] = {
40 /* IDE */
41 { .dev_type = HV_IDE,
42 HV_IDE_GUID,
43 .perf_device = true,
44 },
45
46 /* SCSI */
47 { .dev_type = HV_SCSI,
48 HV_SCSI_GUID,
49 .perf_device = true,
50 },
51
52 /* Fibre Channel */
53 { .dev_type = HV_FC,
54 HV_SYNTHFC_GUID,
55 .perf_device = true,
56 },
57
58 /* Synthetic NIC */
59 { .dev_type = HV_NIC,
60 HV_NIC_GUID,
61 .perf_device = true,
62 },
63
64 /* Network Direct */
65 { .dev_type = HV_ND,
66 HV_ND_GUID,
67 .perf_device = true,
68 },
69
70 /* PCIE */
71 { .dev_type = HV_PCIE,
72 HV_PCIE_GUID,
73 .perf_device = true,
74 },
75
76 /* Synthetic Frame Buffer */
77 { .dev_type = HV_FB,
78 HV_SYNTHVID_GUID,
79 .perf_device = false,
80 },
81
82 /* Synthetic Keyboard */
83 { .dev_type = HV_KBD,
84 HV_KBD_GUID,
85 .perf_device = false,
86 },
87
88 /* Synthetic MOUSE */
89 { .dev_type = HV_MOUSE,
90 HV_MOUSE_GUID,
91 .perf_device = false,
92 },
93
94 /* KVP */
95 { .dev_type = HV_KVP,
96 HV_KVP_GUID,
97 .perf_device = false,
98 },
99
100 /* Time Synch */
101 { .dev_type = HV_TS,
102 HV_TS_GUID,
103 .perf_device = false,
104 },
105
106 /* Heartbeat */
107 { .dev_type = HV_HB,
108 HV_HEART_BEAT_GUID,
109 .perf_device = false,
110 },
111
112 /* Shutdown */
113 { .dev_type = HV_SHUTDOWN,
114 HV_SHUTDOWN_GUID,
115 .perf_device = false,
116 },
117
118 /* File copy */
119 { .dev_type = HV_FCOPY,
120 HV_FCOPY_GUID,
121 .perf_device = false,
122 },
123
124 /* Backup */
125 { .dev_type = HV_BACKUP,
126 HV_VSS_GUID,
127 .perf_device = false,
128 },
129
130 /* Dynamic Memory */
131 { .dev_type = HV_DM,
132 HV_DM_GUID,
133 .perf_device = false,
134 },
135
136 /* Unknown GUID */
137 { .dev_type = HV_UNKNOWN,
138 .perf_device = false,
139 },
140};
141
142static const struct {
143 uuid_le guid;
144} vmbus_unsupported_devs[] = {
145 { HV_AVMA1_GUID },
146 { HV_AVMA2_GUID },
147 { HV_RDV_GUID },
148};
149
150/*
151 * The rescinded channel may be blocked waiting for a response from the host;
152 * take care of that.
153 */
154static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
155{
156 struct vmbus_channel_msginfo *msginfo;
157 unsigned long flags;
158
159
160 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
161
162 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
163 msglistentry) {
164
165 if (msginfo->waiting_channel == channel) {
166 complete(&msginfo->waitevent);
167 break;
168 }
169 }
170 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
171}
172
173static bool is_unsupported_vmbus_devs(const uuid_le *guid)
174{
175 int i;
176
177 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
178 if (!uuid_le_cmp(*guid, vmbus_unsupported_devs[i].guid))
179 return true;
180 return false;
181}
182
183static u16 hv_get_dev_type(const struct vmbus_channel *channel)
184{
185 const uuid_le *guid = &channel->offermsg.offer.if_type;
186 u16 i;
187
188 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
189 return HV_UNKNOWN;
190
191 for (i = HV_IDE; i < HV_UNKNOWN; i++) {
192 if (!uuid_le_cmp(*guid, vmbus_devs[i].guid))
193 return i;
194 }
195 pr_info("Unknown GUID: %pUl\n", guid);
196 return i;
197}
198
199/**
200 * vmbus_prep_negotiate_resp() - Create default response for Hyper-V Negotiate message
201 * @icmsghdrp: Pointer to msg header structure
202 * @icmsg_negotiate: Pointer to negotiate message structure
203 * @buf: Raw buffer channel data
204 *
205 * @icmsghdrp is of type &struct icmsg_hdr.
206 * @negop is of type &struct icmsg_negotiate.
207 * Set up and fill in default negotiate response message.
208 *
209 * The fw_version specifies the framework version that
210 * we can support and srv_version specifies the service
211 * version we can support.
212 *
213 * Mainly used by Hyper-V drivers.
214 */
215bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp,
216 struct icmsg_negotiate *negop, u8 *buf,
217 int fw_version, int srv_version)
218{
219 int icframe_major, icframe_minor;
220 int icmsg_major, icmsg_minor;
221 int fw_major, fw_minor;
222 int srv_major, srv_minor;
223 int i;
224 bool found_match = false;
225
226 icmsghdrp->icmsgsize = 0x10;
227 fw_major = (fw_version >> 16);
228 fw_minor = (fw_version & 0xFFFF);
229
230 srv_major = (srv_version >> 16);
231 srv_minor = (srv_version & 0xFFFF);
232
233 negop = (struct icmsg_negotiate *)&buf[
234 sizeof(struct vmbuspipe_hdr) +
235 sizeof(struct icmsg_hdr)];
236
237 icframe_major = negop->icframe_vercnt;
238 icframe_minor = 0;
239
240 icmsg_major = negop->icmsg_vercnt;
241 icmsg_minor = 0;
242
243 /*
244 * Select the framework version number we will
245 * support.
246 */
247
248 for (i = 0; i < negop->icframe_vercnt; i++) {
249 if ((negop->icversion_data[i].major == fw_major) &&
250 (negop->icversion_data[i].minor == fw_minor)) {
251 icframe_major = negop->icversion_data[i].major;
252 icframe_minor = negop->icversion_data[i].minor;
253 found_match = true;
254 }
255 }
256
257 if (!found_match)
258 goto fw_error;
259
260 found_match = false;
261
262 for (i = negop->icframe_vercnt;
263 (i < negop->icframe_vercnt + negop->icmsg_vercnt); i++) {
264 if ((negop->icversion_data[i].major == srv_major) &&
265 (negop->icversion_data[i].minor == srv_minor)) {
266 icmsg_major = negop->icversion_data[i].major;
267 icmsg_minor = negop->icversion_data[i].minor;
268 found_match = true;
269 }
270 }
271
272 /*
273 * Respond with the framework and service
274 * version numbers we can support.
275 */
276
277fw_error:
278 if (!found_match) {
279 negop->icframe_vercnt = 0;
280 negop->icmsg_vercnt = 0;
281 } else {
282 negop->icframe_vercnt = 1;
283 negop->icmsg_vercnt = 1;
284 }
285
286 negop->icversion_data[0].major = icframe_major;
287 negop->icversion_data[0].minor = icframe_minor;
288 negop->icversion_data[1].major = icmsg_major;
289 negop->icversion_data[1].minor = icmsg_minor;
290 return found_match;
291}
292
293EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
294
295/*
296 * alloc_channel - Allocate and initialize a vmbus channel object
297 */
298static struct vmbus_channel *alloc_channel(void)
299{
300 struct vmbus_channel *channel;
301
302 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
303 if (!channel)
304 return NULL;
305
306 channel->acquire_ring_lock = true;
307 spin_lock_init(&channel->inbound_lock);
308 spin_lock_init(&channel->lock);
309
310 INIT_LIST_HEAD(&channel->sc_list);
311 INIT_LIST_HEAD(&channel->percpu_list);
312
313 return channel;
314}
315
316/*
317 * free_channel - Release the resources used by the vmbus channel object
318 */
319static void free_channel(struct vmbus_channel *channel)
320{
321 kfree(channel);
322}
323
324static void percpu_channel_enq(void *arg)
325{
326 struct vmbus_channel *channel = arg;
327 int cpu = smp_processor_id();
328
329 list_add_tail(&channel->percpu_list, &hv_context.percpu_list[cpu]);
330}
331
332static void percpu_channel_deq(void *arg)
333{
334 struct vmbus_channel *channel = arg;
335
336 list_del(&channel->percpu_list);
337}
338
339
340static void vmbus_release_relid(u32 relid)
341{
342 struct vmbus_channel_relid_released msg;
343
344 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
345 msg.child_relid = relid;
346 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
347 vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
348 true);
349}
350
351void hv_event_tasklet_disable(struct vmbus_channel *channel)
352{
353 struct tasklet_struct *tasklet;
354 tasklet = hv_context.event_dpc[channel->target_cpu];
355 tasklet_disable(tasklet);
356}
357
358void hv_event_tasklet_enable(struct vmbus_channel *channel)
359{
360 struct tasklet_struct *tasklet;
361 tasklet = hv_context.event_dpc[channel->target_cpu];
362 tasklet_enable(tasklet);
363
364 /* In case there is any pending event */
365 tasklet_schedule(tasklet);
366}
367
368void hv_process_channel_removal(struct vmbus_channel *channel, u32 relid)
369{
370 unsigned long flags;
371 struct vmbus_channel *primary_channel;
372
373 BUG_ON(!channel->rescind);
374 BUG_ON(!mutex_is_locked(&vmbus_connection.channel_mutex));
375
376 hv_event_tasklet_disable(channel);
377 if (channel->target_cpu != get_cpu()) {
378 put_cpu();
379 smp_call_function_single(channel->target_cpu,
380 percpu_channel_deq, channel, true);
381 } else {
382 percpu_channel_deq(channel);
383 put_cpu();
384 }
385 hv_event_tasklet_enable(channel);
386
387 if (channel->primary_channel == NULL) {
388 list_del(&channel->listentry);
389
390 primary_channel = channel;
391 } else {
392 primary_channel = channel->primary_channel;
393 spin_lock_irqsave(&primary_channel->lock, flags);
394 list_del(&channel->sc_list);
395 primary_channel->num_sc--;
396 spin_unlock_irqrestore(&primary_channel->lock, flags);
397 }
398
399 /*
400 * We need to free the bit for init_vp_index() to work in the case
401 * of sub-channel, when we reload drivers like hv_netvsc.
402 */
403 if (channel->affinity_policy == HV_LOCALIZED)
404 cpumask_clear_cpu(channel->target_cpu,
405 &primary_channel->alloced_cpus_in_node);
406
407 vmbus_release_relid(relid);
408
409 free_channel(channel);
410}
411
412void vmbus_free_channels(void)
413{
414 struct vmbus_channel *channel, *tmp;
415
416 mutex_lock(&vmbus_connection.channel_mutex);
417 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
418 listentry) {
419 /* hv_process_channel_removal() needs this */
420 channel->rescind = true;
421
422 vmbus_device_unregister(channel->device_obj);
423 }
424 mutex_unlock(&vmbus_connection.channel_mutex);
425}
426
427/*
428 * vmbus_process_offer - Process the offer by creating a channel/device
429 * associated with this offer
430 */
431static void vmbus_process_offer(struct vmbus_channel *newchannel)
432{
433 struct vmbus_channel *channel;
434 bool fnew = true;
435 unsigned long flags;
436 u16 dev_type;
437 int ret;
438
439 /* Make sure this is a new offer */
440 mutex_lock(&vmbus_connection.channel_mutex);
441
442 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
443 if (!uuid_le_cmp(channel->offermsg.offer.if_type,
444 newchannel->offermsg.offer.if_type) &&
445 !uuid_le_cmp(channel->offermsg.offer.if_instance,
446 newchannel->offermsg.offer.if_instance)) {
447 fnew = false;
448 break;
449 }
450 }
451
452 if (fnew)
453 list_add_tail(&newchannel->listentry,
454 &vmbus_connection.chn_list);
455
456 mutex_unlock(&vmbus_connection.channel_mutex);
457
458 if (!fnew) {
459 /*
460 * Check to see if this is a sub-channel.
461 */
462 if (newchannel->offermsg.offer.sub_channel_index != 0) {
463 /*
464 * Process the sub-channel.
465 */
466 newchannel->primary_channel = channel;
467 spin_lock_irqsave(&channel->lock, flags);
468 list_add_tail(&newchannel->sc_list, &channel->sc_list);
469 channel->num_sc++;
470 spin_unlock_irqrestore(&channel->lock, flags);
471 } else
472 goto err_free_chan;
473 }
474
475 dev_type = hv_get_dev_type(newchannel);
476
477 init_vp_index(newchannel, dev_type);
478
479 hv_event_tasklet_disable(newchannel);
480 if (newchannel->target_cpu != get_cpu()) {
481 put_cpu();
482 smp_call_function_single(newchannel->target_cpu,
483 percpu_channel_enq,
484 newchannel, true);
485 } else {
486 percpu_channel_enq(newchannel);
487 put_cpu();
488 }
489 hv_event_tasklet_enable(newchannel);
490
491 /*
492 * This state is used to indicate a successful open
493 * so that when we do close the channel normally, we
494 * can cleanup properly
495 */
496 newchannel->state = CHANNEL_OPEN_STATE;
497
498 if (!fnew) {
499 if (channel->sc_creation_callback != NULL)
500 channel->sc_creation_callback(newchannel);
501 return;
502 }
503
504 /*
505 * Start the process of binding this offer to the driver
506 * We need to set the DeviceObject field before calling
507 * vmbus_child_dev_add()
508 */
509 newchannel->device_obj = vmbus_device_create(
510 &newchannel->offermsg.offer.if_type,
511 &newchannel->offermsg.offer.if_instance,
512 newchannel);
513 if (!newchannel->device_obj)
514 goto err_deq_chan;
515
516 newchannel->device_obj->device_id = dev_type;
517 /*
518 * Add the new device to the bus. This will kick off device-driver
519 * binding which eventually invokes the device driver's AddDevice()
520 * method.
521 */
522 mutex_lock(&vmbus_connection.channel_mutex);
523 ret = vmbus_device_register(newchannel->device_obj);
524 mutex_unlock(&vmbus_connection.channel_mutex);
525
526 if (ret != 0) {
527 pr_err("unable to add child device object (relid %d)\n",
528 newchannel->offermsg.child_relid);
529 kfree(newchannel->device_obj);
530 goto err_deq_chan;
531 }
532 return;
533
534err_deq_chan:
535 mutex_lock(&vmbus_connection.channel_mutex);
536 list_del(&newchannel->listentry);
537 mutex_unlock(&vmbus_connection.channel_mutex);
538
539 hv_event_tasklet_disable(newchannel);
540 if (newchannel->target_cpu != get_cpu()) {
541 put_cpu();
542 smp_call_function_single(newchannel->target_cpu,
543 percpu_channel_deq, newchannel, true);
544 } else {
545 percpu_channel_deq(newchannel);
546 put_cpu();
547 }
548 hv_event_tasklet_enable(newchannel);
549
550 vmbus_release_relid(newchannel->offermsg.child_relid);
551
552err_free_chan:
553 free_channel(newchannel);
554}
555
556/*
557 * We use this state to statically distribute the channel interrupt load.
558 */
559static int next_numa_node_id;
560
561/*
562 * Starting with Win8, we can statically distribute the incoming
563 * channel interrupt load by binding a channel to VCPU.
564 * We do this in a hierarchical fashion:
565 * First distribute the primary channels across available NUMA nodes
566 * and then distribute the subchannels amongst the CPUs in the NUMA
567 * node assigned to the primary channel.
568 *
569 * For pre-win8 hosts or non-performance critical channels we assign the
570 * first CPU in the first NUMA node.
571 */
572static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
573{
574 u32 cur_cpu;
575 bool perf_chn = vmbus_devs[dev_type].perf_device;
576 struct vmbus_channel *primary = channel->primary_channel;
577 int next_node;
578 struct cpumask available_mask;
579 struct cpumask *alloced_mask;
580
581 if ((vmbus_proto_version == VERSION_WS2008) ||
582 (vmbus_proto_version == VERSION_WIN7) || (!perf_chn)) {
583 /*
584 * Prior to win8, all channel interrupts are
585 * delivered on cpu 0.
586 * Also if the channel is not a performance critical
587 * channel, bind it to cpu 0.
588 */
589 channel->numa_node = 0;
590 channel->target_cpu = 0;
591 channel->target_vp = hv_context.vp_index[0];
592 return;
593 }
594
595 /*
596 * Based on the channel affinity policy, we will assign the NUMA
597 * nodes.
598 */
599
600 if ((channel->affinity_policy == HV_BALANCED) || (!primary)) {
601 while (true) {
602 next_node = next_numa_node_id++;
603 if (next_node == nr_node_ids) {
604 next_node = next_numa_node_id = 0;
605 continue;
606 }
607 if (cpumask_empty(cpumask_of_node(next_node)))
608 continue;
609 break;
610 }
611 channel->numa_node = next_node;
612 primary = channel;
613 }
614 alloced_mask = &hv_context.hv_numa_map[primary->numa_node];
615
616 if (cpumask_weight(alloced_mask) ==
617 cpumask_weight(cpumask_of_node(primary->numa_node))) {
618 /*
619 * We have cycled through all the CPUs in the node;
620 * reset the alloced map.
621 */
622 cpumask_clear(alloced_mask);
623 }
624
625 cpumask_xor(&available_mask, alloced_mask,
626 cpumask_of_node(primary->numa_node));
627
628 cur_cpu = -1;
629
630 if (primary->affinity_policy == HV_LOCALIZED) {
631 /*
632 * Normally Hyper-V host doesn't create more subchannels
633 * than there are VCPUs on the node but it is possible when not
634 * all present VCPUs on the node are initialized by guest.
635 * Clear the alloced_cpus_in_node to start over.
636 */
637 if (cpumask_equal(&primary->alloced_cpus_in_node,
638 cpumask_of_node(primary->numa_node)))
639 cpumask_clear(&primary->alloced_cpus_in_node);
640 }
641
642 while (true) {
643 cur_cpu = cpumask_next(cur_cpu, &available_mask);
644 if (cur_cpu >= nr_cpu_ids) {
645 cur_cpu = -1;
646 cpumask_copy(&available_mask,
647 cpumask_of_node(primary->numa_node));
648 continue;
649 }
650
651 if (primary->affinity_policy == HV_LOCALIZED) {
652 /*
653 * NOTE: in the case of sub-channel, we clear the
654 * sub-channel related bit(s) in
655 * primary->alloced_cpus_in_node in
656 * hv_process_channel_removal(), so when we
657 * reload drivers like hv_netvsc in SMP guest, here
658 * we're able to re-allocate
659 * bit from primary->alloced_cpus_in_node.
660 */
661 if (!cpumask_test_cpu(cur_cpu,
662 &primary->alloced_cpus_in_node)) {
663 cpumask_set_cpu(cur_cpu,
664 &primary->alloced_cpus_in_node);
665 cpumask_set_cpu(cur_cpu, alloced_mask);
666 break;
667 }
668 } else {
669 cpumask_set_cpu(cur_cpu, alloced_mask);
670 break;
671 }
672 }
673
674 channel->target_cpu = cur_cpu;
675 channel->target_vp = hv_context.vp_index[cur_cpu];
676}
677
678static void vmbus_wait_for_unload(void)
679{
680 int cpu;
681 void *page_addr;
682 struct hv_message *msg;
683 struct vmbus_channel_message_header *hdr;
684 u32 message_type;
685
686 /*
687 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
688 * used for initial contact or to CPU0 depending on host version. When
689 * we're crashing on a different CPU let's hope that IRQ handler on
690 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
691 * functional and vmbus_unload_response() will complete
692 * vmbus_connection.unload_event. If not, the last thing we can do is
693 * read message pages for all CPUs directly.
694 */
695 while (1) {
696 if (completion_done(&vmbus_connection.unload_event))
697 break;
698
699 for_each_online_cpu(cpu) {
700 page_addr = hv_context.synic_message_page[cpu];
701 msg = (struct hv_message *)page_addr +
702 VMBUS_MESSAGE_SINT;
703
704 message_type = READ_ONCE(msg->header.message_type);
705 if (message_type == HVMSG_NONE)
706 continue;
707
708 hdr = (struct vmbus_channel_message_header *)
709 msg->u.payload;
710
711 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
712 complete(&vmbus_connection.unload_event);
713
714 vmbus_signal_eom(msg, message_type);
715 }
716
717 mdelay(10);
718 }
719
720 /*
721 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
722 * maybe-pending messages on all CPUs to be able to receive new
723 * messages after we reconnect.
724 */
725 for_each_online_cpu(cpu) {
726 page_addr = hv_context.synic_message_page[cpu];
727 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
728 msg->header.message_type = HVMSG_NONE;
729 }
730}
731
732/*
733 * vmbus_unload_response - Handler for the unload response.
734 */
735static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
736{
737 /*
738 * This is a global event; just wakeup the waiting thread.
739 * Once we successfully unload, we can cleanup the monitor state.
740 */
741 complete(&vmbus_connection.unload_event);
742}
743
744void vmbus_initiate_unload(bool crash)
745{
746 struct vmbus_channel_message_header hdr;
747
748 /* Pre-Win2012R2 hosts don't support reconnect */
749 if (vmbus_proto_version < VERSION_WIN8_1)
750 return;
751
752 init_completion(&vmbus_connection.unload_event);
753 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
754 hdr.msgtype = CHANNELMSG_UNLOAD;
755 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
756 !crash);
757
758 /*
759 * vmbus_initiate_unload() is also called on crash and the crash can be
760 * happening in an interrupt context, where scheduling is impossible.
761 */
762 if (!crash)
763 wait_for_completion(&vmbus_connection.unload_event);
764 else
765 vmbus_wait_for_unload();
766}
767
768/*
769 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
770 *
771 */
772static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
773{
774 struct vmbus_channel_offer_channel *offer;
775 struct vmbus_channel *newchannel;
776
777 offer = (struct vmbus_channel_offer_channel *)hdr;
778
779 /* Allocate the channel object and save this offer. */
780 newchannel = alloc_channel();
781 if (!newchannel) {
782 vmbus_release_relid(offer->child_relid);
783 pr_err("Unable to allocate channel object\n");
784 return;
785 }
786
787 /*
788 * By default we setup state to enable batched
789 * reading. A specific service can choose to
790 * disable this prior to opening the channel.
791 */
792 newchannel->batched_reading = true;
793
794 /*
795 * Setup state for signalling the host.
796 */
797 newchannel->sig_event = (struct hv_input_signal_event *)
798 (ALIGN((unsigned long)
799 &newchannel->sig_buf,
800 HV_HYPERCALL_PARAM_ALIGN));
801
802 newchannel->sig_event->connectionid.asu32 = 0;
803 newchannel->sig_event->connectionid.u.id = VMBUS_EVENT_CONNECTION_ID;
804 newchannel->sig_event->flag_number = 0;
805 newchannel->sig_event->rsvdz = 0;
806
807 if (vmbus_proto_version != VERSION_WS2008) {
808 newchannel->is_dedicated_interrupt =
809 (offer->is_dedicated_interrupt != 0);
810 newchannel->sig_event->connectionid.u.id =
811 offer->connection_id;
812 }
813
814 memcpy(&newchannel->offermsg, offer,
815 sizeof(struct vmbus_channel_offer_channel));
816 newchannel->monitor_grp = (u8)offer->monitorid / 32;
817 newchannel->monitor_bit = (u8)offer->monitorid % 32;
818
819 vmbus_process_offer(newchannel);
820}
821
822/*
823 * vmbus_onoffer_rescind - Rescind offer handler.
824 *
825 * We queue a work item to process this offer synchronously
826 */
827static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
828{
829 struct vmbus_channel_rescind_offer *rescind;
830 struct vmbus_channel *channel;
831 unsigned long flags;
832 struct device *dev;
833
834 rescind = (struct vmbus_channel_rescind_offer *)hdr;
835
836 mutex_lock(&vmbus_connection.channel_mutex);
837 channel = relid2channel(rescind->child_relid);
838
839 if (channel == NULL) {
840 /*
841 * This is very impossible, because in
842 * vmbus_process_offer(), we have already invoked
843 * vmbus_release_relid() on error.
844 */
845 goto out;
846 }
847
848 spin_lock_irqsave(&channel->lock, flags);
849 channel->rescind = true;
850 spin_unlock_irqrestore(&channel->lock, flags);
851
852 vmbus_rescind_cleanup(channel);
853
854 if (channel->device_obj) {
855 if (channel->chn_rescind_callback) {
856 channel->chn_rescind_callback(channel);
857 goto out;
858 }
859 /*
860 * We will have to unregister this device from the
861 * driver core.
862 */
863 dev = get_device(&channel->device_obj->device);
864 if (dev) {
865 vmbus_device_unregister(channel->device_obj);
866 put_device(dev);
867 }
868 } else {
869 hv_process_channel_removal(channel,
870 channel->offermsg.child_relid);
871 }
872
873out:
874 mutex_unlock(&vmbus_connection.channel_mutex);
875}
876
877void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
878{
879 mutex_lock(&vmbus_connection.channel_mutex);
880
881 BUG_ON(!is_hvsock_channel(channel));
882
883 channel->rescind = true;
884 vmbus_device_unregister(channel->device_obj);
885
886 mutex_unlock(&vmbus_connection.channel_mutex);
887}
888EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
889
890
891/*
892 * vmbus_onoffers_delivered -
893 * This is invoked when all offers have been delivered.
894 *
895 * Nothing to do here.
896 */
897static void vmbus_onoffers_delivered(
898 struct vmbus_channel_message_header *hdr)
899{
900}
901
902/*
903 * vmbus_onopen_result - Open result handler.
904 *
905 * This is invoked when we received a response to our channel open request.
906 * Find the matching request, copy the response and signal the requesting
907 * thread.
908 */
909static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
910{
911 struct vmbus_channel_open_result *result;
912 struct vmbus_channel_msginfo *msginfo;
913 struct vmbus_channel_message_header *requestheader;
914 struct vmbus_channel_open_channel *openmsg;
915 unsigned long flags;
916
917 result = (struct vmbus_channel_open_result *)hdr;
918
919 /*
920 * Find the open msg, copy the result and signal/unblock the wait event
921 */
922 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
923
924 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
925 msglistentry) {
926 requestheader =
927 (struct vmbus_channel_message_header *)msginfo->msg;
928
929 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
930 openmsg =
931 (struct vmbus_channel_open_channel *)msginfo->msg;
932 if (openmsg->child_relid == result->child_relid &&
933 openmsg->openid == result->openid) {
934 memcpy(&msginfo->response.open_result,
935 result,
936 sizeof(
937 struct vmbus_channel_open_result));
938 complete(&msginfo->waitevent);
939 break;
940 }
941 }
942 }
943 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
944}
945
946/*
947 * vmbus_ongpadl_created - GPADL created handler.
948 *
949 * This is invoked when we received a response to our gpadl create request.
950 * Find the matching request, copy the response and signal the requesting
951 * thread.
952 */
953static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
954{
955 struct vmbus_channel_gpadl_created *gpadlcreated;
956 struct vmbus_channel_msginfo *msginfo;
957 struct vmbus_channel_message_header *requestheader;
958 struct vmbus_channel_gpadl_header *gpadlheader;
959 unsigned long flags;
960
961 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
962
963 /*
964 * Find the establish msg, copy the result and signal/unblock the wait
965 * event
966 */
967 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
968
969 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
970 msglistentry) {
971 requestheader =
972 (struct vmbus_channel_message_header *)msginfo->msg;
973
974 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
975 gpadlheader =
976 (struct vmbus_channel_gpadl_header *)requestheader;
977
978 if ((gpadlcreated->child_relid ==
979 gpadlheader->child_relid) &&
980 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
981 memcpy(&msginfo->response.gpadl_created,
982 gpadlcreated,
983 sizeof(
984 struct vmbus_channel_gpadl_created));
985 complete(&msginfo->waitevent);
986 break;
987 }
988 }
989 }
990 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
991}
992
993/*
994 * vmbus_ongpadl_torndown - GPADL torndown handler.
995 *
996 * This is invoked when we received a response to our gpadl teardown request.
997 * Find the matching request, copy the response and signal the requesting
998 * thread.
999 */
1000static void vmbus_ongpadl_torndown(
1001 struct vmbus_channel_message_header *hdr)
1002{
1003 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1004 struct vmbus_channel_msginfo *msginfo;
1005 struct vmbus_channel_message_header *requestheader;
1006 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1007 unsigned long flags;
1008
1009 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1010
1011 /*
1012 * Find the open msg, copy the result and signal/unblock the wait event
1013 */
1014 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1015
1016 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1017 msglistentry) {
1018 requestheader =
1019 (struct vmbus_channel_message_header *)msginfo->msg;
1020
1021 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1022 gpadl_teardown =
1023 (struct vmbus_channel_gpadl_teardown *)requestheader;
1024
1025 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1026 memcpy(&msginfo->response.gpadl_torndown,
1027 gpadl_torndown,
1028 sizeof(
1029 struct vmbus_channel_gpadl_torndown));
1030 complete(&msginfo->waitevent);
1031 break;
1032 }
1033 }
1034 }
1035 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1036}
1037
1038/*
1039 * vmbus_onversion_response - Version response handler
1040 *
1041 * This is invoked when we received a response to our initiate contact request.
1042 * Find the matching request, copy the response and signal the requesting
1043 * thread.
1044 */
1045static void vmbus_onversion_response(
1046 struct vmbus_channel_message_header *hdr)
1047{
1048 struct vmbus_channel_msginfo *msginfo;
1049 struct vmbus_channel_message_header *requestheader;
1050 struct vmbus_channel_version_response *version_response;
1051 unsigned long flags;
1052
1053 version_response = (struct vmbus_channel_version_response *)hdr;
1054 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1055
1056 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1057 msglistentry) {
1058 requestheader =
1059 (struct vmbus_channel_message_header *)msginfo->msg;
1060
1061 if (requestheader->msgtype ==
1062 CHANNELMSG_INITIATE_CONTACT) {
1063 memcpy(&msginfo->response.version_response,
1064 version_response,
1065 sizeof(struct vmbus_channel_version_response));
1066 complete(&msginfo->waitevent);
1067 }
1068 }
1069 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1070}
1071
1072/* Channel message dispatch table */
1073struct vmbus_channel_message_table_entry
1074 channel_message_table[CHANNELMSG_COUNT] = {
1075 {CHANNELMSG_INVALID, 0, NULL},
1076 {CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer},
1077 {CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind},
1078 {CHANNELMSG_REQUESTOFFERS, 0, NULL},
1079 {CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered},
1080 {CHANNELMSG_OPENCHANNEL, 0, NULL},
1081 {CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result},
1082 {CHANNELMSG_CLOSECHANNEL, 0, NULL},
1083 {CHANNELMSG_GPADL_HEADER, 0, NULL},
1084 {CHANNELMSG_GPADL_BODY, 0, NULL},
1085 {CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created},
1086 {CHANNELMSG_GPADL_TEARDOWN, 0, NULL},
1087 {CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown},
1088 {CHANNELMSG_RELID_RELEASED, 0, NULL},
1089 {CHANNELMSG_INITIATE_CONTACT, 0, NULL},
1090 {CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response},
1091 {CHANNELMSG_UNLOAD, 0, NULL},
1092 {CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response},
1093 {CHANNELMSG_18, 0, NULL},
1094 {CHANNELMSG_19, 0, NULL},
1095 {CHANNELMSG_20, 0, NULL},
1096 {CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL},
1097};
1098
1099/*
1100 * vmbus_onmessage - Handler for channel protocol messages.
1101 *
1102 * This is invoked in the vmbus worker thread context.
1103 */
1104void vmbus_onmessage(void *context)
1105{
1106 struct hv_message *msg = context;
1107 struct vmbus_channel_message_header *hdr;
1108 int size;
1109
1110 hdr = (struct vmbus_channel_message_header *)msg->u.payload;
1111 size = msg->header.payload_size;
1112
1113 if (hdr->msgtype >= CHANNELMSG_COUNT) {
1114 pr_err("Received invalid channel message type %d size %d\n",
1115 hdr->msgtype, size);
1116 print_hex_dump_bytes("", DUMP_PREFIX_NONE,
1117 (unsigned char *)msg->u.payload, size);
1118 return;
1119 }
1120
1121 if (channel_message_table[hdr->msgtype].message_handler)
1122 channel_message_table[hdr->msgtype].message_handler(hdr);
1123 else
1124 pr_err("Unhandled channel message type %d\n", hdr->msgtype);
1125}
1126
1127/*
1128 * vmbus_request_offers - Send a request to get all our pending offers.
1129 */
1130int vmbus_request_offers(void)
1131{
1132 struct vmbus_channel_message_header *msg;
1133 struct vmbus_channel_msginfo *msginfo;
1134 int ret;
1135
1136 msginfo = kmalloc(sizeof(*msginfo) +
1137 sizeof(struct vmbus_channel_message_header),
1138 GFP_KERNEL);
1139 if (!msginfo)
1140 return -ENOMEM;
1141
1142 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1143
1144 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1145
1146
1147 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1148 true);
1149 if (ret != 0) {
1150 pr_err("Unable to request offers - %d\n", ret);
1151
1152 goto cleanup;
1153 }
1154
1155cleanup:
1156 kfree(msginfo);
1157
1158 return ret;
1159}
1160
1161/*
1162 * Retrieve the (sub) channel on which to send an outgoing request.
1163 * When a primary channel has multiple sub-channels, we try to
1164 * distribute the load equally amongst all available channels.
1165 */
1166struct vmbus_channel *vmbus_get_outgoing_channel(struct vmbus_channel *primary)
1167{
1168 struct list_head *cur, *tmp;
1169 int cur_cpu;
1170 struct vmbus_channel *cur_channel;
1171 struct vmbus_channel *outgoing_channel = primary;
1172 int next_channel;
1173 int i = 1;
1174
1175 if (list_empty(&primary->sc_list))
1176 return outgoing_channel;
1177
1178 next_channel = primary->next_oc++;
1179
1180 if (next_channel > (primary->num_sc)) {
1181 primary->next_oc = 0;
1182 return outgoing_channel;
1183 }
1184
1185 cur_cpu = hv_context.vp_index[get_cpu()];
1186 put_cpu();
1187 list_for_each_safe(cur, tmp, &primary->sc_list) {
1188 cur_channel = list_entry(cur, struct vmbus_channel, sc_list);
1189 if (cur_channel->state != CHANNEL_OPENED_STATE)
1190 continue;
1191
1192 if (cur_channel->target_vp == cur_cpu)
1193 return cur_channel;
1194
1195 if (i == next_channel)
1196 return cur_channel;
1197
1198 i++;
1199 }
1200
1201 return outgoing_channel;
1202}
1203EXPORT_SYMBOL_GPL(vmbus_get_outgoing_channel);
1204
1205static void invoke_sc_cb(struct vmbus_channel *primary_channel)
1206{
1207 struct list_head *cur, *tmp;
1208 struct vmbus_channel *cur_channel;
1209
1210 if (primary_channel->sc_creation_callback == NULL)
1211 return;
1212
1213 list_for_each_safe(cur, tmp, &primary_channel->sc_list) {
1214 cur_channel = list_entry(cur, struct vmbus_channel, sc_list);
1215
1216 primary_channel->sc_creation_callback(cur_channel);
1217 }
1218}
1219
1220void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1221 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1222{
1223 primary_channel->sc_creation_callback = sc_cr_cb;
1224}
1225EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1226
1227bool vmbus_are_subchannels_present(struct vmbus_channel *primary)
1228{
1229 bool ret;
1230
1231 ret = !list_empty(&primary->sc_list);
1232
1233 if (ret) {
1234 /*
1235 * Invoke the callback on sub-channel creation.
1236 * This will present a uniform interface to the
1237 * clients.
1238 */
1239 invoke_sc_cb(primary);
1240 }
1241
1242 return ret;
1243}
1244EXPORT_SYMBOL_GPL(vmbus_are_subchannels_present);
1245
1246void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1247 void (*chn_rescind_cb)(struct vmbus_channel *))
1248{
1249 channel->chn_rescind_callback = chn_rescind_cb;
1250}
1251EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);