Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/sched.h>
14#include <linux/wait.h>
15#include <linux/mm.h>
16#include <linux/slab.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/completion.h>
20#include <linux/delay.h>
21#include <linux/cpu.h>
22#include <linux/hyperv.h>
23#include <asm/mshyperv.h>
24#include <linux/sched/isolation.h>
25
26#include "hyperv_vmbus.h"
27
28static void init_vp_index(struct vmbus_channel *channel);
29
30const struct vmbus_device vmbus_devs[] = {
31 /* IDE */
32 { .dev_type = HV_IDE,
33 HV_IDE_GUID,
34 .perf_device = true,
35 .allowed_in_isolated = false,
36 },
37
38 /* SCSI */
39 { .dev_type = HV_SCSI,
40 HV_SCSI_GUID,
41 .perf_device = true,
42 .allowed_in_isolated = true,
43 },
44
45 /* Fibre Channel */
46 { .dev_type = HV_FC,
47 HV_SYNTHFC_GUID,
48 .perf_device = true,
49 .allowed_in_isolated = false,
50 },
51
52 /* Synthetic NIC */
53 { .dev_type = HV_NIC,
54 HV_NIC_GUID,
55 .perf_device = true,
56 .allowed_in_isolated = true,
57 },
58
59 /* Network Direct */
60 { .dev_type = HV_ND,
61 HV_ND_GUID,
62 .perf_device = true,
63 .allowed_in_isolated = false,
64 },
65
66 /* PCIE */
67 { .dev_type = HV_PCIE,
68 HV_PCIE_GUID,
69 .perf_device = false,
70 .allowed_in_isolated = true,
71 },
72
73 /* Synthetic Frame Buffer */
74 { .dev_type = HV_FB,
75 HV_SYNTHVID_GUID,
76 .perf_device = false,
77 .allowed_in_isolated = false,
78 },
79
80 /* Synthetic Keyboard */
81 { .dev_type = HV_KBD,
82 HV_KBD_GUID,
83 .perf_device = false,
84 .allowed_in_isolated = false,
85 },
86
87 /* Synthetic MOUSE */
88 { .dev_type = HV_MOUSE,
89 HV_MOUSE_GUID,
90 .perf_device = false,
91 .allowed_in_isolated = false,
92 },
93
94 /* KVP */
95 { .dev_type = HV_KVP,
96 HV_KVP_GUID,
97 .perf_device = false,
98 .allowed_in_isolated = false,
99 },
100
101 /* Time Synch */
102 { .dev_type = HV_TS,
103 HV_TS_GUID,
104 .perf_device = false,
105 .allowed_in_isolated = true,
106 },
107
108 /* Heartbeat */
109 { .dev_type = HV_HB,
110 HV_HEART_BEAT_GUID,
111 .perf_device = false,
112 .allowed_in_isolated = true,
113 },
114
115 /* Shutdown */
116 { .dev_type = HV_SHUTDOWN,
117 HV_SHUTDOWN_GUID,
118 .perf_device = false,
119 .allowed_in_isolated = true,
120 },
121
122 /* File copy */
123 { .dev_type = HV_FCOPY,
124 HV_FCOPY_GUID,
125 .perf_device = false,
126 .allowed_in_isolated = false,
127 },
128
129 /* Backup */
130 { .dev_type = HV_BACKUP,
131 HV_VSS_GUID,
132 .perf_device = false,
133 .allowed_in_isolated = false,
134 },
135
136 /* Dynamic Memory */
137 { .dev_type = HV_DM,
138 HV_DM_GUID,
139 .perf_device = false,
140 .allowed_in_isolated = false,
141 },
142
143 /* Unknown GUID */
144 { .dev_type = HV_UNKNOWN,
145 .perf_device = false,
146 .allowed_in_isolated = false,
147 },
148};
149
150static const struct {
151 guid_t guid;
152} vmbus_unsupported_devs[] = {
153 { HV_AVMA1_GUID },
154 { HV_AVMA2_GUID },
155 { HV_RDV_GUID },
156 { HV_IMC_GUID },
157};
158
159/*
160 * The rescinded channel may be blocked waiting for a response from the host;
161 * take care of that.
162 */
163static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
164{
165 struct vmbus_channel_msginfo *msginfo;
166 unsigned long flags;
167
168
169 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
170 channel->rescind = true;
171 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
172 msglistentry) {
173
174 if (msginfo->waiting_channel == channel) {
175 complete(&msginfo->waitevent);
176 break;
177 }
178 }
179 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
180}
181
182static bool is_unsupported_vmbus_devs(const guid_t *guid)
183{
184 int i;
185
186 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
187 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))
188 return true;
189 return false;
190}
191
192static u16 hv_get_dev_type(const struct vmbus_channel *channel)
193{
194 const guid_t *guid = &channel->offermsg.offer.if_type;
195 u16 i;
196
197 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
198 return HV_UNKNOWN;
199
200 for (i = HV_IDE; i < HV_UNKNOWN; i++) {
201 if (guid_equal(guid, &vmbus_devs[i].guid))
202 return i;
203 }
204 pr_info("Unknown GUID: %pUl\n", guid);
205 return i;
206}
207
208/**
209 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
210 * @icmsghdrp: Pointer to msg header structure
211 * @buf: Raw buffer channel data
212 * @buflen: Length of the raw buffer channel data.
213 * @fw_version: The framework versions we can support.
214 * @fw_vercnt: The size of @fw_version.
215 * @srv_version: The service versions we can support.
216 * @srv_vercnt: The size of @srv_version.
217 * @nego_fw_version: The selected framework version.
218 * @nego_srv_version: The selected service version.
219 *
220 * Note: Versions are given in decreasing order.
221 *
222 * Set up and fill in default negotiate response message.
223 * Mainly used by Hyper-V drivers.
224 */
225bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf,
226 u32 buflen, const int *fw_version, int fw_vercnt,
227 const int *srv_version, int srv_vercnt,
228 int *nego_fw_version, int *nego_srv_version)
229{
230 int icframe_major, icframe_minor;
231 int icmsg_major, icmsg_minor;
232 int fw_major, fw_minor;
233 int srv_major, srv_minor;
234 int i, j;
235 bool found_match = false;
236 struct icmsg_negotiate *negop;
237
238 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */
239 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) {
240 pr_err_ratelimited("Invalid icmsg negotiate\n");
241 return false;
242 }
243
244 icmsghdrp->icmsgsize = 0x10;
245 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR];
246
247 icframe_major = negop->icframe_vercnt;
248 icframe_minor = 0;
249
250 icmsg_major = negop->icmsg_vercnt;
251 icmsg_minor = 0;
252
253 /* Validate negop packet */
254 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
255 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
256 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) {
257 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",
258 icframe_major, icmsg_major);
259 goto fw_error;
260 }
261
262 /*
263 * Select the framework version number we will
264 * support.
265 */
266
267 for (i = 0; i < fw_vercnt; i++) {
268 fw_major = (fw_version[i] >> 16);
269 fw_minor = (fw_version[i] & 0xFFFF);
270
271 for (j = 0; j < negop->icframe_vercnt; j++) {
272 if ((negop->icversion_data[j].major == fw_major) &&
273 (negop->icversion_data[j].minor == fw_minor)) {
274 icframe_major = negop->icversion_data[j].major;
275 icframe_minor = negop->icversion_data[j].minor;
276 found_match = true;
277 break;
278 }
279 }
280
281 if (found_match)
282 break;
283 }
284
285 if (!found_match)
286 goto fw_error;
287
288 found_match = false;
289
290 for (i = 0; i < srv_vercnt; i++) {
291 srv_major = (srv_version[i] >> 16);
292 srv_minor = (srv_version[i] & 0xFFFF);
293
294 for (j = negop->icframe_vercnt;
295 (j < negop->icframe_vercnt + negop->icmsg_vercnt);
296 j++) {
297
298 if ((negop->icversion_data[j].major == srv_major) &&
299 (negop->icversion_data[j].minor == srv_minor)) {
300
301 icmsg_major = negop->icversion_data[j].major;
302 icmsg_minor = negop->icversion_data[j].minor;
303 found_match = true;
304 break;
305 }
306 }
307
308 if (found_match)
309 break;
310 }
311
312 /*
313 * Respond with the framework and service
314 * version numbers we can support.
315 */
316
317fw_error:
318 if (!found_match) {
319 negop->icframe_vercnt = 0;
320 negop->icmsg_vercnt = 0;
321 } else {
322 negop->icframe_vercnt = 1;
323 negop->icmsg_vercnt = 1;
324 }
325
326 if (nego_fw_version)
327 *nego_fw_version = (icframe_major << 16) | icframe_minor;
328
329 if (nego_srv_version)
330 *nego_srv_version = (icmsg_major << 16) | icmsg_minor;
331
332 negop->icversion_data[0].major = icframe_major;
333 negop->icversion_data[0].minor = icframe_minor;
334 negop->icversion_data[1].major = icmsg_major;
335 negop->icversion_data[1].minor = icmsg_minor;
336 return found_match;
337}
338EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
339
340/*
341 * alloc_channel - Allocate and initialize a vmbus channel object
342 */
343static struct vmbus_channel *alloc_channel(void)
344{
345 struct vmbus_channel *channel;
346
347 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
348 if (!channel)
349 return NULL;
350
351 spin_lock_init(&channel->sched_lock);
352 init_completion(&channel->rescind_event);
353
354 INIT_LIST_HEAD(&channel->sc_list);
355
356 tasklet_init(&channel->callback_event,
357 vmbus_on_event, (unsigned long)channel);
358
359 hv_ringbuffer_pre_init(channel);
360
361 return channel;
362}
363
364/*
365 * free_channel - Release the resources used by the vmbus channel object
366 */
367static void free_channel(struct vmbus_channel *channel)
368{
369 tasklet_kill(&channel->callback_event);
370 vmbus_remove_channel_attr_group(channel);
371
372 kobject_put(&channel->kobj);
373}
374
375void vmbus_channel_map_relid(struct vmbus_channel *channel)
376{
377 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
378 return;
379 /*
380 * The mapping of the channel's relid is visible from the CPUs that
381 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will
382 * execute:
383 *
384 * (a) In the "normal (i.e., not resuming from hibernation)" path,
385 * the full barrier in virt_store_mb() guarantees that the store
386 * is propagated to all CPUs before the add_channel_work work
387 * is queued. In turn, add_channel_work is queued before the
388 * channel's ring buffer is allocated/initialized and the
389 * OPENCHANNEL message for the channel is sent in vmbus_open().
390 * Hyper-V won't start sending the interrupts for the channel
391 * before the OPENCHANNEL message is acked. The memory barrier
392 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures
393 * that vmbus_chan_sched() must find the channel's relid in
394 * recv_int_page before retrieving the channel pointer from the
395 * array of channels.
396 *
397 * (b) In the "resuming from hibernation" path, the virt_store_mb()
398 * guarantees that the store is propagated to all CPUs before
399 * the VMBus connection is marked as ready for the resume event
400 * (cf. check_ready_for_resume_event()). The interrupt handler
401 * of the VMBus driver and vmbus_chan_sched() can not run before
402 * vmbus_bus_resume() has completed execution (cf. resume_noirq).
403 */
404 virt_store_mb(
405 vmbus_connection.channels[channel->offermsg.child_relid],
406 channel);
407}
408
409void vmbus_channel_unmap_relid(struct vmbus_channel *channel)
410{
411 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
412 return;
413 WRITE_ONCE(
414 vmbus_connection.channels[channel->offermsg.child_relid],
415 NULL);
416}
417
418static void vmbus_release_relid(u32 relid)
419{
420 struct vmbus_channel_relid_released msg;
421 int ret;
422
423 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
424 msg.child_relid = relid;
425 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
426 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
427 true);
428
429 trace_vmbus_release_relid(&msg, ret);
430}
431
432void hv_process_channel_removal(struct vmbus_channel *channel)
433{
434 lockdep_assert_held(&vmbus_connection.channel_mutex);
435 BUG_ON(!channel->rescind);
436
437 /*
438 * hv_process_channel_removal() could find INVALID_RELID only for
439 * hv_sock channels. See the inline comments in vmbus_onoffer().
440 */
441 WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&
442 !is_hvsock_channel(channel));
443
444 /*
445 * Upon suspend, an in-use hv_sock channel is removed from the array of
446 * channels and the relid is invalidated. After hibernation, when the
447 * user-space application destroys the channel, it's unnecessary and
448 * unsafe to remove the channel from the array of channels. See also
449 * the inline comments before the call of vmbus_release_relid() below.
450 */
451 if (channel->offermsg.child_relid != INVALID_RELID)
452 vmbus_channel_unmap_relid(channel);
453
454 if (channel->primary_channel == NULL)
455 list_del(&channel->listentry);
456 else
457 list_del(&channel->sc_list);
458
459 /*
460 * If this is a "perf" channel, updates the hv_numa_map[] masks so that
461 * init_vp_index() can (re-)use the CPU.
462 */
463 if (hv_is_perf_channel(channel))
464 hv_clear_allocated_cpu(channel->target_cpu);
465
466 /*
467 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and
468 * the relid is invalidated; after hibernation, when the user-space app
469 * destroys the channel, the relid is INVALID_RELID, and in this case
470 * it's unnecessary and unsafe to release the old relid, since the same
471 * relid can refer to a completely different channel now.
472 */
473 if (channel->offermsg.child_relid != INVALID_RELID)
474 vmbus_release_relid(channel->offermsg.child_relid);
475
476 free_channel(channel);
477}
478
479void vmbus_free_channels(void)
480{
481 struct vmbus_channel *channel, *tmp;
482
483 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
484 listentry) {
485 /* hv_process_channel_removal() needs this */
486 channel->rescind = true;
487
488 vmbus_device_unregister(channel->device_obj);
489 }
490}
491
492/* Note: the function can run concurrently for primary/sub channels. */
493static void vmbus_add_channel_work(struct work_struct *work)
494{
495 struct vmbus_channel *newchannel =
496 container_of(work, struct vmbus_channel, add_channel_work);
497 struct vmbus_channel *primary_channel = newchannel->primary_channel;
498 int ret;
499
500 /*
501 * This state is used to indicate a successful open
502 * so that when we do close the channel normally, we
503 * can cleanup properly.
504 */
505 newchannel->state = CHANNEL_OPEN_STATE;
506
507 if (primary_channel != NULL) {
508 /* newchannel is a sub-channel. */
509 struct hv_device *dev = primary_channel->device_obj;
510
511 if (vmbus_add_channel_kobj(dev, newchannel))
512 goto err_deq_chan;
513
514 if (primary_channel->sc_creation_callback != NULL)
515 primary_channel->sc_creation_callback(newchannel);
516
517 newchannel->probe_done = true;
518 return;
519 }
520
521 /*
522 * Start the process of binding the primary channel to the driver
523 */
524 newchannel->device_obj = vmbus_device_create(
525 &newchannel->offermsg.offer.if_type,
526 &newchannel->offermsg.offer.if_instance,
527 newchannel);
528 if (!newchannel->device_obj)
529 goto err_deq_chan;
530
531 newchannel->device_obj->device_id = newchannel->device_id;
532 /*
533 * Add the new device to the bus. This will kick off device-driver
534 * binding which eventually invokes the device driver's AddDevice()
535 * method.
536 *
537 * If vmbus_device_register() fails, the 'device_obj' is freed in
538 * vmbus_device_release() as called by device_unregister() in the
539 * error path of vmbus_device_register(). In the outside error
540 * path, there's no need to free it.
541 */
542 ret = vmbus_device_register(newchannel->device_obj);
543
544 if (ret != 0) {
545 pr_err("unable to add child device object (relid %d)\n",
546 newchannel->offermsg.child_relid);
547 goto err_deq_chan;
548 }
549
550 newchannel->probe_done = true;
551 return;
552
553err_deq_chan:
554 mutex_lock(&vmbus_connection.channel_mutex);
555
556 /*
557 * We need to set the flag, otherwise
558 * vmbus_onoffer_rescind() can be blocked.
559 */
560 newchannel->probe_done = true;
561
562 if (primary_channel == NULL)
563 list_del(&newchannel->listentry);
564 else
565 list_del(&newchannel->sc_list);
566
567 /* vmbus_process_offer() has mapped the channel. */
568 vmbus_channel_unmap_relid(newchannel);
569
570 mutex_unlock(&vmbus_connection.channel_mutex);
571
572 vmbus_release_relid(newchannel->offermsg.child_relid);
573
574 free_channel(newchannel);
575}
576
577/*
578 * vmbus_process_offer - Process the offer by creating a channel/device
579 * associated with this offer
580 */
581static void vmbus_process_offer(struct vmbus_channel *newchannel)
582{
583 struct vmbus_channel *channel;
584 struct workqueue_struct *wq;
585 bool fnew = true;
586
587 /*
588 * Synchronize vmbus_process_offer() and CPU hotplugging:
589 *
590 * CPU1 CPU2
591 *
592 * [vmbus_process_offer()] [Hot removal of the CPU]
593 *
594 * CPU_READ_LOCK CPUS_WRITE_LOCK
595 * LOAD cpu_online_mask SEARCH chn_list
596 * STORE target_cpu LOAD target_cpu
597 * INSERT chn_list STORE cpu_online_mask
598 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK
599 *
600 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE &&
601 * CPU2's SEARCH from *not* seeing CPU1's INSERT
602 *
603 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT &&
604 * CPU2's LOAD from *not* seing CPU1's STORE
605 */
606 cpus_read_lock();
607
608 /*
609 * Serializes the modifications of the chn_list list as well as
610 * the accesses to next_numa_node_id in init_vp_index().
611 */
612 mutex_lock(&vmbus_connection.channel_mutex);
613
614 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
615 if (guid_equal(&channel->offermsg.offer.if_type,
616 &newchannel->offermsg.offer.if_type) &&
617 guid_equal(&channel->offermsg.offer.if_instance,
618 &newchannel->offermsg.offer.if_instance)) {
619 fnew = false;
620 newchannel->primary_channel = channel;
621 break;
622 }
623 }
624
625 init_vp_index(newchannel);
626
627 /* Remember the channels that should be cleaned up upon suspend. */
628 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))
629 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);
630
631 /*
632 * Now that we have acquired the channel_mutex,
633 * we can release the potentially racing rescind thread.
634 */
635 atomic_dec(&vmbus_connection.offer_in_progress);
636
637 if (fnew) {
638 list_add_tail(&newchannel->listentry,
639 &vmbus_connection.chn_list);
640 } else {
641 /*
642 * Check to see if this is a valid sub-channel.
643 */
644 if (newchannel->offermsg.offer.sub_channel_index == 0) {
645 mutex_unlock(&vmbus_connection.channel_mutex);
646 cpus_read_unlock();
647 /*
648 * Don't call free_channel(), because newchannel->kobj
649 * is not initialized yet.
650 */
651 kfree(newchannel);
652 WARN_ON_ONCE(1);
653 return;
654 }
655 /*
656 * Process the sub-channel.
657 */
658 list_add_tail(&newchannel->sc_list, &channel->sc_list);
659 }
660
661 vmbus_channel_map_relid(newchannel);
662
663 mutex_unlock(&vmbus_connection.channel_mutex);
664 cpus_read_unlock();
665
666 /*
667 * vmbus_process_offer() mustn't call channel->sc_creation_callback()
668 * directly for sub-channels, because sc_creation_callback() ->
669 * vmbus_open() may never get the host's response to the
670 * OPEN_CHANNEL message (the host may rescind a channel at any time,
671 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()
672 * may not wake up the vmbus_open() as it's blocked due to a non-zero
673 * vmbus_connection.offer_in_progress, and finally we have a deadlock.
674 *
675 * The above is also true for primary channels, if the related device
676 * drivers use sync probing mode by default.
677 *
678 * And, usually the handling of primary channels and sub-channels can
679 * depend on each other, so we should offload them to different
680 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,
681 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->
682 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock
683 * and waits for all the sub-channels to appear, but the latter
684 * can't get the rtnl_lock and this blocks the handling of
685 * sub-channels.
686 */
687 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);
688 wq = fnew ? vmbus_connection.handle_primary_chan_wq :
689 vmbus_connection.handle_sub_chan_wq;
690 queue_work(wq, &newchannel->add_channel_work);
691}
692
693/*
694 * Check if CPUs used by other channels of the same device.
695 * It should only be called by init_vp_index().
696 */
697static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn)
698{
699 struct vmbus_channel *primary = chn->primary_channel;
700 struct vmbus_channel *sc;
701
702 lockdep_assert_held(&vmbus_connection.channel_mutex);
703
704 if (!primary)
705 return false;
706
707 if (primary->target_cpu == cpu)
708 return true;
709
710 list_for_each_entry(sc, &primary->sc_list, sc_list)
711 if (sc != chn && sc->target_cpu == cpu)
712 return true;
713
714 return false;
715}
716
717/*
718 * We use this state to statically distribute the channel interrupt load.
719 */
720static int next_numa_node_id;
721
722/*
723 * We can statically distribute the incoming channel interrupt load
724 * by binding a channel to VCPU.
725 *
726 * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.
727 * Performance critical channels will be distributed evenly among all
728 * the available NUMA nodes. Once the node is assigned, we will assign
729 * the CPU based on a simple round robin scheme.
730 */
731static void init_vp_index(struct vmbus_channel *channel)
732{
733 bool perf_chn = hv_is_perf_channel(channel);
734 u32 i, ncpu = num_online_cpus();
735 cpumask_var_t available_mask;
736 struct cpumask *allocated_mask;
737 const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
738 u32 target_cpu;
739 int numa_node;
740
741 if (!perf_chn ||
742 !alloc_cpumask_var(&available_mask, GFP_KERNEL) ||
743 cpumask_empty(hk_mask)) {
744 /*
745 * If the channel is not a performance critical
746 * channel, bind it to VMBUS_CONNECT_CPU.
747 * In case alloc_cpumask_var() fails, bind it to
748 * VMBUS_CONNECT_CPU.
749 * If all the cpus are isolated, bind it to
750 * VMBUS_CONNECT_CPU.
751 */
752 channel->target_cpu = VMBUS_CONNECT_CPU;
753 if (perf_chn)
754 hv_set_allocated_cpu(VMBUS_CONNECT_CPU);
755 return;
756 }
757
758 for (i = 1; i <= ncpu + 1; i++) {
759 while (true) {
760 numa_node = next_numa_node_id++;
761 if (numa_node == nr_node_ids) {
762 next_numa_node_id = 0;
763 continue;
764 }
765 if (cpumask_empty(cpumask_of_node(numa_node)))
766 continue;
767 break;
768 }
769 allocated_mask = &hv_context.hv_numa_map[numa_node];
770
771retry:
772 cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node));
773 cpumask_and(available_mask, available_mask, hk_mask);
774
775 if (cpumask_empty(available_mask)) {
776 /*
777 * We have cycled through all the CPUs in the node;
778 * reset the allocated map.
779 */
780 cpumask_clear(allocated_mask);
781 goto retry;
782 }
783
784 target_cpu = cpumask_first(available_mask);
785 cpumask_set_cpu(target_cpu, allocated_mask);
786
787 if (channel->offermsg.offer.sub_channel_index >= ncpu ||
788 i > ncpu || !hv_cpuself_used(target_cpu, channel))
789 break;
790 }
791
792 channel->target_cpu = target_cpu;
793
794 free_cpumask_var(available_mask);
795}
796
797#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */
798#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */
799#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS)
800#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */
801#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS)
802
803static void vmbus_wait_for_unload(void)
804{
805 int cpu;
806 void *page_addr;
807 struct hv_message *msg;
808 struct vmbus_channel_message_header *hdr;
809 u32 message_type, i;
810
811 /*
812 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
813 * used for initial contact or to CPU0 depending on host version. When
814 * we're crashing on a different CPU let's hope that IRQ handler on
815 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
816 * functional and vmbus_unload_response() will complete
817 * vmbus_connection.unload_event. If not, the last thing we can do is
818 * read message pages for all CPUs directly.
819 *
820 * Wait up to 100 seconds since an Azure host must writeback any dirty
821 * data in its disk cache before the VMbus UNLOAD request will
822 * complete. This flushing has been empirically observed to take up
823 * to 50 seconds in cases with a lot of dirty data, so allow additional
824 * leeway and for inaccuracies in mdelay(). But eventually time out so
825 * that the panic path can't get hung forever in case the response
826 * message isn't seen.
827 */
828 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) {
829 if (completion_done(&vmbus_connection.unload_event))
830 goto completed;
831
832 for_each_present_cpu(cpu) {
833 struct hv_per_cpu_context *hv_cpu
834 = per_cpu_ptr(hv_context.cpu_context, cpu);
835
836 /*
837 * In a CoCo VM the synic_message_page is not allocated
838 * in hv_synic_alloc(). Instead it is set/cleared in
839 * hv_synic_enable_regs() and hv_synic_disable_regs()
840 * such that it is set only when the CPU is online. If
841 * not all present CPUs are online, the message page
842 * might be NULL, so skip such CPUs.
843 */
844 page_addr = hv_cpu->synic_message_page;
845 if (!page_addr)
846 continue;
847
848 msg = (struct hv_message *)page_addr
849 + VMBUS_MESSAGE_SINT;
850
851 message_type = READ_ONCE(msg->header.message_type);
852 if (message_type == HVMSG_NONE)
853 continue;
854
855 hdr = (struct vmbus_channel_message_header *)
856 msg->u.payload;
857
858 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
859 complete(&vmbus_connection.unload_event);
860
861 vmbus_signal_eom(msg, message_type);
862 }
863
864 /*
865 * Give a notice periodically so someone watching the
866 * serial output won't think it is completely hung.
867 */
868 if (!(i % UNLOAD_MSG_LOOPS))
869 pr_notice("Waiting for VMBus UNLOAD to complete\n");
870
871 mdelay(UNLOAD_DELAY_UNIT_MS);
872 }
873 pr_err("Continuing even though VMBus UNLOAD did not complete\n");
874
875completed:
876 /*
877 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
878 * maybe-pending messages on all CPUs to be able to receive new
879 * messages after we reconnect.
880 */
881 for_each_present_cpu(cpu) {
882 struct hv_per_cpu_context *hv_cpu
883 = per_cpu_ptr(hv_context.cpu_context, cpu);
884
885 page_addr = hv_cpu->synic_message_page;
886 if (!page_addr)
887 continue;
888
889 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
890 msg->header.message_type = HVMSG_NONE;
891 }
892}
893
894/*
895 * vmbus_unload_response - Handler for the unload response.
896 */
897static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
898{
899 /*
900 * This is a global event; just wakeup the waiting thread.
901 * Once we successfully unload, we can cleanup the monitor state.
902 *
903 * NB. A malicious or compromised Hyper-V could send a spurious
904 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call
905 * of the complete() below. Make sure that unload_event has been
906 * initialized by the time this complete() is executed.
907 */
908 complete(&vmbus_connection.unload_event);
909}
910
911void vmbus_initiate_unload(bool crash)
912{
913 struct vmbus_channel_message_header hdr;
914
915 if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED)
916 return;
917
918 /* Pre-Win2012R2 hosts don't support reconnect */
919 if (vmbus_proto_version < VERSION_WIN8_1)
920 return;
921
922 reinit_completion(&vmbus_connection.unload_event);
923 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
924 hdr.msgtype = CHANNELMSG_UNLOAD;
925 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
926 !crash);
927
928 /*
929 * vmbus_initiate_unload() is also called on crash and the crash can be
930 * happening in an interrupt context, where scheduling is impossible.
931 */
932 if (!crash)
933 wait_for_completion(&vmbus_connection.unload_event);
934 else
935 vmbus_wait_for_unload();
936}
937
938static void check_ready_for_resume_event(void)
939{
940 /*
941 * If all the old primary channels have been fixed up, then it's safe
942 * to resume.
943 */
944 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))
945 complete(&vmbus_connection.ready_for_resume_event);
946}
947
948static void vmbus_setup_channel_state(struct vmbus_channel *channel,
949 struct vmbus_channel_offer_channel *offer)
950{
951 /*
952 * Setup state for signalling the host.
953 */
954 channel->sig_event = VMBUS_EVENT_CONNECTION_ID;
955
956 channel->is_dedicated_interrupt =
957 (offer->is_dedicated_interrupt != 0);
958 channel->sig_event = offer->connection_id;
959
960 memcpy(&channel->offermsg, offer,
961 sizeof(struct vmbus_channel_offer_channel));
962 channel->monitor_grp = (u8)offer->monitorid / 32;
963 channel->monitor_bit = (u8)offer->monitorid % 32;
964 channel->device_id = hv_get_dev_type(channel);
965}
966
967/*
968 * find_primary_channel_by_offer - Get the channel object given the new offer.
969 * This is only used in the resume path of hibernation.
970 */
971static struct vmbus_channel *
972find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)
973{
974 struct vmbus_channel *channel = NULL, *iter;
975 const guid_t *inst1, *inst2;
976
977 /* Ignore sub-channel offers. */
978 if (offer->offer.sub_channel_index != 0)
979 return NULL;
980
981 mutex_lock(&vmbus_connection.channel_mutex);
982
983 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {
984 inst1 = &iter->offermsg.offer.if_instance;
985 inst2 = &offer->offer.if_instance;
986
987 if (guid_equal(inst1, inst2)) {
988 channel = iter;
989 break;
990 }
991 }
992
993 mutex_unlock(&vmbus_connection.channel_mutex);
994
995 return channel;
996}
997
998static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer)
999{
1000 const guid_t *guid = &offer->offer.if_type;
1001 u16 i;
1002
1003 if (!hv_is_isolation_supported())
1004 return true;
1005
1006 if (is_hvsock_offer(offer))
1007 return true;
1008
1009 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) {
1010 if (guid_equal(guid, &vmbus_devs[i].guid))
1011 return vmbus_devs[i].allowed_in_isolated;
1012 }
1013 return false;
1014}
1015
1016/*
1017 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1018 *
1019 */
1020static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
1021{
1022 struct vmbus_channel_offer_channel *offer;
1023 struct vmbus_channel *oldchannel, *newchannel;
1024 size_t offer_sz;
1025
1026 offer = (struct vmbus_channel_offer_channel *)hdr;
1027
1028 trace_vmbus_onoffer(offer);
1029
1030 if (!vmbus_is_valid_offer(offer)) {
1031 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n",
1032 offer->child_relid);
1033 atomic_dec(&vmbus_connection.offer_in_progress);
1034 return;
1035 }
1036
1037 oldchannel = find_primary_channel_by_offer(offer);
1038
1039 if (oldchannel != NULL) {
1040 /*
1041 * We're resuming from hibernation: all the sub-channel and
1042 * hv_sock channels we had before the hibernation should have
1043 * been cleaned up, and now we must be seeing a re-offered
1044 * primary channel that we had before the hibernation.
1045 */
1046
1047 /*
1048 * { Initially: channel relid = INVALID_RELID,
1049 * channels[valid_relid] = NULL }
1050 *
1051 * CPU1 CPU2
1052 *
1053 * [vmbus_onoffer()] [vmbus_device_release()]
1054 *
1055 * LOCK channel_mutex LOCK channel_mutex
1056 * STORE channel relid = valid_relid LOAD r1 = channel relid
1057 * MAP_RELID channel if (r1 != INVALID_RELID)
1058 * UNLOCK channel_mutex UNMAP_RELID channel
1059 * UNLOCK channel_mutex
1060 *
1061 * Forbids: r1 == valid_relid &&
1062 * channels[valid_relid] == channel
1063 *
1064 * Note. r1 can be INVALID_RELID only for an hv_sock channel.
1065 * None of the hv_sock channels which were present before the
1066 * suspend are re-offered upon the resume. See the WARN_ON()
1067 * in hv_process_channel_removal().
1068 */
1069 mutex_lock(&vmbus_connection.channel_mutex);
1070
1071 atomic_dec(&vmbus_connection.offer_in_progress);
1072
1073 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);
1074 /* Fix up the relid. */
1075 oldchannel->offermsg.child_relid = offer->child_relid;
1076
1077 offer_sz = sizeof(*offer);
1078 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {
1079 /*
1080 * This is not an error, since the host can also change
1081 * the other field(s) of the offer, e.g. on WS RS5
1082 * (Build 17763), the offer->connection_id of the
1083 * Mellanox VF vmbus device can change when the host
1084 * reoffers the device upon resume.
1085 */
1086 pr_debug("vmbus offer changed: relid=%d\n",
1087 offer->child_relid);
1088
1089 print_hex_dump_debug("Old vmbus offer: ",
1090 DUMP_PREFIX_OFFSET, 16, 4,
1091 &oldchannel->offermsg, offer_sz,
1092 false);
1093 print_hex_dump_debug("New vmbus offer: ",
1094 DUMP_PREFIX_OFFSET, 16, 4,
1095 offer, offer_sz, false);
1096
1097 /* Fix up the old channel. */
1098 vmbus_setup_channel_state(oldchannel, offer);
1099 }
1100
1101 /* Add the channel back to the array of channels. */
1102 vmbus_channel_map_relid(oldchannel);
1103 check_ready_for_resume_event();
1104
1105 mutex_unlock(&vmbus_connection.channel_mutex);
1106 return;
1107 }
1108
1109 /* Allocate the channel object and save this offer. */
1110 newchannel = alloc_channel();
1111 if (!newchannel) {
1112 vmbus_release_relid(offer->child_relid);
1113 atomic_dec(&vmbus_connection.offer_in_progress);
1114 pr_err("Unable to allocate channel object\n");
1115 return;
1116 }
1117
1118 vmbus_setup_channel_state(newchannel, offer);
1119
1120 vmbus_process_offer(newchannel);
1121}
1122
1123static void check_ready_for_suspend_event(void)
1124{
1125 /*
1126 * If all the sub-channels or hv_sock channels have been cleaned up,
1127 * then it's safe to suspend.
1128 */
1129 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))
1130 complete(&vmbus_connection.ready_for_suspend_event);
1131}
1132
1133/*
1134 * vmbus_onoffer_rescind - Rescind offer handler.
1135 *
1136 * We queue a work item to process this offer synchronously
1137 */
1138static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
1139{
1140 struct vmbus_channel_rescind_offer *rescind;
1141 struct vmbus_channel *channel;
1142 struct device *dev;
1143 bool clean_up_chan_for_suspend;
1144
1145 rescind = (struct vmbus_channel_rescind_offer *)hdr;
1146
1147 trace_vmbus_onoffer_rescind(rescind);
1148
1149 /*
1150 * The offer msg and the corresponding rescind msg
1151 * from the host are guranteed to be ordered -
1152 * offer comes in first and then the rescind.
1153 * Since we process these events in work elements,
1154 * and with preemption, we may end up processing
1155 * the events out of order. We rely on the synchronization
1156 * provided by offer_in_progress and by channel_mutex for
1157 * ordering these events:
1158 *
1159 * { Initially: offer_in_progress = 1 }
1160 *
1161 * CPU1 CPU2
1162 *
1163 * [vmbus_onoffer()] [vmbus_onoffer_rescind()]
1164 *
1165 * LOCK channel_mutex WAIT_ON offer_in_progress == 0
1166 * DECREMENT offer_in_progress LOCK channel_mutex
1167 * STORE channels[] LOAD channels[]
1168 * UNLOCK channel_mutex UNLOCK channel_mutex
1169 *
1170 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE
1171 */
1172
1173 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {
1174 /*
1175 * We wait here until any channel offer is currently
1176 * being processed.
1177 */
1178 msleep(1);
1179 }
1180
1181 mutex_lock(&vmbus_connection.channel_mutex);
1182 channel = relid2channel(rescind->child_relid);
1183 if (channel != NULL) {
1184 /*
1185 * Guarantee that no other instance of vmbus_onoffer_rescind()
1186 * has got a reference to the channel object. Synchronize on
1187 * &vmbus_connection.channel_mutex.
1188 */
1189 if (channel->rescind_ref) {
1190 mutex_unlock(&vmbus_connection.channel_mutex);
1191 return;
1192 }
1193 channel->rescind_ref = true;
1194 }
1195 mutex_unlock(&vmbus_connection.channel_mutex);
1196
1197 if (channel == NULL) {
1198 /*
1199 * We failed in processing the offer message;
1200 * we would have cleaned up the relid in that
1201 * failure path.
1202 */
1203 return;
1204 }
1205
1206 clean_up_chan_for_suspend = is_hvsock_channel(channel) ||
1207 is_sub_channel(channel);
1208 /*
1209 * Before setting channel->rescind in vmbus_rescind_cleanup(), we
1210 * should make sure the channel callback is not running any more.
1211 */
1212 vmbus_reset_channel_cb(channel);
1213
1214 /*
1215 * Now wait for offer handling to complete.
1216 */
1217 vmbus_rescind_cleanup(channel);
1218 while (READ_ONCE(channel->probe_done) == false) {
1219 /*
1220 * We wait here until any channel offer is currently
1221 * being processed.
1222 */
1223 msleep(1);
1224 }
1225
1226 /*
1227 * At this point, the rescind handling can proceed safely.
1228 */
1229
1230 if (channel->device_obj) {
1231 if (channel->chn_rescind_callback) {
1232 channel->chn_rescind_callback(channel);
1233
1234 if (clean_up_chan_for_suspend)
1235 check_ready_for_suspend_event();
1236
1237 return;
1238 }
1239 /*
1240 * We will have to unregister this device from the
1241 * driver core.
1242 */
1243 dev = get_device(&channel->device_obj->device);
1244 if (dev) {
1245 vmbus_device_unregister(channel->device_obj);
1246 put_device(dev);
1247 }
1248 } else if (channel->primary_channel != NULL) {
1249 /*
1250 * Sub-channel is being rescinded. Following is the channel
1251 * close sequence when initiated from the driveri (refer to
1252 * vmbus_close() for details):
1253 * 1. Close all sub-channels first
1254 * 2. Then close the primary channel.
1255 */
1256 mutex_lock(&vmbus_connection.channel_mutex);
1257 if (channel->state == CHANNEL_OPEN_STATE) {
1258 /*
1259 * The channel is currently not open;
1260 * it is safe for us to cleanup the channel.
1261 */
1262 hv_process_channel_removal(channel);
1263 } else {
1264 complete(&channel->rescind_event);
1265 }
1266 mutex_unlock(&vmbus_connection.channel_mutex);
1267 }
1268
1269 /* The "channel" may have been freed. Do not access it any longer. */
1270
1271 if (clean_up_chan_for_suspend)
1272 check_ready_for_suspend_event();
1273}
1274
1275void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
1276{
1277 BUG_ON(!is_hvsock_channel(channel));
1278
1279 /* We always get a rescind msg when a connection is closed. */
1280 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))
1281 msleep(1);
1282
1283 vmbus_device_unregister(channel->device_obj);
1284}
1285EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
1286
1287
1288/*
1289 * vmbus_onoffers_delivered -
1290 * This is invoked when all offers have been delivered.
1291 *
1292 * Nothing to do here.
1293 */
1294static void vmbus_onoffers_delivered(
1295 struct vmbus_channel_message_header *hdr)
1296{
1297}
1298
1299/*
1300 * vmbus_onopen_result - Open result handler.
1301 *
1302 * This is invoked when we received a response to our channel open request.
1303 * Find the matching request, copy the response and signal the requesting
1304 * thread.
1305 */
1306static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
1307{
1308 struct vmbus_channel_open_result *result;
1309 struct vmbus_channel_msginfo *msginfo;
1310 struct vmbus_channel_message_header *requestheader;
1311 struct vmbus_channel_open_channel *openmsg;
1312 unsigned long flags;
1313
1314 result = (struct vmbus_channel_open_result *)hdr;
1315
1316 trace_vmbus_onopen_result(result);
1317
1318 /*
1319 * Find the open msg, copy the result and signal/unblock the wait event
1320 */
1321 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1322
1323 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1324 msglistentry) {
1325 requestheader =
1326 (struct vmbus_channel_message_header *)msginfo->msg;
1327
1328 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
1329 openmsg =
1330 (struct vmbus_channel_open_channel *)msginfo->msg;
1331 if (openmsg->child_relid == result->child_relid &&
1332 openmsg->openid == result->openid) {
1333 memcpy(&msginfo->response.open_result,
1334 result,
1335 sizeof(
1336 struct vmbus_channel_open_result));
1337 complete(&msginfo->waitevent);
1338 break;
1339 }
1340 }
1341 }
1342 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1343}
1344
1345/*
1346 * vmbus_ongpadl_created - GPADL created handler.
1347 *
1348 * This is invoked when we received a response to our gpadl create request.
1349 * Find the matching request, copy the response and signal the requesting
1350 * thread.
1351 */
1352static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
1353{
1354 struct vmbus_channel_gpadl_created *gpadlcreated;
1355 struct vmbus_channel_msginfo *msginfo;
1356 struct vmbus_channel_message_header *requestheader;
1357 struct vmbus_channel_gpadl_header *gpadlheader;
1358 unsigned long flags;
1359
1360 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
1361
1362 trace_vmbus_ongpadl_created(gpadlcreated);
1363
1364 /*
1365 * Find the establish msg, copy the result and signal/unblock the wait
1366 * event
1367 */
1368 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1369
1370 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1371 msglistentry) {
1372 requestheader =
1373 (struct vmbus_channel_message_header *)msginfo->msg;
1374
1375 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
1376 gpadlheader =
1377 (struct vmbus_channel_gpadl_header *)requestheader;
1378
1379 if ((gpadlcreated->child_relid ==
1380 gpadlheader->child_relid) &&
1381 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
1382 memcpy(&msginfo->response.gpadl_created,
1383 gpadlcreated,
1384 sizeof(
1385 struct vmbus_channel_gpadl_created));
1386 complete(&msginfo->waitevent);
1387 break;
1388 }
1389 }
1390 }
1391 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1392}
1393
1394/*
1395 * vmbus_onmodifychannel_response - Modify Channel response handler.
1396 *
1397 * This is invoked when we received a response to our channel modify request.
1398 * Find the matching request, copy the response and signal the requesting thread.
1399 */
1400static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr)
1401{
1402 struct vmbus_channel_modifychannel_response *response;
1403 struct vmbus_channel_msginfo *msginfo;
1404 unsigned long flags;
1405
1406 response = (struct vmbus_channel_modifychannel_response *)hdr;
1407
1408 trace_vmbus_onmodifychannel_response(response);
1409
1410 /*
1411 * Find the modify msg, copy the response and signal/unblock the wait event.
1412 */
1413 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1414
1415 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) {
1416 struct vmbus_channel_message_header *responseheader =
1417 (struct vmbus_channel_message_header *)msginfo->msg;
1418
1419 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {
1420 struct vmbus_channel_modifychannel *modifymsg;
1421
1422 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;
1423 if (modifymsg->child_relid == response->child_relid) {
1424 memcpy(&msginfo->response.modify_response, response,
1425 sizeof(*response));
1426 complete(&msginfo->waitevent);
1427 break;
1428 }
1429 }
1430 }
1431 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1432}
1433
1434/*
1435 * vmbus_ongpadl_torndown - GPADL torndown handler.
1436 *
1437 * This is invoked when we received a response to our gpadl teardown request.
1438 * Find the matching request, copy the response and signal the requesting
1439 * thread.
1440 */
1441static void vmbus_ongpadl_torndown(
1442 struct vmbus_channel_message_header *hdr)
1443{
1444 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1445 struct vmbus_channel_msginfo *msginfo;
1446 struct vmbus_channel_message_header *requestheader;
1447 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1448 unsigned long flags;
1449
1450 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1451
1452 trace_vmbus_ongpadl_torndown(gpadl_torndown);
1453
1454 /*
1455 * Find the open msg, copy the result and signal/unblock the wait event
1456 */
1457 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1458
1459 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1460 msglistentry) {
1461 requestheader =
1462 (struct vmbus_channel_message_header *)msginfo->msg;
1463
1464 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1465 gpadl_teardown =
1466 (struct vmbus_channel_gpadl_teardown *)requestheader;
1467
1468 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1469 memcpy(&msginfo->response.gpadl_torndown,
1470 gpadl_torndown,
1471 sizeof(
1472 struct vmbus_channel_gpadl_torndown));
1473 complete(&msginfo->waitevent);
1474 break;
1475 }
1476 }
1477 }
1478 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1479}
1480
1481/*
1482 * vmbus_onversion_response - Version response handler
1483 *
1484 * This is invoked when we received a response to our initiate contact request.
1485 * Find the matching request, copy the response and signal the requesting
1486 * thread.
1487 */
1488static void vmbus_onversion_response(
1489 struct vmbus_channel_message_header *hdr)
1490{
1491 struct vmbus_channel_msginfo *msginfo;
1492 struct vmbus_channel_message_header *requestheader;
1493 struct vmbus_channel_version_response *version_response;
1494 unsigned long flags;
1495
1496 version_response = (struct vmbus_channel_version_response *)hdr;
1497
1498 trace_vmbus_onversion_response(version_response);
1499
1500 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1501
1502 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1503 msglistentry) {
1504 requestheader =
1505 (struct vmbus_channel_message_header *)msginfo->msg;
1506
1507 if (requestheader->msgtype ==
1508 CHANNELMSG_INITIATE_CONTACT) {
1509 memcpy(&msginfo->response.version_response,
1510 version_response,
1511 sizeof(struct vmbus_channel_version_response));
1512 complete(&msginfo->waitevent);
1513 }
1514 }
1515 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1516}
1517
1518/* Channel message dispatch table */
1519const struct vmbus_channel_message_table_entry
1520channel_message_table[CHANNELMSG_COUNT] = {
1521 { CHANNELMSG_INVALID, 0, NULL, 0},
1522 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer,
1523 sizeof(struct vmbus_channel_offer_channel)},
1524 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind,
1525 sizeof(struct vmbus_channel_rescind_offer) },
1526 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0},
1527 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0},
1528 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0},
1529 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result,
1530 sizeof(struct vmbus_channel_open_result)},
1531 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0},
1532 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0},
1533 { CHANNELMSG_GPADL_BODY, 0, NULL, 0},
1534 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created,
1535 sizeof(struct vmbus_channel_gpadl_created)},
1536 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0},
1537 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown,
1538 sizeof(struct vmbus_channel_gpadl_torndown) },
1539 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0},
1540 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0},
1541 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response,
1542 sizeof(struct vmbus_channel_version_response)},
1543 { CHANNELMSG_UNLOAD, 0, NULL, 0},
1544 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0},
1545 { CHANNELMSG_18, 0, NULL, 0},
1546 { CHANNELMSG_19, 0, NULL, 0},
1547 { CHANNELMSG_20, 0, NULL, 0},
1548 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0},
1549 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0},
1550 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0},
1551 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response,
1552 sizeof(struct vmbus_channel_modifychannel_response)},
1553};
1554
1555/*
1556 * vmbus_onmessage - Handler for channel protocol messages.
1557 *
1558 * This is invoked in the vmbus worker thread context.
1559 */
1560void vmbus_onmessage(struct vmbus_channel_message_header *hdr)
1561{
1562 trace_vmbus_on_message(hdr);
1563
1564 /*
1565 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go
1566 * out of bound and the message_handler pointer can not be NULL.
1567 */
1568 channel_message_table[hdr->msgtype].message_handler(hdr);
1569}
1570
1571/*
1572 * vmbus_request_offers - Send a request to get all our pending offers.
1573 */
1574int vmbus_request_offers(void)
1575{
1576 struct vmbus_channel_message_header *msg;
1577 struct vmbus_channel_msginfo *msginfo;
1578 int ret;
1579
1580 msginfo = kzalloc(sizeof(*msginfo) +
1581 sizeof(struct vmbus_channel_message_header),
1582 GFP_KERNEL);
1583 if (!msginfo)
1584 return -ENOMEM;
1585
1586 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1587
1588 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1589
1590 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1591 true);
1592
1593 trace_vmbus_request_offers(ret);
1594
1595 if (ret != 0) {
1596 pr_err("Unable to request offers - %d\n", ret);
1597
1598 goto cleanup;
1599 }
1600
1601cleanup:
1602 kfree(msginfo);
1603
1604 return ret;
1605}
1606
1607void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1608 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1609{
1610 primary_channel->sc_creation_callback = sc_cr_cb;
1611}
1612EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1613
1614void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1615 void (*chn_rescind_cb)(struct vmbus_channel *))
1616{
1617 channel->chn_rescind_callback = chn_rescind_cb;
1618}
1619EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/sched.h>
14#include <linux/wait.h>
15#include <linux/mm.h>
16#include <linux/slab.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/completion.h>
20#include <linux/delay.h>
21#include <linux/hyperv.h>
22#include <asm/mshyperv.h>
23
24#include "hyperv_vmbus.h"
25
26static void init_vp_index(struct vmbus_channel *channel, u16 dev_type);
27
28static const struct vmbus_device vmbus_devs[] = {
29 /* IDE */
30 { .dev_type = HV_IDE,
31 HV_IDE_GUID,
32 .perf_device = true,
33 },
34
35 /* SCSI */
36 { .dev_type = HV_SCSI,
37 HV_SCSI_GUID,
38 .perf_device = true,
39 },
40
41 /* Fibre Channel */
42 { .dev_type = HV_FC,
43 HV_SYNTHFC_GUID,
44 .perf_device = true,
45 },
46
47 /* Synthetic NIC */
48 { .dev_type = HV_NIC,
49 HV_NIC_GUID,
50 .perf_device = true,
51 },
52
53 /* Network Direct */
54 { .dev_type = HV_ND,
55 HV_ND_GUID,
56 .perf_device = true,
57 },
58
59 /* PCIE */
60 { .dev_type = HV_PCIE,
61 HV_PCIE_GUID,
62 .perf_device = false,
63 },
64
65 /* Synthetic Frame Buffer */
66 { .dev_type = HV_FB,
67 HV_SYNTHVID_GUID,
68 .perf_device = false,
69 },
70
71 /* Synthetic Keyboard */
72 { .dev_type = HV_KBD,
73 HV_KBD_GUID,
74 .perf_device = false,
75 },
76
77 /* Synthetic MOUSE */
78 { .dev_type = HV_MOUSE,
79 HV_MOUSE_GUID,
80 .perf_device = false,
81 },
82
83 /* KVP */
84 { .dev_type = HV_KVP,
85 HV_KVP_GUID,
86 .perf_device = false,
87 },
88
89 /* Time Synch */
90 { .dev_type = HV_TS,
91 HV_TS_GUID,
92 .perf_device = false,
93 },
94
95 /* Heartbeat */
96 { .dev_type = HV_HB,
97 HV_HEART_BEAT_GUID,
98 .perf_device = false,
99 },
100
101 /* Shutdown */
102 { .dev_type = HV_SHUTDOWN,
103 HV_SHUTDOWN_GUID,
104 .perf_device = false,
105 },
106
107 /* File copy */
108 { .dev_type = HV_FCOPY,
109 HV_FCOPY_GUID,
110 .perf_device = false,
111 },
112
113 /* Backup */
114 { .dev_type = HV_BACKUP,
115 HV_VSS_GUID,
116 .perf_device = false,
117 },
118
119 /* Dynamic Memory */
120 { .dev_type = HV_DM,
121 HV_DM_GUID,
122 .perf_device = false,
123 },
124
125 /* Unknown GUID */
126 { .dev_type = HV_UNKNOWN,
127 .perf_device = false,
128 },
129};
130
131static const struct {
132 guid_t guid;
133} vmbus_unsupported_devs[] = {
134 { HV_AVMA1_GUID },
135 { HV_AVMA2_GUID },
136 { HV_RDV_GUID },
137};
138
139/*
140 * The rescinded channel may be blocked waiting for a response from the host;
141 * take care of that.
142 */
143static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
144{
145 struct vmbus_channel_msginfo *msginfo;
146 unsigned long flags;
147
148
149 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
150 channel->rescind = true;
151 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
152 msglistentry) {
153
154 if (msginfo->waiting_channel == channel) {
155 complete(&msginfo->waitevent);
156 break;
157 }
158 }
159 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
160}
161
162static bool is_unsupported_vmbus_devs(const guid_t *guid)
163{
164 int i;
165
166 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
167 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))
168 return true;
169 return false;
170}
171
172static u16 hv_get_dev_type(const struct vmbus_channel *channel)
173{
174 const guid_t *guid = &channel->offermsg.offer.if_type;
175 u16 i;
176
177 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
178 return HV_UNKNOWN;
179
180 for (i = HV_IDE; i < HV_UNKNOWN; i++) {
181 if (guid_equal(guid, &vmbus_devs[i].guid))
182 return i;
183 }
184 pr_info("Unknown GUID: %pUl\n", guid);
185 return i;
186}
187
188/**
189 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
190 * @icmsghdrp: Pointer to msg header structure
191 * @buf: Raw buffer channel data
192 * @fw_version: The framework versions we can support.
193 * @fw_vercnt: The size of @fw_version.
194 * @srv_version: The service versions we can support.
195 * @srv_vercnt: The size of @srv_version.
196 * @nego_fw_version: The selected framework version.
197 * @nego_srv_version: The selected service version.
198 *
199 * Note: Versions are given in decreasing order.
200 *
201 * Set up and fill in default negotiate response message.
202 * Mainly used by Hyper-V drivers.
203 */
204bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp,
205 u8 *buf, const int *fw_version, int fw_vercnt,
206 const int *srv_version, int srv_vercnt,
207 int *nego_fw_version, int *nego_srv_version)
208{
209 int icframe_major, icframe_minor;
210 int icmsg_major, icmsg_minor;
211 int fw_major, fw_minor;
212 int srv_major, srv_minor;
213 int i, j;
214 bool found_match = false;
215 struct icmsg_negotiate *negop;
216
217 icmsghdrp->icmsgsize = 0x10;
218 negop = (struct icmsg_negotiate *)&buf[
219 sizeof(struct vmbuspipe_hdr) +
220 sizeof(struct icmsg_hdr)];
221
222 icframe_major = negop->icframe_vercnt;
223 icframe_minor = 0;
224
225 icmsg_major = negop->icmsg_vercnt;
226 icmsg_minor = 0;
227
228 /*
229 * Select the framework version number we will
230 * support.
231 */
232
233 for (i = 0; i < fw_vercnt; i++) {
234 fw_major = (fw_version[i] >> 16);
235 fw_minor = (fw_version[i] & 0xFFFF);
236
237 for (j = 0; j < negop->icframe_vercnt; j++) {
238 if ((negop->icversion_data[j].major == fw_major) &&
239 (negop->icversion_data[j].minor == fw_minor)) {
240 icframe_major = negop->icversion_data[j].major;
241 icframe_minor = negop->icversion_data[j].minor;
242 found_match = true;
243 break;
244 }
245 }
246
247 if (found_match)
248 break;
249 }
250
251 if (!found_match)
252 goto fw_error;
253
254 found_match = false;
255
256 for (i = 0; i < srv_vercnt; i++) {
257 srv_major = (srv_version[i] >> 16);
258 srv_minor = (srv_version[i] & 0xFFFF);
259
260 for (j = negop->icframe_vercnt;
261 (j < negop->icframe_vercnt + negop->icmsg_vercnt);
262 j++) {
263
264 if ((negop->icversion_data[j].major == srv_major) &&
265 (negop->icversion_data[j].minor == srv_minor)) {
266
267 icmsg_major = negop->icversion_data[j].major;
268 icmsg_minor = negop->icversion_data[j].minor;
269 found_match = true;
270 break;
271 }
272 }
273
274 if (found_match)
275 break;
276 }
277
278 /*
279 * Respond with the framework and service
280 * version numbers we can support.
281 */
282
283fw_error:
284 if (!found_match) {
285 negop->icframe_vercnt = 0;
286 negop->icmsg_vercnt = 0;
287 } else {
288 negop->icframe_vercnt = 1;
289 negop->icmsg_vercnt = 1;
290 }
291
292 if (nego_fw_version)
293 *nego_fw_version = (icframe_major << 16) | icframe_minor;
294
295 if (nego_srv_version)
296 *nego_srv_version = (icmsg_major << 16) | icmsg_minor;
297
298 negop->icversion_data[0].major = icframe_major;
299 negop->icversion_data[0].minor = icframe_minor;
300 negop->icversion_data[1].major = icmsg_major;
301 negop->icversion_data[1].minor = icmsg_minor;
302 return found_match;
303}
304
305EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
306
307/*
308 * alloc_channel - Allocate and initialize a vmbus channel object
309 */
310static struct vmbus_channel *alloc_channel(void)
311{
312 struct vmbus_channel *channel;
313
314 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
315 if (!channel)
316 return NULL;
317
318 spin_lock_init(&channel->lock);
319 init_completion(&channel->rescind_event);
320
321 INIT_LIST_HEAD(&channel->sc_list);
322 INIT_LIST_HEAD(&channel->percpu_list);
323
324 tasklet_init(&channel->callback_event,
325 vmbus_on_event, (unsigned long)channel);
326
327 hv_ringbuffer_pre_init(channel);
328
329 return channel;
330}
331
332/*
333 * free_channel - Release the resources used by the vmbus channel object
334 */
335static void free_channel(struct vmbus_channel *channel)
336{
337 tasklet_kill(&channel->callback_event);
338 vmbus_remove_channel_attr_group(channel);
339
340 kobject_put(&channel->kobj);
341}
342
343static void percpu_channel_enq(void *arg)
344{
345 struct vmbus_channel *channel = arg;
346 struct hv_per_cpu_context *hv_cpu
347 = this_cpu_ptr(hv_context.cpu_context);
348
349 list_add_tail_rcu(&channel->percpu_list, &hv_cpu->chan_list);
350}
351
352static void percpu_channel_deq(void *arg)
353{
354 struct vmbus_channel *channel = arg;
355
356 list_del_rcu(&channel->percpu_list);
357}
358
359
360static void vmbus_release_relid(u32 relid)
361{
362 struct vmbus_channel_relid_released msg;
363 int ret;
364
365 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
366 msg.child_relid = relid;
367 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
368 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
369 true);
370
371 trace_vmbus_release_relid(&msg, ret);
372}
373
374void hv_process_channel_removal(struct vmbus_channel *channel)
375{
376 struct vmbus_channel *primary_channel;
377 unsigned long flags;
378
379 BUG_ON(!mutex_is_locked(&vmbus_connection.channel_mutex));
380 BUG_ON(!channel->rescind);
381
382 if (channel->target_cpu != get_cpu()) {
383 put_cpu();
384 smp_call_function_single(channel->target_cpu,
385 percpu_channel_deq, channel, true);
386 } else {
387 percpu_channel_deq(channel);
388 put_cpu();
389 }
390
391 if (channel->primary_channel == NULL) {
392 list_del(&channel->listentry);
393
394 primary_channel = channel;
395 } else {
396 primary_channel = channel->primary_channel;
397 spin_lock_irqsave(&primary_channel->lock, flags);
398 list_del(&channel->sc_list);
399 spin_unlock_irqrestore(&primary_channel->lock, flags);
400 }
401
402 /*
403 * We need to free the bit for init_vp_index() to work in the case
404 * of sub-channel, when we reload drivers like hv_netvsc.
405 */
406 if (channel->affinity_policy == HV_LOCALIZED)
407 cpumask_clear_cpu(channel->target_cpu,
408 &primary_channel->alloced_cpus_in_node);
409
410 /*
411 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and
412 * the relid is invalidated; after hibernation, when the user-space app
413 * destroys the channel, the relid is INVALID_RELID, and in this case
414 * it's unnecessary and unsafe to release the old relid, since the same
415 * relid can refer to a completely different channel now.
416 */
417 if (channel->offermsg.child_relid != INVALID_RELID)
418 vmbus_release_relid(channel->offermsg.child_relid);
419
420 free_channel(channel);
421}
422
423void vmbus_free_channels(void)
424{
425 struct vmbus_channel *channel, *tmp;
426
427 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
428 listentry) {
429 /* hv_process_channel_removal() needs this */
430 channel->rescind = true;
431
432 vmbus_device_unregister(channel->device_obj);
433 }
434}
435
436/* Note: the function can run concurrently for primary/sub channels. */
437static void vmbus_add_channel_work(struct work_struct *work)
438{
439 struct vmbus_channel *newchannel =
440 container_of(work, struct vmbus_channel, add_channel_work);
441 struct vmbus_channel *primary_channel = newchannel->primary_channel;
442 unsigned long flags;
443 u16 dev_type;
444 int ret;
445
446 dev_type = hv_get_dev_type(newchannel);
447
448 init_vp_index(newchannel, dev_type);
449
450 if (newchannel->target_cpu != get_cpu()) {
451 put_cpu();
452 smp_call_function_single(newchannel->target_cpu,
453 percpu_channel_enq,
454 newchannel, true);
455 } else {
456 percpu_channel_enq(newchannel);
457 put_cpu();
458 }
459
460 /*
461 * This state is used to indicate a successful open
462 * so that when we do close the channel normally, we
463 * can cleanup properly.
464 */
465 newchannel->state = CHANNEL_OPEN_STATE;
466
467 if (primary_channel != NULL) {
468 /* newchannel is a sub-channel. */
469 struct hv_device *dev = primary_channel->device_obj;
470
471 if (vmbus_add_channel_kobj(dev, newchannel))
472 goto err_deq_chan;
473
474 if (primary_channel->sc_creation_callback != NULL)
475 primary_channel->sc_creation_callback(newchannel);
476
477 newchannel->probe_done = true;
478 return;
479 }
480
481 /*
482 * Start the process of binding the primary channel to the driver
483 */
484 newchannel->device_obj = vmbus_device_create(
485 &newchannel->offermsg.offer.if_type,
486 &newchannel->offermsg.offer.if_instance,
487 newchannel);
488 if (!newchannel->device_obj)
489 goto err_deq_chan;
490
491 newchannel->device_obj->device_id = dev_type;
492 /*
493 * Add the new device to the bus. This will kick off device-driver
494 * binding which eventually invokes the device driver's AddDevice()
495 * method.
496 */
497 ret = vmbus_device_register(newchannel->device_obj);
498
499 if (ret != 0) {
500 pr_err("unable to add child device object (relid %d)\n",
501 newchannel->offermsg.child_relid);
502 kfree(newchannel->device_obj);
503 goto err_deq_chan;
504 }
505
506 newchannel->probe_done = true;
507 return;
508
509err_deq_chan:
510 mutex_lock(&vmbus_connection.channel_mutex);
511
512 /*
513 * We need to set the flag, otherwise
514 * vmbus_onoffer_rescind() can be blocked.
515 */
516 newchannel->probe_done = true;
517
518 if (primary_channel == NULL) {
519 list_del(&newchannel->listentry);
520 } else {
521 spin_lock_irqsave(&primary_channel->lock, flags);
522 list_del(&newchannel->sc_list);
523 spin_unlock_irqrestore(&primary_channel->lock, flags);
524 }
525
526 mutex_unlock(&vmbus_connection.channel_mutex);
527
528 if (newchannel->target_cpu != get_cpu()) {
529 put_cpu();
530 smp_call_function_single(newchannel->target_cpu,
531 percpu_channel_deq,
532 newchannel, true);
533 } else {
534 percpu_channel_deq(newchannel);
535 put_cpu();
536 }
537
538 vmbus_release_relid(newchannel->offermsg.child_relid);
539
540 free_channel(newchannel);
541}
542
543/*
544 * vmbus_process_offer - Process the offer by creating a channel/device
545 * associated with this offer
546 */
547static void vmbus_process_offer(struct vmbus_channel *newchannel)
548{
549 struct vmbus_channel *channel;
550 struct workqueue_struct *wq;
551 unsigned long flags;
552 bool fnew = true;
553
554 mutex_lock(&vmbus_connection.channel_mutex);
555
556 /* Remember the channels that should be cleaned up upon suspend. */
557 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))
558 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);
559
560 /*
561 * Now that we have acquired the channel_mutex,
562 * we can release the potentially racing rescind thread.
563 */
564 atomic_dec(&vmbus_connection.offer_in_progress);
565
566 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
567 if (guid_equal(&channel->offermsg.offer.if_type,
568 &newchannel->offermsg.offer.if_type) &&
569 guid_equal(&channel->offermsg.offer.if_instance,
570 &newchannel->offermsg.offer.if_instance)) {
571 fnew = false;
572 break;
573 }
574 }
575
576 if (fnew)
577 list_add_tail(&newchannel->listentry,
578 &vmbus_connection.chn_list);
579 else {
580 /*
581 * Check to see if this is a valid sub-channel.
582 */
583 if (newchannel->offermsg.offer.sub_channel_index == 0) {
584 mutex_unlock(&vmbus_connection.channel_mutex);
585 /*
586 * Don't call free_channel(), because newchannel->kobj
587 * is not initialized yet.
588 */
589 kfree(newchannel);
590 WARN_ON_ONCE(1);
591 return;
592 }
593 /*
594 * Process the sub-channel.
595 */
596 newchannel->primary_channel = channel;
597 spin_lock_irqsave(&channel->lock, flags);
598 list_add_tail(&newchannel->sc_list, &channel->sc_list);
599 spin_unlock_irqrestore(&channel->lock, flags);
600 }
601
602 mutex_unlock(&vmbus_connection.channel_mutex);
603
604 /*
605 * vmbus_process_offer() mustn't call channel->sc_creation_callback()
606 * directly for sub-channels, because sc_creation_callback() ->
607 * vmbus_open() may never get the host's response to the
608 * OPEN_CHANNEL message (the host may rescind a channel at any time,
609 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()
610 * may not wake up the vmbus_open() as it's blocked due to a non-zero
611 * vmbus_connection.offer_in_progress, and finally we have a deadlock.
612 *
613 * The above is also true for primary channels, if the related device
614 * drivers use sync probing mode by default.
615 *
616 * And, usually the handling of primary channels and sub-channels can
617 * depend on each other, so we should offload them to different
618 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,
619 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->
620 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock
621 * and waits for all the sub-channels to appear, but the latter
622 * can't get the rtnl_lock and this blocks the handling of
623 * sub-channels.
624 */
625 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);
626 wq = fnew ? vmbus_connection.handle_primary_chan_wq :
627 vmbus_connection.handle_sub_chan_wq;
628 queue_work(wq, &newchannel->add_channel_work);
629}
630
631/*
632 * We use this state to statically distribute the channel interrupt load.
633 */
634static int next_numa_node_id;
635/*
636 * init_vp_index() accesses global variables like next_numa_node_id, and
637 * it can run concurrently for primary channels and sub-channels: see
638 * vmbus_process_offer(), so we need the lock to protect the global
639 * variables.
640 */
641static DEFINE_SPINLOCK(bind_channel_to_cpu_lock);
642
643/*
644 * Starting with Win8, we can statically distribute the incoming
645 * channel interrupt load by binding a channel to VCPU.
646 * We distribute the interrupt loads to one or more NUMA nodes based on
647 * the channel's affinity_policy.
648 *
649 * For pre-win8 hosts or non-performance critical channels we assign the
650 * first CPU in the first NUMA node.
651 */
652static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
653{
654 u32 cur_cpu;
655 bool perf_chn = vmbus_devs[dev_type].perf_device;
656 struct vmbus_channel *primary = channel->primary_channel;
657 int next_node;
658 cpumask_var_t available_mask;
659 struct cpumask *alloced_mask;
660
661 if ((vmbus_proto_version == VERSION_WS2008) ||
662 (vmbus_proto_version == VERSION_WIN7) || (!perf_chn) ||
663 !alloc_cpumask_var(&available_mask, GFP_KERNEL)) {
664 /*
665 * Prior to win8, all channel interrupts are
666 * delivered on cpu 0.
667 * Also if the channel is not a performance critical
668 * channel, bind it to cpu 0.
669 * In case alloc_cpumask_var() fails, bind it to cpu 0.
670 */
671 channel->numa_node = 0;
672 channel->target_cpu = 0;
673 channel->target_vp = hv_cpu_number_to_vp_number(0);
674 return;
675 }
676
677 spin_lock(&bind_channel_to_cpu_lock);
678
679 /*
680 * Based on the channel affinity policy, we will assign the NUMA
681 * nodes.
682 */
683
684 if ((channel->affinity_policy == HV_BALANCED) || (!primary)) {
685 while (true) {
686 next_node = next_numa_node_id++;
687 if (next_node == nr_node_ids) {
688 next_node = next_numa_node_id = 0;
689 continue;
690 }
691 if (cpumask_empty(cpumask_of_node(next_node)))
692 continue;
693 break;
694 }
695 channel->numa_node = next_node;
696 primary = channel;
697 }
698 alloced_mask = &hv_context.hv_numa_map[primary->numa_node];
699
700 if (cpumask_weight(alloced_mask) ==
701 cpumask_weight(cpumask_of_node(primary->numa_node))) {
702 /*
703 * We have cycled through all the CPUs in the node;
704 * reset the alloced map.
705 */
706 cpumask_clear(alloced_mask);
707 }
708
709 cpumask_xor(available_mask, alloced_mask,
710 cpumask_of_node(primary->numa_node));
711
712 cur_cpu = -1;
713
714 if (primary->affinity_policy == HV_LOCALIZED) {
715 /*
716 * Normally Hyper-V host doesn't create more subchannels
717 * than there are VCPUs on the node but it is possible when not
718 * all present VCPUs on the node are initialized by guest.
719 * Clear the alloced_cpus_in_node to start over.
720 */
721 if (cpumask_equal(&primary->alloced_cpus_in_node,
722 cpumask_of_node(primary->numa_node)))
723 cpumask_clear(&primary->alloced_cpus_in_node);
724 }
725
726 while (true) {
727 cur_cpu = cpumask_next(cur_cpu, available_mask);
728 if (cur_cpu >= nr_cpu_ids) {
729 cur_cpu = -1;
730 cpumask_copy(available_mask,
731 cpumask_of_node(primary->numa_node));
732 continue;
733 }
734
735 if (primary->affinity_policy == HV_LOCALIZED) {
736 /*
737 * NOTE: in the case of sub-channel, we clear the
738 * sub-channel related bit(s) in
739 * primary->alloced_cpus_in_node in
740 * hv_process_channel_removal(), so when we
741 * reload drivers like hv_netvsc in SMP guest, here
742 * we're able to re-allocate
743 * bit from primary->alloced_cpus_in_node.
744 */
745 if (!cpumask_test_cpu(cur_cpu,
746 &primary->alloced_cpus_in_node)) {
747 cpumask_set_cpu(cur_cpu,
748 &primary->alloced_cpus_in_node);
749 cpumask_set_cpu(cur_cpu, alloced_mask);
750 break;
751 }
752 } else {
753 cpumask_set_cpu(cur_cpu, alloced_mask);
754 break;
755 }
756 }
757
758 channel->target_cpu = cur_cpu;
759 channel->target_vp = hv_cpu_number_to_vp_number(cur_cpu);
760
761 spin_unlock(&bind_channel_to_cpu_lock);
762
763 free_cpumask_var(available_mask);
764}
765
766static void vmbus_wait_for_unload(void)
767{
768 int cpu;
769 void *page_addr;
770 struct hv_message *msg;
771 struct vmbus_channel_message_header *hdr;
772 u32 message_type;
773
774 /*
775 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
776 * used for initial contact or to CPU0 depending on host version. When
777 * we're crashing on a different CPU let's hope that IRQ handler on
778 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
779 * functional and vmbus_unload_response() will complete
780 * vmbus_connection.unload_event. If not, the last thing we can do is
781 * read message pages for all CPUs directly.
782 */
783 while (1) {
784 if (completion_done(&vmbus_connection.unload_event))
785 break;
786
787 for_each_online_cpu(cpu) {
788 struct hv_per_cpu_context *hv_cpu
789 = per_cpu_ptr(hv_context.cpu_context, cpu);
790
791 page_addr = hv_cpu->synic_message_page;
792 msg = (struct hv_message *)page_addr
793 + VMBUS_MESSAGE_SINT;
794
795 message_type = READ_ONCE(msg->header.message_type);
796 if (message_type == HVMSG_NONE)
797 continue;
798
799 hdr = (struct vmbus_channel_message_header *)
800 msg->u.payload;
801
802 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
803 complete(&vmbus_connection.unload_event);
804
805 vmbus_signal_eom(msg, message_type);
806 }
807
808 mdelay(10);
809 }
810
811 /*
812 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
813 * maybe-pending messages on all CPUs to be able to receive new
814 * messages after we reconnect.
815 */
816 for_each_online_cpu(cpu) {
817 struct hv_per_cpu_context *hv_cpu
818 = per_cpu_ptr(hv_context.cpu_context, cpu);
819
820 page_addr = hv_cpu->synic_message_page;
821 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
822 msg->header.message_type = HVMSG_NONE;
823 }
824}
825
826/*
827 * vmbus_unload_response - Handler for the unload response.
828 */
829static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
830{
831 /*
832 * This is a global event; just wakeup the waiting thread.
833 * Once we successfully unload, we can cleanup the monitor state.
834 */
835 complete(&vmbus_connection.unload_event);
836}
837
838void vmbus_initiate_unload(bool crash)
839{
840 struct vmbus_channel_message_header hdr;
841
842 /* Pre-Win2012R2 hosts don't support reconnect */
843 if (vmbus_proto_version < VERSION_WIN8_1)
844 return;
845
846 init_completion(&vmbus_connection.unload_event);
847 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
848 hdr.msgtype = CHANNELMSG_UNLOAD;
849 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
850 !crash);
851
852 /*
853 * vmbus_initiate_unload() is also called on crash and the crash can be
854 * happening in an interrupt context, where scheduling is impossible.
855 */
856 if (!crash)
857 wait_for_completion(&vmbus_connection.unload_event);
858 else
859 vmbus_wait_for_unload();
860}
861
862static void check_ready_for_resume_event(void)
863{
864 /*
865 * If all the old primary channels have been fixed up, then it's safe
866 * to resume.
867 */
868 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))
869 complete(&vmbus_connection.ready_for_resume_event);
870}
871
872static void vmbus_setup_channel_state(struct vmbus_channel *channel,
873 struct vmbus_channel_offer_channel *offer)
874{
875 /*
876 * Setup state for signalling the host.
877 */
878 channel->sig_event = VMBUS_EVENT_CONNECTION_ID;
879
880 if (vmbus_proto_version != VERSION_WS2008) {
881 channel->is_dedicated_interrupt =
882 (offer->is_dedicated_interrupt != 0);
883 channel->sig_event = offer->connection_id;
884 }
885
886 memcpy(&channel->offermsg, offer,
887 sizeof(struct vmbus_channel_offer_channel));
888 channel->monitor_grp = (u8)offer->monitorid / 32;
889 channel->monitor_bit = (u8)offer->monitorid % 32;
890}
891
892/*
893 * find_primary_channel_by_offer - Get the channel object given the new offer.
894 * This is only used in the resume path of hibernation.
895 */
896static struct vmbus_channel *
897find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)
898{
899 struct vmbus_channel *channel = NULL, *iter;
900 const guid_t *inst1, *inst2;
901
902 /* Ignore sub-channel offers. */
903 if (offer->offer.sub_channel_index != 0)
904 return NULL;
905
906 mutex_lock(&vmbus_connection.channel_mutex);
907
908 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {
909 inst1 = &iter->offermsg.offer.if_instance;
910 inst2 = &offer->offer.if_instance;
911
912 if (guid_equal(inst1, inst2)) {
913 channel = iter;
914 break;
915 }
916 }
917
918 mutex_unlock(&vmbus_connection.channel_mutex);
919
920 return channel;
921}
922
923/*
924 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
925 *
926 */
927static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
928{
929 struct vmbus_channel_offer_channel *offer;
930 struct vmbus_channel *oldchannel, *newchannel;
931 size_t offer_sz;
932
933 offer = (struct vmbus_channel_offer_channel *)hdr;
934
935 trace_vmbus_onoffer(offer);
936
937 oldchannel = find_primary_channel_by_offer(offer);
938
939 if (oldchannel != NULL) {
940 atomic_dec(&vmbus_connection.offer_in_progress);
941
942 /*
943 * We're resuming from hibernation: all the sub-channel and
944 * hv_sock channels we had before the hibernation should have
945 * been cleaned up, and now we must be seeing a re-offered
946 * primary channel that we had before the hibernation.
947 */
948
949 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);
950 /* Fix up the relid. */
951 oldchannel->offermsg.child_relid = offer->child_relid;
952
953 offer_sz = sizeof(*offer);
954 if (memcmp(offer, &oldchannel->offermsg, offer_sz) == 0) {
955 check_ready_for_resume_event();
956 return;
957 }
958
959 /*
960 * This is not an error, since the host can also change the
961 * other field(s) of the offer, e.g. on WS RS5 (Build 17763),
962 * the offer->connection_id of the Mellanox VF vmbus device
963 * can change when the host reoffers the device upon resume.
964 */
965 pr_debug("vmbus offer changed: relid=%d\n",
966 offer->child_relid);
967
968 print_hex_dump_debug("Old vmbus offer: ", DUMP_PREFIX_OFFSET,
969 16, 4, &oldchannel->offermsg, offer_sz,
970 false);
971 print_hex_dump_debug("New vmbus offer: ", DUMP_PREFIX_OFFSET,
972 16, 4, offer, offer_sz, false);
973
974 /* Fix up the old channel. */
975 vmbus_setup_channel_state(oldchannel, offer);
976
977 check_ready_for_resume_event();
978
979 return;
980 }
981
982 /* Allocate the channel object and save this offer. */
983 newchannel = alloc_channel();
984 if (!newchannel) {
985 vmbus_release_relid(offer->child_relid);
986 atomic_dec(&vmbus_connection.offer_in_progress);
987 pr_err("Unable to allocate channel object\n");
988 return;
989 }
990
991 vmbus_setup_channel_state(newchannel, offer);
992
993 vmbus_process_offer(newchannel);
994}
995
996static void check_ready_for_suspend_event(void)
997{
998 /*
999 * If all the sub-channels or hv_sock channels have been cleaned up,
1000 * then it's safe to suspend.
1001 */
1002 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))
1003 complete(&vmbus_connection.ready_for_suspend_event);
1004}
1005
1006/*
1007 * vmbus_onoffer_rescind - Rescind offer handler.
1008 *
1009 * We queue a work item to process this offer synchronously
1010 */
1011static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
1012{
1013 struct vmbus_channel_rescind_offer *rescind;
1014 struct vmbus_channel *channel;
1015 struct device *dev;
1016 bool clean_up_chan_for_suspend;
1017
1018 rescind = (struct vmbus_channel_rescind_offer *)hdr;
1019
1020 trace_vmbus_onoffer_rescind(rescind);
1021
1022 /*
1023 * The offer msg and the corresponding rescind msg
1024 * from the host are guranteed to be ordered -
1025 * offer comes in first and then the rescind.
1026 * Since we process these events in work elements,
1027 * and with preemption, we may end up processing
1028 * the events out of order. Given that we handle these
1029 * work elements on the same CPU, this is possible only
1030 * in the case of preemption. In any case wait here
1031 * until the offer processing has moved beyond the
1032 * point where the channel is discoverable.
1033 */
1034
1035 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {
1036 /*
1037 * We wait here until any channel offer is currently
1038 * being processed.
1039 */
1040 msleep(1);
1041 }
1042
1043 mutex_lock(&vmbus_connection.channel_mutex);
1044 channel = relid2channel(rescind->child_relid);
1045 mutex_unlock(&vmbus_connection.channel_mutex);
1046
1047 if (channel == NULL) {
1048 /*
1049 * We failed in processing the offer message;
1050 * we would have cleaned up the relid in that
1051 * failure path.
1052 */
1053 return;
1054 }
1055
1056 clean_up_chan_for_suspend = is_hvsock_channel(channel) ||
1057 is_sub_channel(channel);
1058 /*
1059 * Before setting channel->rescind in vmbus_rescind_cleanup(), we
1060 * should make sure the channel callback is not running any more.
1061 */
1062 vmbus_reset_channel_cb(channel);
1063
1064 /*
1065 * Now wait for offer handling to complete.
1066 */
1067 vmbus_rescind_cleanup(channel);
1068 while (READ_ONCE(channel->probe_done) == false) {
1069 /*
1070 * We wait here until any channel offer is currently
1071 * being processed.
1072 */
1073 msleep(1);
1074 }
1075
1076 /*
1077 * At this point, the rescind handling can proceed safely.
1078 */
1079
1080 if (channel->device_obj) {
1081 if (channel->chn_rescind_callback) {
1082 channel->chn_rescind_callback(channel);
1083
1084 if (clean_up_chan_for_suspend)
1085 check_ready_for_suspend_event();
1086
1087 return;
1088 }
1089 /*
1090 * We will have to unregister this device from the
1091 * driver core.
1092 */
1093 dev = get_device(&channel->device_obj->device);
1094 if (dev) {
1095 vmbus_device_unregister(channel->device_obj);
1096 put_device(dev);
1097 }
1098 }
1099 if (channel->primary_channel != NULL) {
1100 /*
1101 * Sub-channel is being rescinded. Following is the channel
1102 * close sequence when initiated from the driveri (refer to
1103 * vmbus_close() for details):
1104 * 1. Close all sub-channels first
1105 * 2. Then close the primary channel.
1106 */
1107 mutex_lock(&vmbus_connection.channel_mutex);
1108 if (channel->state == CHANNEL_OPEN_STATE) {
1109 /*
1110 * The channel is currently not open;
1111 * it is safe for us to cleanup the channel.
1112 */
1113 hv_process_channel_removal(channel);
1114 } else {
1115 complete(&channel->rescind_event);
1116 }
1117 mutex_unlock(&vmbus_connection.channel_mutex);
1118 }
1119
1120 /* The "channel" may have been freed. Do not access it any longer. */
1121
1122 if (clean_up_chan_for_suspend)
1123 check_ready_for_suspend_event();
1124}
1125
1126void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
1127{
1128 BUG_ON(!is_hvsock_channel(channel));
1129
1130 /* We always get a rescind msg when a connection is closed. */
1131 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))
1132 msleep(1);
1133
1134 vmbus_device_unregister(channel->device_obj);
1135}
1136EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
1137
1138
1139/*
1140 * vmbus_onoffers_delivered -
1141 * This is invoked when all offers have been delivered.
1142 *
1143 * Nothing to do here.
1144 */
1145static void vmbus_onoffers_delivered(
1146 struct vmbus_channel_message_header *hdr)
1147{
1148}
1149
1150/*
1151 * vmbus_onopen_result - Open result handler.
1152 *
1153 * This is invoked when we received a response to our channel open request.
1154 * Find the matching request, copy the response and signal the requesting
1155 * thread.
1156 */
1157static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
1158{
1159 struct vmbus_channel_open_result *result;
1160 struct vmbus_channel_msginfo *msginfo;
1161 struct vmbus_channel_message_header *requestheader;
1162 struct vmbus_channel_open_channel *openmsg;
1163 unsigned long flags;
1164
1165 result = (struct vmbus_channel_open_result *)hdr;
1166
1167 trace_vmbus_onopen_result(result);
1168
1169 /*
1170 * Find the open msg, copy the result and signal/unblock the wait event
1171 */
1172 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1173
1174 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1175 msglistentry) {
1176 requestheader =
1177 (struct vmbus_channel_message_header *)msginfo->msg;
1178
1179 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
1180 openmsg =
1181 (struct vmbus_channel_open_channel *)msginfo->msg;
1182 if (openmsg->child_relid == result->child_relid &&
1183 openmsg->openid == result->openid) {
1184 memcpy(&msginfo->response.open_result,
1185 result,
1186 sizeof(
1187 struct vmbus_channel_open_result));
1188 complete(&msginfo->waitevent);
1189 break;
1190 }
1191 }
1192 }
1193 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1194}
1195
1196/*
1197 * vmbus_ongpadl_created - GPADL created handler.
1198 *
1199 * This is invoked when we received a response to our gpadl create request.
1200 * Find the matching request, copy the response and signal the requesting
1201 * thread.
1202 */
1203static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
1204{
1205 struct vmbus_channel_gpadl_created *gpadlcreated;
1206 struct vmbus_channel_msginfo *msginfo;
1207 struct vmbus_channel_message_header *requestheader;
1208 struct vmbus_channel_gpadl_header *gpadlheader;
1209 unsigned long flags;
1210
1211 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
1212
1213 trace_vmbus_ongpadl_created(gpadlcreated);
1214
1215 /*
1216 * Find the establish msg, copy the result and signal/unblock the wait
1217 * event
1218 */
1219 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1220
1221 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1222 msglistentry) {
1223 requestheader =
1224 (struct vmbus_channel_message_header *)msginfo->msg;
1225
1226 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
1227 gpadlheader =
1228 (struct vmbus_channel_gpadl_header *)requestheader;
1229
1230 if ((gpadlcreated->child_relid ==
1231 gpadlheader->child_relid) &&
1232 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
1233 memcpy(&msginfo->response.gpadl_created,
1234 gpadlcreated,
1235 sizeof(
1236 struct vmbus_channel_gpadl_created));
1237 complete(&msginfo->waitevent);
1238 break;
1239 }
1240 }
1241 }
1242 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1243}
1244
1245/*
1246 * vmbus_ongpadl_torndown - GPADL torndown handler.
1247 *
1248 * This is invoked when we received a response to our gpadl teardown request.
1249 * Find the matching request, copy the response and signal the requesting
1250 * thread.
1251 */
1252static void vmbus_ongpadl_torndown(
1253 struct vmbus_channel_message_header *hdr)
1254{
1255 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1256 struct vmbus_channel_msginfo *msginfo;
1257 struct vmbus_channel_message_header *requestheader;
1258 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1259 unsigned long flags;
1260
1261 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1262
1263 trace_vmbus_ongpadl_torndown(gpadl_torndown);
1264
1265 /*
1266 * Find the open msg, copy the result and signal/unblock the wait event
1267 */
1268 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1269
1270 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1271 msglistentry) {
1272 requestheader =
1273 (struct vmbus_channel_message_header *)msginfo->msg;
1274
1275 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1276 gpadl_teardown =
1277 (struct vmbus_channel_gpadl_teardown *)requestheader;
1278
1279 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1280 memcpy(&msginfo->response.gpadl_torndown,
1281 gpadl_torndown,
1282 sizeof(
1283 struct vmbus_channel_gpadl_torndown));
1284 complete(&msginfo->waitevent);
1285 break;
1286 }
1287 }
1288 }
1289 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1290}
1291
1292/*
1293 * vmbus_onversion_response - Version response handler
1294 *
1295 * This is invoked when we received a response to our initiate contact request.
1296 * Find the matching request, copy the response and signal the requesting
1297 * thread.
1298 */
1299static void vmbus_onversion_response(
1300 struct vmbus_channel_message_header *hdr)
1301{
1302 struct vmbus_channel_msginfo *msginfo;
1303 struct vmbus_channel_message_header *requestheader;
1304 struct vmbus_channel_version_response *version_response;
1305 unsigned long flags;
1306
1307 version_response = (struct vmbus_channel_version_response *)hdr;
1308
1309 trace_vmbus_onversion_response(version_response);
1310
1311 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1312
1313 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1314 msglistentry) {
1315 requestheader =
1316 (struct vmbus_channel_message_header *)msginfo->msg;
1317
1318 if (requestheader->msgtype ==
1319 CHANNELMSG_INITIATE_CONTACT) {
1320 memcpy(&msginfo->response.version_response,
1321 version_response,
1322 sizeof(struct vmbus_channel_version_response));
1323 complete(&msginfo->waitevent);
1324 }
1325 }
1326 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1327}
1328
1329/* Channel message dispatch table */
1330const struct vmbus_channel_message_table_entry
1331channel_message_table[CHANNELMSG_COUNT] = {
1332 { CHANNELMSG_INVALID, 0, NULL },
1333 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer },
1334 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind },
1335 { CHANNELMSG_REQUESTOFFERS, 0, NULL },
1336 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered },
1337 { CHANNELMSG_OPENCHANNEL, 0, NULL },
1338 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result },
1339 { CHANNELMSG_CLOSECHANNEL, 0, NULL },
1340 { CHANNELMSG_GPADL_HEADER, 0, NULL },
1341 { CHANNELMSG_GPADL_BODY, 0, NULL },
1342 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created },
1343 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL },
1344 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown },
1345 { CHANNELMSG_RELID_RELEASED, 0, NULL },
1346 { CHANNELMSG_INITIATE_CONTACT, 0, NULL },
1347 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response },
1348 { CHANNELMSG_UNLOAD, 0, NULL },
1349 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response },
1350 { CHANNELMSG_18, 0, NULL },
1351 { CHANNELMSG_19, 0, NULL },
1352 { CHANNELMSG_20, 0, NULL },
1353 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL },
1354};
1355
1356/*
1357 * vmbus_onmessage - Handler for channel protocol messages.
1358 *
1359 * This is invoked in the vmbus worker thread context.
1360 */
1361void vmbus_onmessage(void *context)
1362{
1363 struct hv_message *msg = context;
1364 struct vmbus_channel_message_header *hdr;
1365 int size;
1366
1367 hdr = (struct vmbus_channel_message_header *)msg->u.payload;
1368 size = msg->header.payload_size;
1369
1370 trace_vmbus_on_message(hdr);
1371
1372 if (hdr->msgtype >= CHANNELMSG_COUNT) {
1373 pr_err("Received invalid channel message type %d size %d\n",
1374 hdr->msgtype, size);
1375 print_hex_dump_bytes("", DUMP_PREFIX_NONE,
1376 (unsigned char *)msg->u.payload, size);
1377 return;
1378 }
1379
1380 if (channel_message_table[hdr->msgtype].message_handler)
1381 channel_message_table[hdr->msgtype].message_handler(hdr);
1382 else
1383 pr_err("Unhandled channel message type %d\n", hdr->msgtype);
1384}
1385
1386/*
1387 * vmbus_request_offers - Send a request to get all our pending offers.
1388 */
1389int vmbus_request_offers(void)
1390{
1391 struct vmbus_channel_message_header *msg;
1392 struct vmbus_channel_msginfo *msginfo;
1393 int ret;
1394
1395 msginfo = kmalloc(sizeof(*msginfo) +
1396 sizeof(struct vmbus_channel_message_header),
1397 GFP_KERNEL);
1398 if (!msginfo)
1399 return -ENOMEM;
1400
1401 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1402
1403 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1404
1405 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1406 true);
1407
1408 trace_vmbus_request_offers(ret);
1409
1410 if (ret != 0) {
1411 pr_err("Unable to request offers - %d\n", ret);
1412
1413 goto cleanup;
1414 }
1415
1416cleanup:
1417 kfree(msginfo);
1418
1419 return ret;
1420}
1421
1422static void invoke_sc_cb(struct vmbus_channel *primary_channel)
1423{
1424 struct list_head *cur, *tmp;
1425 struct vmbus_channel *cur_channel;
1426
1427 if (primary_channel->sc_creation_callback == NULL)
1428 return;
1429
1430 list_for_each_safe(cur, tmp, &primary_channel->sc_list) {
1431 cur_channel = list_entry(cur, struct vmbus_channel, sc_list);
1432
1433 primary_channel->sc_creation_callback(cur_channel);
1434 }
1435}
1436
1437void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1438 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1439{
1440 primary_channel->sc_creation_callback = sc_cr_cb;
1441}
1442EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1443
1444bool vmbus_are_subchannels_present(struct vmbus_channel *primary)
1445{
1446 bool ret;
1447
1448 ret = !list_empty(&primary->sc_list);
1449
1450 if (ret) {
1451 /*
1452 * Invoke the callback on sub-channel creation.
1453 * This will present a uniform interface to the
1454 * clients.
1455 */
1456 invoke_sc_cb(primary);
1457 }
1458
1459 return ret;
1460}
1461EXPORT_SYMBOL_GPL(vmbus_are_subchannels_present);
1462
1463void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1464 void (*chn_rescind_cb)(struct vmbus_channel *))
1465{
1466 channel->chn_rescind_callback = chn_rescind_cb;
1467}
1468EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);