Loading...
1/*
2 * Copyright (c) 2009, Microsoft Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Authors:
18 * Haiyang Zhang <haiyangz@microsoft.com>
19 * Hank Janssen <hjanssen@microsoft.com>
20 */
21#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
22
23#include <linux/kernel.h>
24#include <linux/sched.h>
25#include <linux/wait.h>
26#include <linux/mm.h>
27#include <linux/slab.h>
28#include <linux/list.h>
29#include <linux/module.h>
30#include <linux/completion.h>
31#include <linux/delay.h>
32#include <linux/hyperv.h>
33
34#include "hyperv_vmbus.h"
35
36static void init_vp_index(struct vmbus_channel *channel, u16 dev_type);
37
38static const struct vmbus_device vmbus_devs[] = {
39 /* IDE */
40 { .dev_type = HV_IDE,
41 HV_IDE_GUID,
42 .perf_device = true,
43 },
44
45 /* SCSI */
46 { .dev_type = HV_SCSI,
47 HV_SCSI_GUID,
48 .perf_device = true,
49 },
50
51 /* Fibre Channel */
52 { .dev_type = HV_FC,
53 HV_SYNTHFC_GUID,
54 .perf_device = true,
55 },
56
57 /* Synthetic NIC */
58 { .dev_type = HV_NIC,
59 HV_NIC_GUID,
60 .perf_device = true,
61 },
62
63 /* Network Direct */
64 { .dev_type = HV_ND,
65 HV_ND_GUID,
66 .perf_device = true,
67 },
68
69 /* PCIE */
70 { .dev_type = HV_PCIE,
71 HV_PCIE_GUID,
72 .perf_device = true,
73 },
74
75 /* Synthetic Frame Buffer */
76 { .dev_type = HV_FB,
77 HV_SYNTHVID_GUID,
78 .perf_device = false,
79 },
80
81 /* Synthetic Keyboard */
82 { .dev_type = HV_KBD,
83 HV_KBD_GUID,
84 .perf_device = false,
85 },
86
87 /* Synthetic MOUSE */
88 { .dev_type = HV_MOUSE,
89 HV_MOUSE_GUID,
90 .perf_device = false,
91 },
92
93 /* KVP */
94 { .dev_type = HV_KVP,
95 HV_KVP_GUID,
96 .perf_device = false,
97 },
98
99 /* Time Synch */
100 { .dev_type = HV_TS,
101 HV_TS_GUID,
102 .perf_device = false,
103 },
104
105 /* Heartbeat */
106 { .dev_type = HV_HB,
107 HV_HEART_BEAT_GUID,
108 .perf_device = false,
109 },
110
111 /* Shutdown */
112 { .dev_type = HV_SHUTDOWN,
113 HV_SHUTDOWN_GUID,
114 .perf_device = false,
115 },
116
117 /* File copy */
118 { .dev_type = HV_FCOPY,
119 HV_FCOPY_GUID,
120 .perf_device = false,
121 },
122
123 /* Backup */
124 { .dev_type = HV_BACKUP,
125 HV_VSS_GUID,
126 .perf_device = false,
127 },
128
129 /* Dynamic Memory */
130 { .dev_type = HV_DM,
131 HV_DM_GUID,
132 .perf_device = false,
133 },
134
135 /* Unknown GUID */
136 { .dev_type = HV_UNKOWN,
137 .perf_device = false,
138 },
139};
140
141static u16 hv_get_dev_type(const uuid_le *guid)
142{
143 u16 i;
144
145 for (i = HV_IDE; i < HV_UNKOWN; i++) {
146 if (!uuid_le_cmp(*guid, vmbus_devs[i].guid))
147 return i;
148 }
149 pr_info("Unknown GUID: %pUl\n", guid);
150 return i;
151}
152
153/**
154 * vmbus_prep_negotiate_resp() - Create default response for Hyper-V Negotiate message
155 * @icmsghdrp: Pointer to msg header structure
156 * @icmsg_negotiate: Pointer to negotiate message structure
157 * @buf: Raw buffer channel data
158 *
159 * @icmsghdrp is of type &struct icmsg_hdr.
160 * @negop is of type &struct icmsg_negotiate.
161 * Set up and fill in default negotiate response message.
162 *
163 * The fw_version specifies the framework version that
164 * we can support and srv_version specifies the service
165 * version we can support.
166 *
167 * Mainly used by Hyper-V drivers.
168 */
169bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp,
170 struct icmsg_negotiate *negop, u8 *buf,
171 int fw_version, int srv_version)
172{
173 int icframe_major, icframe_minor;
174 int icmsg_major, icmsg_minor;
175 int fw_major, fw_minor;
176 int srv_major, srv_minor;
177 int i;
178 bool found_match = false;
179
180 icmsghdrp->icmsgsize = 0x10;
181 fw_major = (fw_version >> 16);
182 fw_minor = (fw_version & 0xFFFF);
183
184 srv_major = (srv_version >> 16);
185 srv_minor = (srv_version & 0xFFFF);
186
187 negop = (struct icmsg_negotiate *)&buf[
188 sizeof(struct vmbuspipe_hdr) +
189 sizeof(struct icmsg_hdr)];
190
191 icframe_major = negop->icframe_vercnt;
192 icframe_minor = 0;
193
194 icmsg_major = negop->icmsg_vercnt;
195 icmsg_minor = 0;
196
197 /*
198 * Select the framework version number we will
199 * support.
200 */
201
202 for (i = 0; i < negop->icframe_vercnt; i++) {
203 if ((negop->icversion_data[i].major == fw_major) &&
204 (negop->icversion_data[i].minor == fw_minor)) {
205 icframe_major = negop->icversion_data[i].major;
206 icframe_minor = negop->icversion_data[i].minor;
207 found_match = true;
208 }
209 }
210
211 if (!found_match)
212 goto fw_error;
213
214 found_match = false;
215
216 for (i = negop->icframe_vercnt;
217 (i < negop->icframe_vercnt + negop->icmsg_vercnt); i++) {
218 if ((negop->icversion_data[i].major == srv_major) &&
219 (negop->icversion_data[i].minor == srv_minor)) {
220 icmsg_major = negop->icversion_data[i].major;
221 icmsg_minor = negop->icversion_data[i].minor;
222 found_match = true;
223 }
224 }
225
226 /*
227 * Respond with the framework and service
228 * version numbers we can support.
229 */
230
231fw_error:
232 if (!found_match) {
233 negop->icframe_vercnt = 0;
234 negop->icmsg_vercnt = 0;
235 } else {
236 negop->icframe_vercnt = 1;
237 negop->icmsg_vercnt = 1;
238 }
239
240 negop->icversion_data[0].major = icframe_major;
241 negop->icversion_data[0].minor = icframe_minor;
242 negop->icversion_data[1].major = icmsg_major;
243 negop->icversion_data[1].minor = icmsg_minor;
244 return found_match;
245}
246
247EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
248
249/*
250 * alloc_channel - Allocate and initialize a vmbus channel object
251 */
252static struct vmbus_channel *alloc_channel(void)
253{
254 static atomic_t chan_num = ATOMIC_INIT(0);
255 struct vmbus_channel *channel;
256
257 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
258 if (!channel)
259 return NULL;
260
261 channel->id = atomic_inc_return(&chan_num);
262 channel->acquire_ring_lock = true;
263 spin_lock_init(&channel->inbound_lock);
264 spin_lock_init(&channel->lock);
265
266 INIT_LIST_HEAD(&channel->sc_list);
267 INIT_LIST_HEAD(&channel->percpu_list);
268
269 return channel;
270}
271
272/*
273 * free_channel - Release the resources used by the vmbus channel object
274 */
275static void free_channel(struct vmbus_channel *channel)
276{
277 kfree(channel);
278}
279
280static void percpu_channel_enq(void *arg)
281{
282 struct vmbus_channel *channel = arg;
283 int cpu = smp_processor_id();
284
285 list_add_tail(&channel->percpu_list, &hv_context.percpu_list[cpu]);
286}
287
288static void percpu_channel_deq(void *arg)
289{
290 struct vmbus_channel *channel = arg;
291
292 list_del(&channel->percpu_list);
293}
294
295
296static void vmbus_release_relid(u32 relid)
297{
298 struct vmbus_channel_relid_released msg;
299
300 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
301 msg.child_relid = relid;
302 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
303 vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released));
304}
305
306void hv_process_channel_removal(struct vmbus_channel *channel, u32 relid)
307{
308 unsigned long flags;
309 struct vmbus_channel *primary_channel;
310
311 vmbus_release_relid(relid);
312
313 BUG_ON(!channel->rescind);
314 BUG_ON(!mutex_is_locked(&vmbus_connection.channel_mutex));
315
316 if (channel->target_cpu != get_cpu()) {
317 put_cpu();
318 smp_call_function_single(channel->target_cpu,
319 percpu_channel_deq, channel, true);
320 } else {
321 percpu_channel_deq(channel);
322 put_cpu();
323 }
324
325 if (channel->primary_channel == NULL) {
326 list_del(&channel->listentry);
327
328 primary_channel = channel;
329 } else {
330 primary_channel = channel->primary_channel;
331 spin_lock_irqsave(&primary_channel->lock, flags);
332 list_del(&channel->sc_list);
333 primary_channel->num_sc--;
334 spin_unlock_irqrestore(&primary_channel->lock, flags);
335 }
336
337 /*
338 * We need to free the bit for init_vp_index() to work in the case
339 * of sub-channel, when we reload drivers like hv_netvsc.
340 */
341 cpumask_clear_cpu(channel->target_cpu,
342 &primary_channel->alloced_cpus_in_node);
343
344 free_channel(channel);
345}
346
347void vmbus_free_channels(void)
348{
349 struct vmbus_channel *channel, *tmp;
350
351 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
352 listentry) {
353 /* hv_process_channel_removal() needs this */
354 channel->rescind = true;
355
356 vmbus_device_unregister(channel->device_obj);
357 }
358}
359
360/*
361 * vmbus_process_offer - Process the offer by creating a channel/device
362 * associated with this offer
363 */
364static void vmbus_process_offer(struct vmbus_channel *newchannel)
365{
366 struct vmbus_channel *channel;
367 bool fnew = true;
368 unsigned long flags;
369 u16 dev_type;
370 int ret;
371
372 /* Make sure this is a new offer */
373 mutex_lock(&vmbus_connection.channel_mutex);
374
375 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
376 if (!uuid_le_cmp(channel->offermsg.offer.if_type,
377 newchannel->offermsg.offer.if_type) &&
378 !uuid_le_cmp(channel->offermsg.offer.if_instance,
379 newchannel->offermsg.offer.if_instance)) {
380 fnew = false;
381 break;
382 }
383 }
384
385 if (fnew)
386 list_add_tail(&newchannel->listentry,
387 &vmbus_connection.chn_list);
388
389 mutex_unlock(&vmbus_connection.channel_mutex);
390
391 if (!fnew) {
392 /*
393 * Check to see if this is a sub-channel.
394 */
395 if (newchannel->offermsg.offer.sub_channel_index != 0) {
396 /*
397 * Process the sub-channel.
398 */
399 newchannel->primary_channel = channel;
400 spin_lock_irqsave(&channel->lock, flags);
401 list_add_tail(&newchannel->sc_list, &channel->sc_list);
402 channel->num_sc++;
403 spin_unlock_irqrestore(&channel->lock, flags);
404 } else
405 goto err_free_chan;
406 }
407
408 dev_type = hv_get_dev_type(&newchannel->offermsg.offer.if_type);
409
410 init_vp_index(newchannel, dev_type);
411
412 if (newchannel->target_cpu != get_cpu()) {
413 put_cpu();
414 smp_call_function_single(newchannel->target_cpu,
415 percpu_channel_enq,
416 newchannel, true);
417 } else {
418 percpu_channel_enq(newchannel);
419 put_cpu();
420 }
421
422 /*
423 * This state is used to indicate a successful open
424 * so that when we do close the channel normally, we
425 * can cleanup properly
426 */
427 newchannel->state = CHANNEL_OPEN_STATE;
428
429 if (!fnew) {
430 if (channel->sc_creation_callback != NULL)
431 channel->sc_creation_callback(newchannel);
432 return;
433 }
434
435 /*
436 * Start the process of binding this offer to the driver
437 * We need to set the DeviceObject field before calling
438 * vmbus_child_dev_add()
439 */
440 newchannel->device_obj = vmbus_device_create(
441 &newchannel->offermsg.offer.if_type,
442 &newchannel->offermsg.offer.if_instance,
443 newchannel);
444 if (!newchannel->device_obj)
445 goto err_deq_chan;
446
447 newchannel->device_obj->device_id = dev_type;
448 /*
449 * Add the new device to the bus. This will kick off device-driver
450 * binding which eventually invokes the device driver's AddDevice()
451 * method.
452 */
453 mutex_lock(&vmbus_connection.channel_mutex);
454 ret = vmbus_device_register(newchannel->device_obj);
455 mutex_unlock(&vmbus_connection.channel_mutex);
456
457 if (ret != 0) {
458 pr_err("unable to add child device object (relid %d)\n",
459 newchannel->offermsg.child_relid);
460 kfree(newchannel->device_obj);
461 goto err_deq_chan;
462 }
463 return;
464
465err_deq_chan:
466 vmbus_release_relid(newchannel->offermsg.child_relid);
467
468 mutex_lock(&vmbus_connection.channel_mutex);
469 list_del(&newchannel->listentry);
470 mutex_unlock(&vmbus_connection.channel_mutex);
471
472 if (newchannel->target_cpu != get_cpu()) {
473 put_cpu();
474 smp_call_function_single(newchannel->target_cpu,
475 percpu_channel_deq, newchannel, true);
476 } else {
477 percpu_channel_deq(newchannel);
478 put_cpu();
479 }
480
481err_free_chan:
482 free_channel(newchannel);
483}
484
485/*
486 * We use this state to statically distribute the channel interrupt load.
487 */
488static int next_numa_node_id;
489
490/*
491 * Starting with Win8, we can statically distribute the incoming
492 * channel interrupt load by binding a channel to VCPU.
493 * We do this in a hierarchical fashion:
494 * First distribute the primary channels across available NUMA nodes
495 * and then distribute the subchannels amongst the CPUs in the NUMA
496 * node assigned to the primary channel.
497 *
498 * For pre-win8 hosts or non-performance critical channels we assign the
499 * first CPU in the first NUMA node.
500 */
501static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
502{
503 u32 cur_cpu;
504 bool perf_chn = vmbus_devs[dev_type].perf_device;
505 struct vmbus_channel *primary = channel->primary_channel;
506 int next_node;
507 struct cpumask available_mask;
508 struct cpumask *alloced_mask;
509
510 if ((vmbus_proto_version == VERSION_WS2008) ||
511 (vmbus_proto_version == VERSION_WIN7) || (!perf_chn)) {
512 /*
513 * Prior to win8, all channel interrupts are
514 * delivered on cpu 0.
515 * Also if the channel is not a performance critical
516 * channel, bind it to cpu 0.
517 */
518 channel->numa_node = 0;
519 channel->target_cpu = 0;
520 channel->target_vp = hv_context.vp_index[0];
521 return;
522 }
523
524 /*
525 * We distribute primary channels evenly across all the available
526 * NUMA nodes and within the assigned NUMA node we will assign the
527 * first available CPU to the primary channel.
528 * The sub-channels will be assigned to the CPUs available in the
529 * NUMA node evenly.
530 */
531 if (!primary) {
532 while (true) {
533 next_node = next_numa_node_id++;
534 if (next_node == nr_node_ids)
535 next_node = next_numa_node_id = 0;
536 if (cpumask_empty(cpumask_of_node(next_node)))
537 continue;
538 break;
539 }
540 channel->numa_node = next_node;
541 primary = channel;
542 }
543 alloced_mask = &hv_context.hv_numa_map[primary->numa_node];
544
545 if (cpumask_weight(alloced_mask) ==
546 cpumask_weight(cpumask_of_node(primary->numa_node))) {
547 /*
548 * We have cycled through all the CPUs in the node;
549 * reset the alloced map.
550 */
551 cpumask_clear(alloced_mask);
552 }
553
554 cpumask_xor(&available_mask, alloced_mask,
555 cpumask_of_node(primary->numa_node));
556
557 cur_cpu = -1;
558
559 /*
560 * Normally Hyper-V host doesn't create more subchannels than there
561 * are VCPUs on the node but it is possible when not all present VCPUs
562 * on the node are initialized by guest. Clear the alloced_cpus_in_node
563 * to start over.
564 */
565 if (cpumask_equal(&primary->alloced_cpus_in_node,
566 cpumask_of_node(primary->numa_node)))
567 cpumask_clear(&primary->alloced_cpus_in_node);
568
569 while (true) {
570 cur_cpu = cpumask_next(cur_cpu, &available_mask);
571 if (cur_cpu >= nr_cpu_ids) {
572 cur_cpu = -1;
573 cpumask_copy(&available_mask,
574 cpumask_of_node(primary->numa_node));
575 continue;
576 }
577
578 /*
579 * NOTE: in the case of sub-channel, we clear the sub-channel
580 * related bit(s) in primary->alloced_cpus_in_node in
581 * hv_process_channel_removal(), so when we reload drivers
582 * like hv_netvsc in SMP guest, here we're able to re-allocate
583 * bit from primary->alloced_cpus_in_node.
584 */
585 if (!cpumask_test_cpu(cur_cpu,
586 &primary->alloced_cpus_in_node)) {
587 cpumask_set_cpu(cur_cpu,
588 &primary->alloced_cpus_in_node);
589 cpumask_set_cpu(cur_cpu, alloced_mask);
590 break;
591 }
592 }
593
594 channel->target_cpu = cur_cpu;
595 channel->target_vp = hv_context.vp_index[cur_cpu];
596}
597
598static void vmbus_wait_for_unload(void)
599{
600 int cpu = smp_processor_id();
601 void *page_addr = hv_context.synic_message_page[cpu];
602 struct hv_message *msg = (struct hv_message *)page_addr +
603 VMBUS_MESSAGE_SINT;
604 struct vmbus_channel_message_header *hdr;
605 bool unloaded = false;
606
607 while (1) {
608 if (READ_ONCE(msg->header.message_type) == HVMSG_NONE) {
609 mdelay(10);
610 continue;
611 }
612
613 hdr = (struct vmbus_channel_message_header *)msg->u.payload;
614 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
615 unloaded = true;
616
617 vmbus_signal_eom(msg);
618
619 if (unloaded)
620 break;
621 }
622}
623
624/*
625 * vmbus_unload_response - Handler for the unload response.
626 */
627static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
628{
629 /*
630 * This is a global event; just wakeup the waiting thread.
631 * Once we successfully unload, we can cleanup the monitor state.
632 */
633 complete(&vmbus_connection.unload_event);
634}
635
636void vmbus_initiate_unload(bool crash)
637{
638 struct vmbus_channel_message_header hdr;
639
640 /* Pre-Win2012R2 hosts don't support reconnect */
641 if (vmbus_proto_version < VERSION_WIN8_1)
642 return;
643
644 init_completion(&vmbus_connection.unload_event);
645 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
646 hdr.msgtype = CHANNELMSG_UNLOAD;
647 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header));
648
649 /*
650 * vmbus_initiate_unload() is also called on crash and the crash can be
651 * happening in an interrupt context, where scheduling is impossible.
652 */
653 if (!crash)
654 wait_for_completion(&vmbus_connection.unload_event);
655 else
656 vmbus_wait_for_unload();
657}
658
659/*
660 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
661 *
662 */
663static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
664{
665 struct vmbus_channel_offer_channel *offer;
666 struct vmbus_channel *newchannel;
667
668 offer = (struct vmbus_channel_offer_channel *)hdr;
669
670 /* Allocate the channel object and save this offer. */
671 newchannel = alloc_channel();
672 if (!newchannel) {
673 pr_err("Unable to allocate channel object\n");
674 return;
675 }
676
677 /*
678 * By default we setup state to enable batched
679 * reading. A specific service can choose to
680 * disable this prior to opening the channel.
681 */
682 newchannel->batched_reading = true;
683
684 /*
685 * Setup state for signalling the host.
686 */
687 newchannel->sig_event = (struct hv_input_signal_event *)
688 (ALIGN((unsigned long)
689 &newchannel->sig_buf,
690 HV_HYPERCALL_PARAM_ALIGN));
691
692 newchannel->sig_event->connectionid.asu32 = 0;
693 newchannel->sig_event->connectionid.u.id = VMBUS_EVENT_CONNECTION_ID;
694 newchannel->sig_event->flag_number = 0;
695 newchannel->sig_event->rsvdz = 0;
696
697 if (vmbus_proto_version != VERSION_WS2008) {
698 newchannel->is_dedicated_interrupt =
699 (offer->is_dedicated_interrupt != 0);
700 newchannel->sig_event->connectionid.u.id =
701 offer->connection_id;
702 }
703
704 memcpy(&newchannel->offermsg, offer,
705 sizeof(struct vmbus_channel_offer_channel));
706 newchannel->monitor_grp = (u8)offer->monitorid / 32;
707 newchannel->monitor_bit = (u8)offer->monitorid % 32;
708
709 vmbus_process_offer(newchannel);
710}
711
712/*
713 * vmbus_onoffer_rescind - Rescind offer handler.
714 *
715 * We queue a work item to process this offer synchronously
716 */
717static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
718{
719 struct vmbus_channel_rescind_offer *rescind;
720 struct vmbus_channel *channel;
721 unsigned long flags;
722 struct device *dev;
723
724 rescind = (struct vmbus_channel_rescind_offer *)hdr;
725
726 mutex_lock(&vmbus_connection.channel_mutex);
727 channel = relid2channel(rescind->child_relid);
728
729 if (channel == NULL) {
730 /*
731 * This is very impossible, because in
732 * vmbus_process_offer(), we have already invoked
733 * vmbus_release_relid() on error.
734 */
735 goto out;
736 }
737
738 spin_lock_irqsave(&channel->lock, flags);
739 channel->rescind = true;
740 spin_unlock_irqrestore(&channel->lock, flags);
741
742 if (channel->device_obj) {
743 if (channel->chn_rescind_callback) {
744 channel->chn_rescind_callback(channel);
745 goto out;
746 }
747 /*
748 * We will have to unregister this device from the
749 * driver core.
750 */
751 dev = get_device(&channel->device_obj->device);
752 if (dev) {
753 vmbus_device_unregister(channel->device_obj);
754 put_device(dev);
755 }
756 } else {
757 hv_process_channel_removal(channel,
758 channel->offermsg.child_relid);
759 }
760
761out:
762 mutex_unlock(&vmbus_connection.channel_mutex);
763}
764
765void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
766{
767 mutex_lock(&vmbus_connection.channel_mutex);
768
769 BUG_ON(!is_hvsock_channel(channel));
770
771 channel->rescind = true;
772 vmbus_device_unregister(channel->device_obj);
773
774 mutex_unlock(&vmbus_connection.channel_mutex);
775}
776EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
777
778
779/*
780 * vmbus_onoffers_delivered -
781 * This is invoked when all offers have been delivered.
782 *
783 * Nothing to do here.
784 */
785static void vmbus_onoffers_delivered(
786 struct vmbus_channel_message_header *hdr)
787{
788}
789
790/*
791 * vmbus_onopen_result - Open result handler.
792 *
793 * This is invoked when we received a response to our channel open request.
794 * Find the matching request, copy the response and signal the requesting
795 * thread.
796 */
797static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
798{
799 struct vmbus_channel_open_result *result;
800 struct vmbus_channel_msginfo *msginfo;
801 struct vmbus_channel_message_header *requestheader;
802 struct vmbus_channel_open_channel *openmsg;
803 unsigned long flags;
804
805 result = (struct vmbus_channel_open_result *)hdr;
806
807 /*
808 * Find the open msg, copy the result and signal/unblock the wait event
809 */
810 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
811
812 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
813 msglistentry) {
814 requestheader =
815 (struct vmbus_channel_message_header *)msginfo->msg;
816
817 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
818 openmsg =
819 (struct vmbus_channel_open_channel *)msginfo->msg;
820 if (openmsg->child_relid == result->child_relid &&
821 openmsg->openid == result->openid) {
822 memcpy(&msginfo->response.open_result,
823 result,
824 sizeof(
825 struct vmbus_channel_open_result));
826 complete(&msginfo->waitevent);
827 break;
828 }
829 }
830 }
831 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
832}
833
834/*
835 * vmbus_ongpadl_created - GPADL created handler.
836 *
837 * This is invoked when we received a response to our gpadl create request.
838 * Find the matching request, copy the response and signal the requesting
839 * thread.
840 */
841static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
842{
843 struct vmbus_channel_gpadl_created *gpadlcreated;
844 struct vmbus_channel_msginfo *msginfo;
845 struct vmbus_channel_message_header *requestheader;
846 struct vmbus_channel_gpadl_header *gpadlheader;
847 unsigned long flags;
848
849 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
850
851 /*
852 * Find the establish msg, copy the result and signal/unblock the wait
853 * event
854 */
855 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
856
857 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
858 msglistentry) {
859 requestheader =
860 (struct vmbus_channel_message_header *)msginfo->msg;
861
862 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
863 gpadlheader =
864 (struct vmbus_channel_gpadl_header *)requestheader;
865
866 if ((gpadlcreated->child_relid ==
867 gpadlheader->child_relid) &&
868 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
869 memcpy(&msginfo->response.gpadl_created,
870 gpadlcreated,
871 sizeof(
872 struct vmbus_channel_gpadl_created));
873 complete(&msginfo->waitevent);
874 break;
875 }
876 }
877 }
878 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
879}
880
881/*
882 * vmbus_ongpadl_torndown - GPADL torndown handler.
883 *
884 * This is invoked when we received a response to our gpadl teardown request.
885 * Find the matching request, copy the response and signal the requesting
886 * thread.
887 */
888static void vmbus_ongpadl_torndown(
889 struct vmbus_channel_message_header *hdr)
890{
891 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
892 struct vmbus_channel_msginfo *msginfo;
893 struct vmbus_channel_message_header *requestheader;
894 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
895 unsigned long flags;
896
897 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
898
899 /*
900 * Find the open msg, copy the result and signal/unblock the wait event
901 */
902 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
903
904 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
905 msglistentry) {
906 requestheader =
907 (struct vmbus_channel_message_header *)msginfo->msg;
908
909 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
910 gpadl_teardown =
911 (struct vmbus_channel_gpadl_teardown *)requestheader;
912
913 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
914 memcpy(&msginfo->response.gpadl_torndown,
915 gpadl_torndown,
916 sizeof(
917 struct vmbus_channel_gpadl_torndown));
918 complete(&msginfo->waitevent);
919 break;
920 }
921 }
922 }
923 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
924}
925
926/*
927 * vmbus_onversion_response - Version response handler
928 *
929 * This is invoked when we received a response to our initiate contact request.
930 * Find the matching request, copy the response and signal the requesting
931 * thread.
932 */
933static void vmbus_onversion_response(
934 struct vmbus_channel_message_header *hdr)
935{
936 struct vmbus_channel_msginfo *msginfo;
937 struct vmbus_channel_message_header *requestheader;
938 struct vmbus_channel_version_response *version_response;
939 unsigned long flags;
940
941 version_response = (struct vmbus_channel_version_response *)hdr;
942 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
943
944 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
945 msglistentry) {
946 requestheader =
947 (struct vmbus_channel_message_header *)msginfo->msg;
948
949 if (requestheader->msgtype ==
950 CHANNELMSG_INITIATE_CONTACT) {
951 memcpy(&msginfo->response.version_response,
952 version_response,
953 sizeof(struct vmbus_channel_version_response));
954 complete(&msginfo->waitevent);
955 }
956 }
957 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
958}
959
960/* Channel message dispatch table */
961struct vmbus_channel_message_table_entry
962 channel_message_table[CHANNELMSG_COUNT] = {
963 {CHANNELMSG_INVALID, 0, NULL},
964 {CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer},
965 {CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind},
966 {CHANNELMSG_REQUESTOFFERS, 0, NULL},
967 {CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered},
968 {CHANNELMSG_OPENCHANNEL, 0, NULL},
969 {CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result},
970 {CHANNELMSG_CLOSECHANNEL, 0, NULL},
971 {CHANNELMSG_GPADL_HEADER, 0, NULL},
972 {CHANNELMSG_GPADL_BODY, 0, NULL},
973 {CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created},
974 {CHANNELMSG_GPADL_TEARDOWN, 0, NULL},
975 {CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown},
976 {CHANNELMSG_RELID_RELEASED, 0, NULL},
977 {CHANNELMSG_INITIATE_CONTACT, 0, NULL},
978 {CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response},
979 {CHANNELMSG_UNLOAD, 0, NULL},
980 {CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response},
981 {CHANNELMSG_18, 0, NULL},
982 {CHANNELMSG_19, 0, NULL},
983 {CHANNELMSG_20, 0, NULL},
984 {CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL},
985};
986
987/*
988 * vmbus_onmessage - Handler for channel protocol messages.
989 *
990 * This is invoked in the vmbus worker thread context.
991 */
992void vmbus_onmessage(void *context)
993{
994 struct hv_message *msg = context;
995 struct vmbus_channel_message_header *hdr;
996 int size;
997
998 hdr = (struct vmbus_channel_message_header *)msg->u.payload;
999 size = msg->header.payload_size;
1000
1001 if (hdr->msgtype >= CHANNELMSG_COUNT) {
1002 pr_err("Received invalid channel message type %d size %d\n",
1003 hdr->msgtype, size);
1004 print_hex_dump_bytes("", DUMP_PREFIX_NONE,
1005 (unsigned char *)msg->u.payload, size);
1006 return;
1007 }
1008
1009 if (channel_message_table[hdr->msgtype].message_handler)
1010 channel_message_table[hdr->msgtype].message_handler(hdr);
1011 else
1012 pr_err("Unhandled channel message type %d\n", hdr->msgtype);
1013}
1014
1015/*
1016 * vmbus_request_offers - Send a request to get all our pending offers.
1017 */
1018int vmbus_request_offers(void)
1019{
1020 struct vmbus_channel_message_header *msg;
1021 struct vmbus_channel_msginfo *msginfo;
1022 int ret;
1023
1024 msginfo = kmalloc(sizeof(*msginfo) +
1025 sizeof(struct vmbus_channel_message_header),
1026 GFP_KERNEL);
1027 if (!msginfo)
1028 return -ENOMEM;
1029
1030 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1031
1032 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1033
1034
1035 ret = vmbus_post_msg(msg,
1036 sizeof(struct vmbus_channel_message_header));
1037 if (ret != 0) {
1038 pr_err("Unable to request offers - %d\n", ret);
1039
1040 goto cleanup;
1041 }
1042
1043cleanup:
1044 kfree(msginfo);
1045
1046 return ret;
1047}
1048
1049/*
1050 * Retrieve the (sub) channel on which to send an outgoing request.
1051 * When a primary channel has multiple sub-channels, we try to
1052 * distribute the load equally amongst all available channels.
1053 */
1054struct vmbus_channel *vmbus_get_outgoing_channel(struct vmbus_channel *primary)
1055{
1056 struct list_head *cur, *tmp;
1057 int cur_cpu;
1058 struct vmbus_channel *cur_channel;
1059 struct vmbus_channel *outgoing_channel = primary;
1060 int next_channel;
1061 int i = 1;
1062
1063 if (list_empty(&primary->sc_list))
1064 return outgoing_channel;
1065
1066 next_channel = primary->next_oc++;
1067
1068 if (next_channel > (primary->num_sc)) {
1069 primary->next_oc = 0;
1070 return outgoing_channel;
1071 }
1072
1073 cur_cpu = hv_context.vp_index[get_cpu()];
1074 put_cpu();
1075 list_for_each_safe(cur, tmp, &primary->sc_list) {
1076 cur_channel = list_entry(cur, struct vmbus_channel, sc_list);
1077 if (cur_channel->state != CHANNEL_OPENED_STATE)
1078 continue;
1079
1080 if (cur_channel->target_vp == cur_cpu)
1081 return cur_channel;
1082
1083 if (i == next_channel)
1084 return cur_channel;
1085
1086 i++;
1087 }
1088
1089 return outgoing_channel;
1090}
1091EXPORT_SYMBOL_GPL(vmbus_get_outgoing_channel);
1092
1093static void invoke_sc_cb(struct vmbus_channel *primary_channel)
1094{
1095 struct list_head *cur, *tmp;
1096 struct vmbus_channel *cur_channel;
1097
1098 if (primary_channel->sc_creation_callback == NULL)
1099 return;
1100
1101 list_for_each_safe(cur, tmp, &primary_channel->sc_list) {
1102 cur_channel = list_entry(cur, struct vmbus_channel, sc_list);
1103
1104 primary_channel->sc_creation_callback(cur_channel);
1105 }
1106}
1107
1108void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1109 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1110{
1111 primary_channel->sc_creation_callback = sc_cr_cb;
1112}
1113EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1114
1115bool vmbus_are_subchannels_present(struct vmbus_channel *primary)
1116{
1117 bool ret;
1118
1119 ret = !list_empty(&primary->sc_list);
1120
1121 if (ret) {
1122 /*
1123 * Invoke the callback on sub-channel creation.
1124 * This will present a uniform interface to the
1125 * clients.
1126 */
1127 invoke_sc_cb(primary);
1128 }
1129
1130 return ret;
1131}
1132EXPORT_SYMBOL_GPL(vmbus_are_subchannels_present);
1133
1134void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1135 void (*chn_rescind_cb)(struct vmbus_channel *))
1136{
1137 channel->chn_rescind_callback = chn_rescind_cb;
1138}
1139EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/sched.h>
14#include <linux/wait.h>
15#include <linux/mm.h>
16#include <linux/slab.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/completion.h>
20#include <linux/delay.h>
21#include <linux/cpu.h>
22#include <linux/hyperv.h>
23#include <asm/mshyperv.h>
24#include <linux/sched/isolation.h>
25
26#include "hyperv_vmbus.h"
27
28static void init_vp_index(struct vmbus_channel *channel);
29
30const struct vmbus_device vmbus_devs[] = {
31 /* IDE */
32 { .dev_type = HV_IDE,
33 HV_IDE_GUID,
34 .perf_device = true,
35 .allowed_in_isolated = false,
36 },
37
38 /* SCSI */
39 { .dev_type = HV_SCSI,
40 HV_SCSI_GUID,
41 .perf_device = true,
42 .allowed_in_isolated = true,
43 },
44
45 /* Fibre Channel */
46 { .dev_type = HV_FC,
47 HV_SYNTHFC_GUID,
48 .perf_device = true,
49 .allowed_in_isolated = false,
50 },
51
52 /* Synthetic NIC */
53 { .dev_type = HV_NIC,
54 HV_NIC_GUID,
55 .perf_device = true,
56 .allowed_in_isolated = true,
57 },
58
59 /* Network Direct */
60 { .dev_type = HV_ND,
61 HV_ND_GUID,
62 .perf_device = true,
63 .allowed_in_isolated = false,
64 },
65
66 /* PCIE */
67 { .dev_type = HV_PCIE,
68 HV_PCIE_GUID,
69 .perf_device = false,
70 .allowed_in_isolated = false,
71 },
72
73 /* Synthetic Frame Buffer */
74 { .dev_type = HV_FB,
75 HV_SYNTHVID_GUID,
76 .perf_device = false,
77 .allowed_in_isolated = false,
78 },
79
80 /* Synthetic Keyboard */
81 { .dev_type = HV_KBD,
82 HV_KBD_GUID,
83 .perf_device = false,
84 .allowed_in_isolated = false,
85 },
86
87 /* Synthetic MOUSE */
88 { .dev_type = HV_MOUSE,
89 HV_MOUSE_GUID,
90 .perf_device = false,
91 .allowed_in_isolated = false,
92 },
93
94 /* KVP */
95 { .dev_type = HV_KVP,
96 HV_KVP_GUID,
97 .perf_device = false,
98 .allowed_in_isolated = false,
99 },
100
101 /* Time Synch */
102 { .dev_type = HV_TS,
103 HV_TS_GUID,
104 .perf_device = false,
105 .allowed_in_isolated = true,
106 },
107
108 /* Heartbeat */
109 { .dev_type = HV_HB,
110 HV_HEART_BEAT_GUID,
111 .perf_device = false,
112 .allowed_in_isolated = true,
113 },
114
115 /* Shutdown */
116 { .dev_type = HV_SHUTDOWN,
117 HV_SHUTDOWN_GUID,
118 .perf_device = false,
119 .allowed_in_isolated = true,
120 },
121
122 /* File copy */
123 { .dev_type = HV_FCOPY,
124 HV_FCOPY_GUID,
125 .perf_device = false,
126 .allowed_in_isolated = false,
127 },
128
129 /* Backup */
130 { .dev_type = HV_BACKUP,
131 HV_VSS_GUID,
132 .perf_device = false,
133 .allowed_in_isolated = false,
134 },
135
136 /* Dynamic Memory */
137 { .dev_type = HV_DM,
138 HV_DM_GUID,
139 .perf_device = false,
140 .allowed_in_isolated = false,
141 },
142
143 /* Unknown GUID */
144 { .dev_type = HV_UNKNOWN,
145 .perf_device = false,
146 .allowed_in_isolated = false,
147 },
148};
149
150static const struct {
151 guid_t guid;
152} vmbus_unsupported_devs[] = {
153 { HV_AVMA1_GUID },
154 { HV_AVMA2_GUID },
155 { HV_RDV_GUID },
156 { HV_IMC_GUID },
157};
158
159/*
160 * The rescinded channel may be blocked waiting for a response from the host;
161 * take care of that.
162 */
163static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
164{
165 struct vmbus_channel_msginfo *msginfo;
166 unsigned long flags;
167
168
169 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
170 channel->rescind = true;
171 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
172 msglistentry) {
173
174 if (msginfo->waiting_channel == channel) {
175 complete(&msginfo->waitevent);
176 break;
177 }
178 }
179 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
180}
181
182static bool is_unsupported_vmbus_devs(const guid_t *guid)
183{
184 int i;
185
186 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
187 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))
188 return true;
189 return false;
190}
191
192static u16 hv_get_dev_type(const struct vmbus_channel *channel)
193{
194 const guid_t *guid = &channel->offermsg.offer.if_type;
195 u16 i;
196
197 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
198 return HV_UNKNOWN;
199
200 for (i = HV_IDE; i < HV_UNKNOWN; i++) {
201 if (guid_equal(guid, &vmbus_devs[i].guid))
202 return i;
203 }
204 pr_info("Unknown GUID: %pUl\n", guid);
205 return i;
206}
207
208/**
209 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
210 * @icmsghdrp: Pointer to msg header structure
211 * @buf: Raw buffer channel data
212 * @buflen: Length of the raw buffer channel data.
213 * @fw_version: The framework versions we can support.
214 * @fw_vercnt: The size of @fw_version.
215 * @srv_version: The service versions we can support.
216 * @srv_vercnt: The size of @srv_version.
217 * @nego_fw_version: The selected framework version.
218 * @nego_srv_version: The selected service version.
219 *
220 * Note: Versions are given in decreasing order.
221 *
222 * Set up and fill in default negotiate response message.
223 * Mainly used by Hyper-V drivers.
224 */
225bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf,
226 u32 buflen, const int *fw_version, int fw_vercnt,
227 const int *srv_version, int srv_vercnt,
228 int *nego_fw_version, int *nego_srv_version)
229{
230 int icframe_major, icframe_minor;
231 int icmsg_major, icmsg_minor;
232 int fw_major, fw_minor;
233 int srv_major, srv_minor;
234 int i, j;
235 bool found_match = false;
236 struct icmsg_negotiate *negop;
237
238 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */
239 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) {
240 pr_err_ratelimited("Invalid icmsg negotiate\n");
241 return false;
242 }
243
244 icmsghdrp->icmsgsize = 0x10;
245 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR];
246
247 icframe_major = negop->icframe_vercnt;
248 icframe_minor = 0;
249
250 icmsg_major = negop->icmsg_vercnt;
251 icmsg_minor = 0;
252
253 /* Validate negop packet */
254 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
255 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
256 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) {
257 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",
258 icframe_major, icmsg_major);
259 goto fw_error;
260 }
261
262 /*
263 * Select the framework version number we will
264 * support.
265 */
266
267 for (i = 0; i < fw_vercnt; i++) {
268 fw_major = (fw_version[i] >> 16);
269 fw_minor = (fw_version[i] & 0xFFFF);
270
271 for (j = 0; j < negop->icframe_vercnt; j++) {
272 if ((negop->icversion_data[j].major == fw_major) &&
273 (negop->icversion_data[j].minor == fw_minor)) {
274 icframe_major = negop->icversion_data[j].major;
275 icframe_minor = negop->icversion_data[j].minor;
276 found_match = true;
277 break;
278 }
279 }
280
281 if (found_match)
282 break;
283 }
284
285 if (!found_match)
286 goto fw_error;
287
288 found_match = false;
289
290 for (i = 0; i < srv_vercnt; i++) {
291 srv_major = (srv_version[i] >> 16);
292 srv_minor = (srv_version[i] & 0xFFFF);
293
294 for (j = negop->icframe_vercnt;
295 (j < negop->icframe_vercnt + negop->icmsg_vercnt);
296 j++) {
297
298 if ((negop->icversion_data[j].major == srv_major) &&
299 (negop->icversion_data[j].minor == srv_minor)) {
300
301 icmsg_major = negop->icversion_data[j].major;
302 icmsg_minor = negop->icversion_data[j].minor;
303 found_match = true;
304 break;
305 }
306 }
307
308 if (found_match)
309 break;
310 }
311
312 /*
313 * Respond with the framework and service
314 * version numbers we can support.
315 */
316
317fw_error:
318 if (!found_match) {
319 negop->icframe_vercnt = 0;
320 negop->icmsg_vercnt = 0;
321 } else {
322 negop->icframe_vercnt = 1;
323 negop->icmsg_vercnt = 1;
324 }
325
326 if (nego_fw_version)
327 *nego_fw_version = (icframe_major << 16) | icframe_minor;
328
329 if (nego_srv_version)
330 *nego_srv_version = (icmsg_major << 16) | icmsg_minor;
331
332 negop->icversion_data[0].major = icframe_major;
333 negop->icversion_data[0].minor = icframe_minor;
334 negop->icversion_data[1].major = icmsg_major;
335 negop->icversion_data[1].minor = icmsg_minor;
336 return found_match;
337}
338EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
339
340/*
341 * alloc_channel - Allocate and initialize a vmbus channel object
342 */
343static struct vmbus_channel *alloc_channel(void)
344{
345 struct vmbus_channel *channel;
346
347 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
348 if (!channel)
349 return NULL;
350
351 spin_lock_init(&channel->sched_lock);
352 init_completion(&channel->rescind_event);
353
354 INIT_LIST_HEAD(&channel->sc_list);
355
356 tasklet_init(&channel->callback_event,
357 vmbus_on_event, (unsigned long)channel);
358
359 hv_ringbuffer_pre_init(channel);
360
361 return channel;
362}
363
364/*
365 * free_channel - Release the resources used by the vmbus channel object
366 */
367static void free_channel(struct vmbus_channel *channel)
368{
369 tasklet_kill(&channel->callback_event);
370 vmbus_remove_channel_attr_group(channel);
371
372 kobject_put(&channel->kobj);
373}
374
375void vmbus_channel_map_relid(struct vmbus_channel *channel)
376{
377 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
378 return;
379 /*
380 * The mapping of the channel's relid is visible from the CPUs that
381 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will
382 * execute:
383 *
384 * (a) In the "normal (i.e., not resuming from hibernation)" path,
385 * the full barrier in virt_store_mb() guarantees that the store
386 * is propagated to all CPUs before the add_channel_work work
387 * is queued. In turn, add_channel_work is queued before the
388 * channel's ring buffer is allocated/initialized and the
389 * OPENCHANNEL message for the channel is sent in vmbus_open().
390 * Hyper-V won't start sending the interrupts for the channel
391 * before the OPENCHANNEL message is acked. The memory barrier
392 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures
393 * that vmbus_chan_sched() must find the channel's relid in
394 * recv_int_page before retrieving the channel pointer from the
395 * array of channels.
396 *
397 * (b) In the "resuming from hibernation" path, the virt_store_mb()
398 * guarantees that the store is propagated to all CPUs before
399 * the VMBus connection is marked as ready for the resume event
400 * (cf. check_ready_for_resume_event()). The interrupt handler
401 * of the VMBus driver and vmbus_chan_sched() can not run before
402 * vmbus_bus_resume() has completed execution (cf. resume_noirq).
403 */
404 virt_store_mb(
405 vmbus_connection.channels[channel->offermsg.child_relid],
406 channel);
407}
408
409void vmbus_channel_unmap_relid(struct vmbus_channel *channel)
410{
411 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
412 return;
413 WRITE_ONCE(
414 vmbus_connection.channels[channel->offermsg.child_relid],
415 NULL);
416}
417
418static void vmbus_release_relid(u32 relid)
419{
420 struct vmbus_channel_relid_released msg;
421 int ret;
422
423 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
424 msg.child_relid = relid;
425 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
426 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
427 true);
428
429 trace_vmbus_release_relid(&msg, ret);
430}
431
432void hv_process_channel_removal(struct vmbus_channel *channel)
433{
434 lockdep_assert_held(&vmbus_connection.channel_mutex);
435 BUG_ON(!channel->rescind);
436
437 /*
438 * hv_process_channel_removal() could find INVALID_RELID only for
439 * hv_sock channels. See the inline comments in vmbus_onoffer().
440 */
441 WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&
442 !is_hvsock_channel(channel));
443
444 /*
445 * Upon suspend, an in-use hv_sock channel is removed from the array of
446 * channels and the relid is invalidated. After hibernation, when the
447 * user-space application destroys the channel, it's unnecessary and
448 * unsafe to remove the channel from the array of channels. See also
449 * the inline comments before the call of vmbus_release_relid() below.
450 */
451 if (channel->offermsg.child_relid != INVALID_RELID)
452 vmbus_channel_unmap_relid(channel);
453
454 if (channel->primary_channel == NULL)
455 list_del(&channel->listentry);
456 else
457 list_del(&channel->sc_list);
458
459 /*
460 * If this is a "perf" channel, updates the hv_numa_map[] masks so that
461 * init_vp_index() can (re-)use the CPU.
462 */
463 if (hv_is_perf_channel(channel))
464 hv_clear_allocated_cpu(channel->target_cpu);
465
466 /*
467 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and
468 * the relid is invalidated; after hibernation, when the user-space app
469 * destroys the channel, the relid is INVALID_RELID, and in this case
470 * it's unnecessary and unsafe to release the old relid, since the same
471 * relid can refer to a completely different channel now.
472 */
473 if (channel->offermsg.child_relid != INVALID_RELID)
474 vmbus_release_relid(channel->offermsg.child_relid);
475
476 free_channel(channel);
477}
478
479void vmbus_free_channels(void)
480{
481 struct vmbus_channel *channel, *tmp;
482
483 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
484 listentry) {
485 /* hv_process_channel_removal() needs this */
486 channel->rescind = true;
487
488 vmbus_device_unregister(channel->device_obj);
489 }
490}
491
492/* Note: the function can run concurrently for primary/sub channels. */
493static void vmbus_add_channel_work(struct work_struct *work)
494{
495 struct vmbus_channel *newchannel =
496 container_of(work, struct vmbus_channel, add_channel_work);
497 struct vmbus_channel *primary_channel = newchannel->primary_channel;
498 int ret;
499
500 /*
501 * This state is used to indicate a successful open
502 * so that when we do close the channel normally, we
503 * can cleanup properly.
504 */
505 newchannel->state = CHANNEL_OPEN_STATE;
506
507 if (primary_channel != NULL) {
508 /* newchannel is a sub-channel. */
509 struct hv_device *dev = primary_channel->device_obj;
510
511 if (vmbus_add_channel_kobj(dev, newchannel))
512 goto err_deq_chan;
513
514 if (primary_channel->sc_creation_callback != NULL)
515 primary_channel->sc_creation_callback(newchannel);
516
517 newchannel->probe_done = true;
518 return;
519 }
520
521 /*
522 * Start the process of binding the primary channel to the driver
523 */
524 newchannel->device_obj = vmbus_device_create(
525 &newchannel->offermsg.offer.if_type,
526 &newchannel->offermsg.offer.if_instance,
527 newchannel);
528 if (!newchannel->device_obj)
529 goto err_deq_chan;
530
531 newchannel->device_obj->device_id = newchannel->device_id;
532 /*
533 * Add the new device to the bus. This will kick off device-driver
534 * binding which eventually invokes the device driver's AddDevice()
535 * method.
536 *
537 * If vmbus_device_register() fails, the 'device_obj' is freed in
538 * vmbus_device_release() as called by device_unregister() in the
539 * error path of vmbus_device_register(). In the outside error
540 * path, there's no need to free it.
541 */
542 ret = vmbus_device_register(newchannel->device_obj);
543
544 if (ret != 0) {
545 pr_err("unable to add child device object (relid %d)\n",
546 newchannel->offermsg.child_relid);
547 goto err_deq_chan;
548 }
549
550 newchannel->probe_done = true;
551 return;
552
553err_deq_chan:
554 mutex_lock(&vmbus_connection.channel_mutex);
555
556 /*
557 * We need to set the flag, otherwise
558 * vmbus_onoffer_rescind() can be blocked.
559 */
560 newchannel->probe_done = true;
561
562 if (primary_channel == NULL)
563 list_del(&newchannel->listentry);
564 else
565 list_del(&newchannel->sc_list);
566
567 /* vmbus_process_offer() has mapped the channel. */
568 vmbus_channel_unmap_relid(newchannel);
569
570 mutex_unlock(&vmbus_connection.channel_mutex);
571
572 vmbus_release_relid(newchannel->offermsg.child_relid);
573
574 free_channel(newchannel);
575}
576
577/*
578 * vmbus_process_offer - Process the offer by creating a channel/device
579 * associated with this offer
580 */
581static void vmbus_process_offer(struct vmbus_channel *newchannel)
582{
583 struct vmbus_channel *channel;
584 struct workqueue_struct *wq;
585 bool fnew = true;
586
587 /*
588 * Synchronize vmbus_process_offer() and CPU hotplugging:
589 *
590 * CPU1 CPU2
591 *
592 * [vmbus_process_offer()] [Hot removal of the CPU]
593 *
594 * CPU_READ_LOCK CPUS_WRITE_LOCK
595 * LOAD cpu_online_mask SEARCH chn_list
596 * STORE target_cpu LOAD target_cpu
597 * INSERT chn_list STORE cpu_online_mask
598 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK
599 *
600 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE &&
601 * CPU2's SEARCH from *not* seeing CPU1's INSERT
602 *
603 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT &&
604 * CPU2's LOAD from *not* seing CPU1's STORE
605 */
606 cpus_read_lock();
607
608 /*
609 * Serializes the modifications of the chn_list list as well as
610 * the accesses to next_numa_node_id in init_vp_index().
611 */
612 mutex_lock(&vmbus_connection.channel_mutex);
613
614 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
615 if (guid_equal(&channel->offermsg.offer.if_type,
616 &newchannel->offermsg.offer.if_type) &&
617 guid_equal(&channel->offermsg.offer.if_instance,
618 &newchannel->offermsg.offer.if_instance)) {
619 fnew = false;
620 newchannel->primary_channel = channel;
621 break;
622 }
623 }
624
625 init_vp_index(newchannel);
626
627 /* Remember the channels that should be cleaned up upon suspend. */
628 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))
629 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);
630
631 /*
632 * Now that we have acquired the channel_mutex,
633 * we can release the potentially racing rescind thread.
634 */
635 atomic_dec(&vmbus_connection.offer_in_progress);
636
637 if (fnew) {
638 list_add_tail(&newchannel->listentry,
639 &vmbus_connection.chn_list);
640 } else {
641 /*
642 * Check to see if this is a valid sub-channel.
643 */
644 if (newchannel->offermsg.offer.sub_channel_index == 0) {
645 mutex_unlock(&vmbus_connection.channel_mutex);
646 cpus_read_unlock();
647 /*
648 * Don't call free_channel(), because newchannel->kobj
649 * is not initialized yet.
650 */
651 kfree(newchannel);
652 WARN_ON_ONCE(1);
653 return;
654 }
655 /*
656 * Process the sub-channel.
657 */
658 list_add_tail(&newchannel->sc_list, &channel->sc_list);
659 }
660
661 vmbus_channel_map_relid(newchannel);
662
663 mutex_unlock(&vmbus_connection.channel_mutex);
664 cpus_read_unlock();
665
666 /*
667 * vmbus_process_offer() mustn't call channel->sc_creation_callback()
668 * directly for sub-channels, because sc_creation_callback() ->
669 * vmbus_open() may never get the host's response to the
670 * OPEN_CHANNEL message (the host may rescind a channel at any time,
671 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()
672 * may not wake up the vmbus_open() as it's blocked due to a non-zero
673 * vmbus_connection.offer_in_progress, and finally we have a deadlock.
674 *
675 * The above is also true for primary channels, if the related device
676 * drivers use sync probing mode by default.
677 *
678 * And, usually the handling of primary channels and sub-channels can
679 * depend on each other, so we should offload them to different
680 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,
681 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->
682 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock
683 * and waits for all the sub-channels to appear, but the latter
684 * can't get the rtnl_lock and this blocks the handling of
685 * sub-channels.
686 */
687 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);
688 wq = fnew ? vmbus_connection.handle_primary_chan_wq :
689 vmbus_connection.handle_sub_chan_wq;
690 queue_work(wq, &newchannel->add_channel_work);
691}
692
693/*
694 * Check if CPUs used by other channels of the same device.
695 * It should only be called by init_vp_index().
696 */
697static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn)
698{
699 struct vmbus_channel *primary = chn->primary_channel;
700 struct vmbus_channel *sc;
701
702 lockdep_assert_held(&vmbus_connection.channel_mutex);
703
704 if (!primary)
705 return false;
706
707 if (primary->target_cpu == cpu)
708 return true;
709
710 list_for_each_entry(sc, &primary->sc_list, sc_list)
711 if (sc != chn && sc->target_cpu == cpu)
712 return true;
713
714 return false;
715}
716
717/*
718 * We use this state to statically distribute the channel interrupt load.
719 */
720static int next_numa_node_id;
721
722/*
723 * We can statically distribute the incoming channel interrupt load
724 * by binding a channel to VCPU.
725 *
726 * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.
727 * Performance critical channels will be distributed evenly among all
728 * the available NUMA nodes. Once the node is assigned, we will assign
729 * the CPU based on a simple round robin scheme.
730 */
731static void init_vp_index(struct vmbus_channel *channel)
732{
733 bool perf_chn = hv_is_perf_channel(channel);
734 u32 i, ncpu = num_online_cpus();
735 cpumask_var_t available_mask;
736 struct cpumask *allocated_mask;
737 const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
738 u32 target_cpu;
739 int numa_node;
740
741 if (!perf_chn ||
742 !alloc_cpumask_var(&available_mask, GFP_KERNEL) ||
743 cpumask_empty(hk_mask)) {
744 /*
745 * If the channel is not a performance critical
746 * channel, bind it to VMBUS_CONNECT_CPU.
747 * In case alloc_cpumask_var() fails, bind it to
748 * VMBUS_CONNECT_CPU.
749 * If all the cpus are isolated, bind it to
750 * VMBUS_CONNECT_CPU.
751 */
752 channel->target_cpu = VMBUS_CONNECT_CPU;
753 if (perf_chn)
754 hv_set_allocated_cpu(VMBUS_CONNECT_CPU);
755 return;
756 }
757
758 for (i = 1; i <= ncpu + 1; i++) {
759 while (true) {
760 numa_node = next_numa_node_id++;
761 if (numa_node == nr_node_ids) {
762 next_numa_node_id = 0;
763 continue;
764 }
765 if (cpumask_empty(cpumask_of_node(numa_node)))
766 continue;
767 break;
768 }
769 allocated_mask = &hv_context.hv_numa_map[numa_node];
770
771retry:
772 cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node));
773 cpumask_and(available_mask, available_mask, hk_mask);
774
775 if (cpumask_empty(available_mask)) {
776 /*
777 * We have cycled through all the CPUs in the node;
778 * reset the allocated map.
779 */
780 cpumask_clear(allocated_mask);
781 goto retry;
782 }
783
784 target_cpu = cpumask_first(available_mask);
785 cpumask_set_cpu(target_cpu, allocated_mask);
786
787 if (channel->offermsg.offer.sub_channel_index >= ncpu ||
788 i > ncpu || !hv_cpuself_used(target_cpu, channel))
789 break;
790 }
791
792 channel->target_cpu = target_cpu;
793
794 free_cpumask_var(available_mask);
795}
796
797#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */
798#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */
799#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS)
800#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */
801#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS)
802
803static void vmbus_wait_for_unload(void)
804{
805 int cpu;
806 void *page_addr;
807 struct hv_message *msg;
808 struct vmbus_channel_message_header *hdr;
809 u32 message_type, i;
810
811 /*
812 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
813 * used for initial contact or to CPU0 depending on host version. When
814 * we're crashing on a different CPU let's hope that IRQ handler on
815 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
816 * functional and vmbus_unload_response() will complete
817 * vmbus_connection.unload_event. If not, the last thing we can do is
818 * read message pages for all CPUs directly.
819 *
820 * Wait up to 100 seconds since an Azure host must writeback any dirty
821 * data in its disk cache before the VMbus UNLOAD request will
822 * complete. This flushing has been empirically observed to take up
823 * to 50 seconds in cases with a lot of dirty data, so allow additional
824 * leeway and for inaccuracies in mdelay(). But eventually time out so
825 * that the panic path can't get hung forever in case the response
826 * message isn't seen.
827 */
828 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) {
829 if (completion_done(&vmbus_connection.unload_event))
830 goto completed;
831
832 for_each_online_cpu(cpu) {
833 struct hv_per_cpu_context *hv_cpu
834 = per_cpu_ptr(hv_context.cpu_context, cpu);
835
836 page_addr = hv_cpu->synic_message_page;
837 msg = (struct hv_message *)page_addr
838 + VMBUS_MESSAGE_SINT;
839
840 message_type = READ_ONCE(msg->header.message_type);
841 if (message_type == HVMSG_NONE)
842 continue;
843
844 hdr = (struct vmbus_channel_message_header *)
845 msg->u.payload;
846
847 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
848 complete(&vmbus_connection.unload_event);
849
850 vmbus_signal_eom(msg, message_type);
851 }
852
853 /*
854 * Give a notice periodically so someone watching the
855 * serial output won't think it is completely hung.
856 */
857 if (!(i % UNLOAD_MSG_LOOPS))
858 pr_notice("Waiting for VMBus UNLOAD to complete\n");
859
860 mdelay(UNLOAD_DELAY_UNIT_MS);
861 }
862 pr_err("Continuing even though VMBus UNLOAD did not complete\n");
863
864completed:
865 /*
866 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
867 * maybe-pending messages on all CPUs to be able to receive new
868 * messages after we reconnect.
869 */
870 for_each_online_cpu(cpu) {
871 struct hv_per_cpu_context *hv_cpu
872 = per_cpu_ptr(hv_context.cpu_context, cpu);
873
874 page_addr = hv_cpu->synic_message_page;
875 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
876 msg->header.message_type = HVMSG_NONE;
877 }
878}
879
880/*
881 * vmbus_unload_response - Handler for the unload response.
882 */
883static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
884{
885 /*
886 * This is a global event; just wakeup the waiting thread.
887 * Once we successfully unload, we can cleanup the monitor state.
888 *
889 * NB. A malicious or compromised Hyper-V could send a spurious
890 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call
891 * of the complete() below. Make sure that unload_event has been
892 * initialized by the time this complete() is executed.
893 */
894 complete(&vmbus_connection.unload_event);
895}
896
897void vmbus_initiate_unload(bool crash)
898{
899 struct vmbus_channel_message_header hdr;
900
901 if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED)
902 return;
903
904 /* Pre-Win2012R2 hosts don't support reconnect */
905 if (vmbus_proto_version < VERSION_WIN8_1)
906 return;
907
908 reinit_completion(&vmbus_connection.unload_event);
909 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
910 hdr.msgtype = CHANNELMSG_UNLOAD;
911 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
912 !crash);
913
914 /*
915 * vmbus_initiate_unload() is also called on crash and the crash can be
916 * happening in an interrupt context, where scheduling is impossible.
917 */
918 if (!crash)
919 wait_for_completion(&vmbus_connection.unload_event);
920 else
921 vmbus_wait_for_unload();
922}
923
924static void check_ready_for_resume_event(void)
925{
926 /*
927 * If all the old primary channels have been fixed up, then it's safe
928 * to resume.
929 */
930 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))
931 complete(&vmbus_connection.ready_for_resume_event);
932}
933
934static void vmbus_setup_channel_state(struct vmbus_channel *channel,
935 struct vmbus_channel_offer_channel *offer)
936{
937 /*
938 * Setup state for signalling the host.
939 */
940 channel->sig_event = VMBUS_EVENT_CONNECTION_ID;
941
942 channel->is_dedicated_interrupt =
943 (offer->is_dedicated_interrupt != 0);
944 channel->sig_event = offer->connection_id;
945
946 memcpy(&channel->offermsg, offer,
947 sizeof(struct vmbus_channel_offer_channel));
948 channel->monitor_grp = (u8)offer->monitorid / 32;
949 channel->monitor_bit = (u8)offer->monitorid % 32;
950 channel->device_id = hv_get_dev_type(channel);
951}
952
953/*
954 * find_primary_channel_by_offer - Get the channel object given the new offer.
955 * This is only used in the resume path of hibernation.
956 */
957static struct vmbus_channel *
958find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)
959{
960 struct vmbus_channel *channel = NULL, *iter;
961 const guid_t *inst1, *inst2;
962
963 /* Ignore sub-channel offers. */
964 if (offer->offer.sub_channel_index != 0)
965 return NULL;
966
967 mutex_lock(&vmbus_connection.channel_mutex);
968
969 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {
970 inst1 = &iter->offermsg.offer.if_instance;
971 inst2 = &offer->offer.if_instance;
972
973 if (guid_equal(inst1, inst2)) {
974 channel = iter;
975 break;
976 }
977 }
978
979 mutex_unlock(&vmbus_connection.channel_mutex);
980
981 return channel;
982}
983
984static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer)
985{
986 const guid_t *guid = &offer->offer.if_type;
987 u16 i;
988
989 if (!hv_is_isolation_supported())
990 return true;
991
992 if (is_hvsock_offer(offer))
993 return true;
994
995 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) {
996 if (guid_equal(guid, &vmbus_devs[i].guid))
997 return vmbus_devs[i].allowed_in_isolated;
998 }
999 return false;
1000}
1001
1002/*
1003 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1004 *
1005 */
1006static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
1007{
1008 struct vmbus_channel_offer_channel *offer;
1009 struct vmbus_channel *oldchannel, *newchannel;
1010 size_t offer_sz;
1011
1012 offer = (struct vmbus_channel_offer_channel *)hdr;
1013
1014 trace_vmbus_onoffer(offer);
1015
1016 if (!vmbus_is_valid_offer(offer)) {
1017 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n",
1018 offer->child_relid);
1019 atomic_dec(&vmbus_connection.offer_in_progress);
1020 return;
1021 }
1022
1023 oldchannel = find_primary_channel_by_offer(offer);
1024
1025 if (oldchannel != NULL) {
1026 /*
1027 * We're resuming from hibernation: all the sub-channel and
1028 * hv_sock channels we had before the hibernation should have
1029 * been cleaned up, and now we must be seeing a re-offered
1030 * primary channel that we had before the hibernation.
1031 */
1032
1033 /*
1034 * { Initially: channel relid = INVALID_RELID,
1035 * channels[valid_relid] = NULL }
1036 *
1037 * CPU1 CPU2
1038 *
1039 * [vmbus_onoffer()] [vmbus_device_release()]
1040 *
1041 * LOCK channel_mutex LOCK channel_mutex
1042 * STORE channel relid = valid_relid LOAD r1 = channel relid
1043 * MAP_RELID channel if (r1 != INVALID_RELID)
1044 * UNLOCK channel_mutex UNMAP_RELID channel
1045 * UNLOCK channel_mutex
1046 *
1047 * Forbids: r1 == valid_relid &&
1048 * channels[valid_relid] == channel
1049 *
1050 * Note. r1 can be INVALID_RELID only for an hv_sock channel.
1051 * None of the hv_sock channels which were present before the
1052 * suspend are re-offered upon the resume. See the WARN_ON()
1053 * in hv_process_channel_removal().
1054 */
1055 mutex_lock(&vmbus_connection.channel_mutex);
1056
1057 atomic_dec(&vmbus_connection.offer_in_progress);
1058
1059 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);
1060 /* Fix up the relid. */
1061 oldchannel->offermsg.child_relid = offer->child_relid;
1062
1063 offer_sz = sizeof(*offer);
1064 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {
1065 /*
1066 * This is not an error, since the host can also change
1067 * the other field(s) of the offer, e.g. on WS RS5
1068 * (Build 17763), the offer->connection_id of the
1069 * Mellanox VF vmbus device can change when the host
1070 * reoffers the device upon resume.
1071 */
1072 pr_debug("vmbus offer changed: relid=%d\n",
1073 offer->child_relid);
1074
1075 print_hex_dump_debug("Old vmbus offer: ",
1076 DUMP_PREFIX_OFFSET, 16, 4,
1077 &oldchannel->offermsg, offer_sz,
1078 false);
1079 print_hex_dump_debug("New vmbus offer: ",
1080 DUMP_PREFIX_OFFSET, 16, 4,
1081 offer, offer_sz, false);
1082
1083 /* Fix up the old channel. */
1084 vmbus_setup_channel_state(oldchannel, offer);
1085 }
1086
1087 /* Add the channel back to the array of channels. */
1088 vmbus_channel_map_relid(oldchannel);
1089 check_ready_for_resume_event();
1090
1091 mutex_unlock(&vmbus_connection.channel_mutex);
1092 return;
1093 }
1094
1095 /* Allocate the channel object and save this offer. */
1096 newchannel = alloc_channel();
1097 if (!newchannel) {
1098 vmbus_release_relid(offer->child_relid);
1099 atomic_dec(&vmbus_connection.offer_in_progress);
1100 pr_err("Unable to allocate channel object\n");
1101 return;
1102 }
1103
1104 vmbus_setup_channel_state(newchannel, offer);
1105
1106 vmbus_process_offer(newchannel);
1107}
1108
1109static void check_ready_for_suspend_event(void)
1110{
1111 /*
1112 * If all the sub-channels or hv_sock channels have been cleaned up,
1113 * then it's safe to suspend.
1114 */
1115 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))
1116 complete(&vmbus_connection.ready_for_suspend_event);
1117}
1118
1119/*
1120 * vmbus_onoffer_rescind - Rescind offer handler.
1121 *
1122 * We queue a work item to process this offer synchronously
1123 */
1124static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
1125{
1126 struct vmbus_channel_rescind_offer *rescind;
1127 struct vmbus_channel *channel;
1128 struct device *dev;
1129 bool clean_up_chan_for_suspend;
1130
1131 rescind = (struct vmbus_channel_rescind_offer *)hdr;
1132
1133 trace_vmbus_onoffer_rescind(rescind);
1134
1135 /*
1136 * The offer msg and the corresponding rescind msg
1137 * from the host are guranteed to be ordered -
1138 * offer comes in first and then the rescind.
1139 * Since we process these events in work elements,
1140 * and with preemption, we may end up processing
1141 * the events out of order. We rely on the synchronization
1142 * provided by offer_in_progress and by channel_mutex for
1143 * ordering these events:
1144 *
1145 * { Initially: offer_in_progress = 1 }
1146 *
1147 * CPU1 CPU2
1148 *
1149 * [vmbus_onoffer()] [vmbus_onoffer_rescind()]
1150 *
1151 * LOCK channel_mutex WAIT_ON offer_in_progress == 0
1152 * DECREMENT offer_in_progress LOCK channel_mutex
1153 * STORE channels[] LOAD channels[]
1154 * UNLOCK channel_mutex UNLOCK channel_mutex
1155 *
1156 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE
1157 */
1158
1159 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {
1160 /*
1161 * We wait here until any channel offer is currently
1162 * being processed.
1163 */
1164 msleep(1);
1165 }
1166
1167 mutex_lock(&vmbus_connection.channel_mutex);
1168 channel = relid2channel(rescind->child_relid);
1169 if (channel != NULL) {
1170 /*
1171 * Guarantee that no other instance of vmbus_onoffer_rescind()
1172 * has got a reference to the channel object. Synchronize on
1173 * &vmbus_connection.channel_mutex.
1174 */
1175 if (channel->rescind_ref) {
1176 mutex_unlock(&vmbus_connection.channel_mutex);
1177 return;
1178 }
1179 channel->rescind_ref = true;
1180 }
1181 mutex_unlock(&vmbus_connection.channel_mutex);
1182
1183 if (channel == NULL) {
1184 /*
1185 * We failed in processing the offer message;
1186 * we would have cleaned up the relid in that
1187 * failure path.
1188 */
1189 return;
1190 }
1191
1192 clean_up_chan_for_suspend = is_hvsock_channel(channel) ||
1193 is_sub_channel(channel);
1194 /*
1195 * Before setting channel->rescind in vmbus_rescind_cleanup(), we
1196 * should make sure the channel callback is not running any more.
1197 */
1198 vmbus_reset_channel_cb(channel);
1199
1200 /*
1201 * Now wait for offer handling to complete.
1202 */
1203 vmbus_rescind_cleanup(channel);
1204 while (READ_ONCE(channel->probe_done) == false) {
1205 /*
1206 * We wait here until any channel offer is currently
1207 * being processed.
1208 */
1209 msleep(1);
1210 }
1211
1212 /*
1213 * At this point, the rescind handling can proceed safely.
1214 */
1215
1216 if (channel->device_obj) {
1217 if (channel->chn_rescind_callback) {
1218 channel->chn_rescind_callback(channel);
1219
1220 if (clean_up_chan_for_suspend)
1221 check_ready_for_suspend_event();
1222
1223 return;
1224 }
1225 /*
1226 * We will have to unregister this device from the
1227 * driver core.
1228 */
1229 dev = get_device(&channel->device_obj->device);
1230 if (dev) {
1231 vmbus_device_unregister(channel->device_obj);
1232 put_device(dev);
1233 }
1234 } else if (channel->primary_channel != NULL) {
1235 /*
1236 * Sub-channel is being rescinded. Following is the channel
1237 * close sequence when initiated from the driveri (refer to
1238 * vmbus_close() for details):
1239 * 1. Close all sub-channels first
1240 * 2. Then close the primary channel.
1241 */
1242 mutex_lock(&vmbus_connection.channel_mutex);
1243 if (channel->state == CHANNEL_OPEN_STATE) {
1244 /*
1245 * The channel is currently not open;
1246 * it is safe for us to cleanup the channel.
1247 */
1248 hv_process_channel_removal(channel);
1249 } else {
1250 complete(&channel->rescind_event);
1251 }
1252 mutex_unlock(&vmbus_connection.channel_mutex);
1253 }
1254
1255 /* The "channel" may have been freed. Do not access it any longer. */
1256
1257 if (clean_up_chan_for_suspend)
1258 check_ready_for_suspend_event();
1259}
1260
1261void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
1262{
1263 BUG_ON(!is_hvsock_channel(channel));
1264
1265 /* We always get a rescind msg when a connection is closed. */
1266 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))
1267 msleep(1);
1268
1269 vmbus_device_unregister(channel->device_obj);
1270}
1271EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
1272
1273
1274/*
1275 * vmbus_onoffers_delivered -
1276 * This is invoked when all offers have been delivered.
1277 *
1278 * Nothing to do here.
1279 */
1280static void vmbus_onoffers_delivered(
1281 struct vmbus_channel_message_header *hdr)
1282{
1283}
1284
1285/*
1286 * vmbus_onopen_result - Open result handler.
1287 *
1288 * This is invoked when we received a response to our channel open request.
1289 * Find the matching request, copy the response and signal the requesting
1290 * thread.
1291 */
1292static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
1293{
1294 struct vmbus_channel_open_result *result;
1295 struct vmbus_channel_msginfo *msginfo;
1296 struct vmbus_channel_message_header *requestheader;
1297 struct vmbus_channel_open_channel *openmsg;
1298 unsigned long flags;
1299
1300 result = (struct vmbus_channel_open_result *)hdr;
1301
1302 trace_vmbus_onopen_result(result);
1303
1304 /*
1305 * Find the open msg, copy the result and signal/unblock the wait event
1306 */
1307 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1308
1309 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1310 msglistentry) {
1311 requestheader =
1312 (struct vmbus_channel_message_header *)msginfo->msg;
1313
1314 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
1315 openmsg =
1316 (struct vmbus_channel_open_channel *)msginfo->msg;
1317 if (openmsg->child_relid == result->child_relid &&
1318 openmsg->openid == result->openid) {
1319 memcpy(&msginfo->response.open_result,
1320 result,
1321 sizeof(
1322 struct vmbus_channel_open_result));
1323 complete(&msginfo->waitevent);
1324 break;
1325 }
1326 }
1327 }
1328 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1329}
1330
1331/*
1332 * vmbus_ongpadl_created - GPADL created handler.
1333 *
1334 * This is invoked when we received a response to our gpadl create request.
1335 * Find the matching request, copy the response and signal the requesting
1336 * thread.
1337 */
1338static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
1339{
1340 struct vmbus_channel_gpadl_created *gpadlcreated;
1341 struct vmbus_channel_msginfo *msginfo;
1342 struct vmbus_channel_message_header *requestheader;
1343 struct vmbus_channel_gpadl_header *gpadlheader;
1344 unsigned long flags;
1345
1346 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
1347
1348 trace_vmbus_ongpadl_created(gpadlcreated);
1349
1350 /*
1351 * Find the establish msg, copy the result and signal/unblock the wait
1352 * event
1353 */
1354 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1355
1356 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1357 msglistentry) {
1358 requestheader =
1359 (struct vmbus_channel_message_header *)msginfo->msg;
1360
1361 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
1362 gpadlheader =
1363 (struct vmbus_channel_gpadl_header *)requestheader;
1364
1365 if ((gpadlcreated->child_relid ==
1366 gpadlheader->child_relid) &&
1367 (gpadlcreated->gpadl == gpadlheader->gpadl)) {
1368 memcpy(&msginfo->response.gpadl_created,
1369 gpadlcreated,
1370 sizeof(
1371 struct vmbus_channel_gpadl_created));
1372 complete(&msginfo->waitevent);
1373 break;
1374 }
1375 }
1376 }
1377 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1378}
1379
1380/*
1381 * vmbus_onmodifychannel_response - Modify Channel response handler.
1382 *
1383 * This is invoked when we received a response to our channel modify request.
1384 * Find the matching request, copy the response and signal the requesting thread.
1385 */
1386static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr)
1387{
1388 struct vmbus_channel_modifychannel_response *response;
1389 struct vmbus_channel_msginfo *msginfo;
1390 unsigned long flags;
1391
1392 response = (struct vmbus_channel_modifychannel_response *)hdr;
1393
1394 trace_vmbus_onmodifychannel_response(response);
1395
1396 /*
1397 * Find the modify msg, copy the response and signal/unblock the wait event.
1398 */
1399 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1400
1401 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) {
1402 struct vmbus_channel_message_header *responseheader =
1403 (struct vmbus_channel_message_header *)msginfo->msg;
1404
1405 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {
1406 struct vmbus_channel_modifychannel *modifymsg;
1407
1408 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;
1409 if (modifymsg->child_relid == response->child_relid) {
1410 memcpy(&msginfo->response.modify_response, response,
1411 sizeof(*response));
1412 complete(&msginfo->waitevent);
1413 break;
1414 }
1415 }
1416 }
1417 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1418}
1419
1420/*
1421 * vmbus_ongpadl_torndown - GPADL torndown handler.
1422 *
1423 * This is invoked when we received a response to our gpadl teardown request.
1424 * Find the matching request, copy the response and signal the requesting
1425 * thread.
1426 */
1427static void vmbus_ongpadl_torndown(
1428 struct vmbus_channel_message_header *hdr)
1429{
1430 struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1431 struct vmbus_channel_msginfo *msginfo;
1432 struct vmbus_channel_message_header *requestheader;
1433 struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1434 unsigned long flags;
1435
1436 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1437
1438 trace_vmbus_ongpadl_torndown(gpadl_torndown);
1439
1440 /*
1441 * Find the open msg, copy the result and signal/unblock the wait event
1442 */
1443 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1444
1445 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1446 msglistentry) {
1447 requestheader =
1448 (struct vmbus_channel_message_header *)msginfo->msg;
1449
1450 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1451 gpadl_teardown =
1452 (struct vmbus_channel_gpadl_teardown *)requestheader;
1453
1454 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1455 memcpy(&msginfo->response.gpadl_torndown,
1456 gpadl_torndown,
1457 sizeof(
1458 struct vmbus_channel_gpadl_torndown));
1459 complete(&msginfo->waitevent);
1460 break;
1461 }
1462 }
1463 }
1464 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1465}
1466
1467/*
1468 * vmbus_onversion_response - Version response handler
1469 *
1470 * This is invoked when we received a response to our initiate contact request.
1471 * Find the matching request, copy the response and signal the requesting
1472 * thread.
1473 */
1474static void vmbus_onversion_response(
1475 struct vmbus_channel_message_header *hdr)
1476{
1477 struct vmbus_channel_msginfo *msginfo;
1478 struct vmbus_channel_message_header *requestheader;
1479 struct vmbus_channel_version_response *version_response;
1480 unsigned long flags;
1481
1482 version_response = (struct vmbus_channel_version_response *)hdr;
1483
1484 trace_vmbus_onversion_response(version_response);
1485
1486 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1487
1488 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1489 msglistentry) {
1490 requestheader =
1491 (struct vmbus_channel_message_header *)msginfo->msg;
1492
1493 if (requestheader->msgtype ==
1494 CHANNELMSG_INITIATE_CONTACT) {
1495 memcpy(&msginfo->response.version_response,
1496 version_response,
1497 sizeof(struct vmbus_channel_version_response));
1498 complete(&msginfo->waitevent);
1499 }
1500 }
1501 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1502}
1503
1504/* Channel message dispatch table */
1505const struct vmbus_channel_message_table_entry
1506channel_message_table[CHANNELMSG_COUNT] = {
1507 { CHANNELMSG_INVALID, 0, NULL, 0},
1508 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer,
1509 sizeof(struct vmbus_channel_offer_channel)},
1510 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind,
1511 sizeof(struct vmbus_channel_rescind_offer) },
1512 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0},
1513 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0},
1514 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0},
1515 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result,
1516 sizeof(struct vmbus_channel_open_result)},
1517 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0},
1518 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0},
1519 { CHANNELMSG_GPADL_BODY, 0, NULL, 0},
1520 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created,
1521 sizeof(struct vmbus_channel_gpadl_created)},
1522 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0},
1523 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown,
1524 sizeof(struct vmbus_channel_gpadl_torndown) },
1525 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0},
1526 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0},
1527 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response,
1528 sizeof(struct vmbus_channel_version_response)},
1529 { CHANNELMSG_UNLOAD, 0, NULL, 0},
1530 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0},
1531 { CHANNELMSG_18, 0, NULL, 0},
1532 { CHANNELMSG_19, 0, NULL, 0},
1533 { CHANNELMSG_20, 0, NULL, 0},
1534 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0},
1535 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0},
1536 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0},
1537 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response,
1538 sizeof(struct vmbus_channel_modifychannel_response)},
1539};
1540
1541/*
1542 * vmbus_onmessage - Handler for channel protocol messages.
1543 *
1544 * This is invoked in the vmbus worker thread context.
1545 */
1546void vmbus_onmessage(struct vmbus_channel_message_header *hdr)
1547{
1548 trace_vmbus_on_message(hdr);
1549
1550 /*
1551 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go
1552 * out of bound and the message_handler pointer can not be NULL.
1553 */
1554 channel_message_table[hdr->msgtype].message_handler(hdr);
1555}
1556
1557/*
1558 * vmbus_request_offers - Send a request to get all our pending offers.
1559 */
1560int vmbus_request_offers(void)
1561{
1562 struct vmbus_channel_message_header *msg;
1563 struct vmbus_channel_msginfo *msginfo;
1564 int ret;
1565
1566 msginfo = kzalloc(sizeof(*msginfo) +
1567 sizeof(struct vmbus_channel_message_header),
1568 GFP_KERNEL);
1569 if (!msginfo)
1570 return -ENOMEM;
1571
1572 msg = (struct vmbus_channel_message_header *)msginfo->msg;
1573
1574 msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1575
1576 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1577 true);
1578
1579 trace_vmbus_request_offers(ret);
1580
1581 if (ret != 0) {
1582 pr_err("Unable to request offers - %d\n", ret);
1583
1584 goto cleanup;
1585 }
1586
1587cleanup:
1588 kfree(msginfo);
1589
1590 return ret;
1591}
1592
1593void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1594 void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1595{
1596 primary_channel->sc_creation_callback = sc_cr_cb;
1597}
1598EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1599
1600void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1601 void (*chn_rescind_cb)(struct vmbus_channel *))
1602{
1603 channel->chn_rescind_callback = chn_rescind_cb;
1604}
1605EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);