1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Hyper-V specific APIC code.
  5 *
  6 * Copyright (C) 2018, Microsoft, Inc.
  7 *
  8 * Author : K. Y. Srinivasan <kys@microsoft.com>
  9 *
 10 * This program is free software; you can redistribute it and/or modify it
 11 * under the terms of the GNU General Public License version 2 as published
 12 * by the Free Software Foundation.
 13 *
 14 * This program is distributed in the hope that it will be useful, but
 15 * WITHOUT ANY WARRANTY; without even the implied warranty of
 16 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 17 * NON INFRINGEMENT.  See the GNU General Public License for more
 18 * details.
 19 *
 20 */
 21
 22#include <linux/types.h>
 23#include <linux/vmalloc.h>
 24#include <linux/mm.h>
 25#include <linux/clockchips.h>
 26#include <linux/hyperv.h>
 27#include <linux/slab.h>
 28#include <linux/cpuhotplug.h>
 29#include <asm/hypervisor.h>
 30#include <asm/mshyperv.h>
 31#include <asm/apic.h>
 32
 33#include <asm/trace/hyperv.h>
 34
 35static struct apic orig_apic;
 36
 37static u64 hv_apic_icr_read(void)
 38{
 39	u64 reg_val;
 40
 41	rdmsrl(HV_X64_MSR_ICR, reg_val);
 42	return reg_val;
 43}
 44
 45static void hv_apic_icr_write(u32 low, u32 id)
 46{
 47	u64 reg_val;
 48
 49	reg_val = SET_APIC_DEST_FIELD(id);
 50	reg_val = reg_val << 32;
 51	reg_val |= low;
 52
 53	wrmsrl(HV_X64_MSR_ICR, reg_val);
 54}
 55
 56static u32 hv_apic_read(u32 reg)
 57{
 58	u32 reg_val, hi;
 59
 60	switch (reg) {
 61	case APIC_EOI:
 62		rdmsr(HV_X64_MSR_EOI, reg_val, hi);
 63		return reg_val;
 64	case APIC_TASKPRI:
 65		rdmsr(HV_X64_MSR_TPR, reg_val, hi);
 66		return reg_val;
 67
 68	default:
 69		return native_apic_mem_read(reg);
 70	}
 71}
 72
 73static void hv_apic_write(u32 reg, u32 val)
 74{
 75	switch (reg) {
 76	case APIC_EOI:
 77		wrmsr(HV_X64_MSR_EOI, val, 0);
 78		break;
 79	case APIC_TASKPRI:
 80		wrmsr(HV_X64_MSR_TPR, val, 0);
 81		break;
 82	default:
 83		native_apic_mem_write(reg, val);
 84	}
 85}
 86
 87static void hv_apic_eoi_write(u32 reg, u32 val)
 88{
 89	struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()];
 90
 91	if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1))
 92		return;
 93
 94	wrmsr(HV_X64_MSR_EOI, val, 0);
 95}
 96
 97/*
 98 * IPI implementation on Hyper-V.
 99 */
100static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector)
101{
102	struct hv_send_ipi_ex **arg;
103	struct hv_send_ipi_ex *ipi_arg;
104	unsigned long flags;
105	int nr_bank = 0;
106	int ret = 1;
107
108	if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
109		return false;
110
111	local_irq_save(flags);
112	arg = (struct hv_send_ipi_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);
113
114	ipi_arg = *arg;
115	if (unlikely(!ipi_arg))
116		goto ipi_mask_ex_done;
117
118	ipi_arg->vector = vector;
119	ipi_arg->reserved = 0;
120	ipi_arg->vp_set.valid_bank_mask = 0;
121
122	if (!cpumask_equal(mask, cpu_present_mask)) {
123		ipi_arg->vp_set.format = HV_GENERIC_SET_SPARSE_4K;
124		nr_bank = cpumask_to_vpset(&(ipi_arg->vp_set), mask);
125	}
126	if (nr_bank < 0)
127		goto ipi_mask_ex_done;
128	if (!nr_bank)
129		ipi_arg->vp_set.format = HV_GENERIC_SET_ALL;
130
131	ret = hv_do_rep_hypercall(HVCALL_SEND_IPI_EX, 0, nr_bank,
132			      ipi_arg, NULL);
133
134ipi_mask_ex_done:
135	local_irq_restore(flags);
136	return ((ret == 0) ? true : false);
137}
138
139static bool __send_ipi_mask(const struct cpumask *mask, int vector)
140{
141	int cur_cpu, vcpu;
142	struct hv_send_ipi ipi_arg;
143	int ret = 1;
144
145	trace_hyperv_send_ipi_mask(mask, vector);
146
147	if (cpumask_empty(mask))
148		return true;
149
150	if (!hv_hypercall_pg)
151		return false;
152
153	if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR))
154		return false;
155
156	/*
157	 * From the supplied CPU set we need to figure out if we can get away
158	 * with cheaper HVCALL_SEND_IPI hypercall. This is possible when the
159	 * highest VP number in the set is < 64. As VP numbers are usually in
160	 * ascending order and match Linux CPU ids, here is an optimization:
161	 * we check the VP number for the highest bit in the supplied set first
162	 * so we can quickly find out if using HVCALL_SEND_IPI_EX hypercall is
163	 * a must. We will also check all VP numbers when walking the supplied
164	 * CPU set to remain correct in all cases.
165	 */
166	if (hv_cpu_number_to_vp_number(cpumask_last(mask)) >= 64)
167		goto do_ex_hypercall;
168
169	ipi_arg.vector = vector;
170	ipi_arg.cpu_mask = 0;
171
172	for_each_cpu(cur_cpu, mask) {
173		vcpu = hv_cpu_number_to_vp_number(cur_cpu);
174		if (vcpu == VP_INVAL)
175			return false;
176
177		/*
178		 * This particular version of the IPI hypercall can
179		 * only target upto 64 CPUs.
180		 */
181		if (vcpu >= 64)
182			goto do_ex_hypercall;
183
184		__set_bit(vcpu, (unsigned long *)&ipi_arg.cpu_mask);
185	}
186
187	ret = hv_do_fast_hypercall16(HVCALL_SEND_IPI, ipi_arg.vector,
188				     ipi_arg.cpu_mask);
189	return ((ret == 0) ? true : false);
190
191do_ex_hypercall:
192	return __send_ipi_mask_ex(mask, vector);
193}
194
195static bool __send_ipi_one(int cpu, int vector)
196{
197	struct cpumask mask = CPU_MASK_NONE;
198
199	cpumask_set_cpu(cpu, &mask);
200	return __send_ipi_mask(&mask, vector);
201}
202
203static void hv_send_ipi(int cpu, int vector)
204{
205	if (!__send_ipi_one(cpu, vector))
206		orig_apic.send_IPI(cpu, vector);
207}
208
209static void hv_send_ipi_mask(const struct cpumask *mask, int vector)
210{
211	if (!__send_ipi_mask(mask, vector))
212		orig_apic.send_IPI_mask(mask, vector);
213}
214
215static void hv_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
216{
217	unsigned int this_cpu = smp_processor_id();
218	struct cpumask new_mask;
219	const struct cpumask *local_mask;
220
221	cpumask_copy(&new_mask, mask);
222	cpumask_clear_cpu(this_cpu, &new_mask);
223	local_mask = &new_mask;
224	if (!__send_ipi_mask(local_mask, vector))
225		orig_apic.send_IPI_mask_allbutself(mask, vector);
226}
227
228static void hv_send_ipi_allbutself(int vector)
229{
230	hv_send_ipi_mask_allbutself(cpu_online_mask, vector);
231}
232
233static void hv_send_ipi_all(int vector)
234{
235	if (!__send_ipi_mask(cpu_online_mask, vector))
236		orig_apic.send_IPI_all(vector);
237}
238
239static void hv_send_ipi_self(int vector)
240{
241	if (!__send_ipi_one(smp_processor_id(), vector))
242		orig_apic.send_IPI_self(vector);
243}
244
245void __init hv_apic_init(void)
246{
247	if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) {
248		pr_info("Hyper-V: Using IPI hypercalls\n");
249		/*
250		 * Set the IPI entry points.
251		 */
252		orig_apic = *apic;
253
254		apic->send_IPI = hv_send_ipi;
255		apic->send_IPI_mask = hv_send_ipi_mask;
256		apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself;
257		apic->send_IPI_allbutself = hv_send_ipi_allbutself;
258		apic->send_IPI_all = hv_send_ipi_all;
259		apic->send_IPI_self = hv_send_ipi_self;
260	}
261
262	if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
263		pr_info("Hyper-V: Using enlightened APIC (%s mode)",
264			x2apic_enabled() ? "x2apic" : "xapic");
265		/*
266		 * With x2apic, architectural x2apic MSRs are equivalent to the
267		 * respective synthetic MSRs, so there's no need to override
268		 * the apic accessors.  The only exception is
269		 * hv_apic_eoi_write, because it benefits from lazy EOI when
270		 * available, but it works for both xapic and x2apic modes.
271		 */
272		apic_set_eoi_write(hv_apic_eoi_write);
273		if (!x2apic_enabled()) {
274			apic->read      = hv_apic_read;
275			apic->write     = hv_apic_write;
276			apic->icr_write = hv_apic_icr_write;
277			apic->icr_read  = hv_apic_icr_read;
278		}
279	}
280}