Loading...
1#define pr_fmt(fmt) "Hyper-V: " fmt
2
3#include <linux/hyperv.h>
4#include <linux/log2.h>
5#include <linux/slab.h>
6#include <linux/types.h>
7
8#include <asm/fpu/api.h>
9#include <asm/mshyperv.h>
10#include <asm/msr.h>
11#include <asm/tlbflush.h>
12#include <asm/tlb.h>
13
14#define CREATE_TRACE_POINTS
15#include <asm/trace/hyperv.h>
16
17/* Each gva in gva_list encodes up to 4096 pages to flush */
18#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
19
20static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
21 const struct flush_tlb_info *info);
22
23/*
24 * Fills in gva_list starting from offset. Returns the number of items added.
25 */
26static inline int fill_gva_list(u64 gva_list[], int offset,
27 unsigned long start, unsigned long end)
28{
29 int gva_n = offset;
30 unsigned long cur = start, diff;
31
32 do {
33 diff = end > cur ? end - cur : 0;
34
35 gva_list[gva_n] = cur & PAGE_MASK;
36 /*
37 * Lower 12 bits encode the number of additional
38 * pages to flush (in addition to the 'cur' page).
39 */
40 if (diff >= HV_TLB_FLUSH_UNIT) {
41 gva_list[gva_n] |= ~PAGE_MASK;
42 cur += HV_TLB_FLUSH_UNIT;
43 } else if (diff) {
44 gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
45 cur = end;
46 }
47
48 gva_n++;
49
50 } while (cur < end);
51
52 return gva_n - offset;
53}
54
55static void hyperv_flush_tlb_others(const struct cpumask *cpus,
56 const struct flush_tlb_info *info)
57{
58 int cpu, vcpu, gva_n, max_gvas;
59 struct hv_tlb_flush **flush_pcpu;
60 struct hv_tlb_flush *flush;
61 u64 status = U64_MAX;
62 unsigned long flags;
63
64 trace_hyperv_mmu_flush_tlb_others(cpus, info);
65
66 if (!hv_hypercall_pg)
67 goto do_native;
68
69 if (cpumask_empty(cpus))
70 return;
71
72 local_irq_save(flags);
73
74 flush_pcpu = (struct hv_tlb_flush **)
75 this_cpu_ptr(hyperv_pcpu_input_arg);
76
77 flush = *flush_pcpu;
78
79 if (unlikely(!flush)) {
80 local_irq_restore(flags);
81 goto do_native;
82 }
83
84 if (info->mm) {
85 /*
86 * AddressSpace argument must match the CR3 with PCID bits
87 * stripped out.
88 */
89 flush->address_space = virt_to_phys(info->mm->pgd);
90 flush->address_space &= CR3_ADDR_MASK;
91 flush->flags = 0;
92 } else {
93 flush->address_space = 0;
94 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
95 }
96
97 flush->processor_mask = 0;
98 if (cpumask_equal(cpus, cpu_present_mask)) {
99 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
100 } else {
101 /*
102 * From the supplied CPU set we need to figure out if we can get
103 * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}
104 * hypercalls. This is possible when the highest VP number in
105 * the set is < 64. As VP numbers are usually in ascending order
106 * and match Linux CPU ids, here is an optimization: we check
107 * the VP number for the highest bit in the supplied set first
108 * so we can quickly find out if using *_EX hypercalls is a
109 * must. We will also check all VP numbers when walking the
110 * supplied CPU set to remain correct in all cases.
111 */
112 if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64)
113 goto do_ex_hypercall;
114
115 for_each_cpu(cpu, cpus) {
116 vcpu = hv_cpu_number_to_vp_number(cpu);
117 if (vcpu == VP_INVAL) {
118 local_irq_restore(flags);
119 goto do_native;
120 }
121
122 if (vcpu >= 64)
123 goto do_ex_hypercall;
124
125 __set_bit(vcpu, (unsigned long *)
126 &flush->processor_mask);
127 }
128 }
129
130 /*
131 * We can flush not more than max_gvas with one hypercall. Flush the
132 * whole address space if we were asked to do more.
133 */
134 max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
135
136 if (info->end == TLB_FLUSH_ALL) {
137 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
138 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
139 flush, NULL);
140 } else if (info->end &&
141 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
142 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
143 flush, NULL);
144 } else {
145 gva_n = fill_gva_list(flush->gva_list, 0,
146 info->start, info->end);
147 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
148 gva_n, 0, flush, NULL);
149 }
150 goto check_status;
151
152do_ex_hypercall:
153 status = hyperv_flush_tlb_others_ex(cpus, info);
154
155check_status:
156 local_irq_restore(flags);
157
158 if (!(status & HV_HYPERCALL_RESULT_MASK))
159 return;
160do_native:
161 native_flush_tlb_others(cpus, info);
162}
163
164static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
165 const struct flush_tlb_info *info)
166{
167 int nr_bank = 0, max_gvas, gva_n;
168 struct hv_tlb_flush_ex **flush_pcpu;
169 struct hv_tlb_flush_ex *flush;
170 u64 status;
171
172 if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
173 return U64_MAX;
174
175 flush_pcpu = (struct hv_tlb_flush_ex **)
176 this_cpu_ptr(hyperv_pcpu_input_arg);
177
178 flush = *flush_pcpu;
179
180 if (info->mm) {
181 /*
182 * AddressSpace argument must match the CR3 with PCID bits
183 * stripped out.
184 */
185 flush->address_space = virt_to_phys(info->mm->pgd);
186 flush->address_space &= CR3_ADDR_MASK;
187 flush->flags = 0;
188 } else {
189 flush->address_space = 0;
190 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
191 }
192
193 flush->hv_vp_set.valid_bank_mask = 0;
194
195 flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
196 nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
197 if (nr_bank < 0)
198 return U64_MAX;
199
200 /*
201 * We can flush not more than max_gvas with one hypercall. Flush the
202 * whole address space if we were asked to do more.
203 */
204 max_gvas =
205 (PAGE_SIZE - sizeof(*flush) - nr_bank *
206 sizeof(flush->hv_vp_set.bank_contents[0])) /
207 sizeof(flush->gva_list[0]);
208
209 if (info->end == TLB_FLUSH_ALL) {
210 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
211 status = hv_do_rep_hypercall(
212 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
213 0, nr_bank, flush, NULL);
214 } else if (info->end &&
215 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
216 status = hv_do_rep_hypercall(
217 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
218 0, nr_bank, flush, NULL);
219 } else {
220 gva_n = fill_gva_list(flush->gva_list, nr_bank,
221 info->start, info->end);
222 status = hv_do_rep_hypercall(
223 HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
224 gva_n, nr_bank, flush, NULL);
225 }
226
227 return status;
228}
229
230void hyperv_setup_mmu_ops(void)
231{
232 if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
233 return;
234
235 pr_info("Using hypercall for remote TLB flush\n");
236 pv_ops.mmu.flush_tlb_others = hyperv_flush_tlb_others;
237 pv_ops.mmu.tlb_remove_table = tlb_remove_table;
238}
1#define pr_fmt(fmt) "Hyper-V: " fmt
2
3#include <linux/hyperv.h>
4#include <linux/log2.h>
5#include <linux/slab.h>
6#include <linux/types.h>
7
8#include <asm/fpu/api.h>
9#include <asm/mshyperv.h>
10#include <asm/msr.h>
11#include <asm/tlbflush.h>
12
13#define CREATE_TRACE_POINTS
14#include <asm/trace/hyperv.h>
15
16/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */
17struct hv_flush_pcpu {
18 u64 address_space;
19 u64 flags;
20 u64 processor_mask;
21 u64 gva_list[];
22};
23
24/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */
25struct hv_flush_pcpu_ex {
26 u64 address_space;
27 u64 flags;
28 struct {
29 u64 format;
30 u64 valid_bank_mask;
31 u64 bank_contents[];
32 } hv_vp_set;
33 u64 gva_list[];
34};
35
36/* Each gva in gva_list encodes up to 4096 pages to flush */
37#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
38
39static struct hv_flush_pcpu __percpu **pcpu_flush;
40
41static struct hv_flush_pcpu_ex __percpu **pcpu_flush_ex;
42
43/*
44 * Fills in gva_list starting from offset. Returns the number of items added.
45 */
46static inline int fill_gva_list(u64 gva_list[], int offset,
47 unsigned long start, unsigned long end)
48{
49 int gva_n = offset;
50 unsigned long cur = start, diff;
51
52 do {
53 diff = end > cur ? end - cur : 0;
54
55 gva_list[gva_n] = cur & PAGE_MASK;
56 /*
57 * Lower 12 bits encode the number of additional
58 * pages to flush (in addition to the 'cur' page).
59 */
60 if (diff >= HV_TLB_FLUSH_UNIT)
61 gva_list[gva_n] |= ~PAGE_MASK;
62 else if (diff)
63 gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
64
65 cur += HV_TLB_FLUSH_UNIT;
66 gva_n++;
67
68 } while (cur < end);
69
70 return gva_n - offset;
71}
72
73/* Return the number of banks in the resulting vp_set */
74static inline int cpumask_to_vp_set(struct hv_flush_pcpu_ex *flush,
75 const struct cpumask *cpus)
76{
77 int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1;
78
79 /* valid_bank_mask can represent up to 64 banks */
80 if (hv_max_vp_index / 64 >= 64)
81 return 0;
82
83 /*
84 * Clear all banks up to the maximum possible bank as hv_flush_pcpu_ex
85 * structs are not cleared between calls, we risk flushing unneeded
86 * vCPUs otherwise.
87 */
88 for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++)
89 flush->hv_vp_set.bank_contents[vcpu_bank] = 0;
90
91 /*
92 * Some banks may end up being empty but this is acceptable.
93 */
94 for_each_cpu(cpu, cpus) {
95 vcpu = hv_cpu_number_to_vp_number(cpu);
96 vcpu_bank = vcpu / 64;
97 vcpu_offset = vcpu % 64;
98 __set_bit(vcpu_offset, (unsigned long *)
99 &flush->hv_vp_set.bank_contents[vcpu_bank]);
100 if (vcpu_bank >= nr_bank)
101 nr_bank = vcpu_bank + 1;
102 }
103 flush->hv_vp_set.valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0);
104
105 return nr_bank;
106}
107
108static void hyperv_flush_tlb_others(const struct cpumask *cpus,
109 const struct flush_tlb_info *info)
110{
111 int cpu, vcpu, gva_n, max_gvas;
112 struct hv_flush_pcpu **flush_pcpu;
113 struct hv_flush_pcpu *flush;
114 u64 status = U64_MAX;
115 unsigned long flags;
116
117 trace_hyperv_mmu_flush_tlb_others(cpus, info);
118
119 if (!pcpu_flush || !hv_hypercall_pg)
120 goto do_native;
121
122 if (cpumask_empty(cpus))
123 return;
124
125 local_irq_save(flags);
126
127 flush_pcpu = this_cpu_ptr(pcpu_flush);
128
129 if (unlikely(!*flush_pcpu))
130 *flush_pcpu = page_address(alloc_page(GFP_ATOMIC));
131
132 flush = *flush_pcpu;
133
134 if (unlikely(!flush)) {
135 local_irq_restore(flags);
136 goto do_native;
137 }
138
139 if (info->mm) {
140 /*
141 * AddressSpace argument must match the CR3 with PCID bits
142 * stripped out.
143 */
144 flush->address_space = virt_to_phys(info->mm->pgd);
145 flush->address_space &= CR3_ADDR_MASK;
146 flush->flags = 0;
147 } else {
148 flush->address_space = 0;
149 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
150 }
151
152 flush->processor_mask = 0;
153 if (cpumask_equal(cpus, cpu_present_mask)) {
154 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
155 } else {
156 for_each_cpu(cpu, cpus) {
157 vcpu = hv_cpu_number_to_vp_number(cpu);
158 if (vcpu >= 64)
159 goto do_native;
160
161 __set_bit(vcpu, (unsigned long *)
162 &flush->processor_mask);
163 }
164 }
165
166 /*
167 * We can flush not more than max_gvas with one hypercall. Flush the
168 * whole address space if we were asked to do more.
169 */
170 max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
171
172 if (info->end == TLB_FLUSH_ALL) {
173 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
174 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
175 flush, NULL);
176 } else if (info->end &&
177 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
178 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
179 flush, NULL);
180 } else {
181 gva_n = fill_gva_list(flush->gva_list, 0,
182 info->start, info->end);
183 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
184 gva_n, 0, flush, NULL);
185 }
186
187 local_irq_restore(flags);
188
189 if (!(status & HV_HYPERCALL_RESULT_MASK))
190 return;
191do_native:
192 native_flush_tlb_others(cpus, info);
193}
194
195static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
196 const struct flush_tlb_info *info)
197{
198 int nr_bank = 0, max_gvas, gva_n;
199 struct hv_flush_pcpu_ex **flush_pcpu;
200 struct hv_flush_pcpu_ex *flush;
201 u64 status = U64_MAX;
202 unsigned long flags;
203
204 trace_hyperv_mmu_flush_tlb_others(cpus, info);
205
206 if (!pcpu_flush_ex || !hv_hypercall_pg)
207 goto do_native;
208
209 if (cpumask_empty(cpus))
210 return;
211
212 local_irq_save(flags);
213
214 flush_pcpu = this_cpu_ptr(pcpu_flush_ex);
215
216 if (unlikely(!*flush_pcpu))
217 *flush_pcpu = page_address(alloc_page(GFP_ATOMIC));
218
219 flush = *flush_pcpu;
220
221 if (unlikely(!flush)) {
222 local_irq_restore(flags);
223 goto do_native;
224 }
225
226 if (info->mm) {
227 /*
228 * AddressSpace argument must match the CR3 with PCID bits
229 * stripped out.
230 */
231 flush->address_space = virt_to_phys(info->mm->pgd);
232 flush->address_space &= CR3_ADDR_MASK;
233 flush->flags = 0;
234 } else {
235 flush->address_space = 0;
236 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
237 }
238
239 flush->hv_vp_set.valid_bank_mask = 0;
240
241 if (!cpumask_equal(cpus, cpu_present_mask)) {
242 flush->hv_vp_set.format = HV_GENERIC_SET_SPARCE_4K;
243 nr_bank = cpumask_to_vp_set(flush, cpus);
244 }
245
246 if (!nr_bank) {
247 flush->hv_vp_set.format = HV_GENERIC_SET_ALL;
248 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
249 }
250
251 /*
252 * We can flush not more than max_gvas with one hypercall. Flush the
253 * whole address space if we were asked to do more.
254 */
255 max_gvas =
256 (PAGE_SIZE - sizeof(*flush) - nr_bank *
257 sizeof(flush->hv_vp_set.bank_contents[0])) /
258 sizeof(flush->gva_list[0]);
259
260 if (info->end == TLB_FLUSH_ALL) {
261 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
262 status = hv_do_rep_hypercall(
263 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
264 0, nr_bank, flush, NULL);
265 } else if (info->end &&
266 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
267 status = hv_do_rep_hypercall(
268 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
269 0, nr_bank, flush, NULL);
270 } else {
271 gva_n = fill_gva_list(flush->gva_list, nr_bank,
272 info->start, info->end);
273 status = hv_do_rep_hypercall(
274 HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
275 gva_n, nr_bank, flush, NULL);
276 }
277
278 local_irq_restore(flags);
279
280 if (!(status & HV_HYPERCALL_RESULT_MASK))
281 return;
282do_native:
283 native_flush_tlb_others(cpus, info);
284}
285
286void hyperv_setup_mmu_ops(void)
287{
288 if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
289 return;
290
291 if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) {
292 pr_info("Using hypercall for remote TLB flush\n");
293 pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
294 } else {
295 pr_info("Using ext hypercall for remote TLB flush\n");
296 pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others_ex;
297 }
298}
299
300void hyper_alloc_mmu(void)
301{
302 if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
303 return;
304
305 if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
306 pcpu_flush = alloc_percpu(struct hv_flush_pcpu *);
307 else
308 pcpu_flush_ex = alloc_percpu(struct hv_flush_pcpu_ex *);
309}