Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Virtio-mem device driver.
4 *
5 * Copyright Red Hat, Inc. 2020
6 *
7 * Author(s): David Hildenbrand <david@redhat.com>
8 */
9
10#include <linux/virtio.h>
11#include <linux/virtio_mem.h>
12#include <linux/workqueue.h>
13#include <linux/slab.h>
14#include <linux/module.h>
15#include <linux/mm.h>
16#include <linux/memory_hotplug.h>
17#include <linux/memory.h>
18#include <linux/hrtimer.h>
19#include <linux/crash_dump.h>
20#include <linux/mutex.h>
21#include <linux/bitmap.h>
22#include <linux/lockdep.h>
23#include <linux/log2.h>
24
25#include <acpi/acpi_numa.h>
26
27static bool unplug_online = true;
28module_param(unplug_online, bool, 0644);
29MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
30
31static bool force_bbm;
32module_param(force_bbm, bool, 0444);
33MODULE_PARM_DESC(force_bbm,
34 "Force Big Block Mode. Default is 0 (auto-selection)");
35
36static unsigned long bbm_block_size;
37module_param(bbm_block_size, ulong, 0444);
38MODULE_PARM_DESC(bbm_block_size,
39 "Big Block size in bytes. Default is 0 (auto-detection).");
40
41static bool bbm_safe_unplug = true;
42module_param(bbm_safe_unplug, bool, 0444);
43MODULE_PARM_DESC(bbm_safe_unplug,
44 "Use a safe unplug mechanism in BBM, avoiding long/endless loops");
45
46/*
47 * virtio-mem currently supports the following modes of operation:
48 *
49 * * Sub Block Mode (SBM): A Linux memory block spans 2..X subblocks (SB). The
50 * size of a Sub Block (SB) is determined based on the device block size, the
51 * pageblock size, and the maximum allocation granularity of the buddy.
52 * Subblocks within a Linux memory block might either be plugged or unplugged.
53 * Memory is added/removed to Linux MM in Linux memory block granularity.
54 *
55 * * Big Block Mode (BBM): A Big Block (BB) spans 1..X Linux memory blocks.
56 * Memory is added/removed to Linux MM in Big Block granularity.
57 *
58 * The mode is determined automatically based on the Linux memory block size
59 * and the device block size.
60 *
61 * User space / core MM (auto onlining) is responsible for onlining added
62 * Linux memory blocks - and for selecting a zone. Linux Memory Blocks are
63 * always onlined separately, and all memory within a Linux memory block is
64 * onlined to the same zone - virtio-mem relies on this behavior.
65 */
66
67/*
68 * State of a Linux memory block in SBM.
69 */
70enum virtio_mem_sbm_mb_state {
71 /* Unplugged, not added to Linux. Can be reused later. */
72 VIRTIO_MEM_SBM_MB_UNUSED = 0,
73 /* (Partially) plugged, not added to Linux. Error on add_memory(). */
74 VIRTIO_MEM_SBM_MB_PLUGGED,
75 /* Fully plugged, fully added to Linux, offline. */
76 VIRTIO_MEM_SBM_MB_OFFLINE,
77 /* Partially plugged, fully added to Linux, offline. */
78 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL,
79 /* Fully plugged, fully added to Linux, onlined to a kernel zone. */
80 VIRTIO_MEM_SBM_MB_KERNEL,
81 /* Partially plugged, fully added to Linux, online to a kernel zone */
82 VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL,
83 /* Fully plugged, fully added to Linux, onlined to ZONE_MOVABLE. */
84 VIRTIO_MEM_SBM_MB_MOVABLE,
85 /* Partially plugged, fully added to Linux, onlined to ZONE_MOVABLE. */
86 VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL,
87 VIRTIO_MEM_SBM_MB_COUNT
88};
89
90/*
91 * State of a Big Block (BB) in BBM, covering 1..X Linux memory blocks.
92 */
93enum virtio_mem_bbm_bb_state {
94 /* Unplugged, not added to Linux. Can be reused later. */
95 VIRTIO_MEM_BBM_BB_UNUSED = 0,
96 /* Plugged, not added to Linux. Error on add_memory(). */
97 VIRTIO_MEM_BBM_BB_PLUGGED,
98 /* Plugged and added to Linux. */
99 VIRTIO_MEM_BBM_BB_ADDED,
100 /* All online parts are fake-offline, ready to remove. */
101 VIRTIO_MEM_BBM_BB_FAKE_OFFLINE,
102 VIRTIO_MEM_BBM_BB_COUNT
103};
104
105struct virtio_mem {
106 struct virtio_device *vdev;
107
108 /* We might first have to unplug all memory when starting up. */
109 bool unplug_all_required;
110
111 /* Workqueue that processes the plug/unplug requests. */
112 struct work_struct wq;
113 atomic_t wq_active;
114 atomic_t config_changed;
115
116 /* Virtqueue for guest->host requests. */
117 struct virtqueue *vq;
118
119 /* Wait for a host response to a guest request. */
120 wait_queue_head_t host_resp;
121
122 /* Space for one guest request and the host response. */
123 struct virtio_mem_req req;
124 struct virtio_mem_resp resp;
125
126 /* The current size of the device. */
127 uint64_t plugged_size;
128 /* The requested size of the device. */
129 uint64_t requested_size;
130
131 /* The device block size (for communicating with the device). */
132 uint64_t device_block_size;
133 /* The determined node id for all memory of the device. */
134 int nid;
135 /* Physical start address of the memory region. */
136 uint64_t addr;
137 /* Maximum region size in bytes. */
138 uint64_t region_size;
139
140 /* The parent resource for all memory added via this device. */
141 struct resource *parent_resource;
142 /*
143 * Copy of "System RAM (virtio_mem)" to be used for
144 * add_memory_driver_managed().
145 */
146 const char *resource_name;
147 /* Memory group identification. */
148 int mgid;
149
150 /*
151 * We don't want to add too much memory if it's not getting onlined,
152 * to avoid running OOM. Besides this threshold, we allow to have at
153 * least two offline blocks at a time (whatever is bigger).
154 */
155#define VIRTIO_MEM_DEFAULT_OFFLINE_THRESHOLD (1024 * 1024 * 1024)
156 atomic64_t offline_size;
157 uint64_t offline_threshold;
158
159 /* If set, the driver is in SBM, otherwise in BBM. */
160 bool in_sbm;
161
162 union {
163 struct {
164 /* Id of the first memory block of this device. */
165 unsigned long first_mb_id;
166 /* Id of the last usable memory block of this device. */
167 unsigned long last_usable_mb_id;
168 /* Id of the next memory bock to prepare when needed. */
169 unsigned long next_mb_id;
170
171 /* The subblock size. */
172 uint64_t sb_size;
173 /* The number of subblocks per Linux memory block. */
174 uint32_t sbs_per_mb;
175
176 /* Summary of all memory block states. */
177 unsigned long mb_count[VIRTIO_MEM_SBM_MB_COUNT];
178
179 /*
180 * One byte state per memory block. Allocated via
181 * vmalloc(). Resized (alloc+copy+free) on demand.
182 *
183 * With 128 MiB memory blocks, we have states for 512
184 * GiB of memory in one 4 KiB page.
185 */
186 uint8_t *mb_states;
187
188 /*
189 * Bitmap: one bit per subblock. Allocated similar to
190 * sbm.mb_states.
191 *
192 * A set bit means the corresponding subblock is
193 * plugged, otherwise it's unblocked.
194 *
195 * With 4 MiB subblocks, we manage 128 GiB of memory
196 * in one 4 KiB page.
197 */
198 unsigned long *sb_states;
199 } sbm;
200
201 struct {
202 /* Id of the first big block of this device. */
203 unsigned long first_bb_id;
204 /* Id of the last usable big block of this device. */
205 unsigned long last_usable_bb_id;
206 /* Id of the next device bock to prepare when needed. */
207 unsigned long next_bb_id;
208
209 /* Summary of all big block states. */
210 unsigned long bb_count[VIRTIO_MEM_BBM_BB_COUNT];
211
212 /* One byte state per big block. See sbm.mb_states. */
213 uint8_t *bb_states;
214
215 /* The block size used for plugging/adding/removing. */
216 uint64_t bb_size;
217 } bbm;
218 };
219
220 /*
221 * Mutex that protects the sbm.mb_count, sbm.mb_states,
222 * sbm.sb_states, bbm.bb_count, and bbm.bb_states
223 *
224 * When this lock is held the pointers can't change, ONLINE and
225 * OFFLINE blocks can't change the state and no subblocks will get
226 * plugged/unplugged.
227 *
228 * In kdump mode, used to serialize requests, last_block_addr and
229 * last_block_plugged.
230 */
231 struct mutex hotplug_mutex;
232 bool hotplug_active;
233
234 /* An error occurred we cannot handle - stop processing requests. */
235 bool broken;
236
237 /* Cached valued of is_kdump_kernel() when the device was probed. */
238 bool in_kdump;
239
240 /* The driver is being removed. */
241 spinlock_t removal_lock;
242 bool removing;
243
244 /* Timer for retrying to plug/unplug memory. */
245 struct hrtimer retry_timer;
246 unsigned int retry_timer_ms;
247#define VIRTIO_MEM_RETRY_TIMER_MIN_MS 50000
248#define VIRTIO_MEM_RETRY_TIMER_MAX_MS 300000
249
250 /* Memory notifier (online/offline events). */
251 struct notifier_block memory_notifier;
252
253#ifdef CONFIG_PROC_VMCORE
254 /* vmcore callback for /proc/vmcore handling in kdump mode */
255 struct vmcore_cb vmcore_cb;
256 uint64_t last_block_addr;
257 bool last_block_plugged;
258#endif /* CONFIG_PROC_VMCORE */
259
260 /* Next device in the list of virtio-mem devices. */
261 struct list_head next;
262};
263
264/*
265 * We have to share a single online_page callback among all virtio-mem
266 * devices. We use RCU to iterate the list in the callback.
267 */
268static DEFINE_MUTEX(virtio_mem_mutex);
269static LIST_HEAD(virtio_mem_devices);
270
271static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
272static void virtio_mem_fake_offline_going_offline(unsigned long pfn,
273 unsigned long nr_pages);
274static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn,
275 unsigned long nr_pages);
276static void virtio_mem_retry(struct virtio_mem *vm);
277static int virtio_mem_create_resource(struct virtio_mem *vm);
278static void virtio_mem_delete_resource(struct virtio_mem *vm);
279
280/*
281 * Register a virtio-mem device so it will be considered for the online_page
282 * callback.
283 */
284static int register_virtio_mem_device(struct virtio_mem *vm)
285{
286 int rc = 0;
287
288 /* First device registers the callback. */
289 mutex_lock(&virtio_mem_mutex);
290 if (list_empty(&virtio_mem_devices))
291 rc = set_online_page_callback(&virtio_mem_online_page_cb);
292 if (!rc)
293 list_add_rcu(&vm->next, &virtio_mem_devices);
294 mutex_unlock(&virtio_mem_mutex);
295
296 return rc;
297}
298
299/*
300 * Unregister a virtio-mem device so it will no longer be considered for the
301 * online_page callback.
302 */
303static void unregister_virtio_mem_device(struct virtio_mem *vm)
304{
305 /* Last device unregisters the callback. */
306 mutex_lock(&virtio_mem_mutex);
307 list_del_rcu(&vm->next);
308 if (list_empty(&virtio_mem_devices))
309 restore_online_page_callback(&virtio_mem_online_page_cb);
310 mutex_unlock(&virtio_mem_mutex);
311
312 synchronize_rcu();
313}
314
315/*
316 * Calculate the memory block id of a given address.
317 */
318static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
319{
320 return addr / memory_block_size_bytes();
321}
322
323/*
324 * Calculate the physical start address of a given memory block id.
325 */
326static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
327{
328 return mb_id * memory_block_size_bytes();
329}
330
331/*
332 * Calculate the big block id of a given address.
333 */
334static unsigned long virtio_mem_phys_to_bb_id(struct virtio_mem *vm,
335 uint64_t addr)
336{
337 return addr / vm->bbm.bb_size;
338}
339
340/*
341 * Calculate the physical start address of a given big block id.
342 */
343static uint64_t virtio_mem_bb_id_to_phys(struct virtio_mem *vm,
344 unsigned long bb_id)
345{
346 return bb_id * vm->bbm.bb_size;
347}
348
349/*
350 * Calculate the subblock id of a given address.
351 */
352static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
353 unsigned long addr)
354{
355 const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
356 const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
357
358 return (addr - mb_addr) / vm->sbm.sb_size;
359}
360
361/*
362 * Set the state of a big block, taking care of the state counter.
363 */
364static void virtio_mem_bbm_set_bb_state(struct virtio_mem *vm,
365 unsigned long bb_id,
366 enum virtio_mem_bbm_bb_state state)
367{
368 const unsigned long idx = bb_id - vm->bbm.first_bb_id;
369 enum virtio_mem_bbm_bb_state old_state;
370
371 old_state = vm->bbm.bb_states[idx];
372 vm->bbm.bb_states[idx] = state;
373
374 BUG_ON(vm->bbm.bb_count[old_state] == 0);
375 vm->bbm.bb_count[old_state]--;
376 vm->bbm.bb_count[state]++;
377}
378
379/*
380 * Get the state of a big block.
381 */
382static enum virtio_mem_bbm_bb_state virtio_mem_bbm_get_bb_state(struct virtio_mem *vm,
383 unsigned long bb_id)
384{
385 return vm->bbm.bb_states[bb_id - vm->bbm.first_bb_id];
386}
387
388/*
389 * Prepare the big block state array for the next big block.
390 */
391static int virtio_mem_bbm_bb_states_prepare_next_bb(struct virtio_mem *vm)
392{
393 unsigned long old_bytes = vm->bbm.next_bb_id - vm->bbm.first_bb_id;
394 unsigned long new_bytes = old_bytes + 1;
395 int old_pages = PFN_UP(old_bytes);
396 int new_pages = PFN_UP(new_bytes);
397 uint8_t *new_array;
398
399 if (vm->bbm.bb_states && old_pages == new_pages)
400 return 0;
401
402 new_array = vzalloc(new_pages * PAGE_SIZE);
403 if (!new_array)
404 return -ENOMEM;
405
406 mutex_lock(&vm->hotplug_mutex);
407 if (vm->bbm.bb_states)
408 memcpy(new_array, vm->bbm.bb_states, old_pages * PAGE_SIZE);
409 vfree(vm->bbm.bb_states);
410 vm->bbm.bb_states = new_array;
411 mutex_unlock(&vm->hotplug_mutex);
412
413 return 0;
414}
415
416#define virtio_mem_bbm_for_each_bb(_vm, _bb_id, _state) \
417 for (_bb_id = vm->bbm.first_bb_id; \
418 _bb_id < vm->bbm.next_bb_id && _vm->bbm.bb_count[_state]; \
419 _bb_id++) \
420 if (virtio_mem_bbm_get_bb_state(_vm, _bb_id) == _state)
421
422#define virtio_mem_bbm_for_each_bb_rev(_vm, _bb_id, _state) \
423 for (_bb_id = vm->bbm.next_bb_id - 1; \
424 _bb_id >= vm->bbm.first_bb_id && _vm->bbm.bb_count[_state]; \
425 _bb_id--) \
426 if (virtio_mem_bbm_get_bb_state(_vm, _bb_id) == _state)
427
428/*
429 * Set the state of a memory block, taking care of the state counter.
430 */
431static void virtio_mem_sbm_set_mb_state(struct virtio_mem *vm,
432 unsigned long mb_id, uint8_t state)
433{
434 const unsigned long idx = mb_id - vm->sbm.first_mb_id;
435 uint8_t old_state;
436
437 old_state = vm->sbm.mb_states[idx];
438 vm->sbm.mb_states[idx] = state;
439
440 BUG_ON(vm->sbm.mb_count[old_state] == 0);
441 vm->sbm.mb_count[old_state]--;
442 vm->sbm.mb_count[state]++;
443}
444
445/*
446 * Get the state of a memory block.
447 */
448static uint8_t virtio_mem_sbm_get_mb_state(struct virtio_mem *vm,
449 unsigned long mb_id)
450{
451 const unsigned long idx = mb_id - vm->sbm.first_mb_id;
452
453 return vm->sbm.mb_states[idx];
454}
455
456/*
457 * Prepare the state array for the next memory block.
458 */
459static int virtio_mem_sbm_mb_states_prepare_next_mb(struct virtio_mem *vm)
460{
461 int old_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id);
462 int new_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id + 1);
463 uint8_t *new_array;
464
465 if (vm->sbm.mb_states && old_pages == new_pages)
466 return 0;
467
468 new_array = vzalloc(new_pages * PAGE_SIZE);
469 if (!new_array)
470 return -ENOMEM;
471
472 mutex_lock(&vm->hotplug_mutex);
473 if (vm->sbm.mb_states)
474 memcpy(new_array, vm->sbm.mb_states, old_pages * PAGE_SIZE);
475 vfree(vm->sbm.mb_states);
476 vm->sbm.mb_states = new_array;
477 mutex_unlock(&vm->hotplug_mutex);
478
479 return 0;
480}
481
482#define virtio_mem_sbm_for_each_mb(_vm, _mb_id, _state) \
483 for (_mb_id = _vm->sbm.first_mb_id; \
484 _mb_id < _vm->sbm.next_mb_id && _vm->sbm.mb_count[_state]; \
485 _mb_id++) \
486 if (virtio_mem_sbm_get_mb_state(_vm, _mb_id) == _state)
487
488#define virtio_mem_sbm_for_each_mb_rev(_vm, _mb_id, _state) \
489 for (_mb_id = _vm->sbm.next_mb_id - 1; \
490 _mb_id >= _vm->sbm.first_mb_id && _vm->sbm.mb_count[_state]; \
491 _mb_id--) \
492 if (virtio_mem_sbm_get_mb_state(_vm, _mb_id) == _state)
493
494/*
495 * Calculate the bit number in the subblock bitmap for the given subblock
496 * inside the given memory block.
497 */
498static int virtio_mem_sbm_sb_state_bit_nr(struct virtio_mem *vm,
499 unsigned long mb_id, int sb_id)
500{
501 return (mb_id - vm->sbm.first_mb_id) * vm->sbm.sbs_per_mb + sb_id;
502}
503
504/*
505 * Mark all selected subblocks plugged.
506 *
507 * Will not modify the state of the memory block.
508 */
509static void virtio_mem_sbm_set_sb_plugged(struct virtio_mem *vm,
510 unsigned long mb_id, int sb_id,
511 int count)
512{
513 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
514
515 __bitmap_set(vm->sbm.sb_states, bit, count);
516}
517
518/*
519 * Mark all selected subblocks unplugged.
520 *
521 * Will not modify the state of the memory block.
522 */
523static void virtio_mem_sbm_set_sb_unplugged(struct virtio_mem *vm,
524 unsigned long mb_id, int sb_id,
525 int count)
526{
527 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
528
529 __bitmap_clear(vm->sbm.sb_states, bit, count);
530}
531
532/*
533 * Test if all selected subblocks are plugged.
534 */
535static bool virtio_mem_sbm_test_sb_plugged(struct virtio_mem *vm,
536 unsigned long mb_id, int sb_id,
537 int count)
538{
539 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
540
541 if (count == 1)
542 return test_bit(bit, vm->sbm.sb_states);
543
544 /* TODO: Helper similar to bitmap_set() */
545 return find_next_zero_bit(vm->sbm.sb_states, bit + count, bit) >=
546 bit + count;
547}
548
549/*
550 * Test if all selected subblocks are unplugged.
551 */
552static bool virtio_mem_sbm_test_sb_unplugged(struct virtio_mem *vm,
553 unsigned long mb_id, int sb_id,
554 int count)
555{
556 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id);
557
558 /* TODO: Helper similar to bitmap_set() */
559 return find_next_bit(vm->sbm.sb_states, bit + count, bit) >=
560 bit + count;
561}
562
563/*
564 * Find the first unplugged subblock. Returns vm->sbm.sbs_per_mb in case there is
565 * none.
566 */
567static int virtio_mem_sbm_first_unplugged_sb(struct virtio_mem *vm,
568 unsigned long mb_id)
569{
570 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, 0);
571
572 return find_next_zero_bit(vm->sbm.sb_states,
573 bit + vm->sbm.sbs_per_mb, bit) - bit;
574}
575
576/*
577 * Prepare the subblock bitmap for the next memory block.
578 */
579static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm)
580{
581 const unsigned long old_nb_mb = vm->sbm.next_mb_id - vm->sbm.first_mb_id;
582 const unsigned long old_nb_bits = old_nb_mb * vm->sbm.sbs_per_mb;
583 const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->sbm.sbs_per_mb;
584 int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
585 int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
586 unsigned long *new_bitmap, *old_bitmap;
587
588 if (vm->sbm.sb_states && old_pages == new_pages)
589 return 0;
590
591 new_bitmap = vzalloc(new_pages * PAGE_SIZE);
592 if (!new_bitmap)
593 return -ENOMEM;
594
595 mutex_lock(&vm->hotplug_mutex);
596 if (vm->sbm.sb_states)
597 memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE);
598
599 old_bitmap = vm->sbm.sb_states;
600 vm->sbm.sb_states = new_bitmap;
601 mutex_unlock(&vm->hotplug_mutex);
602
603 vfree(old_bitmap);
604 return 0;
605}
606
607/*
608 * Test if we could add memory without creating too much offline memory -
609 * to avoid running OOM if memory is getting onlined deferred.
610 */
611static bool virtio_mem_could_add_memory(struct virtio_mem *vm, uint64_t size)
612{
613 if (WARN_ON_ONCE(size > vm->offline_threshold))
614 return false;
615
616 return atomic64_read(&vm->offline_size) + size <= vm->offline_threshold;
617}
618
619/*
620 * Try adding memory to Linux. Will usually only fail if out of memory.
621 *
622 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
623 * onlining code).
624 *
625 * Will not modify the state of memory blocks in virtio-mem.
626 */
627static int virtio_mem_add_memory(struct virtio_mem *vm, uint64_t addr,
628 uint64_t size)
629{
630 int rc;
631
632 /*
633 * When force-unloading the driver and we still have memory added to
634 * Linux, the resource name has to stay.
635 */
636 if (!vm->resource_name) {
637 vm->resource_name = kstrdup_const("System RAM (virtio_mem)",
638 GFP_KERNEL);
639 if (!vm->resource_name)
640 return -ENOMEM;
641 }
642
643 dev_dbg(&vm->vdev->dev, "adding memory: 0x%llx - 0x%llx\n", addr,
644 addr + size - 1);
645 /* Memory might get onlined immediately. */
646 atomic64_add(size, &vm->offline_size);
647 rc = add_memory_driver_managed(vm->mgid, addr, size, vm->resource_name,
648 MHP_MERGE_RESOURCE | MHP_NID_IS_MGID);
649 if (rc) {
650 atomic64_sub(size, &vm->offline_size);
651 dev_warn(&vm->vdev->dev, "adding memory failed: %d\n", rc);
652 /*
653 * TODO: Linux MM does not properly clean up yet in all cases
654 * where adding of memory failed - especially on -ENOMEM.
655 */
656 }
657 return rc;
658}
659
660/*
661 * See virtio_mem_add_memory(): Try adding a single Linux memory block.
662 */
663static int virtio_mem_sbm_add_mb(struct virtio_mem *vm, unsigned long mb_id)
664{
665 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
666 const uint64_t size = memory_block_size_bytes();
667
668 return virtio_mem_add_memory(vm, addr, size);
669}
670
671/*
672 * See virtio_mem_add_memory(): Try adding a big block.
673 */
674static int virtio_mem_bbm_add_bb(struct virtio_mem *vm, unsigned long bb_id)
675{
676 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
677 const uint64_t size = vm->bbm.bb_size;
678
679 return virtio_mem_add_memory(vm, addr, size);
680}
681
682/*
683 * Try removing memory from Linux. Will only fail if memory blocks aren't
684 * offline.
685 *
686 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
687 * onlining code).
688 *
689 * Will not modify the state of memory blocks in virtio-mem.
690 */
691static int virtio_mem_remove_memory(struct virtio_mem *vm, uint64_t addr,
692 uint64_t size)
693{
694 int rc;
695
696 dev_dbg(&vm->vdev->dev, "removing memory: 0x%llx - 0x%llx\n", addr,
697 addr + size - 1);
698 rc = remove_memory(addr, size);
699 if (!rc) {
700 atomic64_sub(size, &vm->offline_size);
701 /*
702 * We might have freed up memory we can now unplug, retry
703 * immediately instead of waiting.
704 */
705 virtio_mem_retry(vm);
706 } else {
707 dev_dbg(&vm->vdev->dev, "removing memory failed: %d\n", rc);
708 }
709 return rc;
710}
711
712/*
713 * See virtio_mem_remove_memory(): Try removing a single Linux memory block.
714 */
715static int virtio_mem_sbm_remove_mb(struct virtio_mem *vm, unsigned long mb_id)
716{
717 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
718 const uint64_t size = memory_block_size_bytes();
719
720 return virtio_mem_remove_memory(vm, addr, size);
721}
722
723/*
724 * Try offlining and removing memory from Linux.
725 *
726 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
727 * onlining code).
728 *
729 * Will not modify the state of memory blocks in virtio-mem.
730 */
731static int virtio_mem_offline_and_remove_memory(struct virtio_mem *vm,
732 uint64_t addr,
733 uint64_t size)
734{
735 int rc;
736
737 dev_dbg(&vm->vdev->dev,
738 "offlining and removing memory: 0x%llx - 0x%llx\n", addr,
739 addr + size - 1);
740
741 rc = offline_and_remove_memory(addr, size);
742 if (!rc) {
743 atomic64_sub(size, &vm->offline_size);
744 /*
745 * We might have freed up memory we can now unplug, retry
746 * immediately instead of waiting.
747 */
748 virtio_mem_retry(vm);
749 } else {
750 dev_dbg(&vm->vdev->dev,
751 "offlining and removing memory failed: %d\n", rc);
752 }
753 return rc;
754}
755
756/*
757 * See virtio_mem_offline_and_remove_memory(): Try offlining and removing
758 * a single Linux memory block.
759 */
760static int virtio_mem_sbm_offline_and_remove_mb(struct virtio_mem *vm,
761 unsigned long mb_id)
762{
763 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
764 const uint64_t size = memory_block_size_bytes();
765
766 return virtio_mem_offline_and_remove_memory(vm, addr, size);
767}
768
769/*
770 * See virtio_mem_offline_and_remove_memory(): Try to offline and remove a
771 * all Linux memory blocks covered by the big block.
772 */
773static int virtio_mem_bbm_offline_and_remove_bb(struct virtio_mem *vm,
774 unsigned long bb_id)
775{
776 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
777 const uint64_t size = vm->bbm.bb_size;
778
779 return virtio_mem_offline_and_remove_memory(vm, addr, size);
780}
781
782/*
783 * Trigger the workqueue so the device can perform its magic.
784 */
785static void virtio_mem_retry(struct virtio_mem *vm)
786{
787 unsigned long flags;
788
789 spin_lock_irqsave(&vm->removal_lock, flags);
790 if (!vm->removing)
791 queue_work(system_freezable_wq, &vm->wq);
792 spin_unlock_irqrestore(&vm->removal_lock, flags);
793}
794
795static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
796{
797 int node = NUMA_NO_NODE;
798
799#if defined(CONFIG_ACPI_NUMA)
800 if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
801 node = pxm_to_node(node_id);
802#endif
803 return node;
804}
805
806/*
807 * Test if a virtio-mem device overlaps with the given range. Can be called
808 * from (notifier) callbacks lockless.
809 */
810static bool virtio_mem_overlaps_range(struct virtio_mem *vm, uint64_t start,
811 uint64_t size)
812{
813 return start < vm->addr + vm->region_size && vm->addr < start + size;
814}
815
816/*
817 * Test if a virtio-mem device contains a given range. Can be called from
818 * (notifier) callbacks lockless.
819 */
820static bool virtio_mem_contains_range(struct virtio_mem *vm, uint64_t start,
821 uint64_t size)
822{
823 return start >= vm->addr && start + size <= vm->addr + vm->region_size;
824}
825
826static int virtio_mem_sbm_notify_going_online(struct virtio_mem *vm,
827 unsigned long mb_id)
828{
829 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
830 case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
831 case VIRTIO_MEM_SBM_MB_OFFLINE:
832 return NOTIFY_OK;
833 default:
834 break;
835 }
836 dev_warn_ratelimited(&vm->vdev->dev,
837 "memory block onlining denied\n");
838 return NOTIFY_BAD;
839}
840
841static void virtio_mem_sbm_notify_offline(struct virtio_mem *vm,
842 unsigned long mb_id)
843{
844 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
845 case VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL:
846 case VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL:
847 virtio_mem_sbm_set_mb_state(vm, mb_id,
848 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
849 break;
850 case VIRTIO_MEM_SBM_MB_KERNEL:
851 case VIRTIO_MEM_SBM_MB_MOVABLE:
852 virtio_mem_sbm_set_mb_state(vm, mb_id,
853 VIRTIO_MEM_SBM_MB_OFFLINE);
854 break;
855 default:
856 BUG();
857 break;
858 }
859}
860
861static void virtio_mem_sbm_notify_online(struct virtio_mem *vm,
862 unsigned long mb_id,
863 unsigned long start_pfn)
864{
865 const bool is_movable = is_zone_movable_page(pfn_to_page(start_pfn));
866 int new_state;
867
868 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) {
869 case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
870 new_state = VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL;
871 if (is_movable)
872 new_state = VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL;
873 break;
874 case VIRTIO_MEM_SBM_MB_OFFLINE:
875 new_state = VIRTIO_MEM_SBM_MB_KERNEL;
876 if (is_movable)
877 new_state = VIRTIO_MEM_SBM_MB_MOVABLE;
878 break;
879 default:
880 BUG();
881 break;
882 }
883 virtio_mem_sbm_set_mb_state(vm, mb_id, new_state);
884}
885
886static void virtio_mem_sbm_notify_going_offline(struct virtio_mem *vm,
887 unsigned long mb_id)
888{
889 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size);
890 unsigned long pfn;
891 int sb_id;
892
893 for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) {
894 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
895 continue;
896 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
897 sb_id * vm->sbm.sb_size);
898 virtio_mem_fake_offline_going_offline(pfn, nr_pages);
899 }
900}
901
902static void virtio_mem_sbm_notify_cancel_offline(struct virtio_mem *vm,
903 unsigned long mb_id)
904{
905 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size);
906 unsigned long pfn;
907 int sb_id;
908
909 for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) {
910 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
911 continue;
912 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
913 sb_id * vm->sbm.sb_size);
914 virtio_mem_fake_offline_cancel_offline(pfn, nr_pages);
915 }
916}
917
918static void virtio_mem_bbm_notify_going_offline(struct virtio_mem *vm,
919 unsigned long bb_id,
920 unsigned long pfn,
921 unsigned long nr_pages)
922{
923 /*
924 * When marked as "fake-offline", all online memory of this device block
925 * is allocated by us. Otherwise, we don't have any memory allocated.
926 */
927 if (virtio_mem_bbm_get_bb_state(vm, bb_id) !=
928 VIRTIO_MEM_BBM_BB_FAKE_OFFLINE)
929 return;
930 virtio_mem_fake_offline_going_offline(pfn, nr_pages);
931}
932
933static void virtio_mem_bbm_notify_cancel_offline(struct virtio_mem *vm,
934 unsigned long bb_id,
935 unsigned long pfn,
936 unsigned long nr_pages)
937{
938 if (virtio_mem_bbm_get_bb_state(vm, bb_id) !=
939 VIRTIO_MEM_BBM_BB_FAKE_OFFLINE)
940 return;
941 virtio_mem_fake_offline_cancel_offline(pfn, nr_pages);
942}
943
944/*
945 * This callback will either be called synchronously from add_memory() or
946 * asynchronously (e.g., triggered via user space). We have to be careful
947 * with locking when calling add_memory().
948 */
949static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
950 unsigned long action, void *arg)
951{
952 struct virtio_mem *vm = container_of(nb, struct virtio_mem,
953 memory_notifier);
954 struct memory_notify *mhp = arg;
955 const unsigned long start = PFN_PHYS(mhp->start_pfn);
956 const unsigned long size = PFN_PHYS(mhp->nr_pages);
957 int rc = NOTIFY_OK;
958 unsigned long id;
959
960 if (!virtio_mem_overlaps_range(vm, start, size))
961 return NOTIFY_DONE;
962
963 if (vm->in_sbm) {
964 id = virtio_mem_phys_to_mb_id(start);
965 /*
966 * In SBM, we add memory in separate memory blocks - we expect
967 * it to be onlined/offlined in the same granularity. Bail out
968 * if this ever changes.
969 */
970 if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
971 !IS_ALIGNED(start, memory_block_size_bytes())))
972 return NOTIFY_BAD;
973 } else {
974 id = virtio_mem_phys_to_bb_id(vm, start);
975 /*
976 * In BBM, we only care about onlining/offlining happening
977 * within a single big block, we don't care about the
978 * actual granularity as we don't track individual Linux
979 * memory blocks.
980 */
981 if (WARN_ON_ONCE(id != virtio_mem_phys_to_bb_id(vm, start + size - 1)))
982 return NOTIFY_BAD;
983 }
984
985 /*
986 * Avoid circular locking lockdep warnings. We lock the mutex
987 * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
988 * blocking_notifier_call_chain() has it's own lock, which gets unlocked
989 * between both notifier calls and will bail out. False positive.
990 */
991 lockdep_off();
992
993 switch (action) {
994 case MEM_GOING_OFFLINE:
995 mutex_lock(&vm->hotplug_mutex);
996 if (vm->removing) {
997 rc = notifier_from_errno(-EBUSY);
998 mutex_unlock(&vm->hotplug_mutex);
999 break;
1000 }
1001 vm->hotplug_active = true;
1002 if (vm->in_sbm)
1003 virtio_mem_sbm_notify_going_offline(vm, id);
1004 else
1005 virtio_mem_bbm_notify_going_offline(vm, id,
1006 mhp->start_pfn,
1007 mhp->nr_pages);
1008 break;
1009 case MEM_GOING_ONLINE:
1010 mutex_lock(&vm->hotplug_mutex);
1011 if (vm->removing) {
1012 rc = notifier_from_errno(-EBUSY);
1013 mutex_unlock(&vm->hotplug_mutex);
1014 break;
1015 }
1016 vm->hotplug_active = true;
1017 if (vm->in_sbm)
1018 rc = virtio_mem_sbm_notify_going_online(vm, id);
1019 break;
1020 case MEM_OFFLINE:
1021 if (vm->in_sbm)
1022 virtio_mem_sbm_notify_offline(vm, id);
1023
1024 atomic64_add(size, &vm->offline_size);
1025 /*
1026 * Trigger the workqueue. Now that we have some offline memory,
1027 * maybe we can handle pending unplug requests.
1028 */
1029 if (!unplug_online)
1030 virtio_mem_retry(vm);
1031
1032 vm->hotplug_active = false;
1033 mutex_unlock(&vm->hotplug_mutex);
1034 break;
1035 case MEM_ONLINE:
1036 if (vm->in_sbm)
1037 virtio_mem_sbm_notify_online(vm, id, mhp->start_pfn);
1038
1039 atomic64_sub(size, &vm->offline_size);
1040 /*
1041 * Start adding more memory once we onlined half of our
1042 * threshold. Don't trigger if it's possibly due to our actipn
1043 * (e.g., us adding memory which gets onlined immediately from
1044 * the core).
1045 */
1046 if (!atomic_read(&vm->wq_active) &&
1047 virtio_mem_could_add_memory(vm, vm->offline_threshold / 2))
1048 virtio_mem_retry(vm);
1049
1050 vm->hotplug_active = false;
1051 mutex_unlock(&vm->hotplug_mutex);
1052 break;
1053 case MEM_CANCEL_OFFLINE:
1054 if (!vm->hotplug_active)
1055 break;
1056 if (vm->in_sbm)
1057 virtio_mem_sbm_notify_cancel_offline(vm, id);
1058 else
1059 virtio_mem_bbm_notify_cancel_offline(vm, id,
1060 mhp->start_pfn,
1061 mhp->nr_pages);
1062 vm->hotplug_active = false;
1063 mutex_unlock(&vm->hotplug_mutex);
1064 break;
1065 case MEM_CANCEL_ONLINE:
1066 if (!vm->hotplug_active)
1067 break;
1068 vm->hotplug_active = false;
1069 mutex_unlock(&vm->hotplug_mutex);
1070 break;
1071 default:
1072 break;
1073 }
1074
1075 lockdep_on();
1076
1077 return rc;
1078}
1079
1080/*
1081 * Set a range of pages PG_offline. Remember pages that were never onlined
1082 * (via generic_online_page()) using PageDirty().
1083 */
1084static void virtio_mem_set_fake_offline(unsigned long pfn,
1085 unsigned long nr_pages, bool onlined)
1086{
1087 page_offline_begin();
1088 for (; nr_pages--; pfn++) {
1089 struct page *page = pfn_to_page(pfn);
1090
1091 __SetPageOffline(page);
1092 if (!onlined) {
1093 SetPageDirty(page);
1094 /* FIXME: remove after cleanups */
1095 ClearPageReserved(page);
1096 }
1097 }
1098 page_offline_end();
1099}
1100
1101/*
1102 * Clear PG_offline from a range of pages. If the pages were never onlined,
1103 * (via generic_online_page()), clear PageDirty().
1104 */
1105static void virtio_mem_clear_fake_offline(unsigned long pfn,
1106 unsigned long nr_pages, bool onlined)
1107{
1108 for (; nr_pages--; pfn++) {
1109 struct page *page = pfn_to_page(pfn);
1110
1111 __ClearPageOffline(page);
1112 if (!onlined)
1113 ClearPageDirty(page);
1114 }
1115}
1116
1117/*
1118 * Release a range of fake-offline pages to the buddy, effectively
1119 * fake-onlining them.
1120 */
1121static void virtio_mem_fake_online(unsigned long pfn, unsigned long nr_pages)
1122{
1123 unsigned long order = MAX_ORDER - 1;
1124 unsigned long i;
1125
1126 /*
1127 * We might get called for ranges that don't cover properly aligned
1128 * MAX_ORDER - 1 pages; however, we can only online properly aligned
1129 * pages with an order of MAX_ORDER - 1 at maximum.
1130 */
1131 while (!IS_ALIGNED(pfn | nr_pages, 1 << order))
1132 order--;
1133
1134 for (i = 0; i < nr_pages; i += 1 << order) {
1135 struct page *page = pfn_to_page(pfn + i);
1136
1137 /*
1138 * If the page is PageDirty(), it was kept fake-offline when
1139 * onlining the memory block. Otherwise, it was allocated
1140 * using alloc_contig_range(). All pages in a subblock are
1141 * alike.
1142 */
1143 if (PageDirty(page)) {
1144 virtio_mem_clear_fake_offline(pfn + i, 1 << order, false);
1145 generic_online_page(page, order);
1146 } else {
1147 virtio_mem_clear_fake_offline(pfn + i, 1 << order, true);
1148 free_contig_range(pfn + i, 1 << order);
1149 adjust_managed_page_count(page, 1 << order);
1150 }
1151 }
1152}
1153
1154/*
1155 * Try to allocate a range, marking pages fake-offline, effectively
1156 * fake-offlining them.
1157 */
1158static int virtio_mem_fake_offline(unsigned long pfn, unsigned long nr_pages)
1159{
1160 const bool is_movable = is_zone_movable_page(pfn_to_page(pfn));
1161 int rc, retry_count;
1162
1163 /*
1164 * TODO: We want an alloc_contig_range() mode that tries to allocate
1165 * harder (e.g., dealing with temporarily pinned pages, PCP), especially
1166 * with ZONE_MOVABLE. So for now, retry a couple of times with
1167 * ZONE_MOVABLE before giving up - because that zone is supposed to give
1168 * some guarantees.
1169 */
1170 for (retry_count = 0; retry_count < 5; retry_count++) {
1171 rc = alloc_contig_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE,
1172 GFP_KERNEL);
1173 if (rc == -ENOMEM)
1174 /* whoops, out of memory */
1175 return rc;
1176 else if (rc && !is_movable)
1177 break;
1178 else if (rc)
1179 continue;
1180
1181 virtio_mem_set_fake_offline(pfn, nr_pages, true);
1182 adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
1183 return 0;
1184 }
1185
1186 return -EBUSY;
1187}
1188
1189/*
1190 * Handle fake-offline pages when memory is going offline - such that the
1191 * pages can be skipped by mm-core when offlining.
1192 */
1193static void virtio_mem_fake_offline_going_offline(unsigned long pfn,
1194 unsigned long nr_pages)
1195{
1196 struct page *page;
1197 unsigned long i;
1198
1199 /*
1200 * Drop our reference to the pages so the memory can get offlined
1201 * and add the unplugged pages to the managed page counters (so
1202 * offlining code can correctly subtract them again).
1203 */
1204 adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
1205 /* Drop our reference to the pages so the memory can get offlined. */
1206 for (i = 0; i < nr_pages; i++) {
1207 page = pfn_to_page(pfn + i);
1208 if (WARN_ON(!page_ref_dec_and_test(page)))
1209 dump_page(page, "fake-offline page referenced");
1210 }
1211}
1212
1213/*
1214 * Handle fake-offline pages when memory offlining is canceled - to undo
1215 * what we did in virtio_mem_fake_offline_going_offline().
1216 */
1217static void virtio_mem_fake_offline_cancel_offline(unsigned long pfn,
1218 unsigned long nr_pages)
1219{
1220 unsigned long i;
1221
1222 /*
1223 * Get the reference we dropped when going offline and subtract the
1224 * unplugged pages from the managed page counters.
1225 */
1226 adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
1227 for (i = 0; i < nr_pages; i++)
1228 page_ref_inc(pfn_to_page(pfn + i));
1229}
1230
1231static void virtio_mem_online_page(struct virtio_mem *vm,
1232 struct page *page, unsigned int order)
1233{
1234 const unsigned long start = page_to_phys(page);
1235 const unsigned long end = start + PFN_PHYS(1 << order);
1236 unsigned long addr, next, id, sb_id, count;
1237 bool do_online;
1238
1239 /*
1240 * We can get called with any order up to MAX_ORDER - 1. If our
1241 * subblock size is smaller than that and we have a mixture of plugged
1242 * and unplugged subblocks within such a page, we have to process in
1243 * smaller granularity. In that case we'll adjust the order exactly once
1244 * within the loop.
1245 */
1246 for (addr = start; addr < end; ) {
1247 next = addr + PFN_PHYS(1 << order);
1248
1249 if (vm->in_sbm) {
1250 id = virtio_mem_phys_to_mb_id(addr);
1251 sb_id = virtio_mem_phys_to_sb_id(vm, addr);
1252 count = virtio_mem_phys_to_sb_id(vm, next - 1) - sb_id + 1;
1253
1254 if (virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, count)) {
1255 /* Fully plugged. */
1256 do_online = true;
1257 } else if (count == 1 ||
1258 virtio_mem_sbm_test_sb_unplugged(vm, id, sb_id, count)) {
1259 /* Fully unplugged. */
1260 do_online = false;
1261 } else {
1262 /*
1263 * Mixture, process sub-blocks instead. This
1264 * will be at least the size of a pageblock.
1265 * We'll run into this case exactly once.
1266 */
1267 order = ilog2(vm->sbm.sb_size) - PAGE_SHIFT;
1268 do_online = virtio_mem_sbm_test_sb_plugged(vm, id, sb_id, 1);
1269 continue;
1270 }
1271 } else {
1272 /*
1273 * If the whole block is marked fake offline, keep
1274 * everything that way.
1275 */
1276 id = virtio_mem_phys_to_bb_id(vm, addr);
1277 do_online = virtio_mem_bbm_get_bb_state(vm, id) !=
1278 VIRTIO_MEM_BBM_BB_FAKE_OFFLINE;
1279 }
1280
1281 if (do_online)
1282 generic_online_page(pfn_to_page(PFN_DOWN(addr)), order);
1283 else
1284 virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
1285 false);
1286 addr = next;
1287 }
1288}
1289
1290static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
1291{
1292 const unsigned long addr = page_to_phys(page);
1293 struct virtio_mem *vm;
1294
1295 rcu_read_lock();
1296 list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
1297 /*
1298 * Pages we're onlining will never cross memory blocks and,
1299 * therefore, not virtio-mem devices.
1300 */
1301 if (!virtio_mem_contains_range(vm, addr, PFN_PHYS(1 << order)))
1302 continue;
1303
1304 /*
1305 * virtio_mem_set_fake_offline() might sleep. We can safely
1306 * drop the RCU lock at this point because the device
1307 * cannot go away. See virtio_mem_remove() how races
1308 * between memory onlining and device removal are handled.
1309 */
1310 rcu_read_unlock();
1311
1312 virtio_mem_online_page(vm, page, order);
1313 return;
1314 }
1315 rcu_read_unlock();
1316
1317 /* not virtio-mem memory, but e.g., a DIMM. online it */
1318 generic_online_page(page, order);
1319}
1320
1321static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
1322 const struct virtio_mem_req *req)
1323{
1324 struct scatterlist *sgs[2], sg_req, sg_resp;
1325 unsigned int len;
1326 int rc;
1327
1328 /* don't use the request residing on the stack (vaddr) */
1329 vm->req = *req;
1330
1331 /* out: buffer for request */
1332 sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
1333 sgs[0] = &sg_req;
1334
1335 /* in: buffer for response */
1336 sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
1337 sgs[1] = &sg_resp;
1338
1339 rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
1340 if (rc < 0)
1341 return rc;
1342
1343 virtqueue_kick(vm->vq);
1344
1345 /* wait for a response */
1346 wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
1347
1348 return virtio16_to_cpu(vm->vdev, vm->resp.type);
1349}
1350
1351static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
1352 uint64_t size)
1353{
1354 const uint64_t nb_vm_blocks = size / vm->device_block_size;
1355 const struct virtio_mem_req req = {
1356 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
1357 .u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
1358 .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
1359 };
1360 int rc = -ENOMEM;
1361
1362 if (atomic_read(&vm->config_changed))
1363 return -EAGAIN;
1364
1365 dev_dbg(&vm->vdev->dev, "plugging memory: 0x%llx - 0x%llx\n", addr,
1366 addr + size - 1);
1367
1368 switch (virtio_mem_send_request(vm, &req)) {
1369 case VIRTIO_MEM_RESP_ACK:
1370 vm->plugged_size += size;
1371 return 0;
1372 case VIRTIO_MEM_RESP_NACK:
1373 rc = -EAGAIN;
1374 break;
1375 case VIRTIO_MEM_RESP_BUSY:
1376 rc = -ETXTBSY;
1377 break;
1378 case VIRTIO_MEM_RESP_ERROR:
1379 rc = -EINVAL;
1380 break;
1381 default:
1382 break;
1383 }
1384
1385 dev_dbg(&vm->vdev->dev, "plugging memory failed: %d\n", rc);
1386 return rc;
1387}
1388
1389static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
1390 uint64_t size)
1391{
1392 const uint64_t nb_vm_blocks = size / vm->device_block_size;
1393 const struct virtio_mem_req req = {
1394 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
1395 .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
1396 .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
1397 };
1398 int rc = -ENOMEM;
1399
1400 if (atomic_read(&vm->config_changed))
1401 return -EAGAIN;
1402
1403 dev_dbg(&vm->vdev->dev, "unplugging memory: 0x%llx - 0x%llx\n", addr,
1404 addr + size - 1);
1405
1406 switch (virtio_mem_send_request(vm, &req)) {
1407 case VIRTIO_MEM_RESP_ACK:
1408 vm->plugged_size -= size;
1409 return 0;
1410 case VIRTIO_MEM_RESP_BUSY:
1411 rc = -ETXTBSY;
1412 break;
1413 case VIRTIO_MEM_RESP_ERROR:
1414 rc = -EINVAL;
1415 break;
1416 default:
1417 break;
1418 }
1419
1420 dev_dbg(&vm->vdev->dev, "unplugging memory failed: %d\n", rc);
1421 return rc;
1422}
1423
1424static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
1425{
1426 const struct virtio_mem_req req = {
1427 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
1428 };
1429 int rc = -ENOMEM;
1430
1431 dev_dbg(&vm->vdev->dev, "unplugging all memory");
1432
1433 switch (virtio_mem_send_request(vm, &req)) {
1434 case VIRTIO_MEM_RESP_ACK:
1435 vm->unplug_all_required = false;
1436 vm->plugged_size = 0;
1437 /* usable region might have shrunk */
1438 atomic_set(&vm->config_changed, 1);
1439 return 0;
1440 case VIRTIO_MEM_RESP_BUSY:
1441 rc = -ETXTBSY;
1442 break;
1443 default:
1444 break;
1445 }
1446
1447 dev_dbg(&vm->vdev->dev, "unplugging all memory failed: %d\n", rc);
1448 return rc;
1449}
1450
1451/*
1452 * Plug selected subblocks. Updates the plugged state, but not the state
1453 * of the memory block.
1454 */
1455static int virtio_mem_sbm_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
1456 int sb_id, int count)
1457{
1458 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
1459 sb_id * vm->sbm.sb_size;
1460 const uint64_t size = count * vm->sbm.sb_size;
1461 int rc;
1462
1463 rc = virtio_mem_send_plug_request(vm, addr, size);
1464 if (!rc)
1465 virtio_mem_sbm_set_sb_plugged(vm, mb_id, sb_id, count);
1466 return rc;
1467}
1468
1469/*
1470 * Unplug selected subblocks. Updates the plugged state, but not the state
1471 * of the memory block.
1472 */
1473static int virtio_mem_sbm_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
1474 int sb_id, int count)
1475{
1476 const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
1477 sb_id * vm->sbm.sb_size;
1478 const uint64_t size = count * vm->sbm.sb_size;
1479 int rc;
1480
1481 rc = virtio_mem_send_unplug_request(vm, addr, size);
1482 if (!rc)
1483 virtio_mem_sbm_set_sb_unplugged(vm, mb_id, sb_id, count);
1484 return rc;
1485}
1486
1487/*
1488 * Request to unplug a big block.
1489 *
1490 * Will not modify the state of the big block.
1491 */
1492static int virtio_mem_bbm_unplug_bb(struct virtio_mem *vm, unsigned long bb_id)
1493{
1494 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
1495 const uint64_t size = vm->bbm.bb_size;
1496
1497 return virtio_mem_send_unplug_request(vm, addr, size);
1498}
1499
1500/*
1501 * Request to plug a big block.
1502 *
1503 * Will not modify the state of the big block.
1504 */
1505static int virtio_mem_bbm_plug_bb(struct virtio_mem *vm, unsigned long bb_id)
1506{
1507 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id);
1508 const uint64_t size = vm->bbm.bb_size;
1509
1510 return virtio_mem_send_plug_request(vm, addr, size);
1511}
1512
1513/*
1514 * Unplug the desired number of plugged subblocks of a offline or not-added
1515 * memory block. Will fail if any subblock cannot get unplugged (instead of
1516 * skipping it).
1517 *
1518 * Will not modify the state of the memory block.
1519 *
1520 * Note: can fail after some subblocks were unplugged.
1521 */
1522static int virtio_mem_sbm_unplug_any_sb_raw(struct virtio_mem *vm,
1523 unsigned long mb_id, uint64_t *nb_sb)
1524{
1525 int sb_id, count;
1526 int rc;
1527
1528 sb_id = vm->sbm.sbs_per_mb - 1;
1529 while (*nb_sb) {
1530 /* Find the next candidate subblock */
1531 while (sb_id >= 0 &&
1532 virtio_mem_sbm_test_sb_unplugged(vm, mb_id, sb_id, 1))
1533 sb_id--;
1534 if (sb_id < 0)
1535 break;
1536 /* Try to unplug multiple subblocks at a time */
1537 count = 1;
1538 while (count < *nb_sb && sb_id > 0 &&
1539 virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
1540 count++;
1541 sb_id--;
1542 }
1543
1544 rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count);
1545 if (rc)
1546 return rc;
1547 *nb_sb -= count;
1548 sb_id--;
1549 }
1550
1551 return 0;
1552}
1553
1554/*
1555 * Unplug all plugged subblocks of an offline or not-added memory block.
1556 *
1557 * Will not modify the state of the memory block.
1558 *
1559 * Note: can fail after some subblocks were unplugged.
1560 */
1561static int virtio_mem_sbm_unplug_mb(struct virtio_mem *vm, unsigned long mb_id)
1562{
1563 uint64_t nb_sb = vm->sbm.sbs_per_mb;
1564
1565 return virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, &nb_sb);
1566}
1567
1568/*
1569 * Prepare tracking data for the next memory block.
1570 */
1571static int virtio_mem_sbm_prepare_next_mb(struct virtio_mem *vm,
1572 unsigned long *mb_id)
1573{
1574 int rc;
1575
1576 if (vm->sbm.next_mb_id > vm->sbm.last_usable_mb_id)
1577 return -ENOSPC;
1578
1579 /* Resize the state array if required. */
1580 rc = virtio_mem_sbm_mb_states_prepare_next_mb(vm);
1581 if (rc)
1582 return rc;
1583
1584 /* Resize the subblock bitmap if required. */
1585 rc = virtio_mem_sbm_sb_states_prepare_next_mb(vm);
1586 if (rc)
1587 return rc;
1588
1589 vm->sbm.mb_count[VIRTIO_MEM_SBM_MB_UNUSED]++;
1590 *mb_id = vm->sbm.next_mb_id++;
1591 return 0;
1592}
1593
1594/*
1595 * Try to plug the desired number of subblocks and add the memory block
1596 * to Linux.
1597 *
1598 * Will modify the state of the memory block.
1599 */
1600static int virtio_mem_sbm_plug_and_add_mb(struct virtio_mem *vm,
1601 unsigned long mb_id, uint64_t *nb_sb)
1602{
1603 const int count = min_t(int, *nb_sb, vm->sbm.sbs_per_mb);
1604 int rc;
1605
1606 if (WARN_ON_ONCE(!count))
1607 return -EINVAL;
1608
1609 /*
1610 * Plug the requested number of subblocks before adding it to linux,
1611 * so that onlining will directly online all plugged subblocks.
1612 */
1613 rc = virtio_mem_sbm_plug_sb(vm, mb_id, 0, count);
1614 if (rc)
1615 return rc;
1616
1617 /*
1618 * Mark the block properly offline before adding it to Linux,
1619 * so the memory notifiers will find the block in the right state.
1620 */
1621 if (count == vm->sbm.sbs_per_mb)
1622 virtio_mem_sbm_set_mb_state(vm, mb_id,
1623 VIRTIO_MEM_SBM_MB_OFFLINE);
1624 else
1625 virtio_mem_sbm_set_mb_state(vm, mb_id,
1626 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
1627
1628 /* Add the memory block to linux - if that fails, try to unplug. */
1629 rc = virtio_mem_sbm_add_mb(vm, mb_id);
1630 if (rc) {
1631 int new_state = VIRTIO_MEM_SBM_MB_UNUSED;
1632
1633 if (virtio_mem_sbm_unplug_sb(vm, mb_id, 0, count))
1634 new_state = VIRTIO_MEM_SBM_MB_PLUGGED;
1635 virtio_mem_sbm_set_mb_state(vm, mb_id, new_state);
1636 return rc;
1637 }
1638
1639 *nb_sb -= count;
1640 return 0;
1641}
1642
1643/*
1644 * Try to plug the desired number of subblocks of a memory block that
1645 * is already added to Linux.
1646 *
1647 * Will modify the state of the memory block.
1648 *
1649 * Note: Can fail after some subblocks were successfully plugged.
1650 */
1651static int virtio_mem_sbm_plug_any_sb(struct virtio_mem *vm,
1652 unsigned long mb_id, uint64_t *nb_sb)
1653{
1654 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id);
1655 unsigned long pfn, nr_pages;
1656 int sb_id, count;
1657 int rc;
1658
1659 if (WARN_ON_ONCE(!*nb_sb))
1660 return -EINVAL;
1661
1662 while (*nb_sb) {
1663 sb_id = virtio_mem_sbm_first_unplugged_sb(vm, mb_id);
1664 if (sb_id >= vm->sbm.sbs_per_mb)
1665 break;
1666 count = 1;
1667 while (count < *nb_sb &&
1668 sb_id + count < vm->sbm.sbs_per_mb &&
1669 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id + count, 1))
1670 count++;
1671
1672 rc = virtio_mem_sbm_plug_sb(vm, mb_id, sb_id, count);
1673 if (rc)
1674 return rc;
1675 *nb_sb -= count;
1676 if (old_state == VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL)
1677 continue;
1678
1679 /* fake-online the pages if the memory block is online */
1680 pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1681 sb_id * vm->sbm.sb_size);
1682 nr_pages = PFN_DOWN(count * vm->sbm.sb_size);
1683 virtio_mem_fake_online(pfn, nr_pages);
1684 }
1685
1686 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb))
1687 virtio_mem_sbm_set_mb_state(vm, mb_id, old_state - 1);
1688
1689 return 0;
1690}
1691
1692static int virtio_mem_sbm_plug_request(struct virtio_mem *vm, uint64_t diff)
1693{
1694 const int mb_states[] = {
1695 VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL,
1696 VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL,
1697 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL,
1698 };
1699 uint64_t nb_sb = diff / vm->sbm.sb_size;
1700 unsigned long mb_id;
1701 int rc, i;
1702
1703 if (!nb_sb)
1704 return 0;
1705
1706 /* Don't race with onlining/offlining */
1707 mutex_lock(&vm->hotplug_mutex);
1708
1709 for (i = 0; i < ARRAY_SIZE(mb_states); i++) {
1710 virtio_mem_sbm_for_each_mb(vm, mb_id, mb_states[i]) {
1711 rc = virtio_mem_sbm_plug_any_sb(vm, mb_id, &nb_sb);
1712 if (rc || !nb_sb)
1713 goto out_unlock;
1714 cond_resched();
1715 }
1716 }
1717
1718 /*
1719 * We won't be working on online/offline memory blocks from this point,
1720 * so we can't race with memory onlining/offlining. Drop the mutex.
1721 */
1722 mutex_unlock(&vm->hotplug_mutex);
1723
1724 /* Try to plug and add unused blocks */
1725 virtio_mem_sbm_for_each_mb(vm, mb_id, VIRTIO_MEM_SBM_MB_UNUSED) {
1726 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes()))
1727 return -ENOSPC;
1728
1729 rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb);
1730 if (rc || !nb_sb)
1731 return rc;
1732 cond_resched();
1733 }
1734
1735 /* Try to prepare, plug and add new blocks */
1736 while (nb_sb) {
1737 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes()))
1738 return -ENOSPC;
1739
1740 rc = virtio_mem_sbm_prepare_next_mb(vm, &mb_id);
1741 if (rc)
1742 return rc;
1743 rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb);
1744 if (rc)
1745 return rc;
1746 cond_resched();
1747 }
1748
1749 return 0;
1750out_unlock:
1751 mutex_unlock(&vm->hotplug_mutex);
1752 return rc;
1753}
1754
1755/*
1756 * Plug a big block and add it to Linux.
1757 *
1758 * Will modify the state of the big block.
1759 */
1760static int virtio_mem_bbm_plug_and_add_bb(struct virtio_mem *vm,
1761 unsigned long bb_id)
1762{
1763 int rc;
1764
1765 if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) !=
1766 VIRTIO_MEM_BBM_BB_UNUSED))
1767 return -EINVAL;
1768
1769 rc = virtio_mem_bbm_plug_bb(vm, bb_id);
1770 if (rc)
1771 return rc;
1772 virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED);
1773
1774 rc = virtio_mem_bbm_add_bb(vm, bb_id);
1775 if (rc) {
1776 if (!virtio_mem_bbm_unplug_bb(vm, bb_id))
1777 virtio_mem_bbm_set_bb_state(vm, bb_id,
1778 VIRTIO_MEM_BBM_BB_UNUSED);
1779 else
1780 /* Retry from the main loop. */
1781 virtio_mem_bbm_set_bb_state(vm, bb_id,
1782 VIRTIO_MEM_BBM_BB_PLUGGED);
1783 return rc;
1784 }
1785 return 0;
1786}
1787
1788/*
1789 * Prepare tracking data for the next big block.
1790 */
1791static int virtio_mem_bbm_prepare_next_bb(struct virtio_mem *vm,
1792 unsigned long *bb_id)
1793{
1794 int rc;
1795
1796 if (vm->bbm.next_bb_id > vm->bbm.last_usable_bb_id)
1797 return -ENOSPC;
1798
1799 /* Resize the big block state array if required. */
1800 rc = virtio_mem_bbm_bb_states_prepare_next_bb(vm);
1801 if (rc)
1802 return rc;
1803
1804 vm->bbm.bb_count[VIRTIO_MEM_BBM_BB_UNUSED]++;
1805 *bb_id = vm->bbm.next_bb_id;
1806 vm->bbm.next_bb_id++;
1807 return 0;
1808}
1809
1810static int virtio_mem_bbm_plug_request(struct virtio_mem *vm, uint64_t diff)
1811{
1812 uint64_t nb_bb = diff / vm->bbm.bb_size;
1813 unsigned long bb_id;
1814 int rc;
1815
1816 if (!nb_bb)
1817 return 0;
1818
1819 /* Try to plug and add unused big blocks */
1820 virtio_mem_bbm_for_each_bb(vm, bb_id, VIRTIO_MEM_BBM_BB_UNUSED) {
1821 if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size))
1822 return -ENOSPC;
1823
1824 rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id);
1825 if (!rc)
1826 nb_bb--;
1827 if (rc || !nb_bb)
1828 return rc;
1829 cond_resched();
1830 }
1831
1832 /* Try to prepare, plug and add new big blocks */
1833 while (nb_bb) {
1834 if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size))
1835 return -ENOSPC;
1836
1837 rc = virtio_mem_bbm_prepare_next_bb(vm, &bb_id);
1838 if (rc)
1839 return rc;
1840 rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id);
1841 if (!rc)
1842 nb_bb--;
1843 if (rc)
1844 return rc;
1845 cond_resched();
1846 }
1847
1848 return 0;
1849}
1850
1851/*
1852 * Try to plug the requested amount of memory.
1853 */
1854static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
1855{
1856 if (vm->in_sbm)
1857 return virtio_mem_sbm_plug_request(vm, diff);
1858 return virtio_mem_bbm_plug_request(vm, diff);
1859}
1860
1861/*
1862 * Unplug the desired number of plugged subblocks of an offline memory block.
1863 * Will fail if any subblock cannot get unplugged (instead of skipping it).
1864 *
1865 * Will modify the state of the memory block. Might temporarily drop the
1866 * hotplug_mutex.
1867 *
1868 * Note: Can fail after some subblocks were successfully unplugged.
1869 */
1870static int virtio_mem_sbm_unplug_any_sb_offline(struct virtio_mem *vm,
1871 unsigned long mb_id,
1872 uint64_t *nb_sb)
1873{
1874 int rc;
1875
1876 rc = virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, nb_sb);
1877
1878 /* some subblocks might have been unplugged even on failure */
1879 if (!virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb))
1880 virtio_mem_sbm_set_mb_state(vm, mb_id,
1881 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL);
1882 if (rc)
1883 return rc;
1884
1885 if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) {
1886 /*
1887 * Remove the block from Linux - this should never fail.
1888 * Hinder the block from getting onlined by marking it
1889 * unplugged. Temporarily drop the mutex, so
1890 * any pending GOING_ONLINE requests can be serviced/rejected.
1891 */
1892 virtio_mem_sbm_set_mb_state(vm, mb_id,
1893 VIRTIO_MEM_SBM_MB_UNUSED);
1894
1895 mutex_unlock(&vm->hotplug_mutex);
1896 rc = virtio_mem_sbm_remove_mb(vm, mb_id);
1897 BUG_ON(rc);
1898 mutex_lock(&vm->hotplug_mutex);
1899 }
1900 return 0;
1901}
1902
1903/*
1904 * Unplug the given plugged subblocks of an online memory block.
1905 *
1906 * Will modify the state of the memory block.
1907 */
1908static int virtio_mem_sbm_unplug_sb_online(struct virtio_mem *vm,
1909 unsigned long mb_id, int sb_id,
1910 int count)
1911{
1912 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size) * count;
1913 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id);
1914 unsigned long start_pfn;
1915 int rc;
1916
1917 start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1918 sb_id * vm->sbm.sb_size);
1919
1920 rc = virtio_mem_fake_offline(start_pfn, nr_pages);
1921 if (rc)
1922 return rc;
1923
1924 /* Try to unplug the allocated memory */
1925 rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count);
1926 if (rc) {
1927 /* Return the memory to the buddy. */
1928 virtio_mem_fake_online(start_pfn, nr_pages);
1929 return rc;
1930 }
1931
1932 switch (old_state) {
1933 case VIRTIO_MEM_SBM_MB_KERNEL:
1934 virtio_mem_sbm_set_mb_state(vm, mb_id,
1935 VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL);
1936 break;
1937 case VIRTIO_MEM_SBM_MB_MOVABLE:
1938 virtio_mem_sbm_set_mb_state(vm, mb_id,
1939 VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL);
1940 break;
1941 }
1942
1943 return 0;
1944}
1945
1946/*
1947 * Unplug the desired number of plugged subblocks of an online memory block.
1948 * Will skip subblock that are busy.
1949 *
1950 * Will modify the state of the memory block. Might temporarily drop the
1951 * hotplug_mutex.
1952 *
1953 * Note: Can fail after some subblocks were successfully unplugged. Can
1954 * return 0 even if subblocks were busy and could not get unplugged.
1955 */
1956static int virtio_mem_sbm_unplug_any_sb_online(struct virtio_mem *vm,
1957 unsigned long mb_id,
1958 uint64_t *nb_sb)
1959{
1960 int rc, sb_id;
1961
1962 /* If possible, try to unplug the complete block in one shot. */
1963 if (*nb_sb >= vm->sbm.sbs_per_mb &&
1964 virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) {
1965 rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, 0,
1966 vm->sbm.sbs_per_mb);
1967 if (!rc) {
1968 *nb_sb -= vm->sbm.sbs_per_mb;
1969 goto unplugged;
1970 } else if (rc != -EBUSY)
1971 return rc;
1972 }
1973
1974 /* Fallback to single subblocks. */
1975 for (sb_id = vm->sbm.sbs_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
1976 /* Find the next candidate subblock */
1977 while (sb_id >= 0 &&
1978 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1))
1979 sb_id--;
1980 if (sb_id < 0)
1981 break;
1982
1983 rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, sb_id, 1);
1984 if (rc == -EBUSY)
1985 continue;
1986 else if (rc)
1987 return rc;
1988 *nb_sb -= 1;
1989 }
1990
1991unplugged:
1992 /*
1993 * Once all subblocks of a memory block were unplugged, offline and
1994 * remove it. This will usually not fail, as no memory is in use
1995 * anymore - however some other notifiers might NACK the request.
1996 */
1997 if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) {
1998 mutex_unlock(&vm->hotplug_mutex);
1999 rc = virtio_mem_sbm_offline_and_remove_mb(vm, mb_id);
2000 mutex_lock(&vm->hotplug_mutex);
2001 if (!rc)
2002 virtio_mem_sbm_set_mb_state(vm, mb_id,
2003 VIRTIO_MEM_SBM_MB_UNUSED);
2004 }
2005
2006 return 0;
2007}
2008
2009/*
2010 * Unplug the desired number of plugged subblocks of a memory block that is
2011 * already added to Linux. Will skip subblock of online memory blocks that are
2012 * busy (by the OS). Will fail if any subblock that's not busy cannot get
2013 * unplugged.
2014 *
2015 * Will modify the state of the memory block. Might temporarily drop the
2016 * hotplug_mutex.
2017 *
2018 * Note: Can fail after some subblocks were successfully unplugged. Can
2019 * return 0 even if subblocks were busy and could not get unplugged.
2020 */
2021static int virtio_mem_sbm_unplug_any_sb(struct virtio_mem *vm,
2022 unsigned long mb_id,
2023 uint64_t *nb_sb)
2024{
2025 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id);
2026
2027 switch (old_state) {
2028 case VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL:
2029 case VIRTIO_MEM_SBM_MB_KERNEL:
2030 case VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL:
2031 case VIRTIO_MEM_SBM_MB_MOVABLE:
2032 return virtio_mem_sbm_unplug_any_sb_online(vm, mb_id, nb_sb);
2033 case VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL:
2034 case VIRTIO_MEM_SBM_MB_OFFLINE:
2035 return virtio_mem_sbm_unplug_any_sb_offline(vm, mb_id, nb_sb);
2036 }
2037 return -EINVAL;
2038}
2039
2040static int virtio_mem_sbm_unplug_request(struct virtio_mem *vm, uint64_t diff)
2041{
2042 const int mb_states[] = {
2043 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL,
2044 VIRTIO_MEM_SBM_MB_OFFLINE,
2045 VIRTIO_MEM_SBM_MB_MOVABLE_PARTIAL,
2046 VIRTIO_MEM_SBM_MB_KERNEL_PARTIAL,
2047 VIRTIO_MEM_SBM_MB_MOVABLE,
2048 VIRTIO_MEM_SBM_MB_KERNEL,
2049 };
2050 uint64_t nb_sb = diff / vm->sbm.sb_size;
2051 unsigned long mb_id;
2052 int rc, i;
2053
2054 if (!nb_sb)
2055 return 0;
2056
2057 /*
2058 * We'll drop the mutex a couple of times when it is safe to do so.
2059 * This might result in some blocks switching the state (online/offline)
2060 * and we could miss them in this run - we will retry again later.
2061 */
2062 mutex_lock(&vm->hotplug_mutex);
2063
2064 /*
2065 * We try unplug from partially plugged blocks first, to try removing
2066 * whole memory blocks along with metadata. We prioritize ZONE_MOVABLE
2067 * as it's more reliable to unplug memory and remove whole memory
2068 * blocks, and we don't want to trigger a zone imbalances by
2069 * accidentially removing too much kernel memory.
2070 */
2071 for (i = 0; i < ARRAY_SIZE(mb_states); i++) {
2072 virtio_mem_sbm_for_each_mb_rev(vm, mb_id, mb_states[i]) {
2073 rc = virtio_mem_sbm_unplug_any_sb(vm, mb_id, &nb_sb);
2074 if (rc || !nb_sb)
2075 goto out_unlock;
2076 mutex_unlock(&vm->hotplug_mutex);
2077 cond_resched();
2078 mutex_lock(&vm->hotplug_mutex);
2079 }
2080 if (!unplug_online && i == 1) {
2081 mutex_unlock(&vm->hotplug_mutex);
2082 return 0;
2083 }
2084 }
2085
2086 mutex_unlock(&vm->hotplug_mutex);
2087 return nb_sb ? -EBUSY : 0;
2088out_unlock:
2089 mutex_unlock(&vm->hotplug_mutex);
2090 return rc;
2091}
2092
2093/*
2094 * Try to offline and remove a big block from Linux and unplug it. Will fail
2095 * with -EBUSY if some memory is busy and cannot get unplugged.
2096 *
2097 * Will modify the state of the memory block. Might temporarily drop the
2098 * hotplug_mutex.
2099 */
2100static int virtio_mem_bbm_offline_remove_and_unplug_bb(struct virtio_mem *vm,
2101 unsigned long bb_id)
2102{
2103 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id));
2104 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size);
2105 unsigned long end_pfn = start_pfn + nr_pages;
2106 unsigned long pfn;
2107 struct page *page;
2108 int rc;
2109
2110 if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) !=
2111 VIRTIO_MEM_BBM_BB_ADDED))
2112 return -EINVAL;
2113
2114 if (bbm_safe_unplug) {
2115 /*
2116 * Start by fake-offlining all memory. Once we marked the device
2117 * block as fake-offline, all newly onlined memory will
2118 * automatically be kept fake-offline. Protect from concurrent
2119 * onlining/offlining until we have a consistent state.
2120 */
2121 mutex_lock(&vm->hotplug_mutex);
2122 virtio_mem_bbm_set_bb_state(vm, bb_id,
2123 VIRTIO_MEM_BBM_BB_FAKE_OFFLINE);
2124
2125 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
2126 page = pfn_to_online_page(pfn);
2127 if (!page)
2128 continue;
2129
2130 rc = virtio_mem_fake_offline(pfn, PAGES_PER_SECTION);
2131 if (rc) {
2132 end_pfn = pfn;
2133 goto rollback_safe_unplug;
2134 }
2135 }
2136 mutex_unlock(&vm->hotplug_mutex);
2137 }
2138
2139 rc = virtio_mem_bbm_offline_and_remove_bb(vm, bb_id);
2140 if (rc) {
2141 if (bbm_safe_unplug) {
2142 mutex_lock(&vm->hotplug_mutex);
2143 goto rollback_safe_unplug;
2144 }
2145 return rc;
2146 }
2147
2148 rc = virtio_mem_bbm_unplug_bb(vm, bb_id);
2149 if (rc)
2150 virtio_mem_bbm_set_bb_state(vm, bb_id,
2151 VIRTIO_MEM_BBM_BB_PLUGGED);
2152 else
2153 virtio_mem_bbm_set_bb_state(vm, bb_id,
2154 VIRTIO_MEM_BBM_BB_UNUSED);
2155 return rc;
2156
2157rollback_safe_unplug:
2158 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
2159 page = pfn_to_online_page(pfn);
2160 if (!page)
2161 continue;
2162 virtio_mem_fake_online(pfn, PAGES_PER_SECTION);
2163 }
2164 virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED);
2165 mutex_unlock(&vm->hotplug_mutex);
2166 return rc;
2167}
2168
2169/*
2170 * Test if a big block is completely offline.
2171 */
2172static bool virtio_mem_bbm_bb_is_offline(struct virtio_mem *vm,
2173 unsigned long bb_id)
2174{
2175 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id));
2176 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size);
2177 unsigned long pfn;
2178
2179 for (pfn = start_pfn; pfn < start_pfn + nr_pages;
2180 pfn += PAGES_PER_SECTION) {
2181 if (pfn_to_online_page(pfn))
2182 return false;
2183 }
2184
2185 return true;
2186}
2187
2188/*
2189 * Test if a big block is completely onlined to ZONE_MOVABLE (or offline).
2190 */
2191static bool virtio_mem_bbm_bb_is_movable(struct virtio_mem *vm,
2192 unsigned long bb_id)
2193{
2194 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id));
2195 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size);
2196 struct page *page;
2197 unsigned long pfn;
2198
2199 for (pfn = start_pfn; pfn < start_pfn + nr_pages;
2200 pfn += PAGES_PER_SECTION) {
2201 page = pfn_to_online_page(pfn);
2202 if (!page)
2203 continue;
2204 if (page_zonenum(page) != ZONE_MOVABLE)
2205 return false;
2206 }
2207
2208 return true;
2209}
2210
2211static int virtio_mem_bbm_unplug_request(struct virtio_mem *vm, uint64_t diff)
2212{
2213 uint64_t nb_bb = diff / vm->bbm.bb_size;
2214 uint64_t bb_id;
2215 int rc, i;
2216
2217 if (!nb_bb)
2218 return 0;
2219
2220 /*
2221 * Try to unplug big blocks. Similar to SBM, start with offline
2222 * big blocks.
2223 */
2224 for (i = 0; i < 3; i++) {
2225 virtio_mem_bbm_for_each_bb_rev(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED) {
2226 cond_resched();
2227
2228 /*
2229 * As we're holding no locks, these checks are racy,
2230 * but we don't care.
2231 */
2232 if (i == 0 && !virtio_mem_bbm_bb_is_offline(vm, bb_id))
2233 continue;
2234 if (i == 1 && !virtio_mem_bbm_bb_is_movable(vm, bb_id))
2235 continue;
2236 rc = virtio_mem_bbm_offline_remove_and_unplug_bb(vm, bb_id);
2237 if (rc == -EBUSY)
2238 continue;
2239 if (!rc)
2240 nb_bb--;
2241 if (rc || !nb_bb)
2242 return rc;
2243 }
2244 if (i == 0 && !unplug_online)
2245 return 0;
2246 }
2247
2248 return nb_bb ? -EBUSY : 0;
2249}
2250
2251/*
2252 * Try to unplug the requested amount of memory.
2253 */
2254static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
2255{
2256 if (vm->in_sbm)
2257 return virtio_mem_sbm_unplug_request(vm, diff);
2258 return virtio_mem_bbm_unplug_request(vm, diff);
2259}
2260
2261/*
2262 * Try to unplug all blocks that couldn't be unplugged before, for example,
2263 * because the hypervisor was busy.
2264 */
2265static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
2266{
2267 unsigned long id;
2268 int rc;
2269
2270 if (!vm->in_sbm) {
2271 virtio_mem_bbm_for_each_bb(vm, id,
2272 VIRTIO_MEM_BBM_BB_PLUGGED) {
2273 rc = virtio_mem_bbm_unplug_bb(vm, id);
2274 if (rc)
2275 return rc;
2276 virtio_mem_bbm_set_bb_state(vm, id,
2277 VIRTIO_MEM_BBM_BB_UNUSED);
2278 }
2279 return 0;
2280 }
2281
2282 virtio_mem_sbm_for_each_mb(vm, id, VIRTIO_MEM_SBM_MB_PLUGGED) {
2283 rc = virtio_mem_sbm_unplug_mb(vm, id);
2284 if (rc)
2285 return rc;
2286 virtio_mem_sbm_set_mb_state(vm, id,
2287 VIRTIO_MEM_SBM_MB_UNUSED);
2288 }
2289
2290 return 0;
2291}
2292
2293/*
2294 * Update all parts of the config that could have changed.
2295 */
2296static void virtio_mem_refresh_config(struct virtio_mem *vm)
2297{
2298 const struct range pluggable_range = mhp_get_pluggable_range(true);
2299 uint64_t new_plugged_size, usable_region_size, end_addr;
2300
2301 /* the plugged_size is just a reflection of what _we_ did previously */
2302 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
2303 &new_plugged_size);
2304 if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
2305 vm->plugged_size = new_plugged_size;
2306
2307 /* calculate the last usable memory block id */
2308 virtio_cread_le(vm->vdev, struct virtio_mem_config,
2309 usable_region_size, &usable_region_size);
2310 end_addr = min(vm->addr + usable_region_size - 1,
2311 pluggable_range.end);
2312
2313 if (vm->in_sbm) {
2314 vm->sbm.last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr);
2315 if (!IS_ALIGNED(end_addr + 1, memory_block_size_bytes()))
2316 vm->sbm.last_usable_mb_id--;
2317 } else {
2318 vm->bbm.last_usable_bb_id = virtio_mem_phys_to_bb_id(vm,
2319 end_addr);
2320 if (!IS_ALIGNED(end_addr + 1, vm->bbm.bb_size))
2321 vm->bbm.last_usable_bb_id--;
2322 }
2323 /*
2324 * If we cannot plug any of our device memory (e.g., nothing in the
2325 * usable region is addressable), the last usable memory block id will
2326 * be smaller than the first usable memory block id. We'll stop
2327 * attempting to add memory with -ENOSPC from our main loop.
2328 */
2329
2330 /* see if there is a request to change the size */
2331 virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size,
2332 &vm->requested_size);
2333
2334 dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
2335 dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
2336}
2337
2338/*
2339 * Workqueue function for handling plug/unplug requests and config updates.
2340 */
2341static void virtio_mem_run_wq(struct work_struct *work)
2342{
2343 struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
2344 uint64_t diff;
2345 int rc;
2346
2347 if (unlikely(vm->in_kdump)) {
2348 dev_warn_once(&vm->vdev->dev,
2349 "unexpected workqueue run in kdump kernel\n");
2350 return;
2351 }
2352
2353 hrtimer_cancel(&vm->retry_timer);
2354
2355 if (vm->broken)
2356 return;
2357
2358 atomic_set(&vm->wq_active, 1);
2359retry:
2360 rc = 0;
2361
2362 /* Make sure we start with a clean state if there are leftovers. */
2363 if (unlikely(vm->unplug_all_required))
2364 rc = virtio_mem_send_unplug_all_request(vm);
2365
2366 if (atomic_read(&vm->config_changed)) {
2367 atomic_set(&vm->config_changed, 0);
2368 virtio_mem_refresh_config(vm);
2369 }
2370
2371 /* Unplug any leftovers from previous runs */
2372 if (!rc)
2373 rc = virtio_mem_unplug_pending_mb(vm);
2374
2375 if (!rc && vm->requested_size != vm->plugged_size) {
2376 if (vm->requested_size > vm->plugged_size) {
2377 diff = vm->requested_size - vm->plugged_size;
2378 rc = virtio_mem_plug_request(vm, diff);
2379 } else {
2380 diff = vm->plugged_size - vm->requested_size;
2381 rc = virtio_mem_unplug_request(vm, diff);
2382 }
2383 }
2384
2385 switch (rc) {
2386 case 0:
2387 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
2388 break;
2389 case -ENOSPC:
2390 /*
2391 * We cannot add any more memory (alignment, physical limit)
2392 * or we have too many offline memory blocks.
2393 */
2394 break;
2395 case -ETXTBSY:
2396 /*
2397 * The hypervisor cannot process our request right now
2398 * (e.g., out of memory, migrating);
2399 */
2400 case -EBUSY:
2401 /*
2402 * We cannot free up any memory to unplug it (all plugged memory
2403 * is busy).
2404 */
2405 case -ENOMEM:
2406 /* Out of memory, try again later. */
2407 hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
2408 HRTIMER_MODE_REL);
2409 break;
2410 case -EAGAIN:
2411 /* Retry immediately (e.g., the config changed). */
2412 goto retry;
2413 default:
2414 /* Unknown error, mark as broken */
2415 dev_err(&vm->vdev->dev,
2416 "unknown error, marking device broken: %d\n", rc);
2417 vm->broken = true;
2418 }
2419
2420 atomic_set(&vm->wq_active, 0);
2421}
2422
2423static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
2424{
2425 struct virtio_mem *vm = container_of(timer, struct virtio_mem,
2426 retry_timer);
2427
2428 virtio_mem_retry(vm);
2429 vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
2430 VIRTIO_MEM_RETRY_TIMER_MAX_MS);
2431 return HRTIMER_NORESTART;
2432}
2433
2434static void virtio_mem_handle_response(struct virtqueue *vq)
2435{
2436 struct virtio_mem *vm = vq->vdev->priv;
2437
2438 wake_up(&vm->host_resp);
2439}
2440
2441static int virtio_mem_init_vq(struct virtio_mem *vm)
2442{
2443 struct virtqueue *vq;
2444
2445 vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
2446 "guest-request");
2447 if (IS_ERR(vq))
2448 return PTR_ERR(vq);
2449 vm->vq = vq;
2450
2451 return 0;
2452}
2453
2454static int virtio_mem_init_hotplug(struct virtio_mem *vm)
2455{
2456 const struct range pluggable_range = mhp_get_pluggable_range(true);
2457 uint64_t unit_pages, sb_size, addr;
2458 int rc;
2459
2460 /* bad device setup - warn only */
2461 if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
2462 dev_warn(&vm->vdev->dev,
2463 "The alignment of the physical start address can make some memory unusable.\n");
2464 if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
2465 dev_warn(&vm->vdev->dev,
2466 "The alignment of the physical end address can make some memory unusable.\n");
2467 if (vm->addr < pluggable_range.start ||
2468 vm->addr + vm->region_size - 1 > pluggable_range.end)
2469 dev_warn(&vm->vdev->dev,
2470 "Some device memory is not addressable/pluggable. This can make some memory unusable.\n");
2471
2472 /* Prepare the offline threshold - make sure we can add two blocks. */
2473 vm->offline_threshold = max_t(uint64_t, 2 * memory_block_size_bytes(),
2474 VIRTIO_MEM_DEFAULT_OFFLINE_THRESHOLD);
2475
2476 /*
2477 * alloc_contig_range() works reliably with pageblock
2478 * granularity on ZONE_NORMAL, use pageblock_nr_pages.
2479 */
2480 sb_size = PAGE_SIZE * pageblock_nr_pages;
2481 sb_size = max_t(uint64_t, vm->device_block_size, sb_size);
2482
2483 if (sb_size < memory_block_size_bytes() && !force_bbm) {
2484 /* SBM: At least two subblocks per Linux memory block. */
2485 vm->in_sbm = true;
2486 vm->sbm.sb_size = sb_size;
2487 vm->sbm.sbs_per_mb = memory_block_size_bytes() /
2488 vm->sbm.sb_size;
2489
2490 /* Round up to the next full memory block */
2491 addr = max_t(uint64_t, vm->addr, pluggable_range.start) +
2492 memory_block_size_bytes() - 1;
2493 vm->sbm.first_mb_id = virtio_mem_phys_to_mb_id(addr);
2494 vm->sbm.next_mb_id = vm->sbm.first_mb_id;
2495 } else {
2496 /* BBM: At least one Linux memory block. */
2497 vm->bbm.bb_size = max_t(uint64_t, vm->device_block_size,
2498 memory_block_size_bytes());
2499
2500 if (bbm_block_size) {
2501 if (!is_power_of_2(bbm_block_size)) {
2502 dev_warn(&vm->vdev->dev,
2503 "bbm_block_size is not a power of 2");
2504 } else if (bbm_block_size < vm->bbm.bb_size) {
2505 dev_warn(&vm->vdev->dev,
2506 "bbm_block_size is too small");
2507 } else {
2508 vm->bbm.bb_size = bbm_block_size;
2509 }
2510 }
2511
2512 /* Round up to the next aligned big block */
2513 addr = max_t(uint64_t, vm->addr, pluggable_range.start) +
2514 vm->bbm.bb_size - 1;
2515 vm->bbm.first_bb_id = virtio_mem_phys_to_bb_id(vm, addr);
2516 vm->bbm.next_bb_id = vm->bbm.first_bb_id;
2517
2518 /* Make sure we can add two big blocks. */
2519 vm->offline_threshold = max_t(uint64_t, 2 * vm->bbm.bb_size,
2520 vm->offline_threshold);
2521 }
2522
2523 dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
2524 memory_block_size_bytes());
2525 if (vm->in_sbm)
2526 dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
2527 (unsigned long long)vm->sbm.sb_size);
2528 else
2529 dev_info(&vm->vdev->dev, "big block size: 0x%llx",
2530 (unsigned long long)vm->bbm.bb_size);
2531
2532 /* create the parent resource for all memory */
2533 rc = virtio_mem_create_resource(vm);
2534 if (rc)
2535 return rc;
2536
2537 /* use a single dynamic memory group to cover the whole memory device */
2538 if (vm->in_sbm)
2539 unit_pages = PHYS_PFN(memory_block_size_bytes());
2540 else
2541 unit_pages = PHYS_PFN(vm->bbm.bb_size);
2542 rc = memory_group_register_dynamic(vm->nid, unit_pages);
2543 if (rc < 0)
2544 goto out_del_resource;
2545 vm->mgid = rc;
2546
2547 /*
2548 * If we still have memory plugged, we have to unplug all memory first.
2549 * Registering our parent resource makes sure that this memory isn't
2550 * actually in use (e.g., trying to reload the driver).
2551 */
2552 if (vm->plugged_size) {
2553 vm->unplug_all_required = true;
2554 dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
2555 }
2556
2557 /* register callbacks */
2558 vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
2559 rc = register_memory_notifier(&vm->memory_notifier);
2560 if (rc)
2561 goto out_unreg_group;
2562 rc = register_virtio_mem_device(vm);
2563 if (rc)
2564 goto out_unreg_mem;
2565
2566 return 0;
2567out_unreg_mem:
2568 unregister_memory_notifier(&vm->memory_notifier);
2569out_unreg_group:
2570 memory_group_unregister(vm->mgid);
2571out_del_resource:
2572 virtio_mem_delete_resource(vm);
2573 return rc;
2574}
2575
2576#ifdef CONFIG_PROC_VMCORE
2577static int virtio_mem_send_state_request(struct virtio_mem *vm, uint64_t addr,
2578 uint64_t size)
2579{
2580 const uint64_t nb_vm_blocks = size / vm->device_block_size;
2581 const struct virtio_mem_req req = {
2582 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_STATE),
2583 .u.state.addr = cpu_to_virtio64(vm->vdev, addr),
2584 .u.state.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
2585 };
2586 int rc = -ENOMEM;
2587
2588 dev_dbg(&vm->vdev->dev, "requesting state: 0x%llx - 0x%llx\n", addr,
2589 addr + size - 1);
2590
2591 switch (virtio_mem_send_request(vm, &req)) {
2592 case VIRTIO_MEM_RESP_ACK:
2593 return virtio16_to_cpu(vm->vdev, vm->resp.u.state.state);
2594 case VIRTIO_MEM_RESP_ERROR:
2595 rc = -EINVAL;
2596 break;
2597 default:
2598 break;
2599 }
2600
2601 dev_dbg(&vm->vdev->dev, "requesting state failed: %d\n", rc);
2602 return rc;
2603}
2604
2605static bool virtio_mem_vmcore_pfn_is_ram(struct vmcore_cb *cb,
2606 unsigned long pfn)
2607{
2608 struct virtio_mem *vm = container_of(cb, struct virtio_mem,
2609 vmcore_cb);
2610 uint64_t addr = PFN_PHYS(pfn);
2611 bool is_ram;
2612 int rc;
2613
2614 if (!virtio_mem_contains_range(vm, addr, PAGE_SIZE))
2615 return true;
2616 if (!vm->plugged_size)
2617 return false;
2618
2619 /*
2620 * We have to serialize device requests and access to the information
2621 * about the block queried last.
2622 */
2623 mutex_lock(&vm->hotplug_mutex);
2624
2625 addr = ALIGN_DOWN(addr, vm->device_block_size);
2626 if (addr != vm->last_block_addr) {
2627 rc = virtio_mem_send_state_request(vm, addr,
2628 vm->device_block_size);
2629 /* On any kind of error, we're going to signal !ram. */
2630 if (rc == VIRTIO_MEM_STATE_PLUGGED)
2631 vm->last_block_plugged = true;
2632 else
2633 vm->last_block_plugged = false;
2634 vm->last_block_addr = addr;
2635 }
2636
2637 is_ram = vm->last_block_plugged;
2638 mutex_unlock(&vm->hotplug_mutex);
2639 return is_ram;
2640}
2641#endif /* CONFIG_PROC_VMCORE */
2642
2643static int virtio_mem_init_kdump(struct virtio_mem *vm)
2644{
2645#ifdef CONFIG_PROC_VMCORE
2646 dev_info(&vm->vdev->dev, "memory hot(un)plug disabled in kdump kernel\n");
2647 vm->vmcore_cb.pfn_is_ram = virtio_mem_vmcore_pfn_is_ram;
2648 register_vmcore_cb(&vm->vmcore_cb);
2649 return 0;
2650#else /* CONFIG_PROC_VMCORE */
2651 dev_warn(&vm->vdev->dev, "disabled in kdump kernel without vmcore\n");
2652 return -EBUSY;
2653#endif /* CONFIG_PROC_VMCORE */
2654}
2655
2656static int virtio_mem_init(struct virtio_mem *vm)
2657{
2658 uint16_t node_id;
2659
2660 if (!vm->vdev->config->get) {
2661 dev_err(&vm->vdev->dev, "config access disabled\n");
2662 return -EINVAL;
2663 }
2664
2665 /* Fetch all properties that can't change. */
2666 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
2667 &vm->plugged_size);
2668 virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size,
2669 &vm->device_block_size);
2670 virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id,
2671 &node_id);
2672 vm->nid = virtio_mem_translate_node_id(vm, node_id);
2673 virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
2674 virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size,
2675 &vm->region_size);
2676
2677 /* Determine the nid for the device based on the lowest address. */
2678 if (vm->nid == NUMA_NO_NODE)
2679 vm->nid = memory_add_physaddr_to_nid(vm->addr);
2680
2681 dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
2682 dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
2683 dev_info(&vm->vdev->dev, "device block size: 0x%llx",
2684 (unsigned long long)vm->device_block_size);
2685 if (vm->nid != NUMA_NO_NODE && IS_ENABLED(CONFIG_NUMA))
2686 dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
2687
2688 /*
2689 * We don't want to (un)plug or reuse any memory when in kdump. The
2690 * memory is still accessible (but not exposed to Linux).
2691 */
2692 if (vm->in_kdump)
2693 return virtio_mem_init_kdump(vm);
2694 return virtio_mem_init_hotplug(vm);
2695}
2696
2697static int virtio_mem_create_resource(struct virtio_mem *vm)
2698{
2699 /*
2700 * When force-unloading the driver and removing the device, we
2701 * could have a garbage pointer. Duplicate the string.
2702 */
2703 const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
2704
2705 if (!name)
2706 return -ENOMEM;
2707
2708 /* Disallow mapping device memory via /dev/mem completely. */
2709 vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
2710 name, IORESOURCE_SYSTEM_RAM |
2711 IORESOURCE_EXCLUSIVE);
2712 if (!vm->parent_resource) {
2713 kfree(name);
2714 dev_warn(&vm->vdev->dev, "could not reserve device region\n");
2715 dev_info(&vm->vdev->dev,
2716 "reloading the driver is not supported\n");
2717 return -EBUSY;
2718 }
2719
2720 /* The memory is not actually busy - make add_memory() work. */
2721 vm->parent_resource->flags &= ~IORESOURCE_BUSY;
2722 return 0;
2723}
2724
2725static void virtio_mem_delete_resource(struct virtio_mem *vm)
2726{
2727 const char *name;
2728
2729 if (!vm->parent_resource)
2730 return;
2731
2732 name = vm->parent_resource->name;
2733 release_resource(vm->parent_resource);
2734 kfree(vm->parent_resource);
2735 kfree(name);
2736 vm->parent_resource = NULL;
2737}
2738
2739static int virtio_mem_range_has_system_ram(struct resource *res, void *arg)
2740{
2741 return 1;
2742}
2743
2744static bool virtio_mem_has_memory_added(struct virtio_mem *vm)
2745{
2746 const unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
2747
2748 return walk_iomem_res_desc(IORES_DESC_NONE, flags, vm->addr,
2749 vm->addr + vm->region_size, NULL,
2750 virtio_mem_range_has_system_ram) == 1;
2751}
2752
2753static int virtio_mem_probe(struct virtio_device *vdev)
2754{
2755 struct virtio_mem *vm;
2756 int rc;
2757
2758 BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
2759 BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
2760
2761 vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2762 if (!vm)
2763 return -ENOMEM;
2764
2765 init_waitqueue_head(&vm->host_resp);
2766 vm->vdev = vdev;
2767 INIT_WORK(&vm->wq, virtio_mem_run_wq);
2768 mutex_init(&vm->hotplug_mutex);
2769 INIT_LIST_HEAD(&vm->next);
2770 spin_lock_init(&vm->removal_lock);
2771 hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2772 vm->retry_timer.function = virtio_mem_timer_expired;
2773 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
2774 vm->in_kdump = is_kdump_kernel();
2775
2776 /* register the virtqueue */
2777 rc = virtio_mem_init_vq(vm);
2778 if (rc)
2779 goto out_free_vm;
2780
2781 /* initialize the device by querying the config */
2782 rc = virtio_mem_init(vm);
2783 if (rc)
2784 goto out_del_vq;
2785
2786 virtio_device_ready(vdev);
2787
2788 /* trigger a config update to start processing the requested_size */
2789 if (!vm->in_kdump) {
2790 atomic_set(&vm->config_changed, 1);
2791 queue_work(system_freezable_wq, &vm->wq);
2792 }
2793
2794 return 0;
2795out_del_vq:
2796 vdev->config->del_vqs(vdev);
2797out_free_vm:
2798 kfree(vm);
2799 vdev->priv = NULL;
2800
2801 return rc;
2802}
2803
2804static void virtio_mem_deinit_hotplug(struct virtio_mem *vm)
2805{
2806 unsigned long mb_id;
2807 int rc;
2808
2809 /*
2810 * Make sure the workqueue won't be triggered anymore and no memory
2811 * blocks can be onlined/offlined until we're finished here.
2812 */
2813 mutex_lock(&vm->hotplug_mutex);
2814 spin_lock_irq(&vm->removal_lock);
2815 vm->removing = true;
2816 spin_unlock_irq(&vm->removal_lock);
2817 mutex_unlock(&vm->hotplug_mutex);
2818
2819 /* wait until the workqueue stopped */
2820 cancel_work_sync(&vm->wq);
2821 hrtimer_cancel(&vm->retry_timer);
2822
2823 if (vm->in_sbm) {
2824 /*
2825 * After we unregistered our callbacks, user space can online
2826 * partially plugged offline blocks. Make sure to remove them.
2827 */
2828 virtio_mem_sbm_for_each_mb(vm, mb_id,
2829 VIRTIO_MEM_SBM_MB_OFFLINE_PARTIAL) {
2830 rc = virtio_mem_sbm_remove_mb(vm, mb_id);
2831 BUG_ON(rc);
2832 virtio_mem_sbm_set_mb_state(vm, mb_id,
2833 VIRTIO_MEM_SBM_MB_UNUSED);
2834 }
2835 /*
2836 * After we unregistered our callbacks, user space can no longer
2837 * offline partially plugged online memory blocks. No need to
2838 * worry about them.
2839 */
2840 }
2841
2842 /* unregister callbacks */
2843 unregister_virtio_mem_device(vm);
2844 unregister_memory_notifier(&vm->memory_notifier);
2845
2846 /*
2847 * There is no way we could reliably remove all memory we have added to
2848 * the system. And there is no way to stop the driver/device from going
2849 * away. Warn at least.
2850 */
2851 if (virtio_mem_has_memory_added(vm)) {
2852 dev_warn(&vm->vdev->dev,
2853 "device still has system memory added\n");
2854 } else {
2855 virtio_mem_delete_resource(vm);
2856 kfree_const(vm->resource_name);
2857 memory_group_unregister(vm->mgid);
2858 }
2859
2860 /* remove all tracking data - no locking needed */
2861 if (vm->in_sbm) {
2862 vfree(vm->sbm.mb_states);
2863 vfree(vm->sbm.sb_states);
2864 } else {
2865 vfree(vm->bbm.bb_states);
2866 }
2867}
2868
2869static void virtio_mem_deinit_kdump(struct virtio_mem *vm)
2870{
2871#ifdef CONFIG_PROC_VMCORE
2872 unregister_vmcore_cb(&vm->vmcore_cb);
2873#endif /* CONFIG_PROC_VMCORE */
2874}
2875
2876static void virtio_mem_remove(struct virtio_device *vdev)
2877{
2878 struct virtio_mem *vm = vdev->priv;
2879
2880 if (vm->in_kdump)
2881 virtio_mem_deinit_kdump(vm);
2882 else
2883 virtio_mem_deinit_hotplug(vm);
2884
2885 /* reset the device and cleanup the queues */
2886 virtio_reset_device(vdev);
2887 vdev->config->del_vqs(vdev);
2888
2889 kfree(vm);
2890 vdev->priv = NULL;
2891}
2892
2893static void virtio_mem_config_changed(struct virtio_device *vdev)
2894{
2895 struct virtio_mem *vm = vdev->priv;
2896
2897 if (unlikely(vm->in_kdump))
2898 return;
2899
2900 atomic_set(&vm->config_changed, 1);
2901 virtio_mem_retry(vm);
2902}
2903
2904#ifdef CONFIG_PM_SLEEP
2905static int virtio_mem_freeze(struct virtio_device *vdev)
2906{
2907 /*
2908 * When restarting the VM, all memory is usually unplugged. Don't
2909 * allow to suspend/hibernate.
2910 */
2911 dev_err(&vdev->dev, "save/restore not supported.\n");
2912 return -EPERM;
2913}
2914
2915static int virtio_mem_restore(struct virtio_device *vdev)
2916{
2917 return -EPERM;
2918}
2919#endif
2920
2921static unsigned int virtio_mem_features[] = {
2922#if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
2923 VIRTIO_MEM_F_ACPI_PXM,
2924#endif
2925 VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE,
2926};
2927
2928static const struct virtio_device_id virtio_mem_id_table[] = {
2929 { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
2930 { 0 },
2931};
2932
2933static struct virtio_driver virtio_mem_driver = {
2934 .feature_table = virtio_mem_features,
2935 .feature_table_size = ARRAY_SIZE(virtio_mem_features),
2936 .driver.name = KBUILD_MODNAME,
2937 .driver.owner = THIS_MODULE,
2938 .id_table = virtio_mem_id_table,
2939 .probe = virtio_mem_probe,
2940 .remove = virtio_mem_remove,
2941 .config_changed = virtio_mem_config_changed,
2942#ifdef CONFIG_PM_SLEEP
2943 .freeze = virtio_mem_freeze,
2944 .restore = virtio_mem_restore,
2945#endif
2946};
2947
2948module_virtio_driver(virtio_mem_driver);
2949MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
2950MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
2951MODULE_DESCRIPTION("Virtio-mem driver");
2952MODULE_LICENSE("GPL");