1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Virtio-mem device driver.
   4 *
   5 * Copyright Red Hat, Inc. 2020
   6 *
   7 * Author(s): David Hildenbrand <david@redhat.com>
   8 */
   9
  10#include <linux/virtio.h>
  11#include <linux/virtio_mem.h>
  12#include <linux/workqueue.h>
  13#include <linux/slab.h>
  14#include <linux/module.h>
  15#include <linux/mm.h>
  16#include <linux/memory_hotplug.h>
  17#include <linux/memory.h>
  18#include <linux/hrtimer.h>
  19#include <linux/crash_dump.h>
  20#include <linux/mutex.h>
  21#include <linux/bitmap.h>
  22#include <linux/lockdep.h>
 
  23
  24#include <acpi/acpi_numa.h>
  25
  26static bool unplug_online = true;
  27module_param(unplug_online, bool, 0644);
  28MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
  29
  30enum virtio_mem_mb_state {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  31	/* Unplugged, not added to Linux. Can be reused later. */
  32	VIRTIO_MEM_MB_STATE_UNUSED = 0,
  33	/* (Partially) plugged, not added to Linux. Error on add_memory(). */
  34	VIRTIO_MEM_MB_STATE_PLUGGED,
  35	/* Fully plugged, fully added to Linux, offline. */
  36	VIRTIO_MEM_MB_STATE_OFFLINE,
  37	/* Partially plugged, fully added to Linux, offline. */
  38	VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL,
  39	/* Fully plugged, fully added to Linux, online (!ZONE_MOVABLE). */
  40	VIRTIO_MEM_MB_STATE_ONLINE,
  41	/* Partially plugged, fully added to Linux, online (!ZONE_MOVABLE). */
  42	VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL,
  43	/*
  44	 * Fully plugged, fully added to Linux, online (ZONE_MOVABLE).
  45	 * We are not allowed to allocate (unplug) parts of this block that
  46	 * are not movable (similar to gigantic pages). We will never allow
  47	 * to online OFFLINE_PARTIAL to ZONE_MOVABLE (as they would contain
  48	 * unmovable parts).
  49	 */
  50	VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE,
  51	VIRTIO_MEM_MB_STATE_COUNT
 
 
 
 
 
 
 
 
 
 
 
  52};
  53
  54struct virtio_mem {
  55	struct virtio_device *vdev;
  56
  57	/* We might first have to unplug all memory when starting up. */
  58	bool unplug_all_required;
  59
  60	/* Workqueue that processes the plug/unplug requests. */
  61	struct work_struct wq;
 
  62	atomic_t config_changed;
  63
  64	/* Virtqueue for guest->host requests. */
  65	struct virtqueue *vq;
  66
  67	/* Wait for a host response to a guest request. */
  68	wait_queue_head_t host_resp;
  69
  70	/* Space for one guest request and the host response. */
  71	struct virtio_mem_req req;
  72	struct virtio_mem_resp resp;
  73
  74	/* The current size of the device. */
  75	uint64_t plugged_size;
  76	/* The requested size of the device. */
  77	uint64_t requested_size;
  78
  79	/* The device block size (for communicating with the device). */
  80	uint64_t device_block_size;
  81	/* The translated node id. NUMA_NO_NODE in case not specified. */
  82	int nid;
  83	/* Physical start address of the memory region. */
  84	uint64_t addr;
  85	/* Maximum region size in bytes. */
  86	uint64_t region_size;
  87
  88	/* The subblock size. */
  89	uint64_t subblock_size;
  90	/* The number of subblocks per memory block. */
  91	uint32_t nb_sb_per_mb;
  92
  93	/* Id of the first memory block of this device. */
  94	unsigned long first_mb_id;
  95	/* Id of the last memory block of this device. */
  96	unsigned long last_mb_id;
  97	/* Id of the last usable memory block of this device. */
  98	unsigned long last_usable_mb_id;
  99	/* Id of the next memory bock to prepare when needed. */
 100	unsigned long next_mb_id;
 101
 102	/* The parent resource for all memory added via this device. */
 103	struct resource *parent_resource;
 104	/*
 105	 * Copy of "System RAM (virtio_mem)" to be used for
 106	 * add_memory_driver_managed().
 107	 */
 108	const char *resource_name;
 109
 110	/* Summary of all memory block states. */
 111	unsigned long nb_mb_state[VIRTIO_MEM_MB_STATE_COUNT];
 112#define VIRTIO_MEM_NB_OFFLINE_THRESHOLD		10
 113
 114	/*
 115	 * One byte state per memory block.
 116	 *
 117	 * Allocated via vmalloc(). When preparing new blocks, resized
 118	 * (alloc+copy+free) when needed (crossing pages with the next mb).
 119	 * (when crossing pages).
 120	 *
 121	 * With 128MB memory blocks, we have states for 512GB of memory in one
 122	 * page.
 123	 */
 124	uint8_t *mb_state;
 125
 126	/*
 127	 * $nb_sb_per_mb bit per memory block. Handled similar to mb_state.
 128	 *
 129	 * With 4MB subblocks, we manage 128GB of memory in one page.
 130	 */
 131	unsigned long *sb_bitmap;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 132
 133	/*
 134	 * Mutex that protects the nb_mb_state, mb_state, and sb_bitmap.
 
 135	 *
 136	 * When this lock is held the pointers can't change, ONLINE and
 137	 * OFFLINE blocks can't change the state and no subblocks will get
 138	 * plugged/unplugged.
 
 
 
 139	 */
 140	struct mutex hotplug_mutex;
 141	bool hotplug_active;
 142
 143	/* An error occurred we cannot handle - stop processing requests. */
 144	bool broken;
 145
 
 
 
 146	/* The driver is being removed. */
 147	spinlock_t removal_lock;
 148	bool removing;
 149
 150	/* Timer for retrying to plug/unplug memory. */
 151	struct hrtimer retry_timer;
 152	unsigned int retry_timer_ms;
 153#define VIRTIO_MEM_RETRY_TIMER_MIN_MS		50000
 154#define VIRTIO_MEM_RETRY_TIMER_MAX_MS		300000
 155
 156	/* Memory notifier (online/offline events). */
 157	struct notifier_block memory_notifier;
 158
 
 
 
 
 
 
 
 159	/* Next device in the list of virtio-mem devices. */
 160	struct list_head next;
 161};
 162
 163/*
 164 * We have to share a single online_page callback among all virtio-mem
 165 * devices. We use RCU to iterate the list in the callback.
 166 */
 167static DEFINE_MUTEX(virtio_mem_mutex);
 168static LIST_HEAD(virtio_mem_devices);
 169
 170static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
 
 
 
 
 
 
 
 171
 172/*
 173 * Register a virtio-mem device so it will be considered for the online_page
 174 * callback.
 175 */
 176static int register_virtio_mem_device(struct virtio_mem *vm)
 177{
 178	int rc = 0;
 179
 180	/* First device registers the callback. */
 181	mutex_lock(&virtio_mem_mutex);
 182	if (list_empty(&virtio_mem_devices))
 183		rc = set_online_page_callback(&virtio_mem_online_page_cb);
 184	if (!rc)
 185		list_add_rcu(&vm->next, &virtio_mem_devices);
 186	mutex_unlock(&virtio_mem_mutex);
 187
 188	return rc;
 189}
 190
 191/*
 192 * Unregister a virtio-mem device so it will no longer be considered for the
 193 * online_page callback.
 194 */
 195static void unregister_virtio_mem_device(struct virtio_mem *vm)
 196{
 197	/* Last device unregisters the callback. */
 198	mutex_lock(&virtio_mem_mutex);
 199	list_del_rcu(&vm->next);
 200	if (list_empty(&virtio_mem_devices))
 201		restore_online_page_callback(&virtio_mem_online_page_cb);
 202	mutex_unlock(&virtio_mem_mutex);
 203
 204	synchronize_rcu();
 205}
 206
 207/*
 208 * Calculate the memory block id of a given address.
 209 */
 210static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
 211{
 212	return addr / memory_block_size_bytes();
 213}
 214
 215/*
 216 * Calculate the physical start address of a given memory block id.
 217 */
 218static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
 219{
 220	return mb_id * memory_block_size_bytes();
 221}
 222
 223/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 224 * Calculate the subblock id of a given address.
 225 */
 226static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
 227					      unsigned long addr)
 228{
 229	const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
 230	const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
 231
 232	return (addr - mb_addr) / vm->subblock_size;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 233}
 234
 235/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 236 * Set the state of a memory block, taking care of the state counter.
 237 */
 238static void virtio_mem_mb_set_state(struct virtio_mem *vm, unsigned long mb_id,
 239				    enum virtio_mem_mb_state state)
 240{
 241	const unsigned long idx = mb_id - vm->first_mb_id;
 242	enum virtio_mem_mb_state old_state;
 243
 244	old_state = vm->mb_state[idx];
 245	vm->mb_state[idx] = state;
 246
 247	BUG_ON(vm->nb_mb_state[old_state] == 0);
 248	vm->nb_mb_state[old_state]--;
 249	vm->nb_mb_state[state]++;
 250}
 251
 252/*
 253 * Get the state of a memory block.
 254 */
 255static enum virtio_mem_mb_state virtio_mem_mb_get_state(struct virtio_mem *vm,
 256							unsigned long mb_id)
 257{
 258	const unsigned long idx = mb_id - vm->first_mb_id;
 259
 260	return vm->mb_state[idx];
 261}
 262
 263/*
 264 * Prepare the state array for the next memory block.
 265 */
 266static int virtio_mem_mb_state_prepare_next_mb(struct virtio_mem *vm)
 267{
 268	unsigned long old_bytes = vm->next_mb_id - vm->first_mb_id + 1;
 269	unsigned long new_bytes = vm->next_mb_id - vm->first_mb_id + 2;
 270	int old_pages = PFN_UP(old_bytes);
 271	int new_pages = PFN_UP(new_bytes);
 272	uint8_t *new_mb_state;
 273
 274	if (vm->mb_state && old_pages == new_pages)
 275		return 0;
 276
 277	new_mb_state = vzalloc(new_pages * PAGE_SIZE);
 278	if (!new_mb_state)
 279		return -ENOMEM;
 280
 281	mutex_lock(&vm->hotplug_mutex);
 282	if (vm->mb_state)
 283		memcpy(new_mb_state, vm->mb_state, old_pages * PAGE_SIZE);
 284	vfree(vm->mb_state);
 285	vm->mb_state = new_mb_state;
 286	mutex_unlock(&vm->hotplug_mutex);
 287
 288	return 0;
 289}
 290
 291#define virtio_mem_for_each_mb_state(_vm, _mb_id, _state) \
 292	for (_mb_id = _vm->first_mb_id; \
 293	     _mb_id < _vm->next_mb_id && _vm->nb_mb_state[_state]; \
 294	     _mb_id++) \
 295		if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
 296
 297#define virtio_mem_for_each_mb_state_rev(_vm, _mb_id, _state) \
 298	for (_mb_id = _vm->next_mb_id - 1; \
 299	     _mb_id >= _vm->first_mb_id && _vm->nb_mb_state[_state]; \
 300	     _mb_id--) \
 301		if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
 
 
 
 
 
 
 
 
 
 
 302
 303/*
 304 * Mark all selected subblocks plugged.
 305 *
 306 * Will not modify the state of the memory block.
 307 */
 308static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm,
 309					 unsigned long mb_id, int sb_id,
 310					 int count)
 311{
 312	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
 313
 314	__bitmap_set(vm->sb_bitmap, bit, count);
 315}
 316
 317/*
 318 * Mark all selected subblocks unplugged.
 319 *
 320 * Will not modify the state of the memory block.
 321 */
 322static void virtio_mem_mb_set_sb_unplugged(struct virtio_mem *vm,
 323					   unsigned long mb_id, int sb_id,
 324					   int count)
 325{
 326	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
 327
 328	__bitmap_clear(vm->sb_bitmap, bit, count);
 329}
 330
 331/*
 332 * Test if all selected subblocks are plugged.
 333 */
 334static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm,
 335					  unsigned long mb_id, int sb_id,
 336					  int count)
 337{
 338	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
 339
 340	if (count == 1)
 341		return test_bit(bit, vm->sb_bitmap);
 342
 343	/* TODO: Helper similar to bitmap_set() */
 344	return find_next_zero_bit(vm->sb_bitmap, bit + count, bit) >=
 345	       bit + count;
 346}
 347
 348/*
 349 * Test if all selected subblocks are unplugged.
 350 */
 351static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm,
 352					    unsigned long mb_id, int sb_id,
 353					    int count)
 354{
 355	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
 356
 357	/* TODO: Helper similar to bitmap_set() */
 358	return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count;
 
 359}
 360
 361/*
 362 * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is
 363 * none.
 364 */
 365static int virtio_mem_mb_first_unplugged_sb(struct virtio_mem *vm,
 366					    unsigned long mb_id)
 367{
 368	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb;
 369
 370	return find_next_zero_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) -
 371	       bit;
 372}
 373
 374/*
 375 * Prepare the subblock bitmap for the next memory block.
 376 */
 377static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm)
 378{
 379	const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id;
 380	const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb;
 381	const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb;
 382	int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
 383	int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
 384	unsigned long *new_sb_bitmap, *old_sb_bitmap;
 385
 386	if (vm->sb_bitmap && old_pages == new_pages)
 387		return 0;
 388
 389	new_sb_bitmap = vzalloc(new_pages * PAGE_SIZE);
 390	if (!new_sb_bitmap)
 391		return -ENOMEM;
 392
 393	mutex_lock(&vm->hotplug_mutex);
 394	if (new_sb_bitmap)
 395		memcpy(new_sb_bitmap, vm->sb_bitmap, old_pages * PAGE_SIZE);
 396
 397	old_sb_bitmap = vm->sb_bitmap;
 398	vm->sb_bitmap = new_sb_bitmap;
 399	mutex_unlock(&vm->hotplug_mutex);
 400
 401	vfree(old_sb_bitmap);
 402	return 0;
 403}
 404
 405/*
 406 * Try to add a memory block to Linux. This will usually only fail
 407 * if out of memory.
 
 
 
 
 
 
 
 
 
 
 
 408 *
 409 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
 410 * onlining code).
 411 *
 412 * Will not modify the state of the memory block.
 413 */
 414static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id)
 
 415{
 416	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
 417	int nid = vm->nid;
 418
 419	if (nid == NUMA_NO_NODE)
 420		nid = memory_add_physaddr_to_nid(addr);
 421
 422	/*
 423	 * When force-unloading the driver and we still have memory added to
 424	 * Linux, the resource name has to stay.
 425	 */
 426	if (!vm->resource_name) {
 427		vm->resource_name = kstrdup_const("System RAM (virtio_mem)",
 428						  GFP_KERNEL);
 429		if (!vm->resource_name)
 430			return -ENOMEM;
 431	}
 432
 433	dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id);
 434	return add_memory_driver_managed(nid, addr, memory_block_size_bytes(),
 435					 vm->resource_name);
 
 
 
 
 
 
 
 
 
 
 
 
 436}
 437
 438/*
 439 * Try to remove a memory block from Linux. Will only fail if the memory block
 440 * is not offline.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 441 *
 442 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
 443 * onlining code).
 444 *
 445 * Will not modify the state of the memory block.
 446 */
 447static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id)
 
 448{
 449	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
 450	int nid = vm->nid;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 451
 452	if (nid == NUMA_NO_NODE)
 453		nid = memory_add_physaddr_to_nid(addr);
 
 
 
 
 
 454
 455	dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id);
 456	return remove_memory(nid, addr, memory_block_size_bytes());
 457}
 458
 459/*
 460 * Try to offline and remove a memory block from Linux.
 461 *
 462 * Must not be called with the vm->hotplug_mutex held (possible deadlock with
 463 * onlining code).
 464 *
 465 * Will not modify the state of the memory block.
 466 */
 467static int virtio_mem_mb_offline_and_remove(struct virtio_mem *vm,
 468					    unsigned long mb_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 469{
 470	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
 471	int nid = vm->nid;
 
 
 
 472
 473	if (nid == NUMA_NO_NODE)
 474		nid = memory_add_physaddr_to_nid(addr);
 
 
 
 
 
 
 
 475
 476	dev_dbg(&vm->vdev->dev, "offlining and removing memory block: %lu\n",
 477		mb_id);
 478	return offline_and_remove_memory(nid, addr, memory_block_size_bytes());
 479}
 480
 481/*
 482 * Trigger the workqueue so the device can perform its magic.
 483 */
 484static void virtio_mem_retry(struct virtio_mem *vm)
 485{
 486	unsigned long flags;
 487
 488	spin_lock_irqsave(&vm->removal_lock, flags);
 489	if (!vm->removing)
 490		queue_work(system_freezable_wq, &vm->wq);
 491	spin_unlock_irqrestore(&vm->removal_lock, flags);
 492}
 493
 494static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
 495{
 496	int node = NUMA_NO_NODE;
 497
 498#if defined(CONFIG_ACPI_NUMA)
 499	if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
 500		node = pxm_to_node(node_id);
 501#endif
 502	return node;
 503}
 504
 505/*
 506 * Test if a virtio-mem device overlaps with the given range. Can be called
 507 * from (notifier) callbacks lockless.
 508 */
 509static bool virtio_mem_overlaps_range(struct virtio_mem *vm,
 510				      unsigned long start, unsigned long size)
 511{
 512	unsigned long dev_start = virtio_mem_mb_id_to_phys(vm->first_mb_id);
 513	unsigned long dev_end = virtio_mem_mb_id_to_phys(vm->last_mb_id) +
 514				memory_block_size_bytes();
 515
 516	return start < dev_end && dev_start < start + size;
 517}
 518
 519/*
 520 * Test if a virtio-mem device owns a memory block. Can be called from
 521 * (notifier) callbacks lockless.
 522 */
 523static bool virtio_mem_owned_mb(struct virtio_mem *vm, unsigned long mb_id)
 
 524{
 525	return mb_id >= vm->first_mb_id && mb_id <= vm->last_mb_id;
 526}
 527
 528static int virtio_mem_notify_going_online(struct virtio_mem *vm,
 529					  unsigned long mb_id,
 530					  enum zone_type zone)
 531{
 532	switch (virtio_mem_mb_get_state(vm, mb_id)) {
 533	case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
 534		/*
 535		 * We won't allow to online a partially plugged memory block
 536		 * to the MOVABLE zone - it would contain unmovable parts.
 537		 */
 538		if (zone == ZONE_MOVABLE) {
 539			dev_warn_ratelimited(&vm->vdev->dev,
 540					     "memory block has holes, MOVABLE not supported\n");
 541			return NOTIFY_BAD;
 542		}
 543		return NOTIFY_OK;
 544	case VIRTIO_MEM_MB_STATE_OFFLINE:
 545		return NOTIFY_OK;
 546	default:
 547		break;
 548	}
 549	dev_warn_ratelimited(&vm->vdev->dev,
 550			     "memory block onlining denied\n");
 551	return NOTIFY_BAD;
 552}
 553
 554static void virtio_mem_notify_offline(struct virtio_mem *vm,
 555				      unsigned long mb_id)
 556{
 557	switch (virtio_mem_mb_get_state(vm, mb_id)) {
 558	case VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL:
 559		virtio_mem_mb_set_state(vm, mb_id,
 560					VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
 561		break;
 562	case VIRTIO_MEM_MB_STATE_ONLINE:
 563	case VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE:
 564		virtio_mem_mb_set_state(vm, mb_id,
 565					VIRTIO_MEM_MB_STATE_OFFLINE);
 
 566		break;
 567	default:
 568		BUG();
 569		break;
 570	}
 571
 572	/*
 573	 * Trigger the workqueue, maybe we can now unplug memory. Also,
 574	 * when we offline and remove a memory block, this will re-trigger
 575	 * us immediately - which is often nice because the removal of
 576	 * the memory block (e.g., memmap) might have freed up memory
 577	 * on other memory blocks we manage.
 578	 */
 579	virtio_mem_retry(vm);
 580}
 581
 582static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id,
 583				     enum zone_type zone)
 584{
 585	unsigned long nb_offline;
 586
 587	switch (virtio_mem_mb_get_state(vm, mb_id)) {
 588	case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
 589		BUG_ON(zone == ZONE_MOVABLE);
 590		virtio_mem_mb_set_state(vm, mb_id,
 591					VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
 592		break;
 593	case VIRTIO_MEM_MB_STATE_OFFLINE:
 594		if (zone == ZONE_MOVABLE)
 595			virtio_mem_mb_set_state(vm, mb_id,
 596					    VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE);
 597		else
 598			virtio_mem_mb_set_state(vm, mb_id,
 599						VIRTIO_MEM_MB_STATE_ONLINE);
 600		break;
 601	default:
 602		BUG();
 603		break;
 604	}
 605	nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
 606		     vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
 607
 608	/* see if we can add new blocks now that we onlined one block */
 609	if (nb_offline == VIRTIO_MEM_NB_OFFLINE_THRESHOLD - 1)
 610		virtio_mem_retry(vm);
 611}
 612
 613static void virtio_mem_notify_going_offline(struct virtio_mem *vm,
 614					    unsigned long mb_id)
 615{
 616	const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
 617	struct page *page;
 618	unsigned long pfn;
 619	int sb_id, i;
 620
 621	for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
 622		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
 623			continue;
 624		/*
 625		 * Drop our reference to the pages so the memory can get
 626		 * offlined and add the unplugged pages to the managed
 627		 * page counters (so offlining code can correctly subtract
 628		 * them again).
 629		 */
 630		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
 631			       sb_id * vm->subblock_size);
 632		adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
 633		for (i = 0; i < nr_pages; i++) {
 634			page = pfn_to_page(pfn + i);
 635			if (WARN_ON(!page_ref_dec_and_test(page)))
 636				dump_page(page, "unplugged page referenced");
 637		}
 638	}
 639}
 640
 641static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm,
 642					     unsigned long mb_id)
 643{
 644	const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
 645	unsigned long pfn;
 646	int sb_id, i;
 647
 648	for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
 649		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
 650			continue;
 651		/*
 652		 * Get the reference we dropped when going offline and
 653		 * subtract the unplugged pages from the managed page
 654		 * counters.
 655		 */
 656		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
 657			       sb_id * vm->subblock_size);
 658		adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
 659		for (i = 0; i < nr_pages; i++)
 660			page_ref_inc(pfn_to_page(pfn + i));
 661	}
 662}
 663
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 664/*
 665 * This callback will either be called synchronously from add_memory() or
 666 * asynchronously (e.g., triggered via user space). We have to be careful
 667 * with locking when calling add_memory().
 668 */
 669static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
 670					 unsigned long action, void *arg)
 671{
 672	struct virtio_mem *vm = container_of(nb, struct virtio_mem,
 673					     memory_notifier);
 674	struct memory_notify *mhp = arg;
 675	const unsigned long start = PFN_PHYS(mhp->start_pfn);
 676	const unsigned long size = PFN_PHYS(mhp->nr_pages);
 677	const unsigned long mb_id = virtio_mem_phys_to_mb_id(start);
 678	enum zone_type zone;
 679	int rc = NOTIFY_OK;
 
 680
 681	if (!virtio_mem_overlaps_range(vm, start, size))
 682		return NOTIFY_DONE;
 683
 684	/*
 685	 * Memory is onlined/offlined in memory block granularity. We cannot
 686	 * cross virtio-mem device boundaries and memory block boundaries. Bail
 687	 * out if this ever changes.
 688	 */
 689	if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
 690			 !IS_ALIGNED(start, memory_block_size_bytes())))
 691		return NOTIFY_BAD;
 
 
 
 
 
 
 
 
 
 
 
 
 
 692
 693	/*
 694	 * Avoid circular locking lockdep warnings. We lock the mutex
 695	 * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
 696	 * blocking_notifier_call_chain() has it's own lock, which gets unlocked
 697	 * between both notifier calls and will bail out. False positive.
 698	 */
 699	lockdep_off();
 700
 701	switch (action) {
 702	case MEM_GOING_OFFLINE:
 703		mutex_lock(&vm->hotplug_mutex);
 704		if (vm->removing) {
 705			rc = notifier_from_errno(-EBUSY);
 706			mutex_unlock(&vm->hotplug_mutex);
 707			break;
 708		}
 709		vm->hotplug_active = true;
 710		virtio_mem_notify_going_offline(vm, mb_id);
 
 
 
 
 
 711		break;
 712	case MEM_GOING_ONLINE:
 713		mutex_lock(&vm->hotplug_mutex);
 714		if (vm->removing) {
 715			rc = notifier_from_errno(-EBUSY);
 716			mutex_unlock(&vm->hotplug_mutex);
 717			break;
 718		}
 719		vm->hotplug_active = true;
 720		zone = page_zonenum(pfn_to_page(mhp->start_pfn));
 721		rc = virtio_mem_notify_going_online(vm, mb_id, zone);
 722		break;
 723	case MEM_OFFLINE:
 724		virtio_mem_notify_offline(vm, mb_id);
 
 
 
 
 
 
 
 
 
 
 725		vm->hotplug_active = false;
 726		mutex_unlock(&vm->hotplug_mutex);
 727		break;
 728	case MEM_ONLINE:
 729		zone = page_zonenum(pfn_to_page(mhp->start_pfn));
 730		virtio_mem_notify_online(vm, mb_id, zone);
 
 
 
 
 
 
 
 
 
 
 
 
 731		vm->hotplug_active = false;
 732		mutex_unlock(&vm->hotplug_mutex);
 733		break;
 734	case MEM_CANCEL_OFFLINE:
 735		if (!vm->hotplug_active)
 736			break;
 737		virtio_mem_notify_cancel_offline(vm, mb_id);
 
 
 
 
 
 738		vm->hotplug_active = false;
 739		mutex_unlock(&vm->hotplug_mutex);
 740		break;
 741	case MEM_CANCEL_ONLINE:
 742		if (!vm->hotplug_active)
 743			break;
 744		vm->hotplug_active = false;
 745		mutex_unlock(&vm->hotplug_mutex);
 746		break;
 747	default:
 748		break;
 749	}
 750
 751	lockdep_on();
 752
 753	return rc;
 754}
 755
 756/*
 757 * Set a range of pages PG_offline. Remember pages that were never onlined
 758 * (via generic_online_page()) using PageDirty().
 759 */
 760static void virtio_mem_set_fake_offline(unsigned long pfn,
 761					unsigned int nr_pages, bool onlined)
 762{
 
 763	for (; nr_pages--; pfn++) {
 764		struct page *page = pfn_to_page(pfn);
 765
 766		__SetPageOffline(page);
 767		if (!onlined) {
 768			SetPageDirty(page);
 769			/* FIXME: remove after cleanups */
 770			ClearPageReserved(page);
 771		}
 772	}
 
 773}
 774
 775/*
 776 * Clear PG_offline from a range of pages. If the pages were never onlined,
 777 * (via generic_online_page()), clear PageDirty().
 778 */
 779static void virtio_mem_clear_fake_offline(unsigned long pfn,
 780					  unsigned int nr_pages, bool onlined)
 781{
 782	for (; nr_pages--; pfn++) {
 783		struct page *page = pfn_to_page(pfn);
 784
 785		__ClearPageOffline(page);
 786		if (!onlined)
 787			ClearPageDirty(page);
 788	}
 789}
 790
 791/*
 792 * Release a range of fake-offline pages to the buddy, effectively
 793 * fake-onlining them.
 794 */
 795static void virtio_mem_fake_online(unsigned long pfn, unsigned int nr_pages)
 796{
 797	const int order = MAX_ORDER - 1;
 798	int i;
 799
 800	/*
 801	 * We are always called with subblock granularity, which is at least
 802	 * aligned to MAX_ORDER - 1.
 
 803	 */
 
 
 
 804	for (i = 0; i < nr_pages; i += 1 << order) {
 805		struct page *page = pfn_to_page(pfn + i);
 806
 807		/*
 808		 * If the page is PageDirty(), it was kept fake-offline when
 809		 * onlining the memory block. Otherwise, it was allocated
 810		 * using alloc_contig_range(). All pages in a subblock are
 811		 * alike.
 812		 */
 813		if (PageDirty(page)) {
 814			virtio_mem_clear_fake_offline(pfn + i, 1 << order,
 815						      false);
 816			generic_online_page(page, order);
 817		} else {
 818			virtio_mem_clear_fake_offline(pfn + i, 1 << order,
 819						      true);
 820			free_contig_range(pfn + i, 1 << order);
 821			adjust_managed_page_count(page, 1 << order);
 822		}
 823	}
 824}
 825
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 826static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
 827{
 828	const unsigned long addr = page_to_phys(page);
 829	const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
 830	struct virtio_mem *vm;
 831	int sb_id;
 832
 833	/*
 834	 * We exploit here that subblocks have at least MAX_ORDER - 1
 835	 * size/alignment and that this callback is is called with such a
 836	 * size/alignment. So we cannot cross subblocks and therefore
 837	 * also not memory blocks.
 838	 */
 839	rcu_read_lock();
 840	list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
 841		if (!virtio_mem_owned_mb(vm, mb_id))
 
 
 
 
 842			continue;
 843
 844		sb_id = virtio_mem_phys_to_sb_id(vm, addr);
 845		/*
 846		 * If plugged, online the pages, otherwise, set them fake
 847		 * offline (PageOffline).
 
 
 848		 */
 849		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
 850			generic_online_page(page, order);
 851		else
 852			virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
 853						    false);
 854		rcu_read_unlock();
 
 
 855		return;
 856	}
 857	rcu_read_unlock();
 858
 859	/* not virtio-mem memory, but e.g., a DIMM. online it */
 860	generic_online_page(page, order);
 861}
 862
 863static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
 864					const struct virtio_mem_req *req)
 865{
 866	struct scatterlist *sgs[2], sg_req, sg_resp;
 867	unsigned int len;
 868	int rc;
 869
 870	/* don't use the request residing on the stack (vaddr) */
 871	vm->req = *req;
 872
 873	/* out: buffer for request */
 874	sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
 875	sgs[0] = &sg_req;
 876
 877	/* in: buffer for response */
 878	sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
 879	sgs[1] = &sg_resp;
 880
 881	rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
 882	if (rc < 0)
 883		return rc;
 884
 885	virtqueue_kick(vm->vq);
 886
 887	/* wait for a response */
 888	wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
 889
 890	return virtio16_to_cpu(vm->vdev, vm->resp.type);
 891}
 892
 893static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
 894					uint64_t size)
 895{
 896	const uint64_t nb_vm_blocks = size / vm->device_block_size;
 897	const struct virtio_mem_req req = {
 898		.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
 899		.u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
 900		.u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
 901	};
 
 902
 903	if (atomic_read(&vm->config_changed))
 904		return -EAGAIN;
 905
 
 
 
 906	switch (virtio_mem_send_request(vm, &req)) {
 907	case VIRTIO_MEM_RESP_ACK:
 908		vm->plugged_size += size;
 909		return 0;
 910	case VIRTIO_MEM_RESP_NACK:
 911		return -EAGAIN;
 
 912	case VIRTIO_MEM_RESP_BUSY:
 913		return -ETXTBSY;
 
 914	case VIRTIO_MEM_RESP_ERROR:
 915		return -EINVAL;
 
 916	default:
 917		return -ENOMEM;
 918	}
 
 
 
 919}
 920
 921static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
 922					  uint64_t size)
 923{
 924	const uint64_t nb_vm_blocks = size / vm->device_block_size;
 925	const struct virtio_mem_req req = {
 926		.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
 927		.u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
 928		.u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
 929	};
 
 930
 931	if (atomic_read(&vm->config_changed))
 932		return -EAGAIN;
 933
 
 
 
 934	switch (virtio_mem_send_request(vm, &req)) {
 935	case VIRTIO_MEM_RESP_ACK:
 936		vm->plugged_size -= size;
 937		return 0;
 938	case VIRTIO_MEM_RESP_BUSY:
 939		return -ETXTBSY;
 
 940	case VIRTIO_MEM_RESP_ERROR:
 941		return -EINVAL;
 
 942	default:
 943		return -ENOMEM;
 944	}
 
 
 
 945}
 946
 947static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
 948{
 949	const struct virtio_mem_req req = {
 950		.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
 951	};
 
 
 
 952
 953	switch (virtio_mem_send_request(vm, &req)) {
 954	case VIRTIO_MEM_RESP_ACK:
 955		vm->unplug_all_required = false;
 956		vm->plugged_size = 0;
 957		/* usable region might have shrunk */
 958		atomic_set(&vm->config_changed, 1);
 959		return 0;
 960	case VIRTIO_MEM_RESP_BUSY:
 961		return -ETXTBSY;
 
 962	default:
 963		return -ENOMEM;
 964	}
 
 
 
 965}
 966
 967/*
 968 * Plug selected subblocks. Updates the plugged state, but not the state
 969 * of the memory block.
 970 */
 971static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
 972				 int sb_id, int count)
 973{
 974	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
 975			      sb_id * vm->subblock_size;
 976	const uint64_t size = count * vm->subblock_size;
 977	int rc;
 978
 979	dev_dbg(&vm->vdev->dev, "plugging memory block: %lu : %i - %i\n", mb_id,
 980		sb_id, sb_id + count - 1);
 981
 982	rc = virtio_mem_send_plug_request(vm, addr, size);
 983	if (!rc)
 984		virtio_mem_mb_set_sb_plugged(vm, mb_id, sb_id, count);
 985	return rc;
 986}
 987
 988/*
 989 * Unplug selected subblocks. Updates the plugged state, but not the state
 990 * of the memory block.
 991 */
 992static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
 993				   int sb_id, int count)
 994{
 995	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
 996			      sb_id * vm->subblock_size;
 997	const uint64_t size = count * vm->subblock_size;
 998	int rc;
 999
1000	dev_dbg(&vm->vdev->dev, "unplugging memory block: %lu : %i - %i\n",
1001		mb_id, sb_id, sb_id + count - 1);
1002
1003	rc = virtio_mem_send_unplug_request(vm, addr, size);
1004	if (!rc)
1005		virtio_mem_mb_set_sb_unplugged(vm, mb_id, sb_id, count);
1006	return rc;
1007}
1008
1009/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1010 * Unplug the desired number of plugged subblocks of a offline or not-added
1011 * memory block. Will fail if any subblock cannot get unplugged (instead of
1012 * skipping it).
1013 *
1014 * Will not modify the state of the memory block.
1015 *
1016 * Note: can fail after some subblocks were unplugged.
1017 */
1018static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm,
1019				       unsigned long mb_id, uint64_t *nb_sb)
1020{
1021	int sb_id, count;
1022	int rc;
1023
1024	sb_id = vm->nb_sb_per_mb - 1;
1025	while (*nb_sb) {
1026		/* Find the next candidate subblock */
1027		while (sb_id >= 0 &&
1028		       virtio_mem_mb_test_sb_unplugged(vm, mb_id, sb_id, 1))
1029			sb_id--;
1030		if (sb_id < 0)
1031			break;
1032		/* Try to unplug multiple subblocks at a time */
1033		count = 1;
1034		while (count < *nb_sb && sb_id > 0 &&
1035		       virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
1036			count++;
1037			sb_id--;
1038		}
1039
1040		rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
1041		if (rc)
1042			return rc;
1043		*nb_sb -= count;
1044		sb_id--;
1045	}
1046
1047	return 0;
1048}
1049
1050/*
1051 * Unplug all plugged subblocks of an offline or not-added memory block.
1052 *
1053 * Will not modify the state of the memory block.
1054 *
1055 * Note: can fail after some subblocks were unplugged.
1056 */
1057static int virtio_mem_mb_unplug(struct virtio_mem *vm, unsigned long mb_id)
1058{
1059	uint64_t nb_sb = vm->nb_sb_per_mb;
1060
1061	return virtio_mem_mb_unplug_any_sb(vm, mb_id, &nb_sb);
1062}
1063
1064/*
1065 * Prepare tracking data for the next memory block.
1066 */
1067static int virtio_mem_prepare_next_mb(struct virtio_mem *vm,
1068				      unsigned long *mb_id)
1069{
1070	int rc;
1071
1072	if (vm->next_mb_id > vm->last_usable_mb_id)
1073		return -ENOSPC;
1074
1075	/* Resize the state array if required. */
1076	rc = virtio_mem_mb_state_prepare_next_mb(vm);
1077	if (rc)
1078		return rc;
1079
1080	/* Resize the subblock bitmap if required. */
1081	rc = virtio_mem_sb_bitmap_prepare_next_mb(vm);
1082	if (rc)
1083		return rc;
1084
1085	vm->nb_mb_state[VIRTIO_MEM_MB_STATE_UNUSED]++;
1086	*mb_id = vm->next_mb_id++;
1087	return 0;
1088}
1089
1090/*
1091 * Don't add too many blocks that are not onlined yet to avoid running OOM.
1092 */
1093static bool virtio_mem_too_many_mb_offline(struct virtio_mem *vm)
1094{
1095	unsigned long nb_offline;
1096
1097	nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
1098		     vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
1099	return nb_offline >= VIRTIO_MEM_NB_OFFLINE_THRESHOLD;
1100}
1101
1102/*
1103 * Try to plug the desired number of subblocks and add the memory block
1104 * to Linux.
1105 *
1106 * Will modify the state of the memory block.
1107 */
1108static int virtio_mem_mb_plug_and_add(struct virtio_mem *vm,
1109				      unsigned long mb_id,
1110				      uint64_t *nb_sb)
1111{
1112	const int count = min_t(int, *nb_sb, vm->nb_sb_per_mb);
1113	int rc, rc2;
1114
1115	if (WARN_ON_ONCE(!count))
1116		return -EINVAL;
1117
1118	/*
1119	 * Plug the requested number of subblocks before adding it to linux,
1120	 * so that onlining will directly online all plugged subblocks.
1121	 */
1122	rc = virtio_mem_mb_plug_sb(vm, mb_id, 0, count);
1123	if (rc)
1124		return rc;
1125
1126	/*
1127	 * Mark the block properly offline before adding it to Linux,
1128	 * so the memory notifiers will find the block in the right state.
1129	 */
1130	if (count == vm->nb_sb_per_mb)
1131		virtio_mem_mb_set_state(vm, mb_id,
1132					VIRTIO_MEM_MB_STATE_OFFLINE);
1133	else
1134		virtio_mem_mb_set_state(vm, mb_id,
1135					VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
1136
1137	/* Add the memory block to linux - if that fails, try to unplug. */
1138	rc = virtio_mem_mb_add(vm, mb_id);
1139	if (rc) {
1140		enum virtio_mem_mb_state new_state = VIRTIO_MEM_MB_STATE_UNUSED;
1141
1142		dev_err(&vm->vdev->dev,
1143			"adding memory block %lu failed with %d\n", mb_id, rc);
1144		rc2 = virtio_mem_mb_unplug_sb(vm, mb_id, 0, count);
1145
1146		/*
1147		 * TODO: Linux MM does not properly clean up yet in all cases
1148		 * where adding of memory failed - especially on -ENOMEM.
1149		 */
1150		if (rc2)
1151			new_state = VIRTIO_MEM_MB_STATE_PLUGGED;
1152		virtio_mem_mb_set_state(vm, mb_id, new_state);
1153		return rc;
1154	}
1155
1156	*nb_sb -= count;
1157	return 0;
1158}
1159
1160/*
1161 * Try to plug the desired number of subblocks of a memory block that
1162 * is already added to Linux.
1163 *
1164 * Will modify the state of the memory block.
1165 *
1166 * Note: Can fail after some subblocks were successfully plugged.
1167 */
1168static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id,
1169				     uint64_t *nb_sb, bool online)
1170{
 
1171	unsigned long pfn, nr_pages;
1172	int sb_id, count;
1173	int rc;
1174
1175	if (WARN_ON_ONCE(!*nb_sb))
1176		return -EINVAL;
1177
1178	while (*nb_sb) {
1179		sb_id = virtio_mem_mb_first_unplugged_sb(vm, mb_id);
1180		if (sb_id >= vm->nb_sb_per_mb)
1181			break;
1182		count = 1;
1183		while (count < *nb_sb &&
1184		       sb_id + count < vm->nb_sb_per_mb &&
1185		       !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count,
1186						      1))
1187			count++;
1188
1189		rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count);
1190		if (rc)
1191			return rc;
1192		*nb_sb -= count;
1193		if (!online)
1194			continue;
1195
1196		/* fake-online the pages if the memory block is online */
1197		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1198			       sb_id * vm->subblock_size);
1199		nr_pages = PFN_DOWN(count * vm->subblock_size);
1200		virtio_mem_fake_online(pfn, nr_pages);
1201	}
1202
1203	if (virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1204		if (online)
1205			virtio_mem_mb_set_state(vm, mb_id,
1206						VIRTIO_MEM_MB_STATE_ONLINE);
1207		else
1208			virtio_mem_mb_set_state(vm, mb_id,
1209						VIRTIO_MEM_MB_STATE_OFFLINE);
1210	}
1211
1212	return 0;
1213}
1214
1215/*
1216 * Try to plug the requested amount of memory.
1217 */
1218static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
1219{
1220	uint64_t nb_sb = diff / vm->subblock_size;
 
 
 
 
 
1221	unsigned long mb_id;
1222	int rc;
1223
1224	if (!nb_sb)
1225		return 0;
1226
1227	/* Don't race with onlining/offlining */
1228	mutex_lock(&vm->hotplug_mutex);
1229
1230	/* Try to plug subblocks of partially plugged online blocks. */
1231	virtio_mem_for_each_mb_state(vm, mb_id,
1232				     VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
1233		rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, true);
1234		if (rc || !nb_sb)
1235			goto out_unlock;
1236		cond_resched();
1237	}
1238
1239	/* Try to plug subblocks of partially plugged offline blocks. */
1240	virtio_mem_for_each_mb_state(vm, mb_id,
1241				     VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
1242		rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, false);
1243		if (rc || !nb_sb)
1244			goto out_unlock;
1245		cond_resched();
1246	}
1247
1248	/*
1249	 * We won't be working on online/offline memory blocks from this point,
1250	 * so we can't race with memory onlining/offlining. Drop the mutex.
1251	 */
1252	mutex_unlock(&vm->hotplug_mutex);
1253
1254	/* Try to plug and add unused blocks */
1255	virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED) {
1256		if (virtio_mem_too_many_mb_offline(vm))
1257			return -ENOSPC;
1258
1259		rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
1260		if (rc || !nb_sb)
1261			return rc;
1262		cond_resched();
1263	}
1264
1265	/* Try to prepare, plug and add new blocks */
1266	while (nb_sb) {
1267		if (virtio_mem_too_many_mb_offline(vm))
1268			return -ENOSPC;
1269
1270		rc = virtio_mem_prepare_next_mb(vm, &mb_id);
1271		if (rc)
1272			return rc;
1273		rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
1274		if (rc)
1275			return rc;
1276		cond_resched();
1277	}
1278
1279	return 0;
1280out_unlock:
1281	mutex_unlock(&vm->hotplug_mutex);
1282	return rc;
1283}
1284
1285/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1286 * Unplug the desired number of plugged subblocks of an offline memory block.
1287 * Will fail if any subblock cannot get unplugged (instead of skipping it).
1288 *
1289 * Will modify the state of the memory block. Might temporarily drop the
1290 * hotplug_mutex.
1291 *
1292 * Note: Can fail after some subblocks were successfully unplugged.
1293 */
1294static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm,
1295					       unsigned long mb_id,
1296					       uint64_t *nb_sb)
1297{
1298	int rc;
1299
1300	rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb);
1301
1302	/* some subblocks might have been unplugged even on failure */
1303	if (!virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))
1304		virtio_mem_mb_set_state(vm, mb_id,
1305					VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
1306	if (rc)
1307		return rc;
1308
1309	if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1310		/*
1311		 * Remove the block from Linux - this should never fail.
1312		 * Hinder the block from getting onlined by marking it
1313		 * unplugged. Temporarily drop the mutex, so
1314		 * any pending GOING_ONLINE requests can be serviced/rejected.
1315		 */
1316		virtio_mem_mb_set_state(vm, mb_id,
1317					VIRTIO_MEM_MB_STATE_UNUSED);
1318
1319		mutex_unlock(&vm->hotplug_mutex);
1320		rc = virtio_mem_mb_remove(vm, mb_id);
1321		BUG_ON(rc);
1322		mutex_lock(&vm->hotplug_mutex);
1323	}
1324	return 0;
1325}
1326
1327/*
1328 * Unplug the given plugged subblocks of an online memory block.
1329 *
1330 * Will modify the state of the memory block.
1331 */
1332static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm,
1333					  unsigned long mb_id, int sb_id,
1334					  int count)
1335{
1336	const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count;
 
1337	unsigned long start_pfn;
1338	int rc;
1339
1340	start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
1341			     sb_id * vm->subblock_size);
1342	rc = alloc_contig_range(start_pfn, start_pfn + nr_pages,
1343				MIGRATE_MOVABLE, GFP_KERNEL);
1344	if (rc == -ENOMEM)
1345		/* whoops, out of memory */
1346		return rc;
1347	if (rc)
1348		return -EBUSY;
1349
1350	/* Mark it as fake-offline before unplugging it */
1351	virtio_mem_set_fake_offline(start_pfn, nr_pages, true);
1352	adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages);
1353
1354	/* Try to unplug the allocated memory */
1355	rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
1356	if (rc) {
1357		/* Return the memory to the buddy. */
1358		virtio_mem_fake_online(start_pfn, nr_pages);
1359		return rc;
1360	}
1361
1362	virtio_mem_mb_set_state(vm, mb_id,
1363				VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
 
 
 
 
 
 
 
 
 
1364	return 0;
1365}
1366
1367/*
1368 * Unplug the desired number of plugged subblocks of an online memory block.
1369 * Will skip subblock that are busy.
1370 *
1371 * Will modify the state of the memory block. Might temporarily drop the
1372 * hotplug_mutex.
1373 *
1374 * Note: Can fail after some subblocks were successfully unplugged. Can
1375 *       return 0 even if subblocks were busy and could not get unplugged.
1376 */
1377static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm,
1378					      unsigned long mb_id,
1379					      uint64_t *nb_sb)
1380{
1381	int rc, sb_id;
1382
1383	/* If possible, try to unplug the complete block in one shot. */
1384	if (*nb_sb >= vm->nb_sb_per_mb &&
1385	    virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1386		rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0,
1387						    vm->nb_sb_per_mb);
1388		if (!rc) {
1389			*nb_sb -= vm->nb_sb_per_mb;
1390			goto unplugged;
1391		} else if (rc != -EBUSY)
1392			return rc;
1393	}
1394
1395	/* Fallback to single subblocks. */
1396	for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
1397		/* Find the next candidate subblock */
1398		while (sb_id >= 0 &&
1399		       !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
1400			sb_id--;
1401		if (sb_id < 0)
1402			break;
1403
1404		rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, sb_id, 1);
1405		if (rc == -EBUSY)
1406			continue;
1407		else if (rc)
1408			return rc;
1409		*nb_sb -= 1;
1410	}
1411
1412unplugged:
1413	/*
1414	 * Once all subblocks of a memory block were unplugged, offline and
1415	 * remove it. This will usually not fail, as no memory is in use
1416	 * anymore - however some other notifiers might NACK the request.
1417	 */
1418	if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
1419		mutex_unlock(&vm->hotplug_mutex);
1420		rc = virtio_mem_mb_offline_and_remove(vm, mb_id);
1421		mutex_lock(&vm->hotplug_mutex);
1422		if (!rc)
1423			virtio_mem_mb_set_state(vm, mb_id,
1424						VIRTIO_MEM_MB_STATE_UNUSED);
1425	}
1426
1427	return 0;
1428}
1429
1430/*
1431 * Try to unplug the requested amount of memory.
 
 
 
 
 
 
 
 
 
1432 */
1433static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
1434{
1435	uint64_t nb_sb = diff / vm->subblock_size;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1436	unsigned long mb_id;
1437	int rc;
1438
1439	if (!nb_sb)
1440		return 0;
1441
1442	/*
1443	 * We'll drop the mutex a couple of times when it is safe to do so.
1444	 * This might result in some blocks switching the state (online/offline)
1445	 * and we could miss them in this run - we will retry again later.
1446	 */
1447	mutex_lock(&vm->hotplug_mutex);
1448
1449	/* Try to unplug subblocks of partially plugged offline blocks. */
1450	virtio_mem_for_each_mb_state_rev(vm, mb_id,
1451					 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
1452		rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
1453							 &nb_sb);
1454		if (rc || !nb_sb)
1455			goto out_unlock;
1456		cond_resched();
 
 
 
 
 
 
 
 
 
 
 
 
1457	}
1458
1459	/* Try to unplug subblocks of plugged offline blocks. */
1460	virtio_mem_for_each_mb_state_rev(vm, mb_id,
1461					 VIRTIO_MEM_MB_STATE_OFFLINE) {
1462		rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
1463							 &nb_sb);
1464		if (rc || !nb_sb)
1465			goto out_unlock;
1466		cond_resched();
1467	}
1468
1469	if (!unplug_online) {
1470		mutex_unlock(&vm->hotplug_mutex);
1471		return 0;
1472	}
 
 
 
 
 
 
 
 
 
 
 
 
1473
1474	/* Try to unplug subblocks of partially plugged online blocks. */
1475	virtio_mem_for_each_mb_state_rev(vm, mb_id,
1476					 VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
1477		rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
1478							&nb_sb);
1479		if (rc || !nb_sb)
1480			goto out_unlock;
1481		mutex_unlock(&vm->hotplug_mutex);
1482		cond_resched();
 
 
1483		mutex_lock(&vm->hotplug_mutex);
1484	}
 
1485
1486	/* Try to unplug subblocks of plugged online blocks. */
1487	virtio_mem_for_each_mb_state_rev(vm, mb_id,
1488					 VIRTIO_MEM_MB_STATE_ONLINE) {
1489		rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
1490							&nb_sb);
1491		if (rc || !nb_sb)
1492			goto out_unlock;
 
 
 
 
1493		mutex_unlock(&vm->hotplug_mutex);
1494		cond_resched();
1495		mutex_lock(&vm->hotplug_mutex);
1496	}
1497
1498	mutex_unlock(&vm->hotplug_mutex);
1499	return nb_sb ? -EBUSY : 0;
1500out_unlock:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1501	mutex_unlock(&vm->hotplug_mutex);
1502	return rc;
1503}
1504
1505/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1506 * Try to unplug all blocks that couldn't be unplugged before, for example,
1507 * because the hypervisor was busy.
1508 */
1509static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
1510{
1511	unsigned long mb_id;
1512	int rc;
1513
1514	virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_PLUGGED) {
1515		rc = virtio_mem_mb_unplug(vm, mb_id);
 
 
 
 
 
 
 
 
 
 
 
 
1516		if (rc)
1517			return rc;
1518		virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
 
1519	}
1520
1521	return 0;
1522}
1523
1524/*
1525 * Update all parts of the config that could have changed.
1526 */
1527static void virtio_mem_refresh_config(struct virtio_mem *vm)
1528{
1529	const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
1530	uint64_t new_plugged_size, usable_region_size, end_addr;
1531
1532	/* the plugged_size is just a reflection of what _we_ did previously */
1533	virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
1534			&new_plugged_size);
1535	if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
1536		vm->plugged_size = new_plugged_size;
1537
1538	/* calculate the last usable memory block id */
1539	virtio_cread_le(vm->vdev, struct virtio_mem_config,
1540			usable_region_size, &usable_region_size);
1541	end_addr = vm->addr + usable_region_size;
1542	end_addr = min(end_addr, phys_limit);
1543	vm->last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr) - 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1544
1545	/* see if there is a request to change the size */
1546	virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size,
1547			&vm->requested_size);
1548
1549	dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
1550	dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
1551}
1552
1553/*
1554 * Workqueue function for handling plug/unplug requests and config updates.
1555 */
1556static void virtio_mem_run_wq(struct work_struct *work)
1557{
1558	struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
1559	uint64_t diff;
1560	int rc;
1561
 
 
 
 
 
 
1562	hrtimer_cancel(&vm->retry_timer);
1563
1564	if (vm->broken)
1565		return;
1566
 
1567retry:
1568	rc = 0;
1569
1570	/* Make sure we start with a clean state if there are leftovers. */
1571	if (unlikely(vm->unplug_all_required))
1572		rc = virtio_mem_send_unplug_all_request(vm);
1573
1574	if (atomic_read(&vm->config_changed)) {
1575		atomic_set(&vm->config_changed, 0);
1576		virtio_mem_refresh_config(vm);
1577	}
1578
1579	/* Unplug any leftovers from previous runs */
1580	if (!rc)
1581		rc = virtio_mem_unplug_pending_mb(vm);
1582
1583	if (!rc && vm->requested_size != vm->plugged_size) {
1584		if (vm->requested_size > vm->plugged_size) {
1585			diff = vm->requested_size - vm->plugged_size;
1586			rc = virtio_mem_plug_request(vm, diff);
1587		} else {
1588			diff = vm->plugged_size - vm->requested_size;
1589			rc = virtio_mem_unplug_request(vm, diff);
1590		}
1591	}
1592
1593	switch (rc) {
1594	case 0:
1595		vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
1596		break;
1597	case -ENOSPC:
1598		/*
1599		 * We cannot add any more memory (alignment, physical limit)
1600		 * or we have too many offline memory blocks.
1601		 */
1602		break;
1603	case -ETXTBSY:
1604		/*
1605		 * The hypervisor cannot process our request right now
1606		 * (e.g., out of memory, migrating);
1607		 */
1608	case -EBUSY:
1609		/*
1610		 * We cannot free up any memory to unplug it (all plugged memory
1611		 * is busy).
1612		 */
1613	case -ENOMEM:
1614		/* Out of memory, try again later. */
1615		hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
1616			      HRTIMER_MODE_REL);
1617		break;
1618	case -EAGAIN:
1619		/* Retry immediately (e.g., the config changed). */
1620		goto retry;
1621	default:
1622		/* Unknown error, mark as broken */
1623		dev_err(&vm->vdev->dev,
1624			"unknown error, marking device broken: %d\n", rc);
1625		vm->broken = true;
1626	}
 
 
1627}
1628
1629static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
1630{
1631	struct virtio_mem *vm = container_of(timer, struct virtio_mem,
1632					     retry_timer);
1633
1634	virtio_mem_retry(vm);
1635	vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
1636				   VIRTIO_MEM_RETRY_TIMER_MAX_MS);
1637	return HRTIMER_NORESTART;
1638}
1639
1640static void virtio_mem_handle_response(struct virtqueue *vq)
1641{
1642	struct virtio_mem *vm = vq->vdev->priv;
1643
1644	wake_up(&vm->host_resp);
1645}
1646
1647static int virtio_mem_init_vq(struct virtio_mem *vm)
1648{
1649	struct virtqueue *vq;
1650
1651	vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
1652				   "guest-request");
1653	if (IS_ERR(vq))
1654		return PTR_ERR(vq);
1655	vm->vq = vq;
1656
1657	return 0;
1658}
1659
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1660static int virtio_mem_init(struct virtio_mem *vm)
1661{
1662	const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
1663	uint16_t node_id;
1664
1665	if (!vm->vdev->config->get) {
1666		dev_err(&vm->vdev->dev, "config access disabled\n");
1667		return -EINVAL;
1668	}
1669
1670	/*
1671	 * We don't want to (un)plug or reuse any memory when in kdump. The
1672	 * memory is still accessible (but not mapped).
1673	 */
1674	if (is_kdump_kernel()) {
1675		dev_warn(&vm->vdev->dev, "disabled in kdump kernel\n");
1676		return -EBUSY;
1677	}
1678
1679	/* Fetch all properties that can't change. */
1680	virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
1681			&vm->plugged_size);
1682	virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size,
1683			&vm->device_block_size);
1684	virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id,
1685			&node_id);
1686	vm->nid = virtio_mem_translate_node_id(vm, node_id);
1687	virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
1688	virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size,
1689			&vm->region_size);
1690
1691	/*
1692	 * We always hotplug memory in memory block granularity. This way,
1693	 * we have to wait for exactly one memory block to online.
1694	 */
1695	if (vm->device_block_size > memory_block_size_bytes()) {
1696		dev_err(&vm->vdev->dev,
1697			"The block size is not supported (too big).\n");
1698		return -EINVAL;
1699	}
1700
1701	/* bad device setup - warn only */
1702	if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
1703		dev_warn(&vm->vdev->dev,
1704			 "The alignment of the physical start address can make some memory unusable.\n");
1705	if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
1706		dev_warn(&vm->vdev->dev,
1707			 "The alignment of the physical end address can make some memory unusable.\n");
1708	if (vm->addr + vm->region_size > phys_limit)
1709		dev_warn(&vm->vdev->dev,
1710			 "Some memory is not addressable. This can make some memory unusable.\n");
1711
1712	/*
1713	 * Calculate the subblock size:
1714	 * - At least MAX_ORDER - 1 / pageblock_order.
1715	 * - At least the device block size.
1716	 * In the worst case, a single subblock per memory block.
1717	 */
1718	vm->subblock_size = PAGE_SIZE * 1ul << max_t(uint32_t, MAX_ORDER - 1,
1719						     pageblock_order);
1720	vm->subblock_size = max_t(uint64_t, vm->device_block_size,
1721				  vm->subblock_size);
1722	vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size;
1723
1724	/* Round up to the next full memory block */
1725	vm->first_mb_id = virtio_mem_phys_to_mb_id(vm->addr - 1 +
1726						   memory_block_size_bytes());
1727	vm->next_mb_id = vm->first_mb_id;
1728	vm->last_mb_id = virtio_mem_phys_to_mb_id(vm->addr +
1729			 vm->region_size) - 1;
1730
1731	dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
1732	dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
1733	dev_info(&vm->vdev->dev, "device block size: 0x%llx",
1734		 (unsigned long long)vm->device_block_size);
1735	dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
1736		 memory_block_size_bytes());
1737	dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
1738		 (unsigned long long)vm->subblock_size);
1739	if (vm->nid != NUMA_NO_NODE)
1740		dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
1741
1742	return 0;
 
 
 
 
 
 
1743}
1744
1745static int virtio_mem_create_resource(struct virtio_mem *vm)
1746{
1747	/*
1748	 * When force-unloading the driver and removing the device, we
1749	 * could have a garbage pointer. Duplicate the string.
1750	 */
1751	const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
1752
1753	if (!name)
1754		return -ENOMEM;
1755
 
1756	vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
1757						   name, IORESOURCE_SYSTEM_RAM);
 
1758	if (!vm->parent_resource) {
1759		kfree(name);
1760		dev_warn(&vm->vdev->dev, "could not reserve device region\n");
1761		dev_info(&vm->vdev->dev,
1762			 "reloading the driver is not supported\n");
1763		return -EBUSY;
1764	}
1765
1766	/* The memory is not actually busy - make add_memory() work. */
1767	vm->parent_resource->flags &= ~IORESOURCE_BUSY;
1768	return 0;
1769}
1770
1771static void virtio_mem_delete_resource(struct virtio_mem *vm)
1772{
1773	const char *name;
1774
1775	if (!vm->parent_resource)
1776		return;
1777
1778	name = vm->parent_resource->name;
1779	release_resource(vm->parent_resource);
1780	kfree(vm->parent_resource);
1781	kfree(name);
1782	vm->parent_resource = NULL;
1783}
1784
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1785static int virtio_mem_probe(struct virtio_device *vdev)
1786{
1787	struct virtio_mem *vm;
1788	int rc;
1789
1790	BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
1791	BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
1792
1793	vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1794	if (!vm)
1795		return -ENOMEM;
1796
1797	init_waitqueue_head(&vm->host_resp);
1798	vm->vdev = vdev;
1799	INIT_WORK(&vm->wq, virtio_mem_run_wq);
1800	mutex_init(&vm->hotplug_mutex);
1801	INIT_LIST_HEAD(&vm->next);
1802	spin_lock_init(&vm->removal_lock);
1803	hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1804	vm->retry_timer.function = virtio_mem_timer_expired;
1805	vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
 
1806
1807	/* register the virtqueue */
1808	rc = virtio_mem_init_vq(vm);
1809	if (rc)
1810		goto out_free_vm;
1811
1812	/* initialize the device by querying the config */
1813	rc = virtio_mem_init(vm);
1814	if (rc)
1815		goto out_del_vq;
1816
1817	/* create the parent resource for all memory */
1818	rc = virtio_mem_create_resource(vm);
1819	if (rc)
1820		goto out_del_vq;
1821
1822	/*
1823	 * If we still have memory plugged, we have to unplug all memory first.
1824	 * Registering our parent resource makes sure that this memory isn't
1825	 * actually in use (e.g., trying to reload the driver).
1826	 */
1827	if (vm->plugged_size) {
1828		vm->unplug_all_required = 1;
1829		dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
1830	}
1831
1832	/* register callbacks */
1833	vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
1834	rc = register_memory_notifier(&vm->memory_notifier);
1835	if (rc)
1836		goto out_del_resource;
1837	rc = register_virtio_mem_device(vm);
1838	if (rc)
1839		goto out_unreg_mem;
1840
1841	virtio_device_ready(vdev);
1842
1843	/* trigger a config update to start processing the requested_size */
1844	atomic_set(&vm->config_changed, 1);
1845	queue_work(system_freezable_wq, &vm->wq);
 
 
1846
1847	return 0;
1848out_unreg_mem:
1849	unregister_memory_notifier(&vm->memory_notifier);
1850out_del_resource:
1851	virtio_mem_delete_resource(vm);
1852out_del_vq:
1853	vdev->config->del_vqs(vdev);
1854out_free_vm:
1855	kfree(vm);
1856	vdev->priv = NULL;
1857
1858	return rc;
1859}
1860
1861static void virtio_mem_remove(struct virtio_device *vdev)
1862{
1863	struct virtio_mem *vm = vdev->priv;
1864	unsigned long mb_id;
1865	int rc;
1866
1867	/*
1868	 * Make sure the workqueue won't be triggered anymore and no memory
1869	 * blocks can be onlined/offlined until we're finished here.
1870	 */
1871	mutex_lock(&vm->hotplug_mutex);
1872	spin_lock_irq(&vm->removal_lock);
1873	vm->removing = true;
1874	spin_unlock_irq(&vm->removal_lock);
1875	mutex_unlock(&vm->hotplug_mutex);
1876
1877	/* wait until the workqueue stopped */
1878	cancel_work_sync(&vm->wq);
1879	hrtimer_cancel(&vm->retry_timer);
1880
1881	/*
1882	 * After we unregistered our callbacks, user space can online partially
1883	 * plugged offline blocks. Make sure to remove them.
1884	 */
1885	virtio_mem_for_each_mb_state(vm, mb_id,
1886				     VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
1887		rc = virtio_mem_mb_remove(vm, mb_id);
1888		BUG_ON(rc);
1889		virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
 
 
 
 
 
 
 
 
1890	}
1891	/*
1892	 * After we unregistered our callbacks, user space can no longer
1893	 * offline partially plugged online memory blocks. No need to worry
1894	 * about them.
1895	 */
1896
1897	/* unregister callbacks */
1898	unregister_virtio_mem_device(vm);
1899	unregister_memory_notifier(&vm->memory_notifier);
1900
1901	/*
1902	 * There is no way we could reliably remove all memory we have added to
1903	 * the system. And there is no way to stop the driver/device from going
1904	 * away. Warn at least.
1905	 */
1906	if (vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] ||
1907	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL] ||
1908	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE] ||
1909	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL] ||
1910	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE]) {
1911		dev_warn(&vdev->dev, "device still has system memory added\n");
1912	} else {
1913		virtio_mem_delete_resource(vm);
1914		kfree_const(vm->resource_name);
 
1915	}
1916
1917	/* remove all tracking data - no locking needed */
1918	vfree(vm->mb_state);
1919	vfree(vm->sb_bitmap);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1920
1921	/* reset the device and cleanup the queues */
1922	vdev->config->reset(vdev);
1923	vdev->config->del_vqs(vdev);
1924
1925	kfree(vm);
1926	vdev->priv = NULL;
1927}
1928
1929static void virtio_mem_config_changed(struct virtio_device *vdev)
1930{
1931	struct virtio_mem *vm = vdev->priv;
1932
 
 
 
1933	atomic_set(&vm->config_changed, 1);
1934	virtio_mem_retry(vm);
1935}
1936
1937#ifdef CONFIG_PM_SLEEP
1938static int virtio_mem_freeze(struct virtio_device *vdev)
1939{
1940	/*
1941	 * When restarting the VM, all memory is usually unplugged. Don't
1942	 * allow to suspend/hibernate.
1943	 */
1944	dev_err(&vdev->dev, "save/restore not supported.\n");
1945	return -EPERM;
1946}
1947
1948static int virtio_mem_restore(struct virtio_device *vdev)
1949{
1950	return -EPERM;
1951}
1952#endif
1953
1954static unsigned int virtio_mem_features[] = {
1955#if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
1956	VIRTIO_MEM_F_ACPI_PXM,
1957#endif
 
1958};
1959
1960static struct virtio_device_id virtio_mem_id_table[] = {
1961	{ VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
1962	{ 0 },
1963};
1964
1965static struct virtio_driver virtio_mem_driver = {
1966	.feature_table = virtio_mem_features,
1967	.feature_table_size = ARRAY_SIZE(virtio_mem_features),
1968	.driver.name = KBUILD_MODNAME,
1969	.driver.owner = THIS_MODULE,
1970	.id_table = virtio_mem_id_table,
1971	.probe = virtio_mem_probe,
1972	.remove = virtio_mem_remove,
1973	.config_changed = virtio_mem_config_changed,
1974#ifdef CONFIG_PM_SLEEP
1975	.freeze	=	virtio_mem_freeze,
1976	.restore =	virtio_mem_restore,
1977#endif
1978};
1979
1980module_virtio_driver(virtio_mem_driver);
1981MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
1982MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
1983MODULE_DESCRIPTION("Virtio-mem driver");
1984MODULE_LICENSE("GPL");