1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * VMware Balloon driver.
   4 *
   5 * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved.
   6 *
   7 * This is VMware physical memory management driver for Linux. The driver
   8 * acts like a "balloon" that can be inflated to reclaim physical pages by
   9 * reserving them in the guest and invalidating them in the monitor,
  10 * freeing up the underlying machine pages so they can be allocated to
  11 * other guests.  The balloon can also be deflated to allow the guest to
  12 * use more physical memory. Higher level policies can control the sizes
  13 * of balloons in VMs in order to manage physical memory resources.
  14 */
  15
  16//#define DEBUG
  17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19#include <linux/types.h>
  20#include <linux/io.h>
  21#include <linux/kernel.h>
  22#include <linux/mm.h>
  23#include <linux/vmalloc.h>
  24#include <linux/sched.h>
  25#include <linux/module.h>
  26#include <linux/workqueue.h>
  27#include <linux/debugfs.h>
  28#include <linux/seq_file.h>
  29#include <linux/rwsem.h>
  30#include <linux/slab.h>
  31#include <linux/spinlock.h>
 
 
  32#include <linux/balloon_compaction.h>
  33#include <linux/vmw_vmci_defs.h>
  34#include <linux/vmw_vmci_api.h>
  35#include <asm/hypervisor.h>
  36
  37MODULE_AUTHOR("VMware, Inc.");
  38MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver");
  39MODULE_ALIAS("dmi:*:svnVMware*:*");
  40MODULE_ALIAS("vmware_vmmemctl");
  41MODULE_LICENSE("GPL");
  42
  43static bool __read_mostly vmwballoon_shrinker_enable;
  44module_param(vmwballoon_shrinker_enable, bool, 0444);
  45MODULE_PARM_DESC(vmwballoon_shrinker_enable,
  46	"Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance.");
  47
  48/* Delay in seconds after shrink before inflation. */
  49#define VMBALLOON_SHRINK_DELAY		(5)
  50
  51/* Maximum number of refused pages we accumulate during inflation cycle */
  52#define VMW_BALLOON_MAX_REFUSED		16
  53
  54/* Magic number for the balloon mount-point */
  55#define BALLOON_VMW_MAGIC		0x0ba11007
  56
  57/*
  58 * Hypervisor communication port definitions.
  59 */
  60#define VMW_BALLOON_HV_PORT		0x5670
  61#define VMW_BALLOON_HV_MAGIC		0x456c6d6f
  62#define VMW_BALLOON_GUEST_ID		1	/* Linux */
  63
  64enum vmwballoon_capabilities {
  65	/*
  66	 * Bit 0 is reserved and not associated to any capability.
  67	 */
  68	VMW_BALLOON_BASIC_CMDS			= (1 << 1),
  69	VMW_BALLOON_BATCHED_CMDS		= (1 << 2),
  70	VMW_BALLOON_BATCHED_2M_CMDS		= (1 << 3),
  71	VMW_BALLOON_SIGNALLED_WAKEUP_CMD	= (1 << 4),
  72	VMW_BALLOON_64_BIT_TARGET		= (1 << 5)
  73};
  74
  75#define VMW_BALLOON_CAPABILITIES_COMMON	(VMW_BALLOON_BASIC_CMDS \
  76					| VMW_BALLOON_BATCHED_CMDS \
  77					| VMW_BALLOON_BATCHED_2M_CMDS \
  78					| VMW_BALLOON_SIGNALLED_WAKEUP_CMD)
  79
  80#define VMW_BALLOON_2M_ORDER		(PMD_SHIFT - PAGE_SHIFT)
  81
  82/*
  83 * 64-bit targets are only supported in 64-bit
  84 */
  85#ifdef CONFIG_64BIT
  86#define VMW_BALLOON_CAPABILITIES	(VMW_BALLOON_CAPABILITIES_COMMON \
  87					| VMW_BALLOON_64_BIT_TARGET)
  88#else
  89#define VMW_BALLOON_CAPABILITIES	VMW_BALLOON_CAPABILITIES_COMMON
  90#endif
  91
  92enum vmballoon_page_size_type {
  93	VMW_BALLOON_4K_PAGE,
  94	VMW_BALLOON_2M_PAGE,
  95	VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE
  96};
  97
  98#define VMW_BALLOON_NUM_PAGE_SIZES	(VMW_BALLOON_LAST_SIZE + 1)
  99
 100static const char * const vmballoon_page_size_names[] = {
 101	[VMW_BALLOON_4K_PAGE]			= "4k",
 102	[VMW_BALLOON_2M_PAGE]			= "2M"
 103};
 104
 105enum vmballoon_op {
 106	VMW_BALLOON_INFLATE,
 107	VMW_BALLOON_DEFLATE
 108};
 109
 110enum vmballoon_op_stat_type {
 111	VMW_BALLOON_OP_STAT,
 112	VMW_BALLOON_OP_FAIL_STAT
 113};
 114
 115#define VMW_BALLOON_OP_STAT_TYPES	(VMW_BALLOON_OP_FAIL_STAT + 1)
 116
 117/**
 118 * enum vmballoon_cmd_type - backdoor commands.
 119 *
 120 * Availability of the commands is as followed:
 121 *
 122 * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and
 123 * %VMW_BALLOON_CMD_GUEST_ID are always available.
 124 *
 125 * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then
 126 * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available.
 127 *
 128 * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then
 129 * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands
 130 * are available.
 131 *
 132 * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then
 133 * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK
 134 * are supported.
 135 *
 136 * If the host reports  VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then
 137 * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported.
 138 *
 139 * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor.
 140 * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size.
 141 * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page.
 142 * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about
 143 *			    to be deflated from the balloon.
 144 * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that
 145 *			      runs in the VM.
 146 * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of
 147 *				  ballooned pages (up to 512).
 148 * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of
 149 *				  pages that are about to be deflated from the
 150 *				  balloon (up to 512).
 151 * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK
 152 *				     for 2MB pages.
 153 * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to
 154 *				       @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB
 155 *				       pages.
 156 * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification
 157 *				       that would be invoked when the balloon
 158 *				       size changes.
 159 * @VMW_BALLOON_CMD_LAST: Value of the last command.
 160 */
 161enum vmballoon_cmd_type {
 162	VMW_BALLOON_CMD_START,
 163	VMW_BALLOON_CMD_GET_TARGET,
 164	VMW_BALLOON_CMD_LOCK,
 165	VMW_BALLOON_CMD_UNLOCK,
 166	VMW_BALLOON_CMD_GUEST_ID,
 167	/* No command 5 */
 168	VMW_BALLOON_CMD_BATCHED_LOCK = 6,
 169	VMW_BALLOON_CMD_BATCHED_UNLOCK,
 170	VMW_BALLOON_CMD_BATCHED_2M_LOCK,
 171	VMW_BALLOON_CMD_BATCHED_2M_UNLOCK,
 172	VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
 173	VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
 174};
 175
 176#define VMW_BALLOON_CMD_NUM	(VMW_BALLOON_CMD_LAST + 1)
 177
 178enum vmballoon_error_codes {
 179	VMW_BALLOON_SUCCESS,
 180	VMW_BALLOON_ERROR_CMD_INVALID,
 181	VMW_BALLOON_ERROR_PPN_INVALID,
 182	VMW_BALLOON_ERROR_PPN_LOCKED,
 183	VMW_BALLOON_ERROR_PPN_UNLOCKED,
 184	VMW_BALLOON_ERROR_PPN_PINNED,
 185	VMW_BALLOON_ERROR_PPN_NOTNEEDED,
 186	VMW_BALLOON_ERROR_RESET,
 187	VMW_BALLOON_ERROR_BUSY
 188};
 189
 190#define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES	(0x03000000)
 191
 192#define VMW_BALLOON_CMD_WITH_TARGET_MASK			\
 193	((1UL << VMW_BALLOON_CMD_GET_TARGET)		|	\
 194	 (1UL << VMW_BALLOON_CMD_LOCK)			|	\
 195	 (1UL << VMW_BALLOON_CMD_UNLOCK)		|	\
 196	 (1UL << VMW_BALLOON_CMD_BATCHED_LOCK)		|	\
 197	 (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK)	|	\
 198	 (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK)	|	\
 199	 (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK))
 200
 201static const char * const vmballoon_cmd_names[] = {
 202	[VMW_BALLOON_CMD_START]			= "start",
 203	[VMW_BALLOON_CMD_GET_TARGET]		= "target",
 204	[VMW_BALLOON_CMD_LOCK]			= "lock",
 205	[VMW_BALLOON_CMD_UNLOCK]		= "unlock",
 206	[VMW_BALLOON_CMD_GUEST_ID]		= "guestType",
 207	[VMW_BALLOON_CMD_BATCHED_LOCK]		= "batchLock",
 208	[VMW_BALLOON_CMD_BATCHED_UNLOCK]	= "batchUnlock",
 209	[VMW_BALLOON_CMD_BATCHED_2M_LOCK]	= "2m-lock",
 210	[VMW_BALLOON_CMD_BATCHED_2M_UNLOCK]	= "2m-unlock",
 211	[VMW_BALLOON_CMD_VMCI_DOORBELL_SET]	= "doorbellSet"
 212};
 213
 214enum vmballoon_stat_page {
 215	VMW_BALLOON_PAGE_STAT_ALLOC,
 216	VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
 217	VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
 218	VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
 219	VMW_BALLOON_PAGE_STAT_FREE,
 220	VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE
 221};
 222
 223#define VMW_BALLOON_PAGE_STAT_NUM	(VMW_BALLOON_PAGE_STAT_LAST + 1)
 224
 225enum vmballoon_stat_general {
 226	VMW_BALLOON_STAT_TIMER,
 227	VMW_BALLOON_STAT_DOORBELL,
 228	VMW_BALLOON_STAT_RESET,
 229	VMW_BALLOON_STAT_SHRINK,
 230	VMW_BALLOON_STAT_SHRINK_FREE,
 231	VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE
 232};
 233
 234#define VMW_BALLOON_STAT_NUM		(VMW_BALLOON_STAT_LAST + 1)
 235
 236static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
 237static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
 238
 239struct vmballoon_ctl {
 240	struct list_head pages;
 241	struct list_head refused_pages;
 242	struct list_head prealloc_pages;
 243	unsigned int n_refused_pages;
 244	unsigned int n_pages;
 245	enum vmballoon_page_size_type page_size;
 246	enum vmballoon_op op;
 247};
 248
 249/**
 250 * struct vmballoon_batch_entry - a batch entry for lock or unlock.
 251 *
 252 * @status: the status of the operation, which is written by the hypervisor.
 253 * @reserved: reserved for future use. Must be set to zero.
 254 * @pfn: the physical frame number of the page to be locked or unlocked.
 255 */
 256struct vmballoon_batch_entry {
 257	u64 status : 5;
 258	u64 reserved : PAGE_SHIFT - 5;
 259	u64 pfn : 52;
 260} __packed;
 261
 262struct vmballoon {
 263	/**
 264	 * @max_page_size: maximum supported page size for ballooning.
 265	 *
 266	 * Protected by @conf_sem
 267	 */
 268	enum vmballoon_page_size_type max_page_size;
 269
 270	/**
 271	 * @size: balloon actual size in basic page size (frames).
 272	 *
 273	 * While we currently do not support size which is bigger than 32-bit,
 274	 * in preparation for future support, use 64-bits.
 275	 */
 276	atomic64_t size;
 277
 278	/**
 279	 * @target: balloon target size in basic page size (frames).
 280	 *
 281	 * We do not protect the target under the assumption that setting the
 282	 * value is always done through a single write. If this assumption ever
 283	 * breaks, we would have to use X_ONCE for accesses, and suffer the less
 284	 * optimized code. Although we may read stale target value if multiple
 285	 * accesses happen at once, the performance impact should be minor.
 286	 */
 287	unsigned long target;
 288
 289	/**
 290	 * @reset_required: reset flag
 291	 *
 292	 * Setting this flag may introduce races, but the code is expected to
 293	 * handle them gracefully. In the worst case, another operation will
 294	 * fail as reset did not take place. Clearing the flag is done while
 295	 * holding @conf_sem for write.
 296	 */
 297	bool reset_required;
 298
 299	/**
 300	 * @capabilities: hypervisor balloon capabilities.
 301	 *
 302	 * Protected by @conf_sem.
 303	 */
 304	unsigned long capabilities;
 305
 306	/**
 307	 * @batch_page: pointer to communication batch page.
 308	 *
 309	 * When batching is used, batch_page points to a page, which holds up to
 310	 * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking.
 311	 */
 312	struct vmballoon_batch_entry *batch_page;
 313
 314	/**
 315	 * @batch_max_pages: maximum pages that can be locked/unlocked.
 316	 *
 317	 * Indicates the number of pages that the hypervisor can lock or unlock
 318	 * at once, according to whether batching is enabled. If batching is
 319	 * disabled, only a single page can be locked/unlock on each operation.
 320	 *
 321	 * Protected by @conf_sem.
 322	 */
 323	unsigned int batch_max_pages;
 324
 325	/**
 326	 * @page: page to be locked/unlocked by the hypervisor
 327	 *
 328	 * @page is only used when batching is disabled and a single page is
 329	 * reclaimed on each iteration.
 330	 *
 331	 * Protected by @comm_lock.
 332	 */
 333	struct page *page;
 334
 335	/**
 336	 * @shrink_timeout: timeout until the next inflation.
 337	 *
 338	 * After an shrink event, indicates the time in jiffies after which
 339	 * inflation is allowed again. Can be written concurrently with reads,
 340	 * so must use READ_ONCE/WRITE_ONCE when accessing.
 341	 */
 342	unsigned long shrink_timeout;
 343
 344	/* statistics */
 345	struct vmballoon_stats *stats;
 346
 347	/**
 348	 * @b_dev_info: balloon device information descriptor.
 349	 */
 350	struct balloon_dev_info b_dev_info;
 351
 352	struct delayed_work dwork;
 353
 354	/**
 355	 * @huge_pages - list of the inflated 2MB pages.
 356	 *
 357	 * Protected by @b_dev_info.pages_lock .
 358	 */
 359	struct list_head huge_pages;
 360
 361	/**
 362	 * @vmci_doorbell.
 363	 *
 364	 * Protected by @conf_sem.
 365	 */
 366	struct vmci_handle vmci_doorbell;
 367
 368	/**
 369	 * @conf_sem: semaphore to protect the configuration and the statistics.
 370	 */
 371	struct rw_semaphore conf_sem;
 372
 373	/**
 374	 * @comm_lock: lock to protect the communication with the host.
 375	 *
 376	 * Lock ordering: @conf_sem -> @comm_lock .
 377	 */
 378	spinlock_t comm_lock;
 379
 380	/**
 381	 * @shrinker: shrinker interface that is used to avoid over-inflation.
 382	 */
 383	struct shrinker *shrinker;
 
 
 
 
 
 
 
 
 
 384};
 385
 386static struct vmballoon balloon;
 387
 388struct vmballoon_stats {
 389	/* timer / doorbell operations */
 390	atomic64_t general_stat[VMW_BALLOON_STAT_NUM];
 391
 392	/* allocation statistics for huge and small pages */
 393	atomic64_t
 394	       page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES];
 395
 396	/* Monitor operations: total operations, and failures */
 397	atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES];
 398};
 399
 400static inline bool is_vmballoon_stats_on(void)
 401{
 402	return IS_ENABLED(CONFIG_DEBUG_FS) &&
 403		static_branch_unlikely(&balloon_stat_enabled);
 404}
 405
 406static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,
 407					  enum vmballoon_op_stat_type type)
 408{
 409	if (is_vmballoon_stats_on())
 410		atomic64_inc(&b->stats->ops[op][type]);
 411}
 412
 413static inline void vmballoon_stats_gen_inc(struct vmballoon *b,
 414					   enum vmballoon_stat_general stat)
 415{
 416	if (is_vmballoon_stats_on())
 417		atomic64_inc(&b->stats->general_stat[stat]);
 418}
 419
 420static inline void vmballoon_stats_gen_add(struct vmballoon *b,
 421					   enum vmballoon_stat_general stat,
 422					   unsigned int val)
 423{
 424	if (is_vmballoon_stats_on())
 425		atomic64_add(val, &b->stats->general_stat[stat]);
 426}
 427
 428static inline void vmballoon_stats_page_inc(struct vmballoon *b,
 429					    enum vmballoon_stat_page stat,
 430					    enum vmballoon_page_size_type size)
 431{
 432	if (is_vmballoon_stats_on())
 433		atomic64_inc(&b->stats->page_stat[stat][size]);
 434}
 435
 436static inline void vmballoon_stats_page_add(struct vmballoon *b,
 437					    enum vmballoon_stat_page stat,
 438					    enum vmballoon_page_size_type size,
 439					    unsigned int val)
 440{
 441	if (is_vmballoon_stats_on())
 442		atomic64_add(val, &b->stats->page_stat[stat][size]);
 443}
 444
 445static inline unsigned long
 446__vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
 447		unsigned long arg2, unsigned long *result)
 448{
 449	unsigned long status, dummy1, dummy2, dummy3, local_result;
 450
 451	vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);
 452
 453	asm volatile ("inl %%dx" :
 454		"=a"(status),
 455		"=c"(dummy1),
 456		"=d"(dummy2),
 457		"=b"(local_result),
 458		"=S"(dummy3) :
 459		"0"(VMW_BALLOON_HV_MAGIC),
 460		"1"(cmd),
 461		"2"(VMW_BALLOON_HV_PORT),
 462		"3"(arg1),
 463		"4"(arg2) :
 464		"memory");
 465
 466	/* update the result if needed */
 467	if (result)
 468		*result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 :
 469							   local_result;
 470
 471	/* update target when applicable */
 472	if (status == VMW_BALLOON_SUCCESS &&
 473	    ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))
 474		WRITE_ONCE(b->target, local_result);
 475
 476	if (status != VMW_BALLOON_SUCCESS &&
 477	    status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {
 478		vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);
 479		pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",
 480			 __func__, vmballoon_cmd_names[cmd], arg1, arg2,
 481			 status);
 482	}
 483
 484	/* mark reset required accordingly */
 485	if (status == VMW_BALLOON_ERROR_RESET)
 486		b->reset_required = true;
 487
 488	return status;
 489}
 490
 491static __always_inline unsigned long
 492vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
 493	      unsigned long arg2)
 494{
 495	unsigned long dummy;
 496
 497	return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);
 498}
 499
 500/*
 501 * Send "start" command to the host, communicating supported version
 502 * of the protocol.
 503 */
 504static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
 505{
 506	unsigned long status, capabilities;
 507
 508	status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,
 509				 &capabilities);
 510
 511	switch (status) {
 512	case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES:
 513		b->capabilities = capabilities;
 514		break;
 515	case VMW_BALLOON_SUCCESS:
 516		b->capabilities = VMW_BALLOON_BASIC_CMDS;
 517		break;
 518	default:
 519		return -EIO;
 520	}
 521
 522	/*
 523	 * 2MB pages are only supported with batching. If batching is for some
 524	 * reason disabled, do not use 2MB pages, since otherwise the legacy
 525	 * mechanism is used with 2MB pages, causing a failure.
 526	 */
 527	b->max_page_size = VMW_BALLOON_4K_PAGE;
 528	if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&
 529	    (b->capabilities & VMW_BALLOON_BATCHED_CMDS))
 530		b->max_page_size = VMW_BALLOON_2M_PAGE;
 531
 532
 533	return 0;
 534}
 535
 536/**
 537 * vmballoon_send_guest_id - communicate guest type to the host.
 538 *
 539 * @b: pointer to the balloon.
 540 *
 541 * Communicate guest type to the host so that it can adjust ballooning
 542 * algorithm to the one most appropriate for the guest. This command
 543 * is normally issued after sending "start" command and is part of
 544 * standard reset sequence.
 545 *
 546 * Return: zero on success or appropriate error code.
 547 */
 548static int vmballoon_send_guest_id(struct vmballoon *b)
 549{
 550	unsigned long status;
 551
 552	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,
 553			       VMW_BALLOON_GUEST_ID, 0);
 554
 555	return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
 556}
 557
 558/**
 559 * vmballoon_page_order() - return the order of the page
 560 * @page_size: the size of the page.
 561 *
 562 * Return: the allocation order.
 563 */
 564static inline
 565unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size)
 566{
 567	return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0;
 568}
 569
 570/**
 571 * vmballoon_page_in_frames() - returns the number of frames in a page.
 572 * @page_size: the size of the page.
 573 *
 574 * Return: the number of 4k frames.
 575 */
 576static inline unsigned int
 577vmballoon_page_in_frames(enum vmballoon_page_size_type page_size)
 578{
 579	return 1 << vmballoon_page_order(page_size);
 580}
 581
 582/**
 583 * vmballoon_mark_page_offline() - mark a page as offline
 584 * @page: pointer for the page.
 585 * @page_size: the size of the page.
 586 */
 587static void
 588vmballoon_mark_page_offline(struct page *page,
 589			    enum vmballoon_page_size_type page_size)
 590{
 591	int i;
 592
 593	for (i = 0; i < vmballoon_page_in_frames(page_size); i++)
 594		__SetPageOffline(page + i);
 595}
 596
 597/**
 598 * vmballoon_mark_page_online() - mark a page as online
 599 * @page: pointer for the page.
 600 * @page_size: the size of the page.
 601 */
 602static void
 603vmballoon_mark_page_online(struct page *page,
 604			   enum vmballoon_page_size_type page_size)
 605{
 606	int i;
 607
 608	for (i = 0; i < vmballoon_page_in_frames(page_size); i++)
 609		__ClearPageOffline(page + i);
 610}
 611
 612/**
 613 * vmballoon_send_get_target() - Retrieve desired balloon size from the host.
 614 *
 615 * @b: pointer to the balloon.
 616 *
 617 * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required
 618 * by the host-guest protocol and EIO if an error occurred in communicating with
 619 * the host.
 620 */
 621static int vmballoon_send_get_target(struct vmballoon *b)
 622{
 623	unsigned long status;
 624	unsigned long limit;
 625
 626	limit = totalram_pages();
 627
 628	/* Ensure limit fits in 32-bits if 64-bit targets are not supported */
 629	if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&
 630	    limit != (u32)limit)
 631		return -EINVAL;
 632
 633	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);
 634
 635	return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
 636}
 637
 638/**
 639 * vmballoon_alloc_page_list - allocates a list of pages.
 640 *
 641 * @b: pointer to the balloon.
 642 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
 643 * @req_n_pages: the number of requested pages.
 644 *
 645 * Tries to allocate @req_n_pages. Add them to the list of balloon pages in
 646 * @ctl.pages and updates @ctl.n_pages to reflect the number of pages.
 647 *
 648 * Return: zero on success or error code otherwise.
 649 */
 650static int vmballoon_alloc_page_list(struct vmballoon *b,
 651				     struct vmballoon_ctl *ctl,
 652				     unsigned int req_n_pages)
 653{
 654	struct page *page;
 655	unsigned int i;
 656
 657	for (i = 0; i < req_n_pages; i++) {
 658		/*
 659		 * First check if we happen to have pages that were allocated
 660		 * before. This happens when 2MB page rejected during inflation
 661		 * by the hypervisor, and then split into 4KB pages.
 662		 */
 663		if (!list_empty(&ctl->prealloc_pages)) {
 664			page = list_first_entry(&ctl->prealloc_pages,
 665						struct page, lru);
 666			list_del(&page->lru);
 667		} else {
 668			if (ctl->page_size == VMW_BALLOON_2M_PAGE)
 669				page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
 670					__GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);
 671			else
 672				page = balloon_page_alloc();
 673
 674			vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
 675						 ctl->page_size);
 676		}
 677
 678		if (page) {
 679			/* Success. Add the page to the list and continue. */
 680			list_add(&page->lru, &ctl->pages);
 681			continue;
 682		}
 683
 684		/* Allocation failed. Update statistics and stop. */
 685		vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
 686					 ctl->page_size);
 687		break;
 688	}
 689
 690	ctl->n_pages = i;
 691
 692	return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;
 693}
 694
 695/**
 696 * vmballoon_handle_one_result - Handle lock/unlock result for a single page.
 697 *
 698 * @b: pointer for %struct vmballoon.
 699 * @page: pointer for the page whose result should be handled.
 700 * @page_size: size of the page.
 701 * @status: status of the operation as provided by the hypervisor.
 702 */
 703static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,
 704				       enum vmballoon_page_size_type page_size,
 705				       unsigned long status)
 706{
 707	/* On success do nothing. The page is already on the balloon list. */
 708	if (likely(status == VMW_BALLOON_SUCCESS))
 709		return 0;
 710
 711	pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__,
 712		 page_to_pfn(page), status,
 713		 vmballoon_page_size_names[page_size]);
 714
 715	/* Error occurred */
 716	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
 717				 page_size);
 718
 719	return -EIO;
 720}
 721
 722/**
 723 * vmballoon_status_page - returns the status of (un)lock operation
 724 *
 725 * @b: pointer to the balloon.
 726 * @idx: index for the page for which the operation is performed.
 727 * @p: pointer to where the page struct is returned.
 728 *
 729 * Following a lock or unlock operation, returns the status of the operation for
 730 * an individual page. Provides the page that the operation was performed on on
 731 * the @page argument.
 732 *
 733 * Returns: The status of a lock or unlock operation for an individual page.
 734 */
 735static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,
 736					   struct page **p)
 737{
 738	if (static_branch_likely(&vmw_balloon_batching)) {
 739		/* batching mode */
 740		*p = pfn_to_page(b->batch_page[idx].pfn);
 741		return b->batch_page[idx].status;
 742	}
 743
 744	/* non-batching mode */
 745	*p = b->page;
 746
 747	/*
 748	 * If a failure occurs, the indication will be provided in the status
 749	 * of the entire operation, which is considered before the individual
 750	 * page status. So for non-batching mode, the indication is always of
 751	 * success.
 752	 */
 753	return VMW_BALLOON_SUCCESS;
 754}
 755
 756/**
 757 * vmballoon_lock_op - notifies the host about inflated/deflated pages.
 758 * @b: pointer to the balloon.
 759 * @num_pages: number of inflated/deflated pages.
 760 * @page_size: size of the page.
 761 * @op: the type of operation (lock or unlock).
 762 *
 763 * Notify the host about page(s) that were ballooned (or removed from the
 764 * balloon) so that host can use it without fear that guest will need it (or
 765 * stop using them since the VM does). Host may reject some pages, we need to
 766 * check the return value and maybe submit a different page. The pages that are
 767 * inflated/deflated are pointed by @b->page.
 768 *
 769 * Return: result as provided by the hypervisor.
 770 */
 771static unsigned long vmballoon_lock_op(struct vmballoon *b,
 772				       unsigned int num_pages,
 773				       enum vmballoon_page_size_type page_size,
 774				       enum vmballoon_op op)
 775{
 776	unsigned long cmd, pfn;
 777
 778	lockdep_assert_held(&b->comm_lock);
 779
 780	if (static_branch_likely(&vmw_balloon_batching)) {
 781		if (op == VMW_BALLOON_INFLATE)
 782			cmd = page_size == VMW_BALLOON_2M_PAGE ?
 783				VMW_BALLOON_CMD_BATCHED_2M_LOCK :
 784				VMW_BALLOON_CMD_BATCHED_LOCK;
 785		else
 786			cmd = page_size == VMW_BALLOON_2M_PAGE ?
 787				VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
 788				VMW_BALLOON_CMD_BATCHED_UNLOCK;
 789
 790		pfn = PHYS_PFN(virt_to_phys(b->batch_page));
 791	} else {
 792		cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK :
 793						  VMW_BALLOON_CMD_UNLOCK;
 794		pfn = page_to_pfn(b->page);
 795
 796		/* In non-batching mode, PFNs must fit in 32-bit */
 797		if (unlikely(pfn != (u32)pfn))
 798			return VMW_BALLOON_ERROR_PPN_INVALID;
 799	}
 800
 801	return vmballoon_cmd(b, cmd, pfn, num_pages);
 802}
 803
 804/**
 805 * vmballoon_add_page - adds a page towards lock/unlock operation.
 806 *
 807 * @b: pointer to the balloon.
 808 * @idx: index of the page to be ballooned in this batch.
 809 * @p: pointer to the page that is about to be ballooned.
 810 *
 811 * Adds the page to be ballooned. Must be called while holding @comm_lock.
 812 */
 813static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,
 814			       struct page *p)
 815{
 816	lockdep_assert_held(&b->comm_lock);
 817
 818	if (static_branch_likely(&vmw_balloon_batching))
 819		b->batch_page[idx] = (struct vmballoon_batch_entry)
 820					{ .pfn = page_to_pfn(p) };
 821	else
 822		b->page = p;
 823}
 824
 825/**
 826 * vmballoon_lock - lock or unlock a batch of pages.
 827 *
 828 * @b: pointer to the balloon.
 829 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
 830 *
 831 * Notifies the host of about ballooned pages (after inflation or deflation,
 832 * according to @ctl). If the host rejects the page put it on the
 833 * @ctl refuse list. These refused page are then released when moving to the
 834 * next size of pages.
 835 *
 836 * Note that we neither free any @page here nor put them back on the ballooned
 837 * pages list. Instead we queue it for later processing. We do that for several
 838 * reasons. First, we do not want to free the page under the lock. Second, it
 839 * allows us to unify the handling of lock and unlock. In the inflate case, the
 840 * caller will check if there are too many refused pages and release them.
 841 * Although it is not identical to the past behavior, it should not affect
 842 * performance.
 843 */
 844static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)
 845{
 846	unsigned long batch_status;
 847	struct page *page;
 848	unsigned int i, num_pages;
 849
 850	num_pages = ctl->n_pages;
 851	if (num_pages == 0)
 852		return 0;
 853
 854	/* communication with the host is done under the communication lock */
 855	spin_lock(&b->comm_lock);
 856
 857	i = 0;
 858	list_for_each_entry(page, &ctl->pages, lru)
 859		vmballoon_add_page(b, i++, page);
 860
 861	batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,
 862					 ctl->op);
 863
 864	/*
 865	 * Iterate over the pages in the provided list. Since we are changing
 866	 * @ctl->n_pages we are saving the original value in @num_pages and
 867	 * use this value to bound the loop.
 868	 */
 869	for (i = 0; i < num_pages; i++) {
 870		unsigned long status;
 871
 872		status = vmballoon_status_page(b, i, &page);
 873
 874		/*
 875		 * Failure of the whole batch overrides a single operation
 876		 * results.
 877		 */
 878		if (batch_status != VMW_BALLOON_SUCCESS)
 879			status = batch_status;
 880
 881		/* Continue if no error happened */
 882		if (!vmballoon_handle_one_result(b, page, ctl->page_size,
 883						 status))
 884			continue;
 885
 886		/*
 887		 * Error happened. Move the pages to the refused list and update
 888		 * the pages number.
 889		 */
 890		list_move(&page->lru, &ctl->refused_pages);
 891		ctl->n_pages--;
 892		ctl->n_refused_pages++;
 893	}
 894
 895	spin_unlock(&b->comm_lock);
 896
 897	return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
 898}
 899
 900/**
 901 * vmballoon_release_page_list() - Releases a page list
 902 *
 903 * @page_list: list of pages to release.
 904 * @n_pages: pointer to the number of pages.
 905 * @page_size: whether the pages in the list are 2MB (or else 4KB).
 906 *
 907 * Releases the list of pages and zeros the number of pages.
 908 */
 909static void vmballoon_release_page_list(struct list_head *page_list,
 910				       int *n_pages,
 911				       enum vmballoon_page_size_type page_size)
 912{
 913	struct page *page, *tmp;
 914
 915	list_for_each_entry_safe(page, tmp, page_list, lru) {
 916		list_del(&page->lru);
 917		__free_pages(page, vmballoon_page_order(page_size));
 918	}
 919
 920	if (n_pages)
 921		*n_pages = 0;
 922}
 923
 924
 925/*
 926 * Release pages that were allocated while attempting to inflate the
 927 * balloon but were refused by the host for one reason or another.
 928 */
 929static void vmballoon_release_refused_pages(struct vmballoon *b,
 930					    struct vmballoon_ctl *ctl)
 931{
 932	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
 933				 ctl->page_size);
 934
 935	vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages,
 936				    ctl->page_size);
 937}
 938
 939/**
 940 * vmballoon_change - retrieve the required balloon change
 941 *
 942 * @b: pointer for the balloon.
 943 *
 944 * Return: the required change for the balloon size. A positive number
 945 * indicates inflation, a negative number indicates a deflation.
 946 */
 947static int64_t vmballoon_change(struct vmballoon *b)
 948{
 949	int64_t size, target;
 950
 951	size = atomic64_read(&b->size);
 952	target = READ_ONCE(b->target);
 953
 954	/*
 955	 * We must cast first because of int sizes
 956	 * Otherwise we might get huge positives instead of negatives
 957	 */
 958
 959	if (b->reset_required)
 960		return 0;
 961
 962	/* consider a 2MB slack on deflate, unless the balloon is emptied */
 963	if (target < size && target != 0 &&
 964	    size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))
 965		return 0;
 966
 967	/* If an out-of-memory recently occurred, inflation is disallowed. */
 968	if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))
 969		return 0;
 970
 971	return target - size;
 972}
 973
 974/**
 975 * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.
 976 *
 977 * @b: pointer to balloon.
 978 * @pages: list of pages to enqueue.
 979 * @n_pages: pointer to number of pages in list. The value is zeroed.
 980 * @page_size: whether the pages are 2MB or 4KB pages.
 981 *
 982 * Enqueues the provides list of pages in the ballooned page list, clears the
 983 * list and zeroes the number of pages that was provided.
 984 */
 985static void vmballoon_enqueue_page_list(struct vmballoon *b,
 986					struct list_head *pages,
 987					unsigned int *n_pages,
 988					enum vmballoon_page_size_type page_size)
 989{
 990	unsigned long flags;
 991	struct page *page;
 992
 993	if (page_size == VMW_BALLOON_4K_PAGE) {
 994		balloon_page_list_enqueue(&b->b_dev_info, pages);
 995	} else {
 996		/*
 997		 * Keep the huge pages in a local list which is not available
 998		 * for the balloon compaction mechanism.
 999		 */
1000		spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
1001
1002		list_for_each_entry(page, pages, lru) {
1003			vmballoon_mark_page_offline(page, VMW_BALLOON_2M_PAGE);
1004		}
1005
1006		list_splice_init(pages, &b->huge_pages);
1007		__count_vm_events(BALLOON_INFLATE, *n_pages *
1008				  vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));
1009		spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
1010	}
1011
1012	*n_pages = 0;
1013}
1014
1015/**
1016 * vmballoon_dequeue_page_list() - Dequeues page lists for deflation.
1017 *
1018 * @b: pointer to balloon.
1019 * @pages: list of pages to enqueue.
1020 * @n_pages: pointer to number of pages in list. The value is zeroed.
1021 * @page_size: whether the pages are 2MB or 4KB pages.
1022 * @n_req_pages: the number of requested pages.
1023 *
1024 * Dequeues the number of requested pages from the balloon for deflation. The
1025 * number of dequeued pages may be lower, if not enough pages in the requested
1026 * size are available.
1027 */
1028static void vmballoon_dequeue_page_list(struct vmballoon *b,
1029					struct list_head *pages,
1030					unsigned int *n_pages,
1031					enum vmballoon_page_size_type page_size,
1032					unsigned int n_req_pages)
1033{
1034	struct page *page, *tmp;
1035	unsigned int i = 0;
1036	unsigned long flags;
1037
1038	/* In the case of 4k pages, use the compaction infrastructure */
1039	if (page_size == VMW_BALLOON_4K_PAGE) {
1040		*n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,
1041						     n_req_pages);
1042		return;
1043	}
1044
1045	/* 2MB pages */
1046	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
1047	list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {
1048		vmballoon_mark_page_online(page, VMW_BALLOON_2M_PAGE);
1049
1050		list_move(&page->lru, pages);
1051		if (++i == n_req_pages)
1052			break;
1053	}
1054
1055	__count_vm_events(BALLOON_DEFLATE,
1056			  i * vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));
1057	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
1058	*n_pages = i;
1059}
1060
1061/**
1062 * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k.
1063 *
1064 * If inflation of 2MB pages was denied by the hypervisor, it is likely to be
1065 * due to one or few 4KB pages. These 2MB pages may keep being allocated and
1066 * then being refused. To prevent this case, this function splits the refused
1067 * pages into 4KB pages and adds them into @prealloc_pages list.
1068 *
1069 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
1070 */
1071static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl)
1072{
1073	struct page *page, *tmp;
1074	unsigned int i, order;
1075
1076	order = vmballoon_page_order(ctl->page_size);
1077
1078	list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) {
1079		list_del(&page->lru);
1080		split_page(page, order);
1081		for (i = 0; i < (1 << order); i++)
1082			list_add(&page[i].lru, &ctl->prealloc_pages);
1083	}
1084	ctl->n_refused_pages = 0;
1085}
1086
1087/**
1088 * vmballoon_inflate() - Inflate the balloon towards its target size.
1089 *
1090 * @b: pointer to the balloon.
1091 */
1092static void vmballoon_inflate(struct vmballoon *b)
1093{
1094	int64_t to_inflate_frames;
1095	struct vmballoon_ctl ctl = {
1096		.pages = LIST_HEAD_INIT(ctl.pages),
1097		.refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
1098		.prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages),
1099		.page_size = b->max_page_size,
1100		.op = VMW_BALLOON_INFLATE
1101	};
1102
1103	while ((to_inflate_frames = vmballoon_change(b)) > 0) {
1104		unsigned int to_inflate_pages, page_in_frames;
1105		int alloc_error, lock_error = 0;
1106
1107		VM_BUG_ON(!list_empty(&ctl.pages));
1108		VM_BUG_ON(ctl.n_pages != 0);
1109
1110		page_in_frames = vmballoon_page_in_frames(ctl.page_size);
1111
1112		to_inflate_pages = min_t(unsigned long, b->batch_max_pages,
1113					 DIV_ROUND_UP_ULL(to_inflate_frames,
1114							  page_in_frames));
1115
1116		/* Start by allocating */
1117		alloc_error = vmballoon_alloc_page_list(b, &ctl,
1118							to_inflate_pages);
1119
1120		/* Actually lock the pages by telling the hypervisor */
1121		lock_error = vmballoon_lock(b, &ctl);
1122
1123		/*
1124		 * If an error indicates that something serious went wrong,
1125		 * stop the inflation.
1126		 */
1127		if (lock_error)
1128			break;
1129
1130		/* Update the balloon size */
1131		atomic64_add(ctl.n_pages * page_in_frames, &b->size);
1132
1133		vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,
1134					    ctl.page_size);
1135
1136		/*
1137		 * If allocation failed or the number of refused pages exceeds
1138		 * the maximum allowed, move to the next page size.
1139		 */
1140		if (alloc_error ||
1141		    ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) {
1142			if (ctl.page_size == VMW_BALLOON_4K_PAGE)
1143				break;
1144
1145			/*
1146			 * Split the refused pages to 4k. This will also empty
1147			 * the refused pages list.
1148			 */
1149			vmballoon_split_refused_pages(&ctl);
1150			ctl.page_size--;
1151		}
1152
1153		cond_resched();
1154	}
1155
1156	/*
1157	 * Release pages that were allocated while attempting to inflate the
1158	 * balloon but were refused by the host for one reason or another,
1159	 * and update the statistics.
1160	 */
1161	if (ctl.n_refused_pages != 0)
1162		vmballoon_release_refused_pages(b, &ctl);
1163
1164	vmballoon_release_page_list(&ctl.prealloc_pages, NULL, ctl.page_size);
1165}
1166
1167/**
1168 * vmballoon_deflate() - Decrease the size of the balloon.
1169 *
1170 * @b: pointer to the balloon
1171 * @n_frames: the number of frames to deflate. If zero, automatically
1172 * calculated according to the target size.
1173 * @coordinated: whether to coordinate with the host
1174 *
1175 * Decrease the size of the balloon allowing guest to use more memory.
1176 *
1177 * Return: The number of deflated frames (i.e., basic page size units)
1178 */
1179static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,
1180				       bool coordinated)
1181{
1182	unsigned long deflated_frames = 0;
1183	unsigned long tried_frames = 0;
1184	struct vmballoon_ctl ctl = {
1185		.pages = LIST_HEAD_INIT(ctl.pages),
1186		.refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
1187		.page_size = VMW_BALLOON_4K_PAGE,
1188		.op = VMW_BALLOON_DEFLATE
1189	};
1190
1191	/* free pages to reach target */
1192	while (true) {
1193		unsigned int to_deflate_pages, n_unlocked_frames;
1194		unsigned int page_in_frames;
1195		int64_t to_deflate_frames;
1196		bool deflated_all;
1197
1198		page_in_frames = vmballoon_page_in_frames(ctl.page_size);
1199
1200		VM_BUG_ON(!list_empty(&ctl.pages));
1201		VM_BUG_ON(ctl.n_pages);
1202		VM_BUG_ON(!list_empty(&ctl.refused_pages));
1203		VM_BUG_ON(ctl.n_refused_pages);
1204
1205		/*
1206		 * If we were requested a specific number of frames, we try to
1207		 * deflate this number of frames. Otherwise, deflation is
1208		 * performed according to the target and balloon size.
1209		 */
1210		to_deflate_frames = n_frames ? n_frames - tried_frames :
1211					       -vmballoon_change(b);
1212
1213		/* break if no work to do */
1214		if (to_deflate_frames <= 0)
1215			break;
1216
1217		/*
1218		 * Calculate the number of frames based on current page size,
1219		 * but limit the deflated frames to a single chunk
1220		 */
1221		to_deflate_pages = min_t(unsigned long, b->batch_max_pages,
1222					 DIV_ROUND_UP_ULL(to_deflate_frames,
1223							  page_in_frames));
1224
1225		/* First take the pages from the balloon pages. */
1226		vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,
1227					    ctl.page_size, to_deflate_pages);
1228
1229		/*
1230		 * Before pages are moving to the refused list, count their
1231		 * frames as frames that we tried to deflate.
1232		 */
1233		tried_frames += ctl.n_pages * page_in_frames;
1234
1235		/*
1236		 * Unlock the pages by communicating with the hypervisor if the
1237		 * communication is coordinated (i.e., not pop). We ignore the
1238		 * return code. Instead we check if all the pages we manage to
1239		 * unlock all the pages. If we failed, we will move to the next
1240		 * page size, and would eventually try again later.
1241		 */
1242		if (coordinated)
1243			vmballoon_lock(b, &ctl);
1244
1245		/*
1246		 * Check if we deflated enough. We will move to the next page
1247		 * size if we did not manage to do so. This calculation takes
1248		 * place now, as once the pages are released, the number of
1249		 * pages is zeroed.
1250		 */
1251		deflated_all = (ctl.n_pages == to_deflate_pages);
1252
1253		/* Update local and global counters */
1254		n_unlocked_frames = ctl.n_pages * page_in_frames;
1255		atomic64_sub(n_unlocked_frames, &b->size);
1256		deflated_frames += n_unlocked_frames;
1257
1258		vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,
1259					 ctl.page_size, ctl.n_pages);
1260
1261		/* free the ballooned pages */
1262		vmballoon_release_page_list(&ctl.pages, &ctl.n_pages,
1263					    ctl.page_size);
1264
1265		/* Return the refused pages to the ballooned list. */
1266		vmballoon_enqueue_page_list(b, &ctl.refused_pages,
1267					    &ctl.n_refused_pages,
1268					    ctl.page_size);
1269
1270		/* If we failed to unlock all the pages, move to next size. */
1271		if (!deflated_all) {
1272			if (ctl.page_size == b->max_page_size)
1273				break;
1274			ctl.page_size++;
1275		}
1276
1277		cond_resched();
1278	}
1279
1280	return deflated_frames;
1281}
1282
1283/**
1284 * vmballoon_deinit_batching - disables batching mode.
1285 *
1286 * @b: pointer to &struct vmballoon.
1287 *
1288 * Disables batching, by deallocating the page for communication with the
1289 * hypervisor and disabling the static key to indicate that batching is off.
1290 */
1291static void vmballoon_deinit_batching(struct vmballoon *b)
1292{
1293	free_page((unsigned long)b->batch_page);
1294	b->batch_page = NULL;
1295	static_branch_disable(&vmw_balloon_batching);
1296	b->batch_max_pages = 1;
1297}
1298
1299/**
1300 * vmballoon_init_batching - enable batching mode.
1301 *
1302 * @b: pointer to &struct vmballoon.
1303 *
1304 * Enables batching, by allocating a page for communication with the hypervisor
1305 * and enabling the static_key to use batching.
1306 *
1307 * Return: zero on success or an appropriate error-code.
1308 */
1309static int vmballoon_init_batching(struct vmballoon *b)
1310{
1311	struct page *page;
1312
1313	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1314	if (!page)
1315		return -ENOMEM;
1316
1317	b->batch_page = page_address(page);
1318	b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);
1319
1320	static_branch_enable(&vmw_balloon_batching);
1321
1322	return 0;
1323}
1324
1325/*
1326 * Receive notification and resize balloon
1327 */
1328static void vmballoon_doorbell(void *client_data)
1329{
1330	struct vmballoon *b = client_data;
1331
1332	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL);
1333
1334	mod_delayed_work(system_freezable_wq, &b->dwork, 0);
1335}
1336
1337/*
1338 * Clean up vmci doorbell
1339 */
1340static void vmballoon_vmci_cleanup(struct vmballoon *b)
1341{
1342	vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
1343		      VMCI_INVALID_ID, VMCI_INVALID_ID);
1344
1345	if (!vmci_handle_is_invalid(b->vmci_doorbell)) {
1346		vmci_doorbell_destroy(b->vmci_doorbell);
1347		b->vmci_doorbell = VMCI_INVALID_HANDLE;
1348	}
1349}
1350
1351/**
1352 * vmballoon_vmci_init - Initialize vmci doorbell.
1353 *
1354 * @b: pointer to the balloon.
1355 *
1356 * Return: zero on success or when wakeup command not supported. Error-code
1357 * otherwise.
1358 *
1359 * Initialize vmci doorbell, to get notified as soon as balloon changes.
1360 */
1361static int vmballoon_vmci_init(struct vmballoon *b)
1362{
1363	unsigned long error;
1364
1365	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)
1366		return 0;
1367
1368	error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,
1369				     VMCI_PRIVILEGE_FLAG_RESTRICTED,
1370				     vmballoon_doorbell, b);
1371
1372	if (error != VMCI_SUCCESS)
1373		goto fail;
1374
1375	error =	__vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
1376				b->vmci_doorbell.context,
1377				b->vmci_doorbell.resource, NULL);
1378
1379	if (error != VMW_BALLOON_SUCCESS)
1380		goto fail;
1381
1382	return 0;
1383fail:
1384	vmballoon_vmci_cleanup(b);
1385	return -EIO;
1386}
1387
1388/**
1389 * vmballoon_pop - Quickly release all pages allocate for the balloon.
1390 *
1391 * @b: pointer to the balloon.
1392 *
1393 * This function is called when host decides to "reset" balloon for one reason
1394 * or another. Unlike normal "deflate" we do not (shall not) notify host of the
1395 * pages being released.
1396 */
1397static void vmballoon_pop(struct vmballoon *b)
1398{
1399	unsigned long size;
1400
1401	while ((size = atomic64_read(&b->size)))
1402		vmballoon_deflate(b, size, false);
1403}
1404
1405/*
1406 * Perform standard reset sequence by popping the balloon (in case it
1407 * is not  empty) and then restarting protocol. This operation normally
1408 * happens when host responds with VMW_BALLOON_ERROR_RESET to a command.
1409 */
1410static void vmballoon_reset(struct vmballoon *b)
1411{
1412	int error;
1413
1414	down_write(&b->conf_sem);
1415
1416	vmballoon_vmci_cleanup(b);
1417
1418	/* free all pages, skipping monitor unlock */
1419	vmballoon_pop(b);
1420
1421	if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))
1422		goto unlock;
1423
1424	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {
1425		if (vmballoon_init_batching(b)) {
1426			/*
1427			 * We failed to initialize batching, inform the monitor
1428			 * about it by sending a null capability.
1429			 *
1430			 * The guest will retry in one second.
1431			 */
1432			vmballoon_send_start(b, 0);
1433			goto unlock;
1434		}
1435	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {
1436		vmballoon_deinit_batching(b);
1437	}
1438
1439	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET);
1440	b->reset_required = false;
1441
1442	error = vmballoon_vmci_init(b);
1443	if (error)
1444		pr_err_once("failed to initialize vmci doorbell\n");
1445
1446	if (vmballoon_send_guest_id(b))
1447		pr_err_once("failed to send guest ID to the host\n");
1448
1449unlock:
1450	up_write(&b->conf_sem);
1451}
1452
1453/**
1454 * vmballoon_work - periodic balloon worker for reset, inflation and deflation.
1455 *
1456 * @work: pointer to the &work_struct which is provided by the workqueue.
1457 *
1458 * Resets the protocol if needed, gets the new size and adjusts balloon as
1459 * needed. Repeat in 1 sec.
1460 */
1461static void vmballoon_work(struct work_struct *work)
1462{
1463	struct delayed_work *dwork = to_delayed_work(work);
1464	struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);
1465	int64_t change = 0;
1466
1467	if (b->reset_required)
1468		vmballoon_reset(b);
1469
1470	down_read(&b->conf_sem);
1471
1472	/*
1473	 * Update the stats while holding the semaphore to ensure that
1474	 * @stats_enabled is consistent with whether the stats are actually
1475	 * enabled
1476	 */
1477	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER);
1478
1479	if (!vmballoon_send_get_target(b))
1480		change = vmballoon_change(b);
1481
1482	if (change != 0) {
1483		pr_debug("%s - size: %llu, target %lu\n", __func__,
1484			 atomic64_read(&b->size), READ_ONCE(b->target));
1485
1486		if (change > 0)
1487			vmballoon_inflate(b);
1488		else  /* (change < 0) */
1489			vmballoon_deflate(b, 0, true);
1490	}
1491
1492	up_read(&b->conf_sem);
1493
1494	/*
1495	 * We are using a freezable workqueue so that balloon operations are
1496	 * stopped while the system transitions to/from sleep/hibernation.
1497	 */
1498	queue_delayed_work(system_freezable_wq,
1499			   dwork, round_jiffies_relative(HZ));
1500
1501}
1502
1503/**
1504 * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure.
1505 * @shrinker: pointer to the balloon shrinker.
1506 * @sc: page reclaim information.
1507 *
1508 * Returns: number of pages that were freed during deflation.
1509 */
1510static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker,
1511					     struct shrink_control *sc)
1512{
1513	struct vmballoon *b = &balloon;
1514	unsigned long deflated_frames;
1515
1516	pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));
1517
1518	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK);
1519
1520	/*
1521	 * If the lock is also contended for read, we cannot easily reclaim and
1522	 * we bail out.
1523	 */
1524	if (!down_read_trylock(&b->conf_sem))
1525		return 0;
1526
1527	deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);
1528
1529	vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE,
1530				deflated_frames);
1531
1532	/*
1533	 * Delay future inflation for some time to mitigate the situations in
1534	 * which balloon continuously grows and shrinks. Use WRITE_ONCE() since
1535	 * the access is asynchronous.
1536	 */
1537	WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);
1538
1539	up_read(&b->conf_sem);
1540
1541	return deflated_frames;
1542}
1543
1544/**
1545 * vmballoon_shrinker_count() - return the number of ballooned pages.
1546 * @shrinker: pointer to the balloon shrinker.
1547 * @sc: page reclaim information.
1548 *
1549 * Returns: number of 4k pages that are allocated for the balloon and can
1550 *	    therefore be reclaimed under pressure.
1551 */
1552static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker,
1553					      struct shrink_control *sc)
1554{
1555	struct vmballoon *b = &balloon;
1556
1557	return atomic64_read(&b->size);
1558}
1559
1560static void vmballoon_unregister_shrinker(struct vmballoon *b)
1561{
1562	shrinker_free(b->shrinker);
1563	b->shrinker = NULL;
 
1564}
1565
1566static int vmballoon_register_shrinker(struct vmballoon *b)
1567{
 
 
1568	/* Do nothing if the shrinker is not enabled */
1569	if (!vmwballoon_shrinker_enable)
1570		return 0;
1571
1572	b->shrinker = shrinker_alloc(0, "vmw-balloon");
1573	if (!b->shrinker)
1574		return -ENOMEM;
1575
1576	b->shrinker->scan_objects = vmballoon_shrinker_scan;
1577	b->shrinker->count_objects = vmballoon_shrinker_count;
1578	b->shrinker->private_data = b;
1579
1580	shrinker_register(b->shrinker);
 
1581
1582	return 0;
1583}
1584
1585/*
1586 * DEBUGFS Interface
1587 */
1588#ifdef CONFIG_DEBUG_FS
1589
1590static const char * const vmballoon_stat_page_names[] = {
1591	[VMW_BALLOON_PAGE_STAT_ALLOC]		= "alloc",
1592	[VMW_BALLOON_PAGE_STAT_ALLOC_FAIL]	= "allocFail",
1593	[VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC]	= "errAlloc",
1594	[VMW_BALLOON_PAGE_STAT_REFUSED_FREE]	= "errFree",
1595	[VMW_BALLOON_PAGE_STAT_FREE]		= "free"
1596};
1597
1598static const char * const vmballoon_stat_names[] = {
1599	[VMW_BALLOON_STAT_TIMER]		= "timer",
1600	[VMW_BALLOON_STAT_DOORBELL]		= "doorbell",
1601	[VMW_BALLOON_STAT_RESET]		= "reset",
1602	[VMW_BALLOON_STAT_SHRINK]		= "shrink",
1603	[VMW_BALLOON_STAT_SHRINK_FREE]		= "shrinkFree"
1604};
1605
1606static int vmballoon_enable_stats(struct vmballoon *b)
1607{
1608	int r = 0;
1609
1610	down_write(&b->conf_sem);
1611
1612	/* did we somehow race with another reader which enabled stats? */
1613	if (b->stats)
1614		goto out;
1615
1616	b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);
1617
1618	if (!b->stats) {
1619		/* allocation failed */
1620		r = -ENOMEM;
1621		goto out;
1622	}
1623	static_key_enable(&balloon_stat_enabled.key);
1624out:
1625	up_write(&b->conf_sem);
1626	return r;
1627}
1628
1629/**
1630 * vmballoon_debug_show - shows statistics of balloon operations.
1631 * @f: pointer to the &struct seq_file.
1632 * @offset: ignored.
1633 *
1634 * Provides the statistics that can be accessed in vmmemctl in the debugfs.
1635 * To avoid the overhead - mainly that of memory - of collecting the statistics,
1636 * we only collect statistics after the first time the counters are read.
1637 *
1638 * Return: zero on success or an error code.
1639 */
1640static int vmballoon_debug_show(struct seq_file *f, void *offset)
1641{
1642	struct vmballoon *b = f->private;
1643	int i, j;
1644
1645	/* enables stats if they are disabled */
1646	if (!b->stats) {
1647		int r = vmballoon_enable_stats(b);
1648
1649		if (r)
1650			return r;
1651	}
1652
1653	/* format capabilities info */
1654	seq_printf(f, "%-22s: %#16x\n", "balloon capabilities",
1655		   VMW_BALLOON_CAPABILITIES);
1656	seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);
1657	seq_printf(f, "%-22s: %16s\n", "is resetting",
1658		   b->reset_required ? "y" : "n");
1659
1660	/* format size info */
1661	seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));
1662	seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));
1663
1664	for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {
1665		if (vmballoon_cmd_names[i] == NULL)
1666			continue;
1667
1668		seq_printf(f, "%-22s: %16llu (%llu failed)\n",
1669			   vmballoon_cmd_names[i],
1670			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),
1671			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));
1672	}
1673
1674	for (i = 0; i < VMW_BALLOON_STAT_NUM; i++)
1675		seq_printf(f, "%-22s: %16llu\n",
1676			   vmballoon_stat_names[i],
1677			   atomic64_read(&b->stats->general_stat[i]));
1678
1679	for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) {
1680		for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++)
1681			seq_printf(f, "%-18s(%s): %16llu\n",
1682				   vmballoon_stat_page_names[i],
1683				   vmballoon_page_size_names[j],
1684				   atomic64_read(&b->stats->page_stat[i][j]));
1685	}
1686
1687	return 0;
1688}
1689
1690DEFINE_SHOW_ATTRIBUTE(vmballoon_debug);
1691
1692static void __init vmballoon_debugfs_init(struct vmballoon *b)
1693{
1694	debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,
1695			    &vmballoon_debug_fops);
1696}
1697
1698static void __exit vmballoon_debugfs_exit(struct vmballoon *b)
1699{
1700	static_key_disable(&balloon_stat_enabled.key);
1701	debugfs_lookup_and_remove("vmmemctl", NULL);
1702	kfree(b->stats);
1703	b->stats = NULL;
1704}
1705
1706#else
1707
1708static inline void vmballoon_debugfs_init(struct vmballoon *b)
1709{
1710}
1711
1712static inline void vmballoon_debugfs_exit(struct vmballoon *b)
1713{
1714}
1715
1716#endif	/* CONFIG_DEBUG_FS */
1717
1718
1719#ifdef CONFIG_BALLOON_COMPACTION
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1720/**
1721 * vmballoon_migratepage() - migrates a balloon page.
1722 * @b_dev_info: balloon device information descriptor.
1723 * @newpage: the page to which @page should be migrated.
1724 * @page: a ballooned page that should be migrated.
1725 * @mode: migration mode, ignored.
1726 *
1727 * This function is really open-coded, but that is according to the interface
1728 * that balloon_compaction provides.
1729 *
1730 * Return: zero on success, -EAGAIN when migration cannot be performed
1731 *	   momentarily, and -EBUSY if migration failed and should be retried
1732 *	   with that specific page.
1733 */
1734static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info,
1735				 struct page *newpage, struct page *page,
1736				 enum migrate_mode mode)
1737{
1738	unsigned long status, flags;
1739	struct vmballoon *b;
1740	int ret;
1741
1742	b = container_of(b_dev_info, struct vmballoon, b_dev_info);
1743
1744	/*
1745	 * If the semaphore is taken, there is ongoing configuration change
1746	 * (i.e., balloon reset), so try again.
1747	 */
1748	if (!down_read_trylock(&b->conf_sem))
1749		return -EAGAIN;
1750
1751	spin_lock(&b->comm_lock);
1752	/*
1753	 * We must start by deflating and not inflating, as otherwise the
1754	 * hypervisor may tell us that it has enough memory and the new page is
1755	 * not needed. Since the old page is isolated, we cannot use the list
1756	 * interface to unlock it, as the LRU field is used for isolation.
1757	 * Instead, we use the native interface directly.
1758	 */
1759	vmballoon_add_page(b, 0, page);
1760	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,
1761				   VMW_BALLOON_DEFLATE);
1762
1763	if (status == VMW_BALLOON_SUCCESS)
1764		status = vmballoon_status_page(b, 0, &page);
1765
1766	/*
1767	 * If a failure happened, let the migration mechanism know that it
1768	 * should not retry.
1769	 */
1770	if (status != VMW_BALLOON_SUCCESS) {
1771		spin_unlock(&b->comm_lock);
1772		ret = -EBUSY;
1773		goto out_unlock;
1774	}
1775
1776	/*
1777	 * The page is isolated, so it is safe to delete it without holding
1778	 * @pages_lock . We keep holding @comm_lock since we will need it in a
1779	 * second.
1780	 */
1781	balloon_page_delete(page);
1782
1783	put_page(page);
1784
1785	/* Inflate */
1786	vmballoon_add_page(b, 0, newpage);
1787	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,
1788				   VMW_BALLOON_INFLATE);
1789
1790	if (status == VMW_BALLOON_SUCCESS)
1791		status = vmballoon_status_page(b, 0, &newpage);
1792
1793	spin_unlock(&b->comm_lock);
1794
1795	if (status != VMW_BALLOON_SUCCESS) {
1796		/*
1797		 * A failure happened. While we can deflate the page we just
1798		 * inflated, this deflation can also encounter an error. Instead
1799		 * we will decrease the size of the balloon to reflect the
1800		 * change and report failure.
1801		 */
1802		atomic64_dec(&b->size);
1803		ret = -EBUSY;
1804	} else {
1805		/*
1806		 * Success. Take a reference for the page, and we will add it to
1807		 * the list after acquiring the lock.
1808		 */
1809		get_page(newpage);
1810		ret = MIGRATEPAGE_SUCCESS;
1811	}
1812
1813	/* Update the balloon list under the @pages_lock */
1814	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
1815
1816	/*
1817	 * On inflation success, we already took a reference for the @newpage.
1818	 * If we succeed just insert it to the list and update the statistics
1819	 * under the lock.
1820	 */
1821	if (ret == MIGRATEPAGE_SUCCESS) {
1822		balloon_page_insert(&b->b_dev_info, newpage);
1823		__count_vm_event(BALLOON_MIGRATE);
1824	}
1825
1826	/*
1827	 * We deflated successfully, so regardless to the inflation success, we
1828	 * need to reduce the number of isolated_pages.
1829	 */
1830	b->b_dev_info.isolated_pages--;
1831	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);
1832
1833out_unlock:
1834	up_read(&b->conf_sem);
1835	return ret;
1836}
1837
1838/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1839 * vmballoon_compaction_init() - initialized compaction for the balloon.
1840 *
1841 * @b: pointer to the balloon.
1842 *
1843 * If during the initialization a failure occurred, this function does not
1844 * perform cleanup. The caller must call vmballoon_compaction_deinit() in this
1845 * case.
1846 *
1847 * Return: zero on success or error code on failure.
1848 */
1849static __init void vmballoon_compaction_init(struct vmballoon *b)
1850{
 
 
 
 
1851	b->b_dev_info.migratepage = vmballoon_migratepage;
 
 
 
 
 
 
 
1852}
1853
1854#else /* CONFIG_BALLOON_COMPACTION */
1855static inline void vmballoon_compaction_init(struct vmballoon *b)
 
1856{
1857}
 
 
 
 
 
 
1858#endif /* CONFIG_BALLOON_COMPACTION */
1859
1860static int __init vmballoon_init(void)
1861{
1862	int error;
1863
1864	/*
1865	 * Check if we are running on VMware's hypervisor and bail out
1866	 * if we are not.
1867	 */
1868	if (x86_hyper_type != X86_HYPER_VMWARE)
1869		return -ENODEV;
1870
1871	INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
1872
1873	error = vmballoon_register_shrinker(&balloon);
1874	if (error)
1875		return error;
1876
1877	/*
1878	 * Initialization of compaction must be done after the call to
1879	 * balloon_devinfo_init() .
1880	 */
1881	balloon_devinfo_init(&balloon.b_dev_info);
1882	vmballoon_compaction_init(&balloon);
 
 
1883
1884	INIT_LIST_HEAD(&balloon.huge_pages);
1885	spin_lock_init(&balloon.comm_lock);
1886	init_rwsem(&balloon.conf_sem);
1887	balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
1888	balloon.batch_page = NULL;
1889	balloon.page = NULL;
1890	balloon.reset_required = true;
1891
1892	queue_delayed_work(system_freezable_wq, &balloon.dwork, 0);
1893
1894	vmballoon_debugfs_init(&balloon);
1895
1896	return 0;
 
 
 
 
1897}
1898
1899/*
1900 * Using late_initcall() instead of module_init() allows the balloon to use the
1901 * VMCI doorbell even when the balloon is built into the kernel. Otherwise the
1902 * VMCI is probed only after the balloon is initialized. If the balloon is used
1903 * as a module, late_initcall() is equivalent to module_init().
1904 */
1905late_initcall(vmballoon_init);
1906
1907static void __exit vmballoon_exit(void)
1908{
1909	vmballoon_unregister_shrinker(&balloon);
1910	vmballoon_vmci_cleanup(&balloon);
1911	cancel_delayed_work_sync(&balloon.dwork);
1912
1913	vmballoon_debugfs_exit(&balloon);
1914
1915	/*
1916	 * Deallocate all reserved memory, and reset connection with monitor.
1917	 * Reset connection before deallocating memory to avoid potential for
1918	 * additional spurious resets from guest touching deallocated pages.
1919	 */
1920	vmballoon_send_start(&balloon, 0);
1921	vmballoon_pop(&balloon);
 
 
 
1922}
1923module_exit(vmballoon_exit);