1/******************************************************************************
   2 * grant_table.c
   3 *
   4 * Granting foreign access to our memory reservation.
   5 *
   6 * Copyright (c) 2005-2006, Christopher Clark
   7 * Copyright (c) 2004-2005, K A Fraser
   8 *
   9 * This program is free software; you can redistribute it and/or
  10 * modify it under the terms of the GNU General Public License version 2
  11 * as published by the Free Software Foundation; or, when distributed
  12 * separately from the Linux kernel or incorporated into other
  13 * software packages, subject to the following license:
  14 *
  15 * Permission is hereby granted, free of charge, to any person obtaining a copy
  16 * of this source file (the "Software"), to deal in the Software without
  17 * restriction, including without limitation the rights to use, copy, modify,
  18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19 * and to permit persons to whom the Software is furnished to do so, subject to
  20 * the following conditions:
  21 *
  22 * The above copyright notice and this permission notice shall be included in
  23 * all copies or substantial portions of the Software.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31 * IN THE SOFTWARE.
  32 */
  33
  34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  35
  36#include <linux/bitmap.h>
  37#include <linux/memblock.h>
  38#include <linux/sched.h>
  39#include <linux/mm.h>
  40#include <linux/slab.h>
  41#include <linux/vmalloc.h>
  42#include <linux/uaccess.h>
  43#include <linux/io.h>
  44#include <linux/delay.h>
  45#include <linux/hardirq.h>
  46#include <linux/workqueue.h>
  47#include <linux/ratelimit.h>
  48#include <linux/moduleparam.h>
  49#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
  50#include <linux/dma-mapping.h>
  51#endif
  52
  53#include <xen/xen.h>
  54#include <xen/interface/xen.h>
  55#include <xen/page.h>
  56#include <xen/grant_table.h>
  57#include <xen/interface/memory.h>
  58#include <xen/hvc-console.h>
  59#include <xen/swiotlb-xen.h>
  60#include <xen/balloon.h>
  61#ifdef CONFIG_X86
  62#include <asm/xen/cpuid.h>
  63#endif
  64#include <xen/mem-reservation.h>
  65#include <asm/xen/hypercall.h>
  66#include <asm/xen/interface.h>
  67
  68#include <asm/sync_bitops.h>
  69
  70#define GNTTAB_LIST_END 0xffffffff
  71
  72static grant_ref_t **gnttab_list;
  73static unsigned int nr_grant_frames;
  74
  75/*
  76 * Handling of free grants:
  77 *
  78 * Free grants are in a simple list anchored in gnttab_free_head. They are
  79 * linked by grant ref, the last element contains GNTTAB_LIST_END. The number
  80 * of free entries is stored in gnttab_free_count.
  81 * Additionally there is a bitmap of free entries anchored in
  82 * gnttab_free_bitmap. This is being used for simplifying allocation of
  83 * multiple consecutive grants, which is needed e.g. for support of virtio.
  84 * gnttab_last_free is used to add free entries of new frames at the end of
  85 * the free list.
  86 * gnttab_free_tail_ptr specifies the variable which references the start
  87 * of consecutive free grants ending with gnttab_last_free. This pointer is
  88 * updated in a rather defensive way, in order to avoid performance hits in
  89 * hot paths.
  90 * All those variables are protected by gnttab_list_lock.
  91 */
  92static int gnttab_free_count;
  93static unsigned int gnttab_size;
  94static grant_ref_t gnttab_free_head = GNTTAB_LIST_END;
  95static grant_ref_t gnttab_last_free = GNTTAB_LIST_END;
  96static grant_ref_t *gnttab_free_tail_ptr;
  97static unsigned long *gnttab_free_bitmap;
  98static DEFINE_SPINLOCK(gnttab_list_lock);
  99
 100struct grant_frames xen_auto_xlat_grant_frames;
 101static unsigned int xen_gnttab_version;
 102module_param_named(version, xen_gnttab_version, uint, 0);
 103
 104static union {
 105	struct grant_entry_v1 *v1;
 106	union grant_entry_v2 *v2;
 107	void *addr;
 108} gnttab_shared;
 109
 110/*This is a structure of function pointers for grant table*/
 111struct gnttab_ops {
 112	/*
 113	 * Version of the grant interface.
 114	 */
 115	unsigned int version;
 116	/*
 117	 * Grant refs per grant frame.
 118	 */
 119	unsigned int grefs_per_grant_frame;
 120	/*
 121	 * Mapping a list of frames for storing grant entries. Frames parameter
 122	 * is used to store grant table address when grant table being setup,
 123	 * nr_gframes is the number of frames to map grant table. Returning
 124	 * GNTST_okay means success and negative value means failure.
 125	 */
 126	int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
 127	/*
 128	 * Release a list of frames which are mapped in map_frames for grant
 129	 * entry status.
 130	 */
 131	void (*unmap_frames)(void);
 132	/*
 133	 * Introducing a valid entry into the grant table, granting the frame of
 134	 * this grant entry to domain for accessing. Ref
 135	 * parameter is reference of this introduced grant entry, domid is id of
 136	 * granted domain, frame is the page frame to be granted, and flags is
 137	 * status of the grant entry to be updated.
 138	 */
 139	void (*update_entry)(grant_ref_t ref, domid_t domid,
 140			     unsigned long frame, unsigned flags);
 141	/*
 142	 * Stop granting a grant entry to domain for accessing. Ref parameter is
 143	 * reference of a grant entry whose grant access will be stopped.
 144	 * If the grant entry is currently mapped for reading or writing, just
 145	 * return failure(==0) directly and don't tear down the grant access.
 146	 * Otherwise, stop grant access for this entry and return success(==1).
 147	 */
 148	int (*end_foreign_access_ref)(grant_ref_t ref);
 149	/*
 150	 * Read the frame number related to a given grant reference.
 151	 */
 152	unsigned long (*read_frame)(grant_ref_t ref);
 153};
 154
 155struct unmap_refs_callback_data {
 156	struct completion completion;
 157	int result;
 158};
 159
 160static const struct gnttab_ops *gnttab_interface;
 161
 162/* This reflects status of grant entries, so act as a global value. */
 163static grant_status_t *grstatus;
 164
 165static struct gnttab_free_callback *gnttab_free_callback_list;
 166
 167static int gnttab_expand(unsigned int req_entries);
 168
 169#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 170#define SPP (PAGE_SIZE / sizeof(grant_status_t))
 171
 172static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 173{
 174	return &gnttab_list[(entry) / RPP][(entry) % RPP];
 175}
 176/* This can be used as an l-value */
 177#define gnttab_entry(entry) (*__gnttab_entry(entry))
 178
 179static int get_free_entries(unsigned count)
 180{
 181	unsigned long flags;
 182	int ref, rc = 0;
 183	grant_ref_t head;
 184
 185	spin_lock_irqsave(&gnttab_list_lock, flags);
 186
 187	if ((gnttab_free_count < count) &&
 188	    ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 189		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 190		return rc;
 191	}
 192
 193	ref = head = gnttab_free_head;
 194	gnttab_free_count -= count;
 195	while (count--) {
 196		bitmap_clear(gnttab_free_bitmap, head, 1);
 197		if (gnttab_free_tail_ptr == __gnttab_entry(head))
 198			gnttab_free_tail_ptr = &gnttab_free_head;
 199		if (count)
 200			head = gnttab_entry(head);
 201	}
 202	gnttab_free_head = gnttab_entry(head);
 203	gnttab_entry(head) = GNTTAB_LIST_END;
 204
 205	if (!gnttab_free_count) {
 206		gnttab_last_free = GNTTAB_LIST_END;
 207		gnttab_free_tail_ptr = NULL;
 208	}
 209
 210	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 211
 212	return ref;
 213}
 214
 215static int get_seq_entry_count(void)
 216{
 217	if (gnttab_last_free == GNTTAB_LIST_END || !gnttab_free_tail_ptr ||
 218	    *gnttab_free_tail_ptr == GNTTAB_LIST_END)
 219		return 0;
 220
 221	return gnttab_last_free - *gnttab_free_tail_ptr + 1;
 222}
 223
 224/* Rebuilds the free grant list and tries to find count consecutive entries. */
 225static int get_free_seq(unsigned int count)
 226{
 227	int ret = -ENOSPC;
 228	unsigned int from, to;
 229	grant_ref_t *last;
 230
 231	gnttab_free_tail_ptr = &gnttab_free_head;
 232	last = &gnttab_free_head;
 233
 234	for (from = find_first_bit(gnttab_free_bitmap, gnttab_size);
 235	     from < gnttab_size;
 236	     from = find_next_bit(gnttab_free_bitmap, gnttab_size, to + 1)) {
 237		to = find_next_zero_bit(gnttab_free_bitmap, gnttab_size,
 238					from + 1);
 239		if (ret < 0 && to - from >= count) {
 240			ret = from;
 241			bitmap_clear(gnttab_free_bitmap, ret, count);
 242			from += count;
 243			gnttab_free_count -= count;
 244			if (from == to)
 245				continue;
 246		}
 247
 248		/*
 249		 * Recreate the free list in order to have it properly sorted.
 250		 * This is needed to make sure that the free tail has the maximum
 251		 * possible size.
 252		 */
 253		while (from < to) {
 254			*last = from;
 255			last = __gnttab_entry(from);
 256			gnttab_last_free = from;
 257			from++;
 258		}
 259		if (to < gnttab_size)
 260			gnttab_free_tail_ptr = __gnttab_entry(to - 1);
 261	}
 262
 263	*last = GNTTAB_LIST_END;
 264	if (gnttab_last_free != gnttab_size - 1)
 265		gnttab_free_tail_ptr = NULL;
 266
 267	return ret;
 268}
 269
 270static int get_free_entries_seq(unsigned int count)
 271{
 272	unsigned long flags;
 273	int ret = 0;
 274
 275	spin_lock_irqsave(&gnttab_list_lock, flags);
 276
 277	if (gnttab_free_count < count) {
 278		ret = gnttab_expand(count - gnttab_free_count);
 279		if (ret < 0)
 280			goto out;
 281	}
 282
 283	if (get_seq_entry_count() < count) {
 284		ret = get_free_seq(count);
 285		if (ret >= 0)
 286			goto out;
 287		ret = gnttab_expand(count - get_seq_entry_count());
 288		if (ret < 0)
 289			goto out;
 290	}
 291
 292	ret = *gnttab_free_tail_ptr;
 293	*gnttab_free_tail_ptr = gnttab_entry(ret + count - 1);
 294	gnttab_free_count -= count;
 295	if (!gnttab_free_count)
 296		gnttab_free_tail_ptr = NULL;
 297	bitmap_clear(gnttab_free_bitmap, ret, count);
 298
 299 out:
 300	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 301
 302	return ret;
 303}
 304
 305static void do_free_callbacks(void)
 306{
 307	struct gnttab_free_callback *callback, *next;
 308
 309	callback = gnttab_free_callback_list;
 310	gnttab_free_callback_list = NULL;
 311
 312	while (callback != NULL) {
 313		next = callback->next;
 314		if (gnttab_free_count >= callback->count) {
 315			callback->next = NULL;
 316			callback->fn(callback->arg);
 317		} else {
 318			callback->next = gnttab_free_callback_list;
 319			gnttab_free_callback_list = callback;
 320		}
 321		callback = next;
 322	}
 323}
 324
 325static inline void check_free_callbacks(void)
 326{
 327	if (unlikely(gnttab_free_callback_list))
 328		do_free_callbacks();
 329}
 330
 331static void put_free_entry_locked(grant_ref_t ref)
 332{
 333	if (unlikely(ref < GNTTAB_NR_RESERVED_ENTRIES))
 334		return;
 335
 336	gnttab_entry(ref) = gnttab_free_head;
 337	gnttab_free_head = ref;
 338	if (!gnttab_free_count)
 339		gnttab_last_free = ref;
 340	if (gnttab_free_tail_ptr == &gnttab_free_head)
 341		gnttab_free_tail_ptr = __gnttab_entry(ref);
 342	gnttab_free_count++;
 343	bitmap_set(gnttab_free_bitmap, ref, 1);
 344}
 345
 346static void put_free_entry(grant_ref_t ref)
 347{
 348	unsigned long flags;
 349
 350	spin_lock_irqsave(&gnttab_list_lock, flags);
 351	put_free_entry_locked(ref);
 352	check_free_callbacks();
 353	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 354}
 355
 356static void gnttab_set_free(unsigned int start, unsigned int n)
 357{
 358	unsigned int i;
 359
 360	for (i = start; i < start + n - 1; i++)
 361		gnttab_entry(i) = i + 1;
 362
 363	gnttab_entry(i) = GNTTAB_LIST_END;
 364	if (!gnttab_free_count) {
 365		gnttab_free_head = start;
 366		gnttab_free_tail_ptr = &gnttab_free_head;
 367	} else {
 368		gnttab_entry(gnttab_last_free) = start;
 369	}
 370	gnttab_free_count += n;
 371	gnttab_last_free = i;
 372
 373	bitmap_set(gnttab_free_bitmap, start, n);
 374}
 375
 376/*
 377 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
 378 * Introducing a valid entry into the grant table:
 379 *  1. Write ent->domid.
 380 *  2. Write ent->frame: Frame to which access is permitted.
 381 *  3. Write memory barrier (WMB).
 382 *  4. Write ent->flags, inc. valid type.
 383 */
 384static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 385				   unsigned long frame, unsigned flags)
 386{
 387	gnttab_shared.v1[ref].domid = domid;
 388	gnttab_shared.v1[ref].frame = frame;
 389	wmb();
 390	gnttab_shared.v1[ref].flags = flags;
 391}
 392
 393static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
 394				   unsigned long frame, unsigned int flags)
 395{
 396	gnttab_shared.v2[ref].hdr.domid = domid;
 397	gnttab_shared.v2[ref].full_page.frame = frame;
 398	wmb();	/* Hypervisor concurrent accesses. */
 399	gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
 400}
 401
 402/*
 403 * Public grant-issuing interface functions
 404 */
 405void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 406				     unsigned long frame, int readonly)
 407{
 408	gnttab_interface->update_entry(ref, domid, frame,
 409			   GTF_permit_access | (readonly ? GTF_readonly : 0));
 410}
 411EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 412
 413int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 414				int readonly)
 415{
 416	int ref;
 417
 418	ref = get_free_entries(1);
 419	if (unlikely(ref < 0))
 420		return -ENOSPC;
 421
 422	gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 423
 424	return ref;
 425}
 426EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 427
 428static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref)
 429{
 430	u16 *pflags = &gnttab_shared.v1[ref].flags;
 431	u16 flags;
 432
 433	flags = *pflags;
 
 434	do {
 
 435		if (flags & (GTF_reading|GTF_writing))
 436			return 0;
 437	} while (!sync_try_cmpxchg(pflags, &flags, 0));
 438
 439	return 1;
 440}
 441
 442static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref)
 443{
 444	gnttab_shared.v2[ref].hdr.flags = 0;
 445	mb();	/* Concurrent access by hypervisor. */
 446	if (grstatus[ref] & (GTF_reading|GTF_writing)) {
 447		return 0;
 448	} else {
 449		/*
 450		 * The read of grstatus needs to have acquire semantics.
 451		 *  On x86, reads already have that, and we just need to
 452		 * protect against compiler reorderings.
 453		 * On other architectures we may need a full barrier.
 454		 */
 455#ifdef CONFIG_X86
 456		barrier();
 457#else
 458		mb();
 459#endif
 460	}
 461
 462	return 1;
 463}
 464
 465static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref)
 466{
 467	return gnttab_interface->end_foreign_access_ref(ref);
 468}
 469
 470int gnttab_end_foreign_access_ref(grant_ref_t ref)
 471{
 472	if (_gnttab_end_foreign_access_ref(ref))
 473		return 1;
 474	pr_warn("WARNING: g.e. %#x still in use!\n", ref);
 475	return 0;
 476}
 477EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 478
 479static unsigned long gnttab_read_frame_v1(grant_ref_t ref)
 480{
 481	return gnttab_shared.v1[ref].frame;
 482}
 483
 484static unsigned long gnttab_read_frame_v2(grant_ref_t ref)
 485{
 486	return gnttab_shared.v2[ref].full_page.frame;
 487}
 488
 489struct deferred_entry {
 490	struct list_head list;
 491	grant_ref_t ref;
 492	uint16_t warn_delay;
 493	struct page *page;
 494};
 495static LIST_HEAD(deferred_list);
 496static void gnttab_handle_deferred(struct timer_list *);
 497static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred);
 498
 499static atomic64_t deferred_count;
 500static atomic64_t leaked_count;
 501static unsigned int free_per_iteration = 10;
 502module_param(free_per_iteration, uint, 0600);
 503
 504static void gnttab_handle_deferred(struct timer_list *unused)
 505{
 506	unsigned int nr = READ_ONCE(free_per_iteration);
 507	const bool ignore_limit = nr == 0;
 508	struct deferred_entry *first = NULL;
 509	unsigned long flags;
 510	size_t freed = 0;
 511
 512	spin_lock_irqsave(&gnttab_list_lock, flags);
 513	while ((ignore_limit || nr--) && !list_empty(&deferred_list)) {
 514		struct deferred_entry *entry
 515			= list_first_entry(&deferred_list,
 516					   struct deferred_entry, list);
 517
 518		if (entry == first)
 519			break;
 520		list_del(&entry->list);
 521		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 522		if (_gnttab_end_foreign_access_ref(entry->ref)) {
 523			uint64_t ret = atomic64_dec_return(&deferred_count);
 524
 525			put_free_entry(entry->ref);
 526			pr_debug("freeing g.e. %#x (pfn %#lx), %llu remaining\n",
 527				 entry->ref, page_to_pfn(entry->page),
 528				 (unsigned long long)ret);
 529			put_page(entry->page);
 530			freed++;
 531			kfree(entry);
 532			entry = NULL;
 533		} else {
 534			if (!--entry->warn_delay)
 535				pr_info("g.e. %#x still pending\n", entry->ref);
 536			if (!first)
 537				first = entry;
 538		}
 539		spin_lock_irqsave(&gnttab_list_lock, flags);
 540		if (entry)
 541			list_add_tail(&entry->list, &deferred_list);
 
 
 542	}
 543	if (list_empty(&deferred_list))
 544		WARN_ON(atomic64_read(&deferred_count));
 545	else if (!timer_pending(&deferred_timer)) {
 546		deferred_timer.expires = jiffies + HZ;
 547		add_timer(&deferred_timer);
 548	}
 549	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 550	pr_debug("Freed %zu references", freed);
 551}
 552
 553static void gnttab_add_deferred(grant_ref_t ref, struct page *page)
 554{
 555	struct deferred_entry *entry;
 556	gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL;
 557	uint64_t leaked, deferred;
 558
 559	entry = kmalloc(sizeof(*entry), gfp);
 560	if (!page) {
 561		unsigned long gfn = gnttab_interface->read_frame(ref);
 562
 563		page = pfn_to_page(gfn_to_pfn(gfn));
 564		get_page(page);
 565	}
 566
 567	if (entry) {
 568		unsigned long flags;
 569
 570		entry->ref = ref;
 571		entry->page = page;
 572		entry->warn_delay = 60;
 573		spin_lock_irqsave(&gnttab_list_lock, flags);
 574		list_add_tail(&entry->list, &deferred_list);
 575		if (!timer_pending(&deferred_timer)) {
 576			deferred_timer.expires = jiffies + HZ;
 577			add_timer(&deferred_timer);
 578		}
 579		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 580		deferred = atomic64_inc_return(&deferred_count);
 581		leaked = atomic64_read(&leaked_count);
 582		pr_debug("deferring g.e. %#x (pfn %#lx) (total deferred %llu, total leaked %llu)\n",
 583			 ref, page ? page_to_pfn(page) : -1, deferred, leaked);
 584	} else {
 585		deferred = atomic64_read(&deferred_count);
 586		leaked = atomic64_inc_return(&leaked_count);
 587		pr_warn("leaking g.e. %#x (pfn %#lx) (total deferred %llu, total leaked %llu)\n",
 588			ref, page ? page_to_pfn(page) : -1, deferred, leaked);
 589	}
 
 
 590}
 591
 592int gnttab_try_end_foreign_access(grant_ref_t ref)
 593{
 594	int ret = _gnttab_end_foreign_access_ref(ref);
 595
 596	if (ret)
 597		put_free_entry(ref);
 598
 599	return ret;
 600}
 601EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access);
 602
 603void gnttab_end_foreign_access(grant_ref_t ref, struct page *page)
 604{
 605	if (gnttab_try_end_foreign_access(ref)) {
 606		if (page)
 607			put_page(page);
 608	} else
 609		gnttab_add_deferred(ref, page);
 610}
 611EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 612
 613void gnttab_free_grant_reference(grant_ref_t ref)
 614{
 615	put_free_entry(ref);
 616}
 617EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 618
 619void gnttab_free_grant_references(grant_ref_t head)
 620{
 621	grant_ref_t ref;
 622	unsigned long flags;
 623
 624	spin_lock_irqsave(&gnttab_list_lock, flags);
 625	while (head != GNTTAB_LIST_END) {
 626		ref = gnttab_entry(head);
 627		put_free_entry_locked(head);
 628		head = ref;
 629	}
 630	check_free_callbacks();
 631	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 632}
 633EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 634
 635void gnttab_free_grant_reference_seq(grant_ref_t head, unsigned int count)
 636{
 637	unsigned long flags;
 638	unsigned int i;
 639
 640	spin_lock_irqsave(&gnttab_list_lock, flags);
 641	for (i = count; i > 0; i--)
 642		put_free_entry_locked(head + i - 1);
 643	check_free_callbacks();
 644	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 645}
 646EXPORT_SYMBOL_GPL(gnttab_free_grant_reference_seq);
 647
 648int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 649{
 650	int h = get_free_entries(count);
 651
 652	if (h < 0)
 653		return -ENOSPC;
 654
 655	*head = h;
 656
 657	return 0;
 658}
 659EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 660
 661int gnttab_alloc_grant_reference_seq(unsigned int count, grant_ref_t *first)
 662{
 663	int h;
 664
 665	if (count == 1)
 666		h = get_free_entries(1);
 667	else
 668		h = get_free_entries_seq(count);
 669
 670	if (h < 0)
 671		return -ENOSPC;
 672
 673	*first = h;
 674
 675	return 0;
 676}
 677EXPORT_SYMBOL_GPL(gnttab_alloc_grant_reference_seq);
 678
 679int gnttab_empty_grant_references(const grant_ref_t *private_head)
 680{
 681	return (*private_head == GNTTAB_LIST_END);
 682}
 683EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 684
 685int gnttab_claim_grant_reference(grant_ref_t *private_head)
 686{
 687	grant_ref_t g = *private_head;
 688	if (unlikely(g == GNTTAB_LIST_END))
 689		return -ENOSPC;
 690	*private_head = gnttab_entry(g);
 691	return g;
 692}
 693EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 694
 695void gnttab_release_grant_reference(grant_ref_t *private_head,
 696				    grant_ref_t release)
 697{
 698	gnttab_entry(release) = *private_head;
 699	*private_head = release;
 700}
 701EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 702
 703void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 704				  void (*fn)(void *), void *arg, u16 count)
 705{
 706	unsigned long flags;
 707	struct gnttab_free_callback *cb;
 708
 709	spin_lock_irqsave(&gnttab_list_lock, flags);
 710
 711	/* Check if the callback is already on the list */
 712	cb = gnttab_free_callback_list;
 713	while (cb) {
 714		if (cb == callback)
 715			goto out;
 716		cb = cb->next;
 717	}
 718
 719	callback->fn = fn;
 720	callback->arg = arg;
 721	callback->count = count;
 722	callback->next = gnttab_free_callback_list;
 723	gnttab_free_callback_list = callback;
 724	check_free_callbacks();
 725out:
 726	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 727}
 728EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 729
 730void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 731{
 732	struct gnttab_free_callback **pcb;
 733	unsigned long flags;
 734
 735	spin_lock_irqsave(&gnttab_list_lock, flags);
 736	for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 737		if (*pcb == callback) {
 738			*pcb = callback->next;
 739			break;
 740		}
 741	}
 742	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 743}
 744EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 745
 746static unsigned int gnttab_frames(unsigned int frames, unsigned int align)
 747{
 748	return (frames * gnttab_interface->grefs_per_grant_frame + align - 1) /
 749	       align;
 750}
 751
 752static int grow_gnttab_list(unsigned int more_frames)
 753{
 754	unsigned int new_nr_grant_frames, extra_entries, i;
 755	unsigned int nr_glist_frames, new_nr_glist_frames;
 756	unsigned int grefs_per_frame;
 757
 758	grefs_per_frame = gnttab_interface->grefs_per_grant_frame;
 759
 760	new_nr_grant_frames = nr_grant_frames + more_frames;
 761	extra_entries = more_frames * grefs_per_frame;
 762
 763	nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
 764	new_nr_glist_frames = gnttab_frames(new_nr_grant_frames, RPP);
 765	for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 766		gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 767		if (!gnttab_list[i])
 768			goto grow_nomem;
 769	}
 770
 771	gnttab_set_free(gnttab_size, extra_entries);
 772
 773	if (!gnttab_free_tail_ptr)
 774		gnttab_free_tail_ptr = __gnttab_entry(gnttab_size);
 775
 776	nr_grant_frames = new_nr_grant_frames;
 777	gnttab_size += extra_entries;
 778
 779	check_free_callbacks();
 780
 781	return 0;
 782
 783grow_nomem:
 784	while (i-- > nr_glist_frames)
 785		free_page((unsigned long) gnttab_list[i]);
 786	return -ENOMEM;
 787}
 788
 789static unsigned int __max_nr_grant_frames(void)
 790{
 791	struct gnttab_query_size query;
 792	int rc;
 793
 794	query.dom = DOMID_SELF;
 795
 796	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 797	if ((rc < 0) || (query.status != GNTST_okay))
 798		return 4; /* Legacy max supported number of frames */
 799
 800	return query.max_nr_frames;
 801}
 802
 803unsigned int gnttab_max_grant_frames(void)
 804{
 805	unsigned int xen_max = __max_nr_grant_frames();
 806	static unsigned int boot_max_nr_grant_frames;
 807
 808	/* First time, initialize it properly. */
 809	if (!boot_max_nr_grant_frames)
 810		boot_max_nr_grant_frames = __max_nr_grant_frames();
 811
 812	if (xen_max > boot_max_nr_grant_frames)
 813		return boot_max_nr_grant_frames;
 814	return xen_max;
 815}
 816EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 817
 818int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
 819{
 820	xen_pfn_t *pfn;
 821	unsigned int max_nr_gframes = __max_nr_grant_frames();
 822	unsigned int i;
 823	void *vaddr;
 824
 825	if (xen_auto_xlat_grant_frames.count)
 826		return -EINVAL;
 827
 828	vaddr = memremap(addr, XEN_PAGE_SIZE * max_nr_gframes, MEMREMAP_WB);
 829	if (vaddr == NULL) {
 830		pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",
 831			&addr);
 832		return -ENOMEM;
 833	}
 834	pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL);
 835	if (!pfn) {
 836		memunmap(vaddr);
 837		return -ENOMEM;
 838	}
 839	for (i = 0; i < max_nr_gframes; i++)
 840		pfn[i] = XEN_PFN_DOWN(addr) + i;
 841
 842	xen_auto_xlat_grant_frames.vaddr = vaddr;
 843	xen_auto_xlat_grant_frames.pfn = pfn;
 844	xen_auto_xlat_grant_frames.count = max_nr_gframes;
 845
 846	return 0;
 847}
 848EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames);
 849
 850void gnttab_free_auto_xlat_frames(void)
 851{
 852	if (!xen_auto_xlat_grant_frames.count)
 853		return;
 854	kfree(xen_auto_xlat_grant_frames.pfn);
 855	memunmap(xen_auto_xlat_grant_frames.vaddr);
 856
 857	xen_auto_xlat_grant_frames.pfn = NULL;
 858	xen_auto_xlat_grant_frames.count = 0;
 859	xen_auto_xlat_grant_frames.vaddr = NULL;
 860}
 861EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
 862
 863int gnttab_pages_set_private(int nr_pages, struct page **pages)
 864{
 865	int i;
 866
 867	for (i = 0; i < nr_pages; i++) {
 868#if BITS_PER_LONG < 64
 869		struct xen_page_foreign *foreign;
 870
 871		foreign = kzalloc(sizeof(*foreign), GFP_KERNEL);
 872		if (!foreign)
 873			return -ENOMEM;
 874
 875		set_page_private(pages[i], (unsigned long)foreign);
 876#endif
 877		SetPagePrivate(pages[i]);
 878	}
 879
 880	return 0;
 881}
 882EXPORT_SYMBOL_GPL(gnttab_pages_set_private);
 883
 884/**
 885 * gnttab_alloc_pages - alloc pages suitable for grant mapping into
 886 * @nr_pages: number of pages to alloc
 887 * @pages: returns the pages
 888 */
 889int gnttab_alloc_pages(int nr_pages, struct page **pages)
 890{
 891	int ret;
 892
 893	ret = xen_alloc_unpopulated_pages(nr_pages, pages);
 894	if (ret < 0)
 895		return ret;
 896
 897	ret = gnttab_pages_set_private(nr_pages, pages);
 898	if (ret < 0)
 899		gnttab_free_pages(nr_pages, pages);
 900
 901	return ret;
 902}
 903EXPORT_SYMBOL_GPL(gnttab_alloc_pages);
 904
 905#ifdef CONFIG_XEN_UNPOPULATED_ALLOC
 906static inline void cache_init(struct gnttab_page_cache *cache)
 907{
 908	cache->pages = NULL;
 909}
 910
 911static inline bool cache_empty(struct gnttab_page_cache *cache)
 912{
 913	return !cache->pages;
 914}
 915
 916static inline struct page *cache_deq(struct gnttab_page_cache *cache)
 917{
 918	struct page *page;
 919
 920	page = cache->pages;
 921	cache->pages = page->zone_device_data;
 922
 923	return page;
 924}
 925
 926static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
 927{
 928	page->zone_device_data = cache->pages;
 929	cache->pages = page;
 930}
 931#else
 932static inline void cache_init(struct gnttab_page_cache *cache)
 933{
 934	INIT_LIST_HEAD(&cache->pages);
 935}
 936
 937static inline bool cache_empty(struct gnttab_page_cache *cache)
 938{
 939	return list_empty(&cache->pages);
 940}
 941
 942static inline struct page *cache_deq(struct gnttab_page_cache *cache)
 943{
 944	struct page *page;
 945
 946	page = list_first_entry(&cache->pages, struct page, lru);
 947	list_del(&page->lru);
 948
 949	return page;
 950}
 951
 952static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
 953{
 954	list_add(&page->lru, &cache->pages);
 955}
 956#endif
 957
 958void gnttab_page_cache_init(struct gnttab_page_cache *cache)
 959{
 960	spin_lock_init(&cache->lock);
 961	cache_init(cache);
 962	cache->num_pages = 0;
 963}
 964EXPORT_SYMBOL_GPL(gnttab_page_cache_init);
 965
 966int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page)
 967{
 968	unsigned long flags;
 969
 970	spin_lock_irqsave(&cache->lock, flags);
 971
 972	if (cache_empty(cache)) {
 973		spin_unlock_irqrestore(&cache->lock, flags);
 974		return gnttab_alloc_pages(1, page);
 975	}
 976
 977	page[0] = cache_deq(cache);
 978	cache->num_pages--;
 979
 980	spin_unlock_irqrestore(&cache->lock, flags);
 981
 982	return 0;
 983}
 984EXPORT_SYMBOL_GPL(gnttab_page_cache_get);
 985
 986void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page,
 987			   unsigned int num)
 988{
 989	unsigned long flags;
 990	unsigned int i;
 991
 992	spin_lock_irqsave(&cache->lock, flags);
 993
 994	for (i = 0; i < num; i++)
 995		cache_enq(cache, page[i]);
 996	cache->num_pages += num;
 997
 998	spin_unlock_irqrestore(&cache->lock, flags);
 999}
1000EXPORT_SYMBOL_GPL(gnttab_page_cache_put);
1001
1002void gnttab_page_cache_shrink(struct gnttab_page_cache *cache, unsigned int num)
1003{
1004	struct page *page[10];
1005	unsigned int i = 0;
1006	unsigned long flags;
1007
1008	spin_lock_irqsave(&cache->lock, flags);
1009
1010	while (cache->num_pages > num) {
1011		page[i] = cache_deq(cache);
1012		cache->num_pages--;
1013		if (++i == ARRAY_SIZE(page)) {
1014			spin_unlock_irqrestore(&cache->lock, flags);
1015			gnttab_free_pages(i, page);
1016			i = 0;
1017			spin_lock_irqsave(&cache->lock, flags);
1018		}
1019	}
1020
1021	spin_unlock_irqrestore(&cache->lock, flags);
1022
1023	if (i != 0)
1024		gnttab_free_pages(i, page);
1025}
1026EXPORT_SYMBOL_GPL(gnttab_page_cache_shrink);
1027
1028void gnttab_pages_clear_private(int nr_pages, struct page **pages)
1029{
1030	int i;
1031
1032	for (i = 0; i < nr_pages; i++) {
1033		if (PagePrivate(pages[i])) {
1034#if BITS_PER_LONG < 64
1035			kfree((void *)page_private(pages[i]));
1036#endif
1037			ClearPagePrivate(pages[i]);
1038		}
1039	}
1040}
1041EXPORT_SYMBOL_GPL(gnttab_pages_clear_private);
1042
1043/**
1044 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
1045 * @nr_pages: number of pages to free
1046 * @pages: the pages
1047 */
1048void gnttab_free_pages(int nr_pages, struct page **pages)
1049{
1050	gnttab_pages_clear_private(nr_pages, pages);
1051	xen_free_unpopulated_pages(nr_pages, pages);
1052}
1053EXPORT_SYMBOL_GPL(gnttab_free_pages);
1054
1055#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
1056/**
1057 * gnttab_dma_alloc_pages - alloc DMAable pages suitable for grant mapping into
1058 * @args: arguments to the function
1059 */
1060int gnttab_dma_alloc_pages(struct gnttab_dma_alloc_args *args)
1061{
1062	unsigned long pfn, start_pfn;
1063	size_t size;
1064	int i, ret;
1065
1066	if (args->nr_pages < 0 || args->nr_pages > (INT_MAX >> PAGE_SHIFT))
1067		return -ENOMEM;
1068
1069	size = args->nr_pages << PAGE_SHIFT;
1070	if (args->coherent)
1071		args->vaddr = dma_alloc_coherent(args->dev, size,
1072						 &args->dev_bus_addr,
1073						 GFP_KERNEL | __GFP_NOWARN);
1074	else
1075		args->vaddr = dma_alloc_wc(args->dev, size,
1076					   &args->dev_bus_addr,
1077					   GFP_KERNEL | __GFP_NOWARN);
1078	if (!args->vaddr) {
1079		pr_debug("Failed to allocate DMA buffer of size %zu\n", size);
1080		return -ENOMEM;
1081	}
1082
1083	start_pfn = __phys_to_pfn(args->dev_bus_addr);
1084	for (pfn = start_pfn, i = 0; pfn < start_pfn + args->nr_pages;
1085			pfn++, i++) {
1086		struct page *page = pfn_to_page(pfn);
1087
1088		args->pages[i] = page;
1089		args->frames[i] = xen_page_to_gfn(page);
1090		xenmem_reservation_scrub_page(page);
1091	}
1092
1093	xenmem_reservation_va_mapping_reset(args->nr_pages, args->pages);
1094
1095	ret = xenmem_reservation_decrease(args->nr_pages, args->frames);
1096	if (ret != args->nr_pages) {
1097		pr_debug("Failed to decrease reservation for DMA buffer\n");
1098		ret = -EFAULT;
1099		goto fail;
1100	}
1101
1102	ret = gnttab_pages_set_private(args->nr_pages, args->pages);
1103	if (ret < 0)
1104		goto fail;
1105
1106	return 0;
1107
1108fail:
1109	gnttab_dma_free_pages(args);
1110	return ret;
1111}
1112EXPORT_SYMBOL_GPL(gnttab_dma_alloc_pages);
1113
1114/**
1115 * gnttab_dma_free_pages - free DMAable pages
1116 * @args: arguments to the function
1117 */
1118int gnttab_dma_free_pages(struct gnttab_dma_alloc_args *args)
1119{
1120	size_t size;
1121	int i, ret;
1122
1123	gnttab_pages_clear_private(args->nr_pages, args->pages);
1124
1125	for (i = 0; i < args->nr_pages; i++)
1126		args->frames[i] = page_to_xen_pfn(args->pages[i]);
1127
1128	ret = xenmem_reservation_increase(args->nr_pages, args->frames);
1129	if (ret != args->nr_pages) {
1130		pr_debug("Failed to increase reservation for DMA buffer\n");
1131		ret = -EFAULT;
1132	} else {
1133		ret = 0;
1134	}
1135
1136	xenmem_reservation_va_mapping_update(args->nr_pages, args->pages,
1137					     args->frames);
1138
1139	size = args->nr_pages << PAGE_SHIFT;
1140	if (args->coherent)
1141		dma_free_coherent(args->dev, size,
1142				  args->vaddr, args->dev_bus_addr);
1143	else
1144		dma_free_wc(args->dev, size,
1145			    args->vaddr, args->dev_bus_addr);
1146	return ret;
1147}
1148EXPORT_SYMBOL_GPL(gnttab_dma_free_pages);
1149#endif
1150
1151/* Handling of paged out grant targets (GNTST_eagain) */
1152#define MAX_DELAY 256
1153static inline void
1154gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
1155						const char *func)
1156{
1157	unsigned delay = 1;
1158
1159	do {
1160		BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
1161		if (*status == GNTST_eagain)
1162			msleep(delay++);
1163	} while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
1164
1165	if (delay >= MAX_DELAY) {
1166		pr_err("%s: %s eagain grant\n", func, current->comm);
1167		*status = GNTST_bad_page;
1168	}
1169}
1170
1171void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
1172{
1173	struct gnttab_map_grant_ref *op;
1174
1175	if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
1176		BUG();
1177	for (op = batch; op < batch + count; op++)
1178		if (op->status == GNTST_eagain)
1179			gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
1180						&op->status, __func__);
1181}
1182EXPORT_SYMBOL_GPL(gnttab_batch_map);
1183
1184void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
1185{
1186	struct gnttab_copy *op;
1187
1188	if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
1189		BUG();
1190	for (op = batch; op < batch + count; op++)
1191		if (op->status == GNTST_eagain)
1192			gnttab_retry_eagain_gop(GNTTABOP_copy, op,
1193						&op->status, __func__);
1194}
1195EXPORT_SYMBOL_GPL(gnttab_batch_copy);
1196
1197void gnttab_foreach_grant_in_range(struct page *page,
1198				   unsigned int offset,
1199				   unsigned int len,
1200				   xen_grant_fn_t fn,
1201				   void *data)
1202{
1203	unsigned int goffset;
1204	unsigned int glen;
1205	unsigned long xen_pfn;
1206
1207	len = min_t(unsigned int, PAGE_SIZE - offset, len);
1208	goffset = xen_offset_in_page(offset);
1209
1210	xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset);
1211
1212	while (len) {
1213		glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len);
1214		fn(pfn_to_gfn(xen_pfn), goffset, glen, data);
1215
1216		goffset = 0;
1217		xen_pfn++;
1218		len -= glen;
1219	}
1220}
1221EXPORT_SYMBOL_GPL(gnttab_foreach_grant_in_range);
1222
1223void gnttab_foreach_grant(struct page **pages,
1224			  unsigned int nr_grefs,
1225			  xen_grant_fn_t fn,
1226			  void *data)
1227{
1228	unsigned int goffset = 0;
1229	unsigned long xen_pfn = 0;
1230	unsigned int i;
1231
1232	for (i = 0; i < nr_grefs; i++) {
1233		if ((i % XEN_PFN_PER_PAGE) == 0) {
1234			xen_pfn = page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);
1235			goffset = 0;
1236		}
1237
1238		fn(pfn_to_gfn(xen_pfn), goffset, XEN_PAGE_SIZE, data);
1239
1240		goffset += XEN_PAGE_SIZE;
1241		xen_pfn++;
1242	}
1243}
1244
1245int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
1246		    struct gnttab_map_grant_ref *kmap_ops,
1247		    struct page **pages, unsigned int count)
1248{
1249	int i, ret;
1250
1251	ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
1252	if (ret)
1253		return ret;
1254
1255	for (i = 0; i < count; i++) {
1256		switch (map_ops[i].status) {
1257		case GNTST_okay:
1258		{
1259			struct xen_page_foreign *foreign;
1260
1261			SetPageForeign(pages[i]);
1262			foreign = xen_page_foreign(pages[i]);
1263			foreign->domid = map_ops[i].dom;
1264			foreign->gref = map_ops[i].ref;
1265			break;
1266		}
1267
1268		case GNTST_no_device_space:
1269			pr_warn_ratelimited("maptrack limit reached, can't map all guest pages\n");
1270			break;
1271
1272		case GNTST_eagain:
1273			/* Retry eagain maps */
1274			gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref,
1275						map_ops + i,
1276						&map_ops[i].status, __func__);
1277			/* Test status in next loop iteration. */
1278			i--;
1279			break;
1280
1281		default:
1282			break;
1283		}
1284	}
1285
1286	return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);
1287}
1288EXPORT_SYMBOL_GPL(gnttab_map_refs);
1289
1290int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
1291		      struct gnttab_unmap_grant_ref *kunmap_ops,
1292		      struct page **pages, unsigned int count)
1293{
1294	unsigned int i;
1295	int ret;
1296
1297	ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
1298	if (ret)
1299		return ret;
1300
1301	for (i = 0; i < count; i++)
1302		ClearPageForeign(pages[i]);
1303
1304	return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count);
1305}
1306EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
1307
1308#define GNTTAB_UNMAP_REFS_DELAY 5
1309
1310static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
1311
1312static void gnttab_unmap_work(struct work_struct *work)
1313{
1314	struct gntab_unmap_queue_data
1315		*unmap_data = container_of(work, 
1316					   struct gntab_unmap_queue_data,
1317					   gnttab_work.work);
1318	if (unmap_data->age != UINT_MAX)
1319		unmap_data->age++;
1320	__gnttab_unmap_refs_async(unmap_data);
1321}
1322
1323static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
1324{
1325	int ret;
1326	int pc;
1327
1328	for (pc = 0; pc < item->count; pc++) {
1329		if (page_count(item->pages[pc]) > 1) {
1330			unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);
1331			schedule_delayed_work(&item->gnttab_work,
1332					      msecs_to_jiffies(delay));
1333			return;
1334		}
1335	}
1336
1337	ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,
1338				item->pages, item->count);
1339	item->done(ret, item);
1340}
1341
1342void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
1343{
1344	INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);
1345	item->age = 0;
1346
1347	__gnttab_unmap_refs_async(item);
1348}
1349EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async);
1350
1351static void unmap_refs_callback(int result,
1352		struct gntab_unmap_queue_data *data)
1353{
1354	struct unmap_refs_callback_data *d = data->data;
1355
1356	d->result = result;
1357	complete(&d->completion);
1358}
1359
1360int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item)
1361{
1362	struct unmap_refs_callback_data data;
1363
1364	init_completion(&data.completion);
1365	item->data = &data;
1366	item->done = &unmap_refs_callback;
1367	gnttab_unmap_refs_async(item);
1368	wait_for_completion(&data.completion);
1369
1370	return data.result;
1371}
1372EXPORT_SYMBOL_GPL(gnttab_unmap_refs_sync);
1373
1374static unsigned int nr_status_frames(unsigned int nr_grant_frames)
1375{
1376	return gnttab_frames(nr_grant_frames, SPP);
1377}
1378
1379static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
1380{
1381	int rc;
1382
1383	rc = arch_gnttab_map_shared(frames, nr_gframes,
1384				    gnttab_max_grant_frames(),
1385				    &gnttab_shared.addr);
1386	BUG_ON(rc);
1387
1388	return 0;
1389}
1390
1391static void gnttab_unmap_frames_v1(void)
1392{
1393	arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1394}
1395
1396static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
1397{
1398	uint64_t *sframes;
1399	unsigned int nr_sframes;
1400	struct gnttab_get_status_frames getframes;
1401	int rc;
1402
1403	nr_sframes = nr_status_frames(nr_gframes);
1404
1405	/* No need for kzalloc as it is initialized in following hypercall
1406	 * GNTTABOP_get_status_frames.
1407	 */
1408	sframes = kmalloc_array(nr_sframes, sizeof(uint64_t), GFP_ATOMIC);
1409	if (!sframes)
1410		return -ENOMEM;
1411
1412	getframes.dom        = DOMID_SELF;
1413	getframes.nr_frames  = nr_sframes;
1414	set_xen_guest_handle(getframes.frame_list, sframes);
1415
1416	rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1417				       &getframes, 1);
1418	if (rc == -ENOSYS) {
1419		kfree(sframes);
1420		return -ENOSYS;
1421	}
1422
1423	BUG_ON(rc || getframes.status);
1424
1425	rc = arch_gnttab_map_status(sframes, nr_sframes,
1426				    nr_status_frames(gnttab_max_grant_frames()),
1427				    &grstatus);
1428	BUG_ON(rc);
1429	kfree(sframes);
1430
1431	rc = arch_gnttab_map_shared(frames, nr_gframes,
1432				    gnttab_max_grant_frames(),
1433				    &gnttab_shared.addr);
1434	BUG_ON(rc);
1435
1436	return 0;
1437}
1438
1439static void gnttab_unmap_frames_v2(void)
1440{
1441	arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1442	arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
1443}
1444
1445static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1446{
1447	struct gnttab_setup_table setup;
1448	xen_pfn_t *frames;
1449	unsigned int nr_gframes = end_idx + 1;
1450	int rc;
1451
1452	if (xen_feature(XENFEAT_auto_translated_physmap)) {
1453		struct xen_add_to_physmap xatp;
1454		unsigned int i = end_idx;
1455		rc = 0;
1456		BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes);
1457		/*
1458		 * Loop backwards, so that the first hypercall has the largest
1459		 * index, ensuring that the table will grow only once.
1460		 */
1461		do {
1462			xatp.domid = DOMID_SELF;
1463			xatp.idx = i;
1464			xatp.space = XENMAPSPACE_grant_table;
1465			xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i];
1466			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1467			if (rc != 0) {
1468				pr_warn("grant table add_to_physmap failed, err=%d\n",
1469					rc);
1470				break;
1471			}
1472		} while (i-- > start_idx);
1473
1474		return rc;
1475	}
1476
1477	/* No need for kzalloc as it is initialized in following hypercall
1478	 * GNTTABOP_setup_table.
1479	 */
1480	frames = kmalloc_array(nr_gframes, sizeof(unsigned long), GFP_ATOMIC);
1481	if (!frames)
1482		return -ENOMEM;
1483
1484	setup.dom        = DOMID_SELF;
1485	setup.nr_frames  = nr_gframes;
1486	set_xen_guest_handle(setup.frame_list, frames);
1487
1488	rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1489	if (rc == -ENOSYS) {
1490		kfree(frames);
1491		return -ENOSYS;
1492	}
1493
1494	BUG_ON(rc || setup.status);
1495
1496	rc = gnttab_interface->map_frames(frames, nr_gframes);
1497
1498	kfree(frames);
1499
1500	return rc;
1501}
1502
1503static const struct gnttab_ops gnttab_v1_ops = {
1504	.version			= 1,
1505	.grefs_per_grant_frame		= XEN_PAGE_SIZE /
1506					  sizeof(struct grant_entry_v1),
1507	.map_frames			= gnttab_map_frames_v1,
1508	.unmap_frames			= gnttab_unmap_frames_v1,
1509	.update_entry			= gnttab_update_entry_v1,
1510	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v1,
1511	.read_frame			= gnttab_read_frame_v1,
1512};
1513
1514static const struct gnttab_ops gnttab_v2_ops = {
1515	.version			= 2,
1516	.grefs_per_grant_frame		= XEN_PAGE_SIZE /
1517					  sizeof(union grant_entry_v2),
1518	.map_frames			= gnttab_map_frames_v2,
1519	.unmap_frames			= gnttab_unmap_frames_v2,
1520	.update_entry			= gnttab_update_entry_v2,
1521	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v2,
1522	.read_frame			= gnttab_read_frame_v2,
1523};
1524
1525static bool gnttab_need_v2(void)
1526{
1527#ifdef CONFIG_X86
1528	uint32_t base, width;
1529
1530	if (xen_pv_domain()) {
1531		base = xen_cpuid_base();
1532		if (cpuid_eax(base) < 5)
1533			return false;	/* Information not available, use V1. */
1534		width = cpuid_ebx(base + 5) &
1535			XEN_CPUID_MACHINE_ADDRESS_WIDTH_MASK;
1536		return width > 32 + PAGE_SHIFT;
1537	}
1538#endif
1539	return !!(max_possible_pfn >> 32);
1540}
1541
1542static void gnttab_request_version(void)
1543{
1544	long rc;
1545	struct gnttab_set_version gsv;
1546
1547	if (gnttab_need_v2())
1548		gsv.version = 2;
1549	else
1550		gsv.version = 1;
1551
1552	/* Boot parameter overrides automatic selection. */
1553	if (xen_gnttab_version >= 1 && xen_gnttab_version <= 2)
1554		gsv.version = xen_gnttab_version;
1555
1556	rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1557	if (rc == 0 && gsv.version == 2)
1558		gnttab_interface = &gnttab_v2_ops;
1559	else
1560		gnttab_interface = &gnttab_v1_ops;
1561	pr_info("Grant tables using version %d layout\n",
1562		gnttab_interface->version);
1563}
1564
1565static int gnttab_setup(void)
1566{
1567	unsigned int max_nr_gframes;
1568
1569	max_nr_gframes = gnttab_max_grant_frames();
1570	if (max_nr_gframes < nr_grant_frames)
1571		return -ENOSYS;
1572
1573	if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {
1574		gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;
1575		if (gnttab_shared.addr == NULL) {
1576			pr_warn("gnttab share frames is not mapped!\n");
1577			return -ENOMEM;
1578		}
1579	}
1580	return gnttab_map(0, nr_grant_frames - 1);
1581}
1582
1583int gnttab_resume(void)
1584{
1585	gnttab_request_version();
1586	return gnttab_setup();
1587}
1588
1589int gnttab_suspend(void)
1590{
1591	if (!xen_feature(XENFEAT_auto_translated_physmap))
1592		gnttab_interface->unmap_frames();
1593	return 0;
1594}
1595
1596static int gnttab_expand(unsigned int req_entries)
1597{
1598	int rc;
1599	unsigned int cur, extra;
1600
1601	cur = nr_grant_frames;
1602	extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) /
1603		 gnttab_interface->grefs_per_grant_frame);
1604	if (cur + extra > gnttab_max_grant_frames()) {
1605		pr_warn_ratelimited("xen/grant-table: max_grant_frames reached"
1606				    " cur=%u extra=%u limit=%u"
1607				    " gnttab_free_count=%u req_entries=%u\n",
1608				    cur, extra, gnttab_max_grant_frames(),
1609				    gnttab_free_count, req_entries);
1610		return -ENOSPC;
1611	}
1612
1613	rc = gnttab_map(cur, cur + extra - 1);
1614	if (rc == 0)
1615		rc = grow_gnttab_list(extra);
1616
1617	return rc;
1618}
1619
1620int gnttab_init(void)
1621{
1622	int i;
1623	unsigned long max_nr_grant_frames, max_nr_grefs;
1624	unsigned int max_nr_glist_frames, nr_glist_frames;
1625	int ret;
1626
1627	gnttab_request_version();
1628	max_nr_grant_frames = gnttab_max_grant_frames();
1629	max_nr_grefs = max_nr_grant_frames *
1630			gnttab_interface->grefs_per_grant_frame;
1631	nr_grant_frames = 1;
1632
1633	/* Determine the maximum number of frames required for the
1634	 * grant reference free list on the current hypervisor.
1635	 */
1636	max_nr_glist_frames = max_nr_grefs / RPP;
1637
1638	gnttab_list = kmalloc_array(max_nr_glist_frames,
1639				    sizeof(grant_ref_t *),
1640				    GFP_KERNEL);
1641	if (gnttab_list == NULL)
1642		return -ENOMEM;
1643
1644	nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
1645	for (i = 0; i < nr_glist_frames; i++) {
1646		gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1647		if (gnttab_list[i] == NULL) {
1648			ret = -ENOMEM;
1649			goto ini_nomem;
1650		}
1651	}
1652
1653	gnttab_free_bitmap = bitmap_zalloc(max_nr_grefs, GFP_KERNEL);
1654	if (!gnttab_free_bitmap) {
1655		ret = -ENOMEM;
1656		goto ini_nomem;
1657	}
1658
1659	ret = arch_gnttab_init(max_nr_grant_frames,
1660			       nr_status_frames(max_nr_grant_frames));
1661	if (ret < 0)
1662		goto ini_nomem;
1663
1664	if (gnttab_setup() < 0) {
1665		ret = -ENODEV;
1666		goto ini_nomem;
1667	}
1668
1669	gnttab_size = nr_grant_frames * gnttab_interface->grefs_per_grant_frame;
1670
1671	gnttab_set_free(GNTTAB_NR_RESERVED_ENTRIES,
1672			gnttab_size - GNTTAB_NR_RESERVED_ENTRIES);
1673
1674	printk("Grant table initialized\n");
1675	return 0;
1676
1677 ini_nomem:
1678	for (i--; i >= 0; i--)
1679		free_page((unsigned long)gnttab_list[i]);
1680	kfree(gnttab_list);
1681	bitmap_free(gnttab_free_bitmap);
1682	return ret;
1683}
1684EXPORT_SYMBOL_GPL(gnttab_init);
1685
1686static int __gnttab_init(void)
1687{
1688	if (!xen_domain())
1689		return -ENODEV;
1690
1691	/* Delay grant-table initialization in the PV on HVM case */
1692	if (xen_hvm_domain() && !xen_pvh_domain())
1693		return 0;
1694
1695	return gnttab_init();
1696}
1697/* Starts after core_initcall so that xen_pvh_gnttab_setup can be called
1698 * beforehand to initialize xen_auto_xlat_grant_frames. */
1699core_initcall_sync(__gnttab_init);