Linux Audio

Check our new training course

Loading...
v6.13.7
   1/******************************************************************************
   2 * grant_table.c
   3 *
   4 * Granting foreign access to our memory reservation.
   5 *
   6 * Copyright (c) 2005-2006, Christopher Clark
   7 * Copyright (c) 2004-2005, K A Fraser
   8 *
   9 * This program is free software; you can redistribute it and/or
  10 * modify it under the terms of the GNU General Public License version 2
  11 * as published by the Free Software Foundation; or, when distributed
  12 * separately from the Linux kernel or incorporated into other
  13 * software packages, subject to the following license:
  14 *
  15 * Permission is hereby granted, free of charge, to any person obtaining a copy
  16 * of this source file (the "Software"), to deal in the Software without
  17 * restriction, including without limitation the rights to use, copy, modify,
  18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19 * and to permit persons to whom the Software is furnished to do so, subject to
  20 * the following conditions:
  21 *
  22 * The above copyright notice and this permission notice shall be included in
  23 * all copies or substantial portions of the Software.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31 * IN THE SOFTWARE.
  32 */
  33
  34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  35
  36#include <linux/bitmap.h>
  37#include <linux/memblock.h>
  38#include <linux/sched.h>
  39#include <linux/mm.h>
  40#include <linux/slab.h>
  41#include <linux/vmalloc.h>
  42#include <linux/uaccess.h>
  43#include <linux/io.h>
  44#include <linux/delay.h>
  45#include <linux/hardirq.h>
  46#include <linux/workqueue.h>
  47#include <linux/ratelimit.h>
  48#include <linux/moduleparam.h>
  49#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
  50#include <linux/dma-mapping.h>
  51#endif
  52
  53#include <xen/xen.h>
  54#include <xen/interface/xen.h>
  55#include <xen/page.h>
  56#include <xen/grant_table.h>
  57#include <xen/interface/memory.h>
  58#include <xen/hvc-console.h>
  59#include <xen/swiotlb-xen.h>
  60#include <xen/balloon.h>
  61#ifdef CONFIG_X86
  62#include <asm/xen/cpuid.h>
  63#endif
  64#include <xen/mem-reservation.h>
  65#include <asm/xen/hypercall.h>
  66#include <asm/xen/interface.h>
  67
 
  68#include <asm/sync_bitops.h>
  69
 
 
  70#define GNTTAB_LIST_END 0xffffffff
 
 
 
 
  71
  72static grant_ref_t **gnttab_list;
  73static unsigned int nr_grant_frames;
  74
  75/*
  76 * Handling of free grants:
  77 *
  78 * Free grants are in a simple list anchored in gnttab_free_head. They are
  79 * linked by grant ref, the last element contains GNTTAB_LIST_END. The number
  80 * of free entries is stored in gnttab_free_count.
  81 * Additionally there is a bitmap of free entries anchored in
  82 * gnttab_free_bitmap. This is being used for simplifying allocation of
  83 * multiple consecutive grants, which is needed e.g. for support of virtio.
  84 * gnttab_last_free is used to add free entries of new frames at the end of
  85 * the free list.
  86 * gnttab_free_tail_ptr specifies the variable which references the start
  87 * of consecutive free grants ending with gnttab_last_free. This pointer is
  88 * updated in a rather defensive way, in order to avoid performance hits in
  89 * hot paths.
  90 * All those variables are protected by gnttab_list_lock.
  91 */
  92static int gnttab_free_count;
  93static unsigned int gnttab_size;
  94static grant_ref_t gnttab_free_head = GNTTAB_LIST_END;
  95static grant_ref_t gnttab_last_free = GNTTAB_LIST_END;
  96static grant_ref_t *gnttab_free_tail_ptr;
  97static unsigned long *gnttab_free_bitmap;
  98static DEFINE_SPINLOCK(gnttab_list_lock);
  99
 100struct grant_frames xen_auto_xlat_grant_frames;
 101static unsigned int xen_gnttab_version;
 102module_param_named(version, xen_gnttab_version, uint, 0);
 103
 104static union {
 105	struct grant_entry_v1 *v1;
 106	union grant_entry_v2 *v2;
 107	void *addr;
 108} gnttab_shared;
 109
 110/*This is a structure of function pointers for grant table*/
 111struct gnttab_ops {
 112	/*
 113	 * Version of the grant interface.
 114	 */
 115	unsigned int version;
 116	/*
 117	 * Grant refs per grant frame.
 118	 */
 119	unsigned int grefs_per_grant_frame;
 120	/*
 121	 * Mapping a list of frames for storing grant entries. Frames parameter
 122	 * is used to store grant table address when grant table being setup,
 123	 * nr_gframes is the number of frames to map grant table. Returning
 124	 * GNTST_okay means success and negative value means failure.
 125	 */
 126	int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
 127	/*
 128	 * Release a list of frames which are mapped in map_frames for grant
 129	 * entry status.
 130	 */
 131	void (*unmap_frames)(void);
 132	/*
 133	 * Introducing a valid entry into the grant table, granting the frame of
 134	 * this grant entry to domain for accessing. Ref
 135	 * parameter is reference of this introduced grant entry, domid is id of
 136	 * granted domain, frame is the page frame to be granted, and flags is
 137	 * status of the grant entry to be updated.
 138	 */
 139	void (*update_entry)(grant_ref_t ref, domid_t domid,
 140			     unsigned long frame, unsigned flags);
 141	/*
 142	 * Stop granting a grant entry to domain for accessing. Ref parameter is
 143	 * reference of a grant entry whose grant access will be stopped.
 144	 * If the grant entry is currently mapped for reading or writing, just
 145	 * return failure(==0) directly and don't tear down the grant access.
 146	 * Otherwise, stop grant access for this entry and return success(==1).
 
 147	 */
 148	int (*end_foreign_access_ref)(grant_ref_t ref);
 149	/*
 150	 * Read the frame number related to a given grant reference.
 
 
 
 
 151	 */
 152	unsigned long (*read_frame)(grant_ref_t ref);
 153};
 154
 155struct unmap_refs_callback_data {
 156	struct completion completion;
 157	int result;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 158};
 159
 160static const struct gnttab_ops *gnttab_interface;
 161
 162/* This reflects status of grant entries, so act as a global value. */
 163static grant_status_t *grstatus;
 164
 
 
 165static struct gnttab_free_callback *gnttab_free_callback_list;
 166
 167static int gnttab_expand(unsigned int req_entries);
 168
 169#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 170#define SPP (PAGE_SIZE / sizeof(grant_status_t))
 171
 172static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 173{
 174	return &gnttab_list[(entry) / RPP][(entry) % RPP];
 175}
 176/* This can be used as an l-value */
 177#define gnttab_entry(entry) (*__gnttab_entry(entry))
 178
 179static int get_free_entries(unsigned count)
 180{
 181	unsigned long flags;
 182	int ref, rc = 0;
 183	grant_ref_t head;
 184
 185	spin_lock_irqsave(&gnttab_list_lock, flags);
 186
 187	if ((gnttab_free_count < count) &&
 188	    ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 189		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 190		return rc;
 191	}
 192
 193	ref = head = gnttab_free_head;
 194	gnttab_free_count -= count;
 195	while (count--) {
 196		bitmap_clear(gnttab_free_bitmap, head, 1);
 197		if (gnttab_free_tail_ptr == __gnttab_entry(head))
 198			gnttab_free_tail_ptr = &gnttab_free_head;
 199		if (count)
 200			head = gnttab_entry(head);
 201	}
 202	gnttab_free_head = gnttab_entry(head);
 203	gnttab_entry(head) = GNTTAB_LIST_END;
 204
 205	if (!gnttab_free_count) {
 206		gnttab_last_free = GNTTAB_LIST_END;
 207		gnttab_free_tail_ptr = NULL;
 208	}
 209
 210	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 211
 212	return ref;
 213}
 214
 215static int get_seq_entry_count(void)
 216{
 217	if (gnttab_last_free == GNTTAB_LIST_END || !gnttab_free_tail_ptr ||
 218	    *gnttab_free_tail_ptr == GNTTAB_LIST_END)
 219		return 0;
 220
 221	return gnttab_last_free - *gnttab_free_tail_ptr + 1;
 222}
 223
 224/* Rebuilds the free grant list and tries to find count consecutive entries. */
 225static int get_free_seq(unsigned int count)
 226{
 227	int ret = -ENOSPC;
 228	unsigned int from, to;
 229	grant_ref_t *last;
 230
 231	gnttab_free_tail_ptr = &gnttab_free_head;
 232	last = &gnttab_free_head;
 233
 234	for (from = find_first_bit(gnttab_free_bitmap, gnttab_size);
 235	     from < gnttab_size;
 236	     from = find_next_bit(gnttab_free_bitmap, gnttab_size, to + 1)) {
 237		to = find_next_zero_bit(gnttab_free_bitmap, gnttab_size,
 238					from + 1);
 239		if (ret < 0 && to - from >= count) {
 240			ret = from;
 241			bitmap_clear(gnttab_free_bitmap, ret, count);
 242			from += count;
 243			gnttab_free_count -= count;
 244			if (from == to)
 245				continue;
 246		}
 247
 248		/*
 249		 * Recreate the free list in order to have it properly sorted.
 250		 * This is needed to make sure that the free tail has the maximum
 251		 * possible size.
 252		 */
 253		while (from < to) {
 254			*last = from;
 255			last = __gnttab_entry(from);
 256			gnttab_last_free = from;
 257			from++;
 258		}
 259		if (to < gnttab_size)
 260			gnttab_free_tail_ptr = __gnttab_entry(to - 1);
 261	}
 262
 263	*last = GNTTAB_LIST_END;
 264	if (gnttab_last_free != gnttab_size - 1)
 265		gnttab_free_tail_ptr = NULL;
 266
 267	return ret;
 268}
 269
 270static int get_free_entries_seq(unsigned int count)
 271{
 272	unsigned long flags;
 273	int ret = 0;
 274
 275	spin_lock_irqsave(&gnttab_list_lock, flags);
 276
 277	if (gnttab_free_count < count) {
 278		ret = gnttab_expand(count - gnttab_free_count);
 279		if (ret < 0)
 280			goto out;
 281	}
 282
 283	if (get_seq_entry_count() < count) {
 284		ret = get_free_seq(count);
 285		if (ret >= 0)
 286			goto out;
 287		ret = gnttab_expand(count - get_seq_entry_count());
 288		if (ret < 0)
 289			goto out;
 290	}
 291
 292	ret = *gnttab_free_tail_ptr;
 293	*gnttab_free_tail_ptr = gnttab_entry(ret + count - 1);
 294	gnttab_free_count -= count;
 295	if (!gnttab_free_count)
 296		gnttab_free_tail_ptr = NULL;
 297	bitmap_clear(gnttab_free_bitmap, ret, count);
 298
 299 out:
 300	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 301
 302	return ret;
 303}
 304
 305static void do_free_callbacks(void)
 306{
 307	struct gnttab_free_callback *callback, *next;
 308
 309	callback = gnttab_free_callback_list;
 310	gnttab_free_callback_list = NULL;
 311
 312	while (callback != NULL) {
 313		next = callback->next;
 314		if (gnttab_free_count >= callback->count) {
 315			callback->next = NULL;
 316			callback->fn(callback->arg);
 317		} else {
 318			callback->next = gnttab_free_callback_list;
 319			gnttab_free_callback_list = callback;
 320		}
 321		callback = next;
 322	}
 323}
 324
 325static inline void check_free_callbacks(void)
 326{
 327	if (unlikely(gnttab_free_callback_list))
 328		do_free_callbacks();
 329}
 330
 331static void put_free_entry_locked(grant_ref_t ref)
 332{
 333	if (unlikely(ref < GNTTAB_NR_RESERVED_ENTRIES))
 334		return;
 335
 336	gnttab_entry(ref) = gnttab_free_head;
 337	gnttab_free_head = ref;
 338	if (!gnttab_free_count)
 339		gnttab_last_free = ref;
 340	if (gnttab_free_tail_ptr == &gnttab_free_head)
 341		gnttab_free_tail_ptr = __gnttab_entry(ref);
 342	gnttab_free_count++;
 343	bitmap_set(gnttab_free_bitmap, ref, 1);
 344}
 345
 346static void put_free_entry(grant_ref_t ref)
 347{
 348	unsigned long flags;
 349
 350	spin_lock_irqsave(&gnttab_list_lock, flags);
 351	put_free_entry_locked(ref);
 
 
 352	check_free_callbacks();
 353	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 354}
 355
 356static void gnttab_set_free(unsigned int start, unsigned int n)
 357{
 358	unsigned int i;
 359
 360	for (i = start; i < start + n - 1; i++)
 361		gnttab_entry(i) = i + 1;
 362
 363	gnttab_entry(i) = GNTTAB_LIST_END;
 364	if (!gnttab_free_count) {
 365		gnttab_free_head = start;
 366		gnttab_free_tail_ptr = &gnttab_free_head;
 367	} else {
 368		gnttab_entry(gnttab_last_free) = start;
 369	}
 370	gnttab_free_count += n;
 371	gnttab_last_free = i;
 372
 373	bitmap_set(gnttab_free_bitmap, start, n);
 374}
 375
 376/*
 377 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
 378 * Introducing a valid entry into the grant table:
 379 *  1. Write ent->domid.
 380 *  2. Write ent->frame: Frame to which access is permitted.
 
 
 
 381 *  3. Write memory barrier (WMB).
 382 *  4. Write ent->flags, inc. valid type.
 383 */
 384static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 385				   unsigned long frame, unsigned flags)
 386{
 387	gnttab_shared.v1[ref].domid = domid;
 388	gnttab_shared.v1[ref].frame = frame;
 389	wmb();
 390	gnttab_shared.v1[ref].flags = flags;
 391}
 392
 393static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
 394				   unsigned long frame, unsigned int flags)
 395{
 396	gnttab_shared.v2[ref].hdr.domid = domid;
 397	gnttab_shared.v2[ref].full_page.frame = frame;
 398	wmb();	/* Hypervisor concurrent accesses. */
 399	gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
 400}
 401
 402/*
 403 * Public grant-issuing interface functions
 404 */
 405void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 406				     unsigned long frame, int readonly)
 407{
 408	gnttab_interface->update_entry(ref, domid, frame,
 409			   GTF_permit_access | (readonly ? GTF_readonly : 0));
 410}
 411EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 412
 413int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 414				int readonly)
 415{
 416	int ref;
 417
 418	ref = get_free_entries(1);
 419	if (unlikely(ref < 0))
 420		return -ENOSPC;
 421
 422	gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 423
 424	return ref;
 425}
 426EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 427
 428static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 429{
 430	u16 *pflags = &gnttab_shared.v1[ref].flags;
 431	u16 flags;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 432
 433	flags = *pflags;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 434	do {
 
 435		if (flags & (GTF_reading|GTF_writing))
 436			return 0;
 437	} while (!sync_try_cmpxchg(pflags, &flags, 0));
 438
 439	return 1;
 440}
 441
 442static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref)
 443{
 444	gnttab_shared.v2[ref].hdr.flags = 0;
 445	mb();	/* Concurrent access by hypervisor. */
 446	if (grstatus[ref] & (GTF_reading|GTF_writing)) {
 447		return 0;
 448	} else {
 449		/*
 450		 * The read of grstatus needs to have acquire semantics.
 451		 *  On x86, reads already have that, and we just need to
 452		 * protect against compiler reorderings.
 453		 * On other architectures we may need a full barrier.
 454		 */
 455#ifdef CONFIG_X86
 456		barrier();
 457#else
 458		mb();
 459#endif
 460	}
 461
 462	return 1;
 463}
 464
 465static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref)
 466{
 467	return gnttab_interface->end_foreign_access_ref(ref);
 468}
 469
 470int gnttab_end_foreign_access_ref(grant_ref_t ref)
 471{
 472	if (_gnttab_end_foreign_access_ref(ref))
 473		return 1;
 474	pr_warn("WARNING: g.e. %#x still in use!\n", ref);
 475	return 0;
 476}
 477EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 478
 479static unsigned long gnttab_read_frame_v1(grant_ref_t ref)
 480{
 481	return gnttab_shared.v1[ref].frame;
 482}
 483
 484static unsigned long gnttab_read_frame_v2(grant_ref_t ref)
 485{
 486	return gnttab_shared.v2[ref].full_page.frame;
 487}
 488
 489struct deferred_entry {
 490	struct list_head list;
 491	grant_ref_t ref;
 
 492	uint16_t warn_delay;
 493	struct page *page;
 494};
 495static LIST_HEAD(deferred_list);
 496static void gnttab_handle_deferred(struct timer_list *);
 497static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred);
 498
 499static atomic64_t deferred_count;
 500static atomic64_t leaked_count;
 501static unsigned int free_per_iteration = 10;
 502module_param(free_per_iteration, uint, 0600);
 503
 504static void gnttab_handle_deferred(struct timer_list *unused)
 505{
 506	unsigned int nr = READ_ONCE(free_per_iteration);
 507	const bool ignore_limit = nr == 0;
 508	struct deferred_entry *first = NULL;
 509	unsigned long flags;
 510	size_t freed = 0;
 511
 512	spin_lock_irqsave(&gnttab_list_lock, flags);
 513	while ((ignore_limit || nr--) && !list_empty(&deferred_list)) {
 514		struct deferred_entry *entry
 515			= list_first_entry(&deferred_list,
 516					   struct deferred_entry, list);
 517
 518		if (entry == first)
 519			break;
 520		list_del(&entry->list);
 521		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 522		if (_gnttab_end_foreign_access_ref(entry->ref)) {
 523			uint64_t ret = atomic64_dec_return(&deferred_count);
 524
 525			put_free_entry(entry->ref);
 526			pr_debug("freeing g.e. %#x (pfn %#lx), %llu remaining\n",
 527				 entry->ref, page_to_pfn(entry->page),
 528				 (unsigned long long)ret);
 529			put_page(entry->page);
 530			freed++;
 
 531			kfree(entry);
 532			entry = NULL;
 533		} else {
 534			if (!--entry->warn_delay)
 535				pr_info("g.e. %#x still pending\n", entry->ref);
 
 536			if (!first)
 537				first = entry;
 538		}
 539		spin_lock_irqsave(&gnttab_list_lock, flags);
 540		if (entry)
 541			list_add_tail(&entry->list, &deferred_list);
 
 
 542	}
 543	if (list_empty(&deferred_list))
 544		WARN_ON(atomic64_read(&deferred_count));
 545	else if (!timer_pending(&deferred_timer)) {
 546		deferred_timer.expires = jiffies + HZ;
 547		add_timer(&deferred_timer);
 548	}
 549	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 550	pr_debug("Freed %zu references", freed);
 551}
 552
 553static void gnttab_add_deferred(grant_ref_t ref, struct page *page)
 
 554{
 555	struct deferred_entry *entry;
 556	gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL;
 557	uint64_t leaked, deferred;
 558
 559	entry = kmalloc(sizeof(*entry), gfp);
 560	if (!page) {
 561		unsigned long gfn = gnttab_interface->read_frame(ref);
 562
 563		page = pfn_to_page(gfn_to_pfn(gfn));
 564		get_page(page);
 565	}
 566
 567	if (entry) {
 568		unsigned long flags;
 569
 570		entry->ref = ref;
 
 571		entry->page = page;
 572		entry->warn_delay = 60;
 573		spin_lock_irqsave(&gnttab_list_lock, flags);
 574		list_add_tail(&entry->list, &deferred_list);
 575		if (!timer_pending(&deferred_timer)) {
 576			deferred_timer.expires = jiffies + HZ;
 577			add_timer(&deferred_timer);
 578		}
 579		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 580		deferred = atomic64_inc_return(&deferred_count);
 581		leaked = atomic64_read(&leaked_count);
 582		pr_debug("deferring g.e. %#x (pfn %#lx) (total deferred %llu, total leaked %llu)\n",
 583			 ref, page ? page_to_pfn(page) : -1, deferred, leaked);
 584	} else {
 585		deferred = atomic64_read(&deferred_count);
 586		leaked = atomic64_inc_return(&leaked_count);
 587		pr_warn("leaking g.e. %#x (pfn %#lx) (total deferred %llu, total leaked %llu)\n",
 588			ref, page ? page_to_pfn(page) : -1, deferred, leaked);
 589	}
 
 
 590}
 591
 592int gnttab_try_end_foreign_access(grant_ref_t ref)
 
 593{
 594	int ret = _gnttab_end_foreign_access_ref(ref);
 595
 596	if (ret)
 597		put_free_entry(ref);
 
 
 
 
 
 
 
 598
 599	return ret;
 
 
 
 
 
 
 
 
 
 600}
 601EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access);
 602
 603void gnttab_end_foreign_access(grant_ref_t ref, struct page *page)
 
 604{
 605	if (gnttab_try_end_foreign_access(ref)) {
 606		if (page)
 607			put_page(page);
 608	} else
 609		gnttab_add_deferred(ref, page);
 610}
 611EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 612
 613void gnttab_free_grant_reference(grant_ref_t ref)
 614{
 615	put_free_entry(ref);
 616}
 617EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 618
 619void gnttab_free_grant_references(grant_ref_t head)
 620{
 621	grant_ref_t ref;
 622	unsigned long flags;
 623
 
 
 624	spin_lock_irqsave(&gnttab_list_lock, flags);
 625	while (head != GNTTAB_LIST_END) {
 626		ref = gnttab_entry(head);
 627		put_free_entry_locked(head);
 628		head = ref;
 629	}
 
 
 
 630	check_free_callbacks();
 631	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 632}
 633EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 634
 635void gnttab_free_grant_reference_seq(grant_ref_t head, unsigned int count)
 636{
 637	unsigned long flags;
 638	unsigned int i;
 639
 640	spin_lock_irqsave(&gnttab_list_lock, flags);
 641	for (i = count; i > 0; i--)
 642		put_free_entry_locked(head + i - 1);
 643	check_free_callbacks();
 644	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 645}
 646EXPORT_SYMBOL_GPL(gnttab_free_grant_reference_seq);
 647
 648int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 649{
 650	int h = get_free_entries(count);
 651
 652	if (h < 0)
 653		return -ENOSPC;
 654
 655	*head = h;
 656
 657	return 0;
 658}
 659EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 660
 661int gnttab_alloc_grant_reference_seq(unsigned int count, grant_ref_t *first)
 662{
 663	int h;
 664
 665	if (count == 1)
 666		h = get_free_entries(1);
 667	else
 668		h = get_free_entries_seq(count);
 669
 670	if (h < 0)
 671		return -ENOSPC;
 672
 673	*first = h;
 674
 675	return 0;
 676}
 677EXPORT_SYMBOL_GPL(gnttab_alloc_grant_reference_seq);
 678
 679int gnttab_empty_grant_references(const grant_ref_t *private_head)
 680{
 681	return (*private_head == GNTTAB_LIST_END);
 682}
 683EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 684
 685int gnttab_claim_grant_reference(grant_ref_t *private_head)
 686{
 687	grant_ref_t g = *private_head;
 688	if (unlikely(g == GNTTAB_LIST_END))
 689		return -ENOSPC;
 690	*private_head = gnttab_entry(g);
 691	return g;
 692}
 693EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 694
 695void gnttab_release_grant_reference(grant_ref_t *private_head,
 696				    grant_ref_t release)
 697{
 698	gnttab_entry(release) = *private_head;
 699	*private_head = release;
 700}
 701EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 702
 703void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 704				  void (*fn)(void *), void *arg, u16 count)
 705{
 706	unsigned long flags;
 707	struct gnttab_free_callback *cb;
 708
 709	spin_lock_irqsave(&gnttab_list_lock, flags);
 710
 711	/* Check if the callback is already on the list */
 712	cb = gnttab_free_callback_list;
 713	while (cb) {
 714		if (cb == callback)
 715			goto out;
 716		cb = cb->next;
 717	}
 718
 719	callback->fn = fn;
 720	callback->arg = arg;
 721	callback->count = count;
 722	callback->next = gnttab_free_callback_list;
 723	gnttab_free_callback_list = callback;
 724	check_free_callbacks();
 725out:
 726	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 727}
 728EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 729
 730void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 731{
 732	struct gnttab_free_callback **pcb;
 733	unsigned long flags;
 734
 735	spin_lock_irqsave(&gnttab_list_lock, flags);
 736	for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 737		if (*pcb == callback) {
 738			*pcb = callback->next;
 739			break;
 740		}
 741	}
 742	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 743}
 744EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 745
 746static unsigned int gnttab_frames(unsigned int frames, unsigned int align)
 747{
 748	return (frames * gnttab_interface->grefs_per_grant_frame + align - 1) /
 749	       align;
 750}
 751
 752static int grow_gnttab_list(unsigned int more_frames)
 753{
 754	unsigned int new_nr_grant_frames, extra_entries, i;
 755	unsigned int nr_glist_frames, new_nr_glist_frames;
 756	unsigned int grefs_per_frame;
 757
 758	grefs_per_frame = gnttab_interface->grefs_per_grant_frame;
 759
 760	new_nr_grant_frames = nr_grant_frames + more_frames;
 761	extra_entries = more_frames * grefs_per_frame;
 762
 763	nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
 764	new_nr_glist_frames = gnttab_frames(new_nr_grant_frames, RPP);
 
 765	for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 766		gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 767		if (!gnttab_list[i])
 768			goto grow_nomem;
 769	}
 770
 771	gnttab_set_free(gnttab_size, extra_entries);
 772
 773	if (!gnttab_free_tail_ptr)
 774		gnttab_free_tail_ptr = __gnttab_entry(gnttab_size);
 
 
 
 
 
 775
 776	nr_grant_frames = new_nr_grant_frames;
 777	gnttab_size += extra_entries;
 778
 779	check_free_callbacks();
 780
 781	return 0;
 782
 783grow_nomem:
 784	while (i-- > nr_glist_frames)
 785		free_page((unsigned long) gnttab_list[i]);
 786	return -ENOMEM;
 787}
 788
 789static unsigned int __max_nr_grant_frames(void)
 790{
 791	struct gnttab_query_size query;
 792	int rc;
 793
 794	query.dom = DOMID_SELF;
 795
 796	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 797	if ((rc < 0) || (query.status != GNTST_okay))
 798		return 4; /* Legacy max supported number of frames */
 799
 800	return query.max_nr_frames;
 801}
 802
 803unsigned int gnttab_max_grant_frames(void)
 804{
 805	unsigned int xen_max = __max_nr_grant_frames();
 806	static unsigned int boot_max_nr_grant_frames;
 807
 808	/* First time, initialize it properly. */
 809	if (!boot_max_nr_grant_frames)
 810		boot_max_nr_grant_frames = __max_nr_grant_frames();
 811
 812	if (xen_max > boot_max_nr_grant_frames)
 813		return boot_max_nr_grant_frames;
 814	return xen_max;
 815}
 816EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 817
 818int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
 819{
 820	xen_pfn_t *pfn;
 821	unsigned int max_nr_gframes = __max_nr_grant_frames();
 822	unsigned int i;
 823	void *vaddr;
 824
 825	if (xen_auto_xlat_grant_frames.count)
 826		return -EINVAL;
 827
 828	vaddr = memremap(addr, XEN_PAGE_SIZE * max_nr_gframes, MEMREMAP_WB);
 829	if (vaddr == NULL) {
 830		pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",
 831			&addr);
 832		return -ENOMEM;
 833	}
 834	pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL);
 835	if (!pfn) {
 836		memunmap(vaddr);
 837		return -ENOMEM;
 838	}
 839	for (i = 0; i < max_nr_gframes; i++)
 840		pfn[i] = XEN_PFN_DOWN(addr) + i;
 841
 842	xen_auto_xlat_grant_frames.vaddr = vaddr;
 843	xen_auto_xlat_grant_frames.pfn = pfn;
 844	xen_auto_xlat_grant_frames.count = max_nr_gframes;
 845
 846	return 0;
 847}
 848EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames);
 849
 850void gnttab_free_auto_xlat_frames(void)
 851{
 852	if (!xen_auto_xlat_grant_frames.count)
 853		return;
 854	kfree(xen_auto_xlat_grant_frames.pfn);
 855	memunmap(xen_auto_xlat_grant_frames.vaddr);
 856
 857	xen_auto_xlat_grant_frames.pfn = NULL;
 858	xen_auto_xlat_grant_frames.count = 0;
 859	xen_auto_xlat_grant_frames.vaddr = NULL;
 860}
 861EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
 862
 863int gnttab_pages_set_private(int nr_pages, struct page **pages)
 864{
 865	int i;
 866
 867	for (i = 0; i < nr_pages; i++) {
 868#if BITS_PER_LONG < 64
 869		struct xen_page_foreign *foreign;
 870
 871		foreign = kzalloc(sizeof(*foreign), GFP_KERNEL);
 872		if (!foreign)
 873			return -ENOMEM;
 874
 875		set_page_private(pages[i], (unsigned long)foreign);
 876#endif
 877		SetPagePrivate(pages[i]);
 878	}
 879
 880	return 0;
 881}
 882EXPORT_SYMBOL_GPL(gnttab_pages_set_private);
 883
 884/**
 885 * gnttab_alloc_pages - alloc pages suitable for grant mapping into
 886 * @nr_pages: number of pages to alloc
 887 * @pages: returns the pages
 888 */
 889int gnttab_alloc_pages(int nr_pages, struct page **pages)
 890{
 891	int ret;
 892
 893	ret = xen_alloc_unpopulated_pages(nr_pages, pages);
 894	if (ret < 0)
 895		return ret;
 896
 897	ret = gnttab_pages_set_private(nr_pages, pages);
 898	if (ret < 0)
 899		gnttab_free_pages(nr_pages, pages);
 900
 901	return ret;
 902}
 903EXPORT_SYMBOL_GPL(gnttab_alloc_pages);
 904
 905#ifdef CONFIG_XEN_UNPOPULATED_ALLOC
 906static inline void cache_init(struct gnttab_page_cache *cache)
 907{
 908	cache->pages = NULL;
 909}
 910
 911static inline bool cache_empty(struct gnttab_page_cache *cache)
 912{
 913	return !cache->pages;
 914}
 915
 916static inline struct page *cache_deq(struct gnttab_page_cache *cache)
 917{
 918	struct page *page;
 919
 920	page = cache->pages;
 921	cache->pages = page->zone_device_data;
 922
 923	return page;
 924}
 925
 926static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
 927{
 928	page->zone_device_data = cache->pages;
 929	cache->pages = page;
 930}
 931#else
 932static inline void cache_init(struct gnttab_page_cache *cache)
 933{
 934	INIT_LIST_HEAD(&cache->pages);
 935}
 936
 937static inline bool cache_empty(struct gnttab_page_cache *cache)
 938{
 939	return list_empty(&cache->pages);
 940}
 941
 942static inline struct page *cache_deq(struct gnttab_page_cache *cache)
 943{
 944	struct page *page;
 945
 946	page = list_first_entry(&cache->pages, struct page, lru);
 947	list_del(&page->lru);
 948
 949	return page;
 950}
 951
 952static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
 953{
 954	list_add(&page->lru, &cache->pages);
 955}
 956#endif
 957
 958void gnttab_page_cache_init(struct gnttab_page_cache *cache)
 959{
 960	spin_lock_init(&cache->lock);
 961	cache_init(cache);
 962	cache->num_pages = 0;
 963}
 964EXPORT_SYMBOL_GPL(gnttab_page_cache_init);
 965
 966int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page)
 967{
 968	unsigned long flags;
 969
 970	spin_lock_irqsave(&cache->lock, flags);
 971
 972	if (cache_empty(cache)) {
 973		spin_unlock_irqrestore(&cache->lock, flags);
 974		return gnttab_alloc_pages(1, page);
 975	}
 976
 977	page[0] = cache_deq(cache);
 978	cache->num_pages--;
 979
 980	spin_unlock_irqrestore(&cache->lock, flags);
 981
 982	return 0;
 983}
 984EXPORT_SYMBOL_GPL(gnttab_page_cache_get);
 985
 986void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page,
 987			   unsigned int num)
 988{
 989	unsigned long flags;
 990	unsigned int i;
 991
 992	spin_lock_irqsave(&cache->lock, flags);
 993
 994	for (i = 0; i < num; i++)
 995		cache_enq(cache, page[i]);
 996	cache->num_pages += num;
 997
 998	spin_unlock_irqrestore(&cache->lock, flags);
 999}
1000EXPORT_SYMBOL_GPL(gnttab_page_cache_put);
1001
1002void gnttab_page_cache_shrink(struct gnttab_page_cache *cache, unsigned int num)
1003{
1004	struct page *page[10];
1005	unsigned int i = 0;
1006	unsigned long flags;
1007
1008	spin_lock_irqsave(&cache->lock, flags);
1009
1010	while (cache->num_pages > num) {
1011		page[i] = cache_deq(cache);
1012		cache->num_pages--;
1013		if (++i == ARRAY_SIZE(page)) {
1014			spin_unlock_irqrestore(&cache->lock, flags);
1015			gnttab_free_pages(i, page);
1016			i = 0;
1017			spin_lock_irqsave(&cache->lock, flags);
1018		}
1019	}
1020
1021	spin_unlock_irqrestore(&cache->lock, flags);
1022
1023	if (i != 0)
1024		gnttab_free_pages(i, page);
1025}
1026EXPORT_SYMBOL_GPL(gnttab_page_cache_shrink);
1027
1028void gnttab_pages_clear_private(int nr_pages, struct page **pages)
1029{
1030	int i;
1031
1032	for (i = 0; i < nr_pages; i++) {
1033		if (PagePrivate(pages[i])) {
1034#if BITS_PER_LONG < 64
1035			kfree((void *)page_private(pages[i]));
1036#endif
1037			ClearPagePrivate(pages[i]);
1038		}
1039	}
1040}
1041EXPORT_SYMBOL_GPL(gnttab_pages_clear_private);
1042
1043/**
1044 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
1045 * @nr_pages: number of pages to free
1046 * @pages: the pages
1047 */
1048void gnttab_free_pages(int nr_pages, struct page **pages)
1049{
1050	gnttab_pages_clear_private(nr_pages, pages);
1051	xen_free_unpopulated_pages(nr_pages, pages);
1052}
1053EXPORT_SYMBOL_GPL(gnttab_free_pages);
1054
1055#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
1056/**
1057 * gnttab_dma_alloc_pages - alloc DMAable pages suitable for grant mapping into
1058 * @args: arguments to the function
1059 */
1060int gnttab_dma_alloc_pages(struct gnttab_dma_alloc_args *args)
1061{
1062	unsigned long pfn, start_pfn;
1063	size_t size;
1064	int i, ret;
1065
1066	if (args->nr_pages < 0 || args->nr_pages > (INT_MAX >> PAGE_SHIFT))
1067		return -ENOMEM;
1068
1069	size = args->nr_pages << PAGE_SHIFT;
1070	if (args->coherent)
1071		args->vaddr = dma_alloc_coherent(args->dev, size,
1072						 &args->dev_bus_addr,
1073						 GFP_KERNEL | __GFP_NOWARN);
1074	else
1075		args->vaddr = dma_alloc_wc(args->dev, size,
1076					   &args->dev_bus_addr,
1077					   GFP_KERNEL | __GFP_NOWARN);
1078	if (!args->vaddr) {
1079		pr_debug("Failed to allocate DMA buffer of size %zu\n", size);
1080		return -ENOMEM;
1081	}
1082
1083	start_pfn = __phys_to_pfn(args->dev_bus_addr);
1084	for (pfn = start_pfn, i = 0; pfn < start_pfn + args->nr_pages;
1085			pfn++, i++) {
1086		struct page *page = pfn_to_page(pfn);
1087
1088		args->pages[i] = page;
1089		args->frames[i] = xen_page_to_gfn(page);
1090		xenmem_reservation_scrub_page(page);
1091	}
1092
1093	xenmem_reservation_va_mapping_reset(args->nr_pages, args->pages);
1094
1095	ret = xenmem_reservation_decrease(args->nr_pages, args->frames);
1096	if (ret != args->nr_pages) {
1097		pr_debug("Failed to decrease reservation for DMA buffer\n");
1098		ret = -EFAULT;
1099		goto fail;
1100	}
1101
1102	ret = gnttab_pages_set_private(args->nr_pages, args->pages);
1103	if (ret < 0)
1104		goto fail;
1105
1106	return 0;
1107
1108fail:
1109	gnttab_dma_free_pages(args);
1110	return ret;
1111}
1112EXPORT_SYMBOL_GPL(gnttab_dma_alloc_pages);
1113
1114/**
1115 * gnttab_dma_free_pages - free DMAable pages
1116 * @args: arguments to the function
1117 */
1118int gnttab_dma_free_pages(struct gnttab_dma_alloc_args *args)
1119{
1120	size_t size;
1121	int i, ret;
1122
1123	gnttab_pages_clear_private(args->nr_pages, args->pages);
1124
1125	for (i = 0; i < args->nr_pages; i++)
1126		args->frames[i] = page_to_xen_pfn(args->pages[i]);
1127
1128	ret = xenmem_reservation_increase(args->nr_pages, args->frames);
1129	if (ret != args->nr_pages) {
1130		pr_debug("Failed to increase reservation for DMA buffer\n");
1131		ret = -EFAULT;
1132	} else {
1133		ret = 0;
1134	}
1135
1136	xenmem_reservation_va_mapping_update(args->nr_pages, args->pages,
1137					     args->frames);
1138
1139	size = args->nr_pages << PAGE_SHIFT;
1140	if (args->coherent)
1141		dma_free_coherent(args->dev, size,
1142				  args->vaddr, args->dev_bus_addr);
1143	else
1144		dma_free_wc(args->dev, size,
1145			    args->vaddr, args->dev_bus_addr);
1146	return ret;
1147}
1148EXPORT_SYMBOL_GPL(gnttab_dma_free_pages);
1149#endif
1150
1151/* Handling of paged out grant targets (GNTST_eagain) */
1152#define MAX_DELAY 256
1153static inline void
1154gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
1155						const char *func)
1156{
1157	unsigned delay = 1;
1158
1159	do {
1160		BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
1161		if (*status == GNTST_eagain)
1162			msleep(delay++);
1163	} while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
1164
1165	if (delay >= MAX_DELAY) {
1166		pr_err("%s: %s eagain grant\n", func, current->comm);
1167		*status = GNTST_bad_page;
1168	}
1169}
1170
1171void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
1172{
1173	struct gnttab_map_grant_ref *op;
1174
1175	if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
1176		BUG();
1177	for (op = batch; op < batch + count; op++)
1178		if (op->status == GNTST_eagain)
1179			gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
1180						&op->status, __func__);
1181}
1182EXPORT_SYMBOL_GPL(gnttab_batch_map);
1183
1184void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
1185{
1186	struct gnttab_copy *op;
1187
1188	if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
1189		BUG();
1190	for (op = batch; op < batch + count; op++)
1191		if (op->status == GNTST_eagain)
1192			gnttab_retry_eagain_gop(GNTTABOP_copy, op,
1193						&op->status, __func__);
1194}
1195EXPORT_SYMBOL_GPL(gnttab_batch_copy);
1196
1197void gnttab_foreach_grant_in_range(struct page *page,
1198				   unsigned int offset,
1199				   unsigned int len,
1200				   xen_grant_fn_t fn,
1201				   void *data)
1202{
1203	unsigned int goffset;
1204	unsigned int glen;
1205	unsigned long xen_pfn;
1206
1207	len = min_t(unsigned int, PAGE_SIZE - offset, len);
1208	goffset = xen_offset_in_page(offset);
1209
1210	xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset);
1211
1212	while (len) {
1213		glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len);
1214		fn(pfn_to_gfn(xen_pfn), goffset, glen, data);
1215
1216		goffset = 0;
1217		xen_pfn++;
1218		len -= glen;
1219	}
1220}
1221EXPORT_SYMBOL_GPL(gnttab_foreach_grant_in_range);
1222
1223void gnttab_foreach_grant(struct page **pages,
1224			  unsigned int nr_grefs,
1225			  xen_grant_fn_t fn,
1226			  void *data)
1227{
1228	unsigned int goffset = 0;
1229	unsigned long xen_pfn = 0;
1230	unsigned int i;
1231
1232	for (i = 0; i < nr_grefs; i++) {
1233		if ((i % XEN_PFN_PER_PAGE) == 0) {
1234			xen_pfn = page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);
1235			goffset = 0;
1236		}
1237
1238		fn(pfn_to_gfn(xen_pfn), goffset, XEN_PAGE_SIZE, data);
1239
1240		goffset += XEN_PAGE_SIZE;
1241		xen_pfn++;
1242	}
1243}
1244
1245int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
1246		    struct gnttab_map_grant_ref *kmap_ops,
1247		    struct page **pages, unsigned int count)
1248{
1249	int i, ret;
 
 
 
1250
1251	ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
1252	if (ret)
1253		return ret;
1254
1255	for (i = 0; i < count; i++) {
1256		switch (map_ops[i].status) {
1257		case GNTST_okay:
1258		{
1259			struct xen_page_foreign *foreign;
1260
1261			SetPageForeign(pages[i]);
1262			foreign = xen_page_foreign(pages[i]);
1263			foreign->domid = map_ops[i].dom;
1264			foreign->gref = map_ops[i].ref;
1265			break;
1266		}
1267
1268		case GNTST_no_device_space:
1269			pr_warn_ratelimited("maptrack limit reached, can't map all guest pages\n");
1270			break;
1271
1272		case GNTST_eagain:
1273			/* Retry eagain maps */
1274			gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref,
1275						map_ops + i,
1276						&map_ops[i].status, __func__);
1277			/* Test status in next loop iteration. */
1278			i--;
1279			break;
1280
1281		default:
1282			break;
 
 
 
 
 
 
 
 
 
1283		}
 
 
 
 
1284	}
1285
1286	return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);
 
 
 
1287}
1288EXPORT_SYMBOL_GPL(gnttab_map_refs);
1289
1290int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
1291		      struct gnttab_unmap_grant_ref *kunmap_ops,
1292		      struct page **pages, unsigned int count)
1293{
1294	unsigned int i;
1295	int ret;
1296
1297	ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
1298	if (ret)
1299		return ret;
1300
1301	for (i = 0; i < count; i++)
1302		ClearPageForeign(pages[i]);
1303
1304	return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count);
1305}
1306EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
1307
1308#define GNTTAB_UNMAP_REFS_DELAY 5
1309
1310static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
1311
1312static void gnttab_unmap_work(struct work_struct *work)
1313{
1314	struct gntab_unmap_queue_data
1315		*unmap_data = container_of(work, 
1316					   struct gntab_unmap_queue_data,
1317					   gnttab_work.work);
1318	if (unmap_data->age != UINT_MAX)
1319		unmap_data->age++;
1320	__gnttab_unmap_refs_async(unmap_data);
1321}
1322
1323static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
1324{
1325	int ret;
1326	int pc;
1327
1328	for (pc = 0; pc < item->count; pc++) {
1329		if (page_count(item->pages[pc]) > 1) {
1330			unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);
1331			schedule_delayed_work(&item->gnttab_work,
1332					      msecs_to_jiffies(delay));
1333			return;
1334		}
1335	}
1336
1337	ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,
1338				item->pages, item->count);
1339	item->done(ret, item);
1340}
1341
1342void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
1343{
1344	INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);
1345	item->age = 0;
1346
1347	__gnttab_unmap_refs_async(item);
1348}
1349EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async);
1350
1351static void unmap_refs_callback(int result,
1352		struct gntab_unmap_queue_data *data)
1353{
1354	struct unmap_refs_callback_data *d = data->data;
1355
1356	d->result = result;
1357	complete(&d->completion);
1358}
1359
1360int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item)
1361{
1362	struct unmap_refs_callback_data data;
1363
1364	init_completion(&data.completion);
1365	item->data = &data;
1366	item->done = &unmap_refs_callback;
1367	gnttab_unmap_refs_async(item);
1368	wait_for_completion(&data.completion);
1369
1370	return data.result;
1371}
1372EXPORT_SYMBOL_GPL(gnttab_unmap_refs_sync);
1373
1374static unsigned int nr_status_frames(unsigned int nr_grant_frames)
1375{
1376	return gnttab_frames(nr_grant_frames, SPP);
1377}
1378
1379static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
1380{
1381	int rc;
1382
1383	rc = arch_gnttab_map_shared(frames, nr_gframes,
1384				    gnttab_max_grant_frames(),
1385				    &gnttab_shared.addr);
1386	BUG_ON(rc);
1387
1388	return 0;
1389}
1390
1391static void gnttab_unmap_frames_v1(void)
1392{
1393	arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1394}
1395
1396static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
1397{
1398	uint64_t *sframes;
1399	unsigned int nr_sframes;
1400	struct gnttab_get_status_frames getframes;
1401	int rc;
1402
1403	nr_sframes = nr_status_frames(nr_gframes);
1404
1405	/* No need for kzalloc as it is initialized in following hypercall
1406	 * GNTTABOP_get_status_frames.
1407	 */
1408	sframes = kmalloc_array(nr_sframes, sizeof(uint64_t), GFP_ATOMIC);
1409	if (!sframes)
1410		return -ENOMEM;
1411
1412	getframes.dom        = DOMID_SELF;
1413	getframes.nr_frames  = nr_sframes;
1414	set_xen_guest_handle(getframes.frame_list, sframes);
1415
1416	rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1417				       &getframes, 1);
1418	if (rc == -ENOSYS) {
1419		kfree(sframes);
1420		return -ENOSYS;
1421	}
1422
1423	BUG_ON(rc || getframes.status);
1424
1425	rc = arch_gnttab_map_status(sframes, nr_sframes,
1426				    nr_status_frames(gnttab_max_grant_frames()),
1427				    &grstatus);
1428	BUG_ON(rc);
1429	kfree(sframes);
1430
1431	rc = arch_gnttab_map_shared(frames, nr_gframes,
1432				    gnttab_max_grant_frames(),
1433				    &gnttab_shared.addr);
1434	BUG_ON(rc);
1435
1436	return 0;
1437}
1438
1439static void gnttab_unmap_frames_v2(void)
1440{
1441	arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1442	arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
1443}
1444
1445static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1446{
1447	struct gnttab_setup_table setup;
1448	xen_pfn_t *frames;
1449	unsigned int nr_gframes = end_idx + 1;
1450	int rc;
1451
1452	if (xen_feature(XENFEAT_auto_translated_physmap)) {
1453		struct xen_add_to_physmap xatp;
1454		unsigned int i = end_idx;
1455		rc = 0;
1456		BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes);
1457		/*
1458		 * Loop backwards, so that the first hypercall has the largest
1459		 * index, ensuring that the table will grow only once.
1460		 */
1461		do {
1462			xatp.domid = DOMID_SELF;
1463			xatp.idx = i;
1464			xatp.space = XENMAPSPACE_grant_table;
1465			xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i];
1466			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1467			if (rc != 0) {
1468				pr_warn("grant table add_to_physmap failed, err=%d\n",
1469					rc);
1470				break;
1471			}
1472		} while (i-- > start_idx);
1473
1474		return rc;
1475	}
1476
1477	/* No need for kzalloc as it is initialized in following hypercall
1478	 * GNTTABOP_setup_table.
1479	 */
1480	frames = kmalloc_array(nr_gframes, sizeof(unsigned long), GFP_ATOMIC);
1481	if (!frames)
1482		return -ENOMEM;
1483
1484	setup.dom        = DOMID_SELF;
1485	setup.nr_frames  = nr_gframes;
1486	set_xen_guest_handle(setup.frame_list, frames);
1487
1488	rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1489	if (rc == -ENOSYS) {
1490		kfree(frames);
1491		return -ENOSYS;
1492	}
1493
1494	BUG_ON(rc || setup.status);
1495
1496	rc = gnttab_interface->map_frames(frames, nr_gframes);
1497
1498	kfree(frames);
1499
1500	return rc;
1501}
1502
1503static const struct gnttab_ops gnttab_v1_ops = {
1504	.version			= 1,
1505	.grefs_per_grant_frame		= XEN_PAGE_SIZE /
1506					  sizeof(struct grant_entry_v1),
1507	.map_frames			= gnttab_map_frames_v1,
1508	.unmap_frames			= gnttab_unmap_frames_v1,
1509	.update_entry			= gnttab_update_entry_v1,
1510	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v1,
1511	.read_frame			= gnttab_read_frame_v1,
 
1512};
1513
1514static const struct gnttab_ops gnttab_v2_ops = {
1515	.version			= 2,
1516	.grefs_per_grant_frame		= XEN_PAGE_SIZE /
1517					  sizeof(union grant_entry_v2),
1518	.map_frames			= gnttab_map_frames_v2,
1519	.unmap_frames			= gnttab_unmap_frames_v2,
1520	.update_entry			= gnttab_update_entry_v2,
1521	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v2,
1522	.read_frame			= gnttab_read_frame_v2,
 
 
 
1523};
1524
1525static bool gnttab_need_v2(void)
1526{
1527#ifdef CONFIG_X86
1528	uint32_t base, width;
1529
1530	if (xen_pv_domain()) {
1531		base = xen_cpuid_base();
1532		if (cpuid_eax(base) < 5)
1533			return false;	/* Information not available, use V1. */
1534		width = cpuid_ebx(base + 5) &
1535			XEN_CPUID_MACHINE_ADDRESS_WIDTH_MASK;
1536		return width > 32 + PAGE_SHIFT;
1537	}
1538#endif
1539	return !!(max_possible_pfn >> 32);
1540}
1541
1542static void gnttab_request_version(void)
1543{
1544	long rc;
1545	struct gnttab_set_version gsv;
1546
1547	if (gnttab_need_v2())
1548		gsv.version = 2;
1549	else
1550		gsv.version = 1;
1551
1552	/* Boot parameter overrides automatic selection. */
1553	if (xen_gnttab_version >= 1 && xen_gnttab_version <= 2)
1554		gsv.version = xen_gnttab_version;
1555
1556	rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1557	if (rc == 0 && gsv.version == 2)
 
1558		gnttab_interface = &gnttab_v2_ops;
1559	else
 
 
 
 
 
 
 
 
 
 
1560		gnttab_interface = &gnttab_v1_ops;
1561	pr_info("Grant tables using version %d layout\n",
1562		gnttab_interface->version);
 
1563}
1564
1565static int gnttab_setup(void)
1566{
1567	unsigned int max_nr_gframes;
1568
 
1569	max_nr_gframes = gnttab_max_grant_frames();
1570	if (max_nr_gframes < nr_grant_frames)
1571		return -ENOSYS;
1572
1573	if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {
1574		gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;
 
 
 
 
1575		if (gnttab_shared.addr == NULL) {
1576			pr_warn("gnttab share frames is not mapped!\n");
 
1577			return -ENOMEM;
1578		}
1579	}
1580	return gnttab_map(0, nr_grant_frames - 1);
1581}
1582
1583int gnttab_resume(void)
1584{
1585	gnttab_request_version();
1586	return gnttab_setup();
1587}
1588
1589int gnttab_suspend(void)
1590{
1591	if (!xen_feature(XENFEAT_auto_translated_physmap))
1592		gnttab_interface->unmap_frames();
1593	return 0;
1594}
1595
1596static int gnttab_expand(unsigned int req_entries)
1597{
1598	int rc;
1599	unsigned int cur, extra;
1600
1601	cur = nr_grant_frames;
1602	extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) /
1603		 gnttab_interface->grefs_per_grant_frame);
1604	if (cur + extra > gnttab_max_grant_frames()) {
1605		pr_warn_ratelimited("xen/grant-table: max_grant_frames reached"
1606				    " cur=%u extra=%u limit=%u"
1607				    " gnttab_free_count=%u req_entries=%u\n",
1608				    cur, extra, gnttab_max_grant_frames(),
1609				    gnttab_free_count, req_entries);
1610		return -ENOSPC;
1611	}
1612
1613	rc = gnttab_map(cur, cur + extra - 1);
1614	if (rc == 0)
1615		rc = grow_gnttab_list(extra);
1616
1617	return rc;
1618}
1619
1620int gnttab_init(void)
1621{
1622	int i;
1623	unsigned long max_nr_grant_frames, max_nr_grefs;
1624	unsigned int max_nr_glist_frames, nr_glist_frames;
 
1625	int ret;
1626
1627	gnttab_request_version();
1628	max_nr_grant_frames = gnttab_max_grant_frames();
1629	max_nr_grefs = max_nr_grant_frames *
1630			gnttab_interface->grefs_per_grant_frame;
1631	nr_grant_frames = 1;
 
1632
1633	/* Determine the maximum number of frames required for the
1634	 * grant reference free list on the current hypervisor.
1635	 */
1636	max_nr_glist_frames = max_nr_grefs / RPP;
 
1637
1638	gnttab_list = kmalloc_array(max_nr_glist_frames,
1639				    sizeof(grant_ref_t *),
1640				    GFP_KERNEL);
1641	if (gnttab_list == NULL)
1642		return -ENOMEM;
1643
1644	nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
1645	for (i = 0; i < nr_glist_frames; i++) {
1646		gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1647		if (gnttab_list[i] == NULL) {
1648			ret = -ENOMEM;
1649			goto ini_nomem;
1650		}
1651	}
1652
1653	gnttab_free_bitmap = bitmap_zalloc(max_nr_grefs, GFP_KERNEL);
1654	if (!gnttab_free_bitmap) {
1655		ret = -ENOMEM;
1656		goto ini_nomem;
1657	}
1658
1659	ret = arch_gnttab_init(max_nr_grant_frames,
1660			       nr_status_frames(max_nr_grant_frames));
1661	if (ret < 0)
1662		goto ini_nomem;
1663
1664	if (gnttab_setup() < 0) {
1665		ret = -ENODEV;
1666		goto ini_nomem;
1667	}
1668
1669	gnttab_size = nr_grant_frames * gnttab_interface->grefs_per_grant_frame;
1670
1671	gnttab_set_free(GNTTAB_NR_RESERVED_ENTRIES,
1672			gnttab_size - GNTTAB_NR_RESERVED_ENTRIES);
 
 
 
 
1673
1674	printk("Grant table initialized\n");
1675	return 0;
1676
1677 ini_nomem:
1678	for (i--; i >= 0; i--)
1679		free_page((unsigned long)gnttab_list[i]);
1680	kfree(gnttab_list);
1681	bitmap_free(gnttab_free_bitmap);
1682	return ret;
1683}
1684EXPORT_SYMBOL_GPL(gnttab_init);
1685
1686static int __gnttab_init(void)
1687{
1688	if (!xen_domain())
1689		return -ENODEV;
1690
1691	/* Delay grant-table initialization in the PV on HVM case */
1692	if (xen_hvm_domain() && !xen_pvh_domain())
1693		return 0;
1694
 
 
 
1695	return gnttab_init();
1696}
1697/* Starts after core_initcall so that xen_pvh_gnttab_setup can be called
1698 * beforehand to initialize xen_auto_xlat_grant_frames. */
1699core_initcall_sync(__gnttab_init);
v3.5.6
   1/******************************************************************************
   2 * grant_table.c
   3 *
   4 * Granting foreign access to our memory reservation.
   5 *
   6 * Copyright (c) 2005-2006, Christopher Clark
   7 * Copyright (c) 2004-2005, K A Fraser
   8 *
   9 * This program is free software; you can redistribute it and/or
  10 * modify it under the terms of the GNU General Public License version 2
  11 * as published by the Free Software Foundation; or, when distributed
  12 * separately from the Linux kernel or incorporated into other
  13 * software packages, subject to the following license:
  14 *
  15 * Permission is hereby granted, free of charge, to any person obtaining a copy
  16 * of this source file (the "Software"), to deal in the Software without
  17 * restriction, including without limitation the rights to use, copy, modify,
  18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19 * and to permit persons to whom the Software is furnished to do so, subject to
  20 * the following conditions:
  21 *
  22 * The above copyright notice and this permission notice shall be included in
  23 * all copies or substantial portions of the Software.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31 * IN THE SOFTWARE.
  32 */
  33
  34#include <linux/module.h>
 
 
 
  35#include <linux/sched.h>
  36#include <linux/mm.h>
  37#include <linux/slab.h>
  38#include <linux/vmalloc.h>
  39#include <linux/uaccess.h>
  40#include <linux/io.h>
 
  41#include <linux/hardirq.h>
 
 
 
 
 
 
  42
  43#include <xen/xen.h>
  44#include <xen/interface/xen.h>
  45#include <xen/page.h>
  46#include <xen/grant_table.h>
  47#include <xen/interface/memory.h>
  48#include <xen/hvc-console.h>
 
 
 
 
 
 
  49#include <asm/xen/hypercall.h>
 
  50
  51#include <asm/pgtable.h>
  52#include <asm/sync_bitops.h>
  53
  54/* External tools reserve first few grant table entries. */
  55#define NR_RESERVED_ENTRIES 8
  56#define GNTTAB_LIST_END 0xffffffff
  57#define GREFS_PER_GRANT_FRAME \
  58(grant_table_version == 1 ?                      \
  59(PAGE_SIZE / sizeof(struct grant_entry_v1)) :   \
  60(PAGE_SIZE / sizeof(union grant_entry_v2)))
  61
  62static grant_ref_t **gnttab_list;
  63static unsigned int nr_grant_frames;
  64static unsigned int boot_max_nr_grant_frames;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  65static int gnttab_free_count;
  66static grant_ref_t gnttab_free_head;
 
 
 
 
  67static DEFINE_SPINLOCK(gnttab_list_lock);
  68unsigned long xen_hvm_resume_frames;
  69EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
 
 
  70
  71static union {
  72	struct grant_entry_v1 *v1;
  73	union grant_entry_v2 *v2;
  74	void *addr;
  75} gnttab_shared;
  76
  77/*This is a structure of function pointers for grant table*/
  78struct gnttab_ops {
  79	/*
 
 
 
 
 
 
 
 
  80	 * Mapping a list of frames for storing grant entries. Frames parameter
  81	 * is used to store grant table address when grant table being setup,
  82	 * nr_gframes is the number of frames to map grant table. Returning
  83	 * GNTST_okay means success and negative value means failure.
  84	 */
  85	int (*map_frames)(unsigned long *frames, unsigned int nr_gframes);
  86	/*
  87	 * Release a list of frames which are mapped in map_frames for grant
  88	 * entry status.
  89	 */
  90	void (*unmap_frames)(void);
  91	/*
  92	 * Introducing a valid entry into the grant table, granting the frame of
  93	 * this grant entry to domain for accessing or transfering. Ref
  94	 * parameter is reference of this introduced grant entry, domid is id of
  95	 * granted domain, frame is the page frame to be granted, and flags is
  96	 * status of the grant entry to be updated.
  97	 */
  98	void (*update_entry)(grant_ref_t ref, domid_t domid,
  99			     unsigned long frame, unsigned flags);
 100	/*
 101	 * Stop granting a grant entry to domain for accessing. Ref parameter is
 102	 * reference of a grant entry whose grant access will be stopped,
 103	 * readonly is not in use in this function. If the grant entry is
 104	 * currently mapped for reading or writing, just return failure(==0)
 105	 * directly and don't tear down the grant access. Otherwise, stop grant
 106	 * access for this entry and return success(==1).
 107	 */
 108	int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
 109	/*
 110	 * Stop granting a grant entry to domain for transfer. Ref parameter is
 111	 * reference of a grant entry whose grant transfer will be stopped. If
 112	 * tranfer has not started, just reclaim the grant entry and return
 113	 * failure(==0). Otherwise, wait for the transfer to complete and then
 114	 * return the frame.
 115	 */
 116	unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
 117	/*
 118	 * Query the status of a grant entry. Ref parameter is reference of
 119	 * queried grant entry, return value is the status of queried entry.
 120	 * Detailed status(writing/reading) can be gotten from the return value
 121	 * by bit operations.
 122	 */
 123	int (*query_foreign_access)(grant_ref_t ref);
 124	/*
 125	 * Grant a domain to access a range of bytes within the page referred by
 126	 * an available grant entry. Ref parameter is reference of a grant entry
 127	 * which will be sub-page accessed, domid is id of grantee domain, frame
 128	 * is frame address of subpage grant, flags is grant type and flag
 129	 * information, page_off is offset of the range of bytes, and length is
 130	 * length of bytes to be accessed.
 131	 */
 132	void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
 133				     unsigned long frame, int flags,
 134				     unsigned page_off, unsigned length);
 135	/*
 136	 * Redirect an available grant entry on domain A to another grant
 137	 * reference of domain B, then allow domain C to use grant reference
 138	 * of domain B transitively. Ref parameter is an available grant entry
 139	 * reference on domain A, domid is id of domain C which accesses grant
 140	 * entry transitively, flags is grant type and flag information,
 141	 * trans_domid is id of domain B whose grant entry is finally accessed
 142	 * transitively, trans_gref is grant entry transitive reference of
 143	 * domain B.
 144	 */
 145	void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
 146				   domid_t trans_domid, grant_ref_t trans_gref);
 147};
 148
 149static struct gnttab_ops *gnttab_interface;
 150
 151/*This reflects status of grant entries, so act as a global value*/
 152static grant_status_t *grstatus;
 153
 154static int grant_table_version;
 155
 156static struct gnttab_free_callback *gnttab_free_callback_list;
 157
 158static int gnttab_expand(unsigned int req_entries);
 159
 160#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 161#define SPP (PAGE_SIZE / sizeof(grant_status_t))
 162
 163static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 164{
 165	return &gnttab_list[(entry) / RPP][(entry) % RPP];
 166}
 167/* This can be used as an l-value */
 168#define gnttab_entry(entry) (*__gnttab_entry(entry))
 169
 170static int get_free_entries(unsigned count)
 171{
 172	unsigned long flags;
 173	int ref, rc = 0;
 174	grant_ref_t head;
 175
 176	spin_lock_irqsave(&gnttab_list_lock, flags);
 177
 178	if ((gnttab_free_count < count) &&
 179	    ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 180		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 181		return rc;
 182	}
 183
 184	ref = head = gnttab_free_head;
 185	gnttab_free_count -= count;
 186	while (count-- > 1)
 187		head = gnttab_entry(head);
 
 
 
 
 
 188	gnttab_free_head = gnttab_entry(head);
 189	gnttab_entry(head) = GNTTAB_LIST_END;
 190
 
 
 
 
 
 191	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 192
 193	return ref;
 194}
 195
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 196static void do_free_callbacks(void)
 197{
 198	struct gnttab_free_callback *callback, *next;
 199
 200	callback = gnttab_free_callback_list;
 201	gnttab_free_callback_list = NULL;
 202
 203	while (callback != NULL) {
 204		next = callback->next;
 205		if (gnttab_free_count >= callback->count) {
 206			callback->next = NULL;
 207			callback->fn(callback->arg);
 208		} else {
 209			callback->next = gnttab_free_callback_list;
 210			gnttab_free_callback_list = callback;
 211		}
 212		callback = next;
 213	}
 214}
 215
 216static inline void check_free_callbacks(void)
 217{
 218	if (unlikely(gnttab_free_callback_list))
 219		do_free_callbacks();
 220}
 221
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 222static void put_free_entry(grant_ref_t ref)
 223{
 224	unsigned long flags;
 
 225	spin_lock_irqsave(&gnttab_list_lock, flags);
 226	gnttab_entry(ref) = gnttab_free_head;
 227	gnttab_free_head = ref;
 228	gnttab_free_count++;
 229	check_free_callbacks();
 230	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 231}
 232
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 233/*
 234 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
 235 * Introducing a valid entry into the grant table:
 236 *  1. Write ent->domid.
 237 *  2. Write ent->frame:
 238 *      GTF_permit_access:   Frame to which access is permitted.
 239 *      GTF_accept_transfer: Pseudo-phys frame slot being filled by new
 240 *                           frame, or zero if none.
 241 *  3. Write memory barrier (WMB).
 242 *  4. Write ent->flags, inc. valid type.
 243 */
 244static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 245				   unsigned long frame, unsigned flags)
 246{
 247	gnttab_shared.v1[ref].domid = domid;
 248	gnttab_shared.v1[ref].frame = frame;
 249	wmb();
 250	gnttab_shared.v1[ref].flags = flags;
 251}
 252
 253static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
 254				   unsigned long frame, unsigned flags)
 255{
 256	gnttab_shared.v2[ref].hdr.domid = domid;
 257	gnttab_shared.v2[ref].full_page.frame = frame;
 258	wmb();
 259	gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
 260}
 261
 262/*
 263 * Public grant-issuing interface functions
 264 */
 265void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 266				     unsigned long frame, int readonly)
 267{
 268	gnttab_interface->update_entry(ref, domid, frame,
 269			   GTF_permit_access | (readonly ? GTF_readonly : 0));
 270}
 271EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 272
 273int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 274				int readonly)
 275{
 276	int ref;
 277
 278	ref = get_free_entries(1);
 279	if (unlikely(ref < 0))
 280		return -ENOSPC;
 281
 282	gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 283
 284	return ref;
 285}
 286EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 287
 288void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
 289				    unsigned long frame, int flags,
 290				    unsigned page_off,
 291				    unsigned length)
 292{
 293	gnttab_shared.v2[ref].sub_page.frame = frame;
 294	gnttab_shared.v2[ref].sub_page.page_off = page_off;
 295	gnttab_shared.v2[ref].sub_page.length = length;
 296	gnttab_shared.v2[ref].hdr.domid = domid;
 297	wmb();
 298	gnttab_shared.v2[ref].hdr.flags =
 299				GTF_permit_access | GTF_sub_page | flags;
 300}
 301
 302int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
 303					    unsigned long frame, int flags,
 304					    unsigned page_off,
 305					    unsigned length)
 306{
 307	if (flags & (GTF_accept_transfer | GTF_reading |
 308		     GTF_writing | GTF_transitive))
 309		return -EPERM;
 310
 311	if (gnttab_interface->update_subpage_entry == NULL)
 312		return -ENOSYS;
 313
 314	gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
 315					       page_off, length);
 316
 317	return 0;
 318}
 319EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
 320
 321int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
 322					int flags, unsigned page_off,
 323					unsigned length)
 324{
 325	int ref, rc;
 326
 327	ref = get_free_entries(1);
 328	if (unlikely(ref < 0))
 329		return -ENOSPC;
 330
 331	rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
 332						     page_off, length);
 333	if (rc < 0) {
 334		put_free_entry(ref);
 335		return rc;
 336	}
 337
 338	return ref;
 339}
 340EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
 341
 342bool gnttab_subpage_grants_available(void)
 343{
 344	return gnttab_interface->update_subpage_entry != NULL;
 345}
 346EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
 347
 348void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
 349				  int flags, domid_t trans_domid,
 350				  grant_ref_t trans_gref)
 351{
 352	gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
 353	gnttab_shared.v2[ref].transitive.gref = trans_gref;
 354	gnttab_shared.v2[ref].hdr.domid = domid;
 355	wmb();
 356	gnttab_shared.v2[ref].hdr.flags =
 357				GTF_permit_access | GTF_transitive | flags;
 358}
 359
 360int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
 361					  int flags, domid_t trans_domid,
 362					  grant_ref_t trans_gref)
 363{
 364	if (flags & (GTF_accept_transfer | GTF_reading |
 365		     GTF_writing | GTF_sub_page))
 366		return -EPERM;
 367
 368	if (gnttab_interface->update_trans_entry == NULL)
 369		return -ENOSYS;
 370
 371	gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
 372					     trans_gref);
 373
 374	return 0;
 375}
 376EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
 377
 378int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
 379				      domid_t trans_domid,
 380				      grant_ref_t trans_gref)
 381{
 382	int ref, rc;
 383
 384	ref = get_free_entries(1);
 385	if (unlikely(ref < 0))
 386		return -ENOSPC;
 387
 388	rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
 389						   trans_domid, trans_gref);
 390	if (rc < 0) {
 391		put_free_entry(ref);
 392		return rc;
 393	}
 394
 395	return ref;
 396}
 397EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
 398
 399bool gnttab_trans_grants_available(void)
 400{
 401	return gnttab_interface->update_trans_entry != NULL;
 402}
 403EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
 404
 405static int gnttab_query_foreign_access_v1(grant_ref_t ref)
 406{
 407	return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
 408}
 409
 410static int gnttab_query_foreign_access_v2(grant_ref_t ref)
 411{
 412	return grstatus[ref] & (GTF_reading|GTF_writing);
 413}
 414
 415int gnttab_query_foreign_access(grant_ref_t ref)
 416{
 417	return gnttab_interface->query_foreign_access(ref);
 418}
 419EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
 420
 421static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
 422{
 423	u16 flags, nflags;
 424	u16 *pflags;
 425
 426	pflags = &gnttab_shared.v1[ref].flags;
 427	nflags = *pflags;
 428	do {
 429		flags = nflags;
 430		if (flags & (GTF_reading|GTF_writing))
 431			return 0;
 432	} while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
 433
 434	return 1;
 435}
 436
 437static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
 438{
 439	gnttab_shared.v2[ref].hdr.flags = 0;
 440	mb();
 441	if (grstatus[ref] & (GTF_reading|GTF_writing)) {
 442		return 0;
 443	} else {
 444		/* The read of grstatus needs to have acquire
 445		semantics.  On x86, reads already have
 446		that, and we just need to protect against
 447		compiler reorderings.  On other
 448		architectures we may need a full
 449		barrier. */
 450#ifdef CONFIG_X86
 451		barrier();
 452#else
 453		mb();
 454#endif
 455	}
 456
 457	return 1;
 458}
 459
 460static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 461{
 462	return gnttab_interface->end_foreign_access_ref(ref, readonly);
 463}
 464
 465int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 466{
 467	if (_gnttab_end_foreign_access_ref(ref, readonly))
 468		return 1;
 469	pr_warn("WARNING: g.e. %#x still in use!\n", ref);
 470	return 0;
 471}
 472EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 473
 
 
 
 
 
 
 
 
 
 
 474struct deferred_entry {
 475	struct list_head list;
 476	grant_ref_t ref;
 477	bool ro;
 478	uint16_t warn_delay;
 479	struct page *page;
 480};
 481static LIST_HEAD(deferred_list);
 482static void gnttab_handle_deferred(unsigned long);
 483static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
 
 
 
 
 
 484
 485static void gnttab_handle_deferred(unsigned long unused)
 486{
 487	unsigned int nr = 10;
 
 488	struct deferred_entry *first = NULL;
 489	unsigned long flags;
 
 490
 491	spin_lock_irqsave(&gnttab_list_lock, flags);
 492	while (nr--) {
 493		struct deferred_entry *entry
 494			= list_first_entry(&deferred_list,
 495					   struct deferred_entry, list);
 496
 497		if (entry == first)
 498			break;
 499		list_del(&entry->list);
 500		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 501		if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
 
 
 502			put_free_entry(entry->ref);
 503			if (entry->page) {
 504				pr_debug("freeing g.e. %#x (pfn %#lx)\n",
 505					 entry->ref, page_to_pfn(entry->page));
 506				__free_page(entry->page);
 507			} else
 508				pr_info("freeing g.e. %#x\n", entry->ref);
 509			kfree(entry);
 510			entry = NULL;
 511		} else {
 512			if (!--entry->warn_delay)
 513				pr_info("g.e. %#x still pending\n",
 514					entry->ref);
 515			if (!first)
 516				first = entry;
 517		}
 518		spin_lock_irqsave(&gnttab_list_lock, flags);
 519		if (entry)
 520			list_add_tail(&entry->list, &deferred_list);
 521		else if (list_empty(&deferred_list))
 522			break;
 523	}
 524	if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
 
 
 525		deferred_timer.expires = jiffies + HZ;
 526		add_timer(&deferred_timer);
 527	}
 528	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 
 529}
 530
 531static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
 532				struct page *page)
 533{
 534	struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 535	const char *what = KERN_WARNING "leaking";
 
 
 
 
 
 
 
 
 
 536
 537	if (entry) {
 538		unsigned long flags;
 539
 540		entry->ref = ref;
 541		entry->ro = readonly;
 542		entry->page = page;
 543		entry->warn_delay = 60;
 544		spin_lock_irqsave(&gnttab_list_lock, flags);
 545		list_add_tail(&entry->list, &deferred_list);
 546		if (!timer_pending(&deferred_timer)) {
 547			deferred_timer.expires = jiffies + HZ;
 548			add_timer(&deferred_timer);
 549		}
 550		spin_unlock_irqrestore(&gnttab_list_lock, flags);
 551		what = KERN_DEBUG "deferring";
 
 
 
 
 
 
 
 
 552	}
 553	printk("%s g.e. %#x (pfn %#lx)\n",
 554	       what, ref, page ? page_to_pfn(page) : -1);
 555}
 556
 557void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
 558			       unsigned long page)
 559{
 560	if (gnttab_end_foreign_access_ref(ref, readonly)) {
 
 
 561		put_free_entry(ref);
 562		if (page != 0)
 563			free_page(page);
 564	} else
 565		gnttab_add_deferred(ref, readonly,
 566				    page ? virt_to_page(page) : NULL);
 567}
 568EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 569
 570int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 571{
 572	int ref;
 573
 574	ref = get_free_entries(1);
 575	if (unlikely(ref < 0))
 576		return -ENOSPC;
 577	gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 578
 579	return ref;
 580}
 581EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
 582
 583void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
 584				       unsigned long pfn)
 585{
 586	gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
 
 
 
 
 587}
 588EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
 589
 590static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
 591{
 592	unsigned long frame;
 593	u16           flags;
 594	u16          *pflags;
 595
 596	pflags = &gnttab_shared.v1[ref].flags;
 597
 598	/*
 599	 * If a transfer is not even yet started, try to reclaim the grant
 600	 * reference and return failure (== 0).
 601	 */
 602	while (!((flags = *pflags) & GTF_transfer_committed)) {
 603		if (sync_cmpxchg(pflags, flags, 0) == flags)
 604			return 0;
 605		cpu_relax();
 606	}
 607
 608	/* If a transfer is in progress then wait until it is completed. */
 609	while (!(flags & GTF_transfer_completed)) {
 610		flags = *pflags;
 611		cpu_relax();
 612	}
 613
 614	rmb();	/* Read the frame number /after/ reading completion status. */
 615	frame = gnttab_shared.v1[ref].frame;
 616	BUG_ON(frame == 0);
 617
 618	return frame;
 619}
 620
 621static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
 622{
 623	unsigned long frame;
 624	u16           flags;
 625	u16          *pflags;
 626
 627	pflags = &gnttab_shared.v2[ref].hdr.flags;
 628
 629	/*
 630	 * If a transfer is not even yet started, try to reclaim the grant
 631	 * reference and return failure (== 0).
 632	 */
 633	while (!((flags = *pflags) & GTF_transfer_committed)) {
 634		if (sync_cmpxchg(pflags, flags, 0) == flags)
 635			return 0;
 636		cpu_relax();
 637	}
 638
 639	/* If a transfer is in progress then wait until it is completed. */
 640	while (!(flags & GTF_transfer_completed)) {
 641		flags = *pflags;
 642		cpu_relax();
 643	}
 644
 645	rmb();  /* Read the frame number /after/ reading completion status. */
 646	frame = gnttab_shared.v2[ref].full_page.frame;
 647	BUG_ON(frame == 0);
 648
 649	return frame;
 650}
 651
 652unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 653{
 654	return gnttab_interface->end_foreign_transfer_ref(ref);
 655}
 656EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
 657
 658unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
 659{
 660	unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
 661	put_free_entry(ref);
 662	return frame;
 663}
 664EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
 665
 666void gnttab_free_grant_reference(grant_ref_t ref)
 667{
 668	put_free_entry(ref);
 669}
 670EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 671
 672void gnttab_free_grant_references(grant_ref_t head)
 673{
 674	grant_ref_t ref;
 675	unsigned long flags;
 676	int count = 1;
 677	if (head == GNTTAB_LIST_END)
 678		return;
 679	spin_lock_irqsave(&gnttab_list_lock, flags);
 680	ref = head;
 681	while (gnttab_entry(ref) != GNTTAB_LIST_END) {
 682		ref = gnttab_entry(ref);
 683		count++;
 684	}
 685	gnttab_entry(ref) = gnttab_free_head;
 686	gnttab_free_head = head;
 687	gnttab_free_count += count;
 688	check_free_callbacks();
 689	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 690}
 691EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 692
 
 
 
 
 
 
 
 
 
 
 
 
 
 693int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 694{
 695	int h = get_free_entries(count);
 696
 697	if (h < 0)
 698		return -ENOSPC;
 699
 700	*head = h;
 701
 702	return 0;
 703}
 704EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 705
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 706int gnttab_empty_grant_references(const grant_ref_t *private_head)
 707{
 708	return (*private_head == GNTTAB_LIST_END);
 709}
 710EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 711
 712int gnttab_claim_grant_reference(grant_ref_t *private_head)
 713{
 714	grant_ref_t g = *private_head;
 715	if (unlikely(g == GNTTAB_LIST_END))
 716		return -ENOSPC;
 717	*private_head = gnttab_entry(g);
 718	return g;
 719}
 720EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 721
 722void gnttab_release_grant_reference(grant_ref_t *private_head,
 723				    grant_ref_t release)
 724{
 725	gnttab_entry(release) = *private_head;
 726	*private_head = release;
 727}
 728EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 729
 730void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 731				  void (*fn)(void *), void *arg, u16 count)
 732{
 733	unsigned long flags;
 
 
 734	spin_lock_irqsave(&gnttab_list_lock, flags);
 735	if (callback->next)
 736		goto out;
 
 
 
 
 
 
 
 737	callback->fn = fn;
 738	callback->arg = arg;
 739	callback->count = count;
 740	callback->next = gnttab_free_callback_list;
 741	gnttab_free_callback_list = callback;
 742	check_free_callbacks();
 743out:
 744	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 745}
 746EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 747
 748void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 749{
 750	struct gnttab_free_callback **pcb;
 751	unsigned long flags;
 752
 753	spin_lock_irqsave(&gnttab_list_lock, flags);
 754	for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 755		if (*pcb == callback) {
 756			*pcb = callback->next;
 757			break;
 758		}
 759	}
 760	spin_unlock_irqrestore(&gnttab_list_lock, flags);
 761}
 762EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 763
 
 
 
 
 
 
 764static int grow_gnttab_list(unsigned int more_frames)
 765{
 766	unsigned int new_nr_grant_frames, extra_entries, i;
 767	unsigned int nr_glist_frames, new_nr_glist_frames;
 
 
 
 768
 769	new_nr_grant_frames = nr_grant_frames + more_frames;
 770	extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
 771
 772	nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 773	new_nr_glist_frames =
 774		(new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 775	for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 776		gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 777		if (!gnttab_list[i])
 778			goto grow_nomem;
 779	}
 780
 
 781
 782	for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 783	     i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
 784		gnttab_entry(i) = i + 1;
 785
 786	gnttab_entry(i) = gnttab_free_head;
 787	gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 788	gnttab_free_count += extra_entries;
 789
 790	nr_grant_frames = new_nr_grant_frames;
 
 791
 792	check_free_callbacks();
 793
 794	return 0;
 795
 796grow_nomem:
 797	for ( ; i >= nr_glist_frames; i--)
 798		free_page((unsigned long) gnttab_list[i]);
 799	return -ENOMEM;
 800}
 801
 802static unsigned int __max_nr_grant_frames(void)
 803{
 804	struct gnttab_query_size query;
 805	int rc;
 806
 807	query.dom = DOMID_SELF;
 808
 809	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 810	if ((rc < 0) || (query.status != GNTST_okay))
 811		return 4; /* Legacy max supported number of frames */
 812
 813	return query.max_nr_frames;
 814}
 815
 816unsigned int gnttab_max_grant_frames(void)
 817{
 818	unsigned int xen_max = __max_nr_grant_frames();
 
 
 
 
 
 819
 820	if (xen_max > boot_max_nr_grant_frames)
 821		return boot_max_nr_grant_frames;
 822	return xen_max;
 823}
 824EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 825
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 826int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 827		    struct gnttab_map_grant_ref *kmap_ops,
 828		    struct page **pages, unsigned int count)
 829{
 830	int i, ret;
 831	bool lazy = false;
 832	pte_t *pte;
 833	unsigned long mfn;
 834
 835	ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
 836	if (ret)
 837		return ret;
 838
 839	if (xen_feature(XENFEAT_auto_translated_physmap))
 840		return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 841
 842	if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 843		arch_enter_lazy_mmu_mode();
 844		lazy = true;
 845	}
 
 
 
 
 846
 847	for (i = 0; i < count; i++) {
 848		/* Do not add to override if the map failed. */
 849		if (map_ops[i].status)
 850			continue;
 851
 852		if (map_ops[i].flags & GNTMAP_contains_pte) {
 853			pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
 854				(map_ops[i].host_addr & ~PAGE_MASK));
 855			mfn = pte_mfn(*pte);
 856		} else {
 857			mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
 858		}
 859		ret = m2p_add_override(mfn, pages[i], kmap_ops ?
 860				       &kmap_ops[i] : NULL);
 861		if (ret)
 862			return ret;
 863	}
 864
 865	if (lazy)
 866		arch_leave_lazy_mmu_mode();
 867
 868	return ret;
 869}
 870EXPORT_SYMBOL_GPL(gnttab_map_refs);
 871
 872int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
 873		      struct gnttab_map_grant_ref *kmap_ops,
 874		      struct page **pages, unsigned int count)
 875{
 876	int i, ret;
 877	bool lazy = false;
 878
 879	ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
 880	if (ret)
 881		return ret;
 882
 883	if (xen_feature(XENFEAT_auto_translated_physmap))
 884		return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 885
 886	if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 887		arch_enter_lazy_mmu_mode();
 888		lazy = true;
 
 
 
 
 889	}
 890
 891	for (i = 0; i < count; i++) {
 892		ret = m2p_remove_override(pages[i], kmap_ops ?
 893				       &kmap_ops[i] : NULL);
 894		if (ret)
 895			return ret;
 896	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 897
 898	if (lazy)
 899		arch_leave_lazy_mmu_mode();
 
 
 
 900
 901	return ret;
 902}
 903EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
 904
 905static unsigned nr_status_frames(unsigned nr_grant_frames)
 906{
 907	return (nr_grant_frames * GREFS_PER_GRANT_FRAME + SPP - 1) / SPP;
 908}
 909
 910static int gnttab_map_frames_v1(unsigned long *frames, unsigned int nr_gframes)
 911{
 912	int rc;
 913
 914	rc = arch_gnttab_map_shared(frames, nr_gframes,
 915				    gnttab_max_grant_frames(),
 916				    &gnttab_shared.addr);
 917	BUG_ON(rc);
 918
 919	return 0;
 920}
 921
 922static void gnttab_unmap_frames_v1(void)
 923{
 924	arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
 925}
 926
 927static int gnttab_map_frames_v2(unsigned long *frames, unsigned int nr_gframes)
 928{
 929	uint64_t *sframes;
 930	unsigned int nr_sframes;
 931	struct gnttab_get_status_frames getframes;
 932	int rc;
 933
 934	nr_sframes = nr_status_frames(nr_gframes);
 935
 936	/* No need for kzalloc as it is initialized in following hypercall
 937	 * GNTTABOP_get_status_frames.
 938	 */
 939	sframes = kmalloc(nr_sframes  * sizeof(uint64_t), GFP_ATOMIC);
 940	if (!sframes)
 941		return -ENOMEM;
 942
 943	getframes.dom        = DOMID_SELF;
 944	getframes.nr_frames  = nr_sframes;
 945	set_xen_guest_handle(getframes.frame_list, sframes);
 946
 947	rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
 948				       &getframes, 1);
 949	if (rc == -ENOSYS) {
 950		kfree(sframes);
 951		return -ENOSYS;
 952	}
 953
 954	BUG_ON(rc || getframes.status);
 955
 956	rc = arch_gnttab_map_status(sframes, nr_sframes,
 957				    nr_status_frames(gnttab_max_grant_frames()),
 958				    &grstatus);
 959	BUG_ON(rc);
 960	kfree(sframes);
 961
 962	rc = arch_gnttab_map_shared(frames, nr_gframes,
 963				    gnttab_max_grant_frames(),
 964				    &gnttab_shared.addr);
 965	BUG_ON(rc);
 966
 967	return 0;
 968}
 969
 970static void gnttab_unmap_frames_v2(void)
 971{
 972	arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
 973	arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
 974}
 975
 976static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 977{
 978	struct gnttab_setup_table setup;
 979	unsigned long *frames;
 980	unsigned int nr_gframes = end_idx + 1;
 981	int rc;
 982
 983	if (xen_hvm_domain()) {
 984		struct xen_add_to_physmap xatp;
 985		unsigned int i = end_idx;
 986		rc = 0;
 
 987		/*
 988		 * Loop backwards, so that the first hypercall has the largest
 989		 * index, ensuring that the table will grow only once.
 990		 */
 991		do {
 992			xatp.domid = DOMID_SELF;
 993			xatp.idx = i;
 994			xatp.space = XENMAPSPACE_grant_table;
 995			xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
 996			rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 997			if (rc != 0) {
 998				printk(KERN_WARNING
 999						"grant table add_to_physmap failed, err=%d\n", rc);
1000				break;
1001			}
1002		} while (i-- > start_idx);
1003
1004		return rc;
1005	}
1006
1007	/* No need for kzalloc as it is initialized in following hypercall
1008	 * GNTTABOP_setup_table.
1009	 */
1010	frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1011	if (!frames)
1012		return -ENOMEM;
1013
1014	setup.dom        = DOMID_SELF;
1015	setup.nr_frames  = nr_gframes;
1016	set_xen_guest_handle(setup.frame_list, frames);
1017
1018	rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1019	if (rc == -ENOSYS) {
1020		kfree(frames);
1021		return -ENOSYS;
1022	}
1023
1024	BUG_ON(rc || setup.status);
1025
1026	rc = gnttab_interface->map_frames(frames, nr_gframes);
1027
1028	kfree(frames);
1029
1030	return rc;
1031}
1032
1033static struct gnttab_ops gnttab_v1_ops = {
 
 
 
1034	.map_frames			= gnttab_map_frames_v1,
1035	.unmap_frames			= gnttab_unmap_frames_v1,
1036	.update_entry			= gnttab_update_entry_v1,
1037	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v1,
1038	.end_foreign_transfer_ref	= gnttab_end_foreign_transfer_ref_v1,
1039	.query_foreign_access		= gnttab_query_foreign_access_v1,
1040};
1041
1042static struct gnttab_ops gnttab_v2_ops = {
 
 
 
1043	.map_frames			= gnttab_map_frames_v2,
1044	.unmap_frames			= gnttab_unmap_frames_v2,
1045	.update_entry			= gnttab_update_entry_v2,
1046	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v2,
1047	.end_foreign_transfer_ref	= gnttab_end_foreign_transfer_ref_v2,
1048	.query_foreign_access		= gnttab_query_foreign_access_v2,
1049	.update_subpage_entry		= gnttab_update_subpage_entry_v2,
1050	.update_trans_entry		= gnttab_update_trans_entry_v2,
1051};
1052
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1053static void gnttab_request_version(void)
1054{
1055	int rc;
1056	struct gnttab_set_version gsv;
1057
1058	if (xen_hvm_domain())
 
 
1059		gsv.version = 1;
1060	else
1061		gsv.version = 2;
 
 
 
1062	rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1063	if (rc == 0 && gsv.version == 2) {
1064		grant_table_version = 2;
1065		gnttab_interface = &gnttab_v2_ops;
1066	} else if (grant_table_version == 2) {
1067		/*
1068		 * If we've already used version 2 features,
1069		 * but then suddenly discover that they're not
1070		 * available (e.g. migrating to an older
1071		 * version of Xen), almost unbounded badness
1072		 * can happen.
1073		 */
1074		panic("we need grant tables version 2, but only version 1 is available");
1075	} else {
1076		grant_table_version = 1;
1077		gnttab_interface = &gnttab_v1_ops;
1078	}
1079	printk(KERN_INFO "Grant tables using version %d layout.\n",
1080		grant_table_version);
1081}
1082
1083int gnttab_resume(void)
1084{
1085	unsigned int max_nr_gframes;
1086
1087	gnttab_request_version();
1088	max_nr_gframes = gnttab_max_grant_frames();
1089	if (max_nr_gframes < nr_grant_frames)
1090		return -ENOSYS;
1091
1092	if (xen_pv_domain())
1093		return gnttab_map(0, nr_grant_frames - 1);
1094
1095	if (gnttab_shared.addr == NULL) {
1096		gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
1097						PAGE_SIZE * max_nr_gframes);
1098		if (gnttab_shared.addr == NULL) {
1099			printk(KERN_WARNING
1100					"Failed to ioremap gnttab share frames!");
1101			return -ENOMEM;
1102		}
1103	}
 
 
1104
1105	gnttab_map(0, nr_grant_frames - 1);
1106
1107	return 0;
 
1108}
1109
1110int gnttab_suspend(void)
1111{
1112	gnttab_interface->unmap_frames();
 
1113	return 0;
1114}
1115
1116static int gnttab_expand(unsigned int req_entries)
1117{
1118	int rc;
1119	unsigned int cur, extra;
1120
1121	cur = nr_grant_frames;
1122	extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
1123		 GREFS_PER_GRANT_FRAME);
1124	if (cur + extra > gnttab_max_grant_frames())
 
 
 
 
 
1125		return -ENOSPC;
 
1126
1127	rc = gnttab_map(cur, cur + extra - 1);
1128	if (rc == 0)
1129		rc = grow_gnttab_list(extra);
1130
1131	return rc;
1132}
1133
1134int gnttab_init(void)
1135{
1136	int i;
 
1137	unsigned int max_nr_glist_frames, nr_glist_frames;
1138	unsigned int nr_init_grefs;
1139	int ret;
1140
 
 
 
 
1141	nr_grant_frames = 1;
1142	boot_max_nr_grant_frames = __max_nr_grant_frames();
1143
1144	/* Determine the maximum number of frames required for the
1145	 * grant reference free list on the current hypervisor.
1146	 */
1147	max_nr_glist_frames = (boot_max_nr_grant_frames *
1148			       GREFS_PER_GRANT_FRAME / RPP);
1149
1150	gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1151			      GFP_KERNEL);
 
1152	if (gnttab_list == NULL)
1153		return -ENOMEM;
1154
1155	nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
1156	for (i = 0; i < nr_glist_frames; i++) {
1157		gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1158		if (gnttab_list[i] == NULL) {
1159			ret = -ENOMEM;
1160			goto ini_nomem;
1161		}
1162	}
1163
1164	if (gnttab_resume() < 0) {
 
 
 
 
 
 
 
 
 
 
 
1165		ret = -ENODEV;
1166		goto ini_nomem;
1167	}
1168
1169	nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
1170
1171	for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1172		gnttab_entry(i) = i + 1;
1173
1174	gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1175	gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1176	gnttab_free_head  = NR_RESERVED_ENTRIES;
1177
1178	printk("Grant table initialized\n");
1179	return 0;
1180
1181 ini_nomem:
1182	for (i--; i >= 0; i--)
1183		free_page((unsigned long)gnttab_list[i]);
1184	kfree(gnttab_list);
 
1185	return ret;
1186}
1187EXPORT_SYMBOL_GPL(gnttab_init);
1188
1189static int __devinit __gnttab_init(void)
1190{
 
 
 
1191	/* Delay grant-table initialization in the PV on HVM case */
1192	if (xen_hvm_domain())
1193		return 0;
1194
1195	if (!xen_pv_domain())
1196		return -ENODEV;
1197
1198	return gnttab_init();
1199}
1200
1201core_initcall(__gnttab_init);