Loading...
1/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
35
36#include <linux/bootmem.h>
37#include <linux/sched.h>
38#include <linux/mm.h>
39#include <linux/slab.h>
40#include <linux/vmalloc.h>
41#include <linux/uaccess.h>
42#include <linux/io.h>
43#include <linux/delay.h>
44#include <linux/hardirq.h>
45#include <linux/workqueue.h>
46#include <linux/ratelimit.h>
47#include <linux/moduleparam.h>
48
49#include <xen/xen.h>
50#include <xen/interface/xen.h>
51#include <xen/page.h>
52#include <xen/grant_table.h>
53#include <xen/interface/memory.h>
54#include <xen/hvc-console.h>
55#include <xen/swiotlb-xen.h>
56#include <xen/balloon.h>
57#ifdef CONFIG_X86
58#include <asm/xen/cpuid.h>
59#endif
60#include <asm/xen/hypercall.h>
61#include <asm/xen/interface.h>
62
63#include <asm/pgtable.h>
64#include <asm/sync_bitops.h>
65
66/* External tools reserve first few grant table entries. */
67#define NR_RESERVED_ENTRIES 8
68#define GNTTAB_LIST_END 0xffffffff
69
70static grant_ref_t **gnttab_list;
71static unsigned int nr_grant_frames;
72static int gnttab_free_count;
73static grant_ref_t gnttab_free_head;
74static DEFINE_SPINLOCK(gnttab_list_lock);
75struct grant_frames xen_auto_xlat_grant_frames;
76static unsigned int xen_gnttab_version;
77module_param_named(version, xen_gnttab_version, uint, 0);
78
79static union {
80 struct grant_entry_v1 *v1;
81 union grant_entry_v2 *v2;
82 void *addr;
83} gnttab_shared;
84
85/*This is a structure of function pointers for grant table*/
86struct gnttab_ops {
87 /*
88 * Version of the grant interface.
89 */
90 unsigned int version;
91 /*
92 * Grant refs per grant frame.
93 */
94 unsigned int grefs_per_grant_frame;
95 /*
96 * Mapping a list of frames for storing grant entries. Frames parameter
97 * is used to store grant table address when grant table being setup,
98 * nr_gframes is the number of frames to map grant table. Returning
99 * GNTST_okay means success and negative value means failure.
100 */
101 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
102 /*
103 * Release a list of frames which are mapped in map_frames for grant
104 * entry status.
105 */
106 void (*unmap_frames)(void);
107 /*
108 * Introducing a valid entry into the grant table, granting the frame of
109 * this grant entry to domain for accessing or transfering. Ref
110 * parameter is reference of this introduced grant entry, domid is id of
111 * granted domain, frame is the page frame to be granted, and flags is
112 * status of the grant entry to be updated.
113 */
114 void (*update_entry)(grant_ref_t ref, domid_t domid,
115 unsigned long frame, unsigned flags);
116 /*
117 * Stop granting a grant entry to domain for accessing. Ref parameter is
118 * reference of a grant entry whose grant access will be stopped,
119 * readonly is not in use in this function. If the grant entry is
120 * currently mapped for reading or writing, just return failure(==0)
121 * directly and don't tear down the grant access. Otherwise, stop grant
122 * access for this entry and return success(==1).
123 */
124 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
125 /*
126 * Stop granting a grant entry to domain for transfer. Ref parameter is
127 * reference of a grant entry whose grant transfer will be stopped. If
128 * tranfer has not started, just reclaim the grant entry and return
129 * failure(==0). Otherwise, wait for the transfer to complete and then
130 * return the frame.
131 */
132 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
133 /*
134 * Query the status of a grant entry. Ref parameter is reference of
135 * queried grant entry, return value is the status of queried entry.
136 * Detailed status(writing/reading) can be gotten from the return value
137 * by bit operations.
138 */
139 int (*query_foreign_access)(grant_ref_t ref);
140};
141
142struct unmap_refs_callback_data {
143 struct completion completion;
144 int result;
145};
146
147static const struct gnttab_ops *gnttab_interface;
148
149/* This reflects status of grant entries, so act as a global value. */
150static grant_status_t *grstatus;
151
152static struct gnttab_free_callback *gnttab_free_callback_list;
153
154static int gnttab_expand(unsigned int req_entries);
155
156#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
157#define SPP (PAGE_SIZE / sizeof(grant_status_t))
158
159static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
160{
161 return &gnttab_list[(entry) / RPP][(entry) % RPP];
162}
163/* This can be used as an l-value */
164#define gnttab_entry(entry) (*__gnttab_entry(entry))
165
166static int get_free_entries(unsigned count)
167{
168 unsigned long flags;
169 int ref, rc = 0;
170 grant_ref_t head;
171
172 spin_lock_irqsave(&gnttab_list_lock, flags);
173
174 if ((gnttab_free_count < count) &&
175 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
176 spin_unlock_irqrestore(&gnttab_list_lock, flags);
177 return rc;
178 }
179
180 ref = head = gnttab_free_head;
181 gnttab_free_count -= count;
182 while (count-- > 1)
183 head = gnttab_entry(head);
184 gnttab_free_head = gnttab_entry(head);
185 gnttab_entry(head) = GNTTAB_LIST_END;
186
187 spin_unlock_irqrestore(&gnttab_list_lock, flags);
188
189 return ref;
190}
191
192static void do_free_callbacks(void)
193{
194 struct gnttab_free_callback *callback, *next;
195
196 callback = gnttab_free_callback_list;
197 gnttab_free_callback_list = NULL;
198
199 while (callback != NULL) {
200 next = callback->next;
201 if (gnttab_free_count >= callback->count) {
202 callback->next = NULL;
203 callback->fn(callback->arg);
204 } else {
205 callback->next = gnttab_free_callback_list;
206 gnttab_free_callback_list = callback;
207 }
208 callback = next;
209 }
210}
211
212static inline void check_free_callbacks(void)
213{
214 if (unlikely(gnttab_free_callback_list))
215 do_free_callbacks();
216}
217
218static void put_free_entry(grant_ref_t ref)
219{
220 unsigned long flags;
221 spin_lock_irqsave(&gnttab_list_lock, flags);
222 gnttab_entry(ref) = gnttab_free_head;
223 gnttab_free_head = ref;
224 gnttab_free_count++;
225 check_free_callbacks();
226 spin_unlock_irqrestore(&gnttab_list_lock, flags);
227}
228
229/*
230 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
231 * Introducing a valid entry into the grant table:
232 * 1. Write ent->domid.
233 * 2. Write ent->frame:
234 * GTF_permit_access: Frame to which access is permitted.
235 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
236 * frame, or zero if none.
237 * 3. Write memory barrier (WMB).
238 * 4. Write ent->flags, inc. valid type.
239 */
240static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
241 unsigned long frame, unsigned flags)
242{
243 gnttab_shared.v1[ref].domid = domid;
244 gnttab_shared.v1[ref].frame = frame;
245 wmb();
246 gnttab_shared.v1[ref].flags = flags;
247}
248
249static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
250 unsigned long frame, unsigned int flags)
251{
252 gnttab_shared.v2[ref].hdr.domid = domid;
253 gnttab_shared.v2[ref].full_page.frame = frame;
254 wmb(); /* Hypervisor concurrent accesses. */
255 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
256}
257
258/*
259 * Public grant-issuing interface functions
260 */
261void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
262 unsigned long frame, int readonly)
263{
264 gnttab_interface->update_entry(ref, domid, frame,
265 GTF_permit_access | (readonly ? GTF_readonly : 0));
266}
267EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
268
269int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
270 int readonly)
271{
272 int ref;
273
274 ref = get_free_entries(1);
275 if (unlikely(ref < 0))
276 return -ENOSPC;
277
278 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
279
280 return ref;
281}
282EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
283
284static int gnttab_query_foreign_access_v1(grant_ref_t ref)
285{
286 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
287}
288
289static int gnttab_query_foreign_access_v2(grant_ref_t ref)
290{
291 return grstatus[ref] & (GTF_reading|GTF_writing);
292}
293
294int gnttab_query_foreign_access(grant_ref_t ref)
295{
296 return gnttab_interface->query_foreign_access(ref);
297}
298EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
299
300static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
301{
302 u16 flags, nflags;
303 u16 *pflags;
304
305 pflags = &gnttab_shared.v1[ref].flags;
306 nflags = *pflags;
307 do {
308 flags = nflags;
309 if (flags & (GTF_reading|GTF_writing))
310 return 0;
311 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
312
313 return 1;
314}
315
316static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
317{
318 gnttab_shared.v2[ref].hdr.flags = 0;
319 mb(); /* Concurrent access by hypervisor. */
320 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
321 return 0;
322 } else {
323 /*
324 * The read of grstatus needs to have acquire semantics.
325 * On x86, reads already have that, and we just need to
326 * protect against compiler reorderings.
327 * On other architectures we may need a full barrier.
328 */
329#ifdef CONFIG_X86
330 barrier();
331#else
332 mb();
333#endif
334 }
335
336 return 1;
337}
338
339static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
340{
341 return gnttab_interface->end_foreign_access_ref(ref, readonly);
342}
343
344int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
345{
346 if (_gnttab_end_foreign_access_ref(ref, readonly))
347 return 1;
348 pr_warn("WARNING: g.e. %#x still in use!\n", ref);
349 return 0;
350}
351EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
352
353struct deferred_entry {
354 struct list_head list;
355 grant_ref_t ref;
356 bool ro;
357 uint16_t warn_delay;
358 struct page *page;
359};
360static LIST_HEAD(deferred_list);
361static void gnttab_handle_deferred(struct timer_list *);
362static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred);
363
364static void gnttab_handle_deferred(struct timer_list *unused)
365{
366 unsigned int nr = 10;
367 struct deferred_entry *first = NULL;
368 unsigned long flags;
369
370 spin_lock_irqsave(&gnttab_list_lock, flags);
371 while (nr--) {
372 struct deferred_entry *entry
373 = list_first_entry(&deferred_list,
374 struct deferred_entry, list);
375
376 if (entry == first)
377 break;
378 list_del(&entry->list);
379 spin_unlock_irqrestore(&gnttab_list_lock, flags);
380 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
381 put_free_entry(entry->ref);
382 if (entry->page) {
383 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
384 entry->ref, page_to_pfn(entry->page));
385 put_page(entry->page);
386 } else
387 pr_info("freeing g.e. %#x\n", entry->ref);
388 kfree(entry);
389 entry = NULL;
390 } else {
391 if (!--entry->warn_delay)
392 pr_info("g.e. %#x still pending\n", entry->ref);
393 if (!first)
394 first = entry;
395 }
396 spin_lock_irqsave(&gnttab_list_lock, flags);
397 if (entry)
398 list_add_tail(&entry->list, &deferred_list);
399 else if (list_empty(&deferred_list))
400 break;
401 }
402 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
403 deferred_timer.expires = jiffies + HZ;
404 add_timer(&deferred_timer);
405 }
406 spin_unlock_irqrestore(&gnttab_list_lock, flags);
407}
408
409static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
410 struct page *page)
411{
412 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
413 const char *what = KERN_WARNING "leaking";
414
415 if (entry) {
416 unsigned long flags;
417
418 entry->ref = ref;
419 entry->ro = readonly;
420 entry->page = page;
421 entry->warn_delay = 60;
422 spin_lock_irqsave(&gnttab_list_lock, flags);
423 list_add_tail(&entry->list, &deferred_list);
424 if (!timer_pending(&deferred_timer)) {
425 deferred_timer.expires = jiffies + HZ;
426 add_timer(&deferred_timer);
427 }
428 spin_unlock_irqrestore(&gnttab_list_lock, flags);
429 what = KERN_DEBUG "deferring";
430 }
431 printk("%s g.e. %#x (pfn %#lx)\n",
432 what, ref, page ? page_to_pfn(page) : -1);
433}
434
435void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
436 unsigned long page)
437{
438 if (gnttab_end_foreign_access_ref(ref, readonly)) {
439 put_free_entry(ref);
440 if (page != 0)
441 put_page(virt_to_page(page));
442 } else
443 gnttab_add_deferred(ref, readonly,
444 page ? virt_to_page(page) : NULL);
445}
446EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
447
448int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
449{
450 int ref;
451
452 ref = get_free_entries(1);
453 if (unlikely(ref < 0))
454 return -ENOSPC;
455 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
456
457 return ref;
458}
459EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
460
461void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
462 unsigned long pfn)
463{
464 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
465}
466EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
467
468static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
469{
470 unsigned long frame;
471 u16 flags;
472 u16 *pflags;
473
474 pflags = &gnttab_shared.v1[ref].flags;
475
476 /*
477 * If a transfer is not even yet started, try to reclaim the grant
478 * reference and return failure (== 0).
479 */
480 while (!((flags = *pflags) & GTF_transfer_committed)) {
481 if (sync_cmpxchg(pflags, flags, 0) == flags)
482 return 0;
483 cpu_relax();
484 }
485
486 /* If a transfer is in progress then wait until it is completed. */
487 while (!(flags & GTF_transfer_completed)) {
488 flags = *pflags;
489 cpu_relax();
490 }
491
492 rmb(); /* Read the frame number /after/ reading completion status. */
493 frame = gnttab_shared.v1[ref].frame;
494 BUG_ON(frame == 0);
495
496 return frame;
497}
498
499static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
500{
501 unsigned long frame;
502 u16 flags;
503 u16 *pflags;
504
505 pflags = &gnttab_shared.v2[ref].hdr.flags;
506
507 /*
508 * If a transfer is not even yet started, try to reclaim the grant
509 * reference and return failure (== 0).
510 */
511 while (!((flags = *pflags) & GTF_transfer_committed)) {
512 if (sync_cmpxchg(pflags, flags, 0) == flags)
513 return 0;
514 cpu_relax();
515 }
516
517 /* If a transfer is in progress then wait until it is completed. */
518 while (!(flags & GTF_transfer_completed)) {
519 flags = *pflags;
520 cpu_relax();
521 }
522
523 rmb(); /* Read the frame number /after/ reading completion status. */
524 frame = gnttab_shared.v2[ref].full_page.frame;
525 BUG_ON(frame == 0);
526
527 return frame;
528}
529
530unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
531{
532 return gnttab_interface->end_foreign_transfer_ref(ref);
533}
534EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
535
536unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
537{
538 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
539 put_free_entry(ref);
540 return frame;
541}
542EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
543
544void gnttab_free_grant_reference(grant_ref_t ref)
545{
546 put_free_entry(ref);
547}
548EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
549
550void gnttab_free_grant_references(grant_ref_t head)
551{
552 grant_ref_t ref;
553 unsigned long flags;
554 int count = 1;
555 if (head == GNTTAB_LIST_END)
556 return;
557 spin_lock_irqsave(&gnttab_list_lock, flags);
558 ref = head;
559 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
560 ref = gnttab_entry(ref);
561 count++;
562 }
563 gnttab_entry(ref) = gnttab_free_head;
564 gnttab_free_head = head;
565 gnttab_free_count += count;
566 check_free_callbacks();
567 spin_unlock_irqrestore(&gnttab_list_lock, flags);
568}
569EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
570
571int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
572{
573 int h = get_free_entries(count);
574
575 if (h < 0)
576 return -ENOSPC;
577
578 *head = h;
579
580 return 0;
581}
582EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
583
584int gnttab_empty_grant_references(const grant_ref_t *private_head)
585{
586 return (*private_head == GNTTAB_LIST_END);
587}
588EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
589
590int gnttab_claim_grant_reference(grant_ref_t *private_head)
591{
592 grant_ref_t g = *private_head;
593 if (unlikely(g == GNTTAB_LIST_END))
594 return -ENOSPC;
595 *private_head = gnttab_entry(g);
596 return g;
597}
598EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
599
600void gnttab_release_grant_reference(grant_ref_t *private_head,
601 grant_ref_t release)
602{
603 gnttab_entry(release) = *private_head;
604 *private_head = release;
605}
606EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
607
608void gnttab_request_free_callback(struct gnttab_free_callback *callback,
609 void (*fn)(void *), void *arg, u16 count)
610{
611 unsigned long flags;
612 struct gnttab_free_callback *cb;
613
614 spin_lock_irqsave(&gnttab_list_lock, flags);
615
616 /* Check if the callback is already on the list */
617 cb = gnttab_free_callback_list;
618 while (cb) {
619 if (cb == callback)
620 goto out;
621 cb = cb->next;
622 }
623
624 callback->fn = fn;
625 callback->arg = arg;
626 callback->count = count;
627 callback->next = gnttab_free_callback_list;
628 gnttab_free_callback_list = callback;
629 check_free_callbacks();
630out:
631 spin_unlock_irqrestore(&gnttab_list_lock, flags);
632}
633EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
634
635void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
636{
637 struct gnttab_free_callback **pcb;
638 unsigned long flags;
639
640 spin_lock_irqsave(&gnttab_list_lock, flags);
641 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
642 if (*pcb == callback) {
643 *pcb = callback->next;
644 break;
645 }
646 }
647 spin_unlock_irqrestore(&gnttab_list_lock, flags);
648}
649EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
650
651static unsigned int gnttab_frames(unsigned int frames, unsigned int align)
652{
653 return (frames * gnttab_interface->grefs_per_grant_frame + align - 1) /
654 align;
655}
656
657static int grow_gnttab_list(unsigned int more_frames)
658{
659 unsigned int new_nr_grant_frames, extra_entries, i;
660 unsigned int nr_glist_frames, new_nr_glist_frames;
661 unsigned int grefs_per_frame;
662
663 BUG_ON(gnttab_interface == NULL);
664 grefs_per_frame = gnttab_interface->grefs_per_grant_frame;
665
666 new_nr_grant_frames = nr_grant_frames + more_frames;
667 extra_entries = more_frames * grefs_per_frame;
668
669 nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
670 new_nr_glist_frames = gnttab_frames(new_nr_grant_frames, RPP);
671 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
672 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
673 if (!gnttab_list[i])
674 goto grow_nomem;
675 }
676
677
678 for (i = grefs_per_frame * nr_grant_frames;
679 i < grefs_per_frame * new_nr_grant_frames - 1; i++)
680 gnttab_entry(i) = i + 1;
681
682 gnttab_entry(i) = gnttab_free_head;
683 gnttab_free_head = grefs_per_frame * nr_grant_frames;
684 gnttab_free_count += extra_entries;
685
686 nr_grant_frames = new_nr_grant_frames;
687
688 check_free_callbacks();
689
690 return 0;
691
692grow_nomem:
693 while (i-- > nr_glist_frames)
694 free_page((unsigned long) gnttab_list[i]);
695 return -ENOMEM;
696}
697
698static unsigned int __max_nr_grant_frames(void)
699{
700 struct gnttab_query_size query;
701 int rc;
702
703 query.dom = DOMID_SELF;
704
705 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
706 if ((rc < 0) || (query.status != GNTST_okay))
707 return 4; /* Legacy max supported number of frames */
708
709 return query.max_nr_frames;
710}
711
712unsigned int gnttab_max_grant_frames(void)
713{
714 unsigned int xen_max = __max_nr_grant_frames();
715 static unsigned int boot_max_nr_grant_frames;
716
717 /* First time, initialize it properly. */
718 if (!boot_max_nr_grant_frames)
719 boot_max_nr_grant_frames = __max_nr_grant_frames();
720
721 if (xen_max > boot_max_nr_grant_frames)
722 return boot_max_nr_grant_frames;
723 return xen_max;
724}
725EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
726
727int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
728{
729 xen_pfn_t *pfn;
730 unsigned int max_nr_gframes = __max_nr_grant_frames();
731 unsigned int i;
732 void *vaddr;
733
734 if (xen_auto_xlat_grant_frames.count)
735 return -EINVAL;
736
737 vaddr = xen_remap(addr, XEN_PAGE_SIZE * max_nr_gframes);
738 if (vaddr == NULL) {
739 pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",
740 &addr);
741 return -ENOMEM;
742 }
743 pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL);
744 if (!pfn) {
745 xen_unmap(vaddr);
746 return -ENOMEM;
747 }
748 for (i = 0; i < max_nr_gframes; i++)
749 pfn[i] = XEN_PFN_DOWN(addr) + i;
750
751 xen_auto_xlat_grant_frames.vaddr = vaddr;
752 xen_auto_xlat_grant_frames.pfn = pfn;
753 xen_auto_xlat_grant_frames.count = max_nr_gframes;
754
755 return 0;
756}
757EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames);
758
759void gnttab_free_auto_xlat_frames(void)
760{
761 if (!xen_auto_xlat_grant_frames.count)
762 return;
763 kfree(xen_auto_xlat_grant_frames.pfn);
764 xen_unmap(xen_auto_xlat_grant_frames.vaddr);
765
766 xen_auto_xlat_grant_frames.pfn = NULL;
767 xen_auto_xlat_grant_frames.count = 0;
768 xen_auto_xlat_grant_frames.vaddr = NULL;
769}
770EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
771
772/**
773 * gnttab_alloc_pages - alloc pages suitable for grant mapping into
774 * @nr_pages: number of pages to alloc
775 * @pages: returns the pages
776 */
777int gnttab_alloc_pages(int nr_pages, struct page **pages)
778{
779 int i;
780 int ret;
781
782 ret = alloc_xenballooned_pages(nr_pages, pages);
783 if (ret < 0)
784 return ret;
785
786 for (i = 0; i < nr_pages; i++) {
787#if BITS_PER_LONG < 64
788 struct xen_page_foreign *foreign;
789
790 foreign = kzalloc(sizeof(*foreign), GFP_KERNEL);
791 if (!foreign) {
792 gnttab_free_pages(nr_pages, pages);
793 return -ENOMEM;
794 }
795 set_page_private(pages[i], (unsigned long)foreign);
796#endif
797 SetPagePrivate(pages[i]);
798 }
799
800 return 0;
801}
802EXPORT_SYMBOL(gnttab_alloc_pages);
803
804/**
805 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
806 * @nr_pages; number of pages to free
807 * @pages: the pages
808 */
809void gnttab_free_pages(int nr_pages, struct page **pages)
810{
811 int i;
812
813 for (i = 0; i < nr_pages; i++) {
814 if (PagePrivate(pages[i])) {
815#if BITS_PER_LONG < 64
816 kfree((void *)page_private(pages[i]));
817#endif
818 ClearPagePrivate(pages[i]);
819 }
820 }
821 free_xenballooned_pages(nr_pages, pages);
822}
823EXPORT_SYMBOL(gnttab_free_pages);
824
825/* Handling of paged out grant targets (GNTST_eagain) */
826#define MAX_DELAY 256
827static inline void
828gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
829 const char *func)
830{
831 unsigned delay = 1;
832
833 do {
834 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
835 if (*status == GNTST_eagain)
836 msleep(delay++);
837 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
838
839 if (delay >= MAX_DELAY) {
840 pr_err("%s: %s eagain grant\n", func, current->comm);
841 *status = GNTST_bad_page;
842 }
843}
844
845void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
846{
847 struct gnttab_map_grant_ref *op;
848
849 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
850 BUG();
851 for (op = batch; op < batch + count; op++)
852 if (op->status == GNTST_eagain)
853 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
854 &op->status, __func__);
855}
856EXPORT_SYMBOL_GPL(gnttab_batch_map);
857
858void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
859{
860 struct gnttab_copy *op;
861
862 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
863 BUG();
864 for (op = batch; op < batch + count; op++)
865 if (op->status == GNTST_eagain)
866 gnttab_retry_eagain_gop(GNTTABOP_copy, op,
867 &op->status, __func__);
868}
869EXPORT_SYMBOL_GPL(gnttab_batch_copy);
870
871void gnttab_foreach_grant_in_range(struct page *page,
872 unsigned int offset,
873 unsigned int len,
874 xen_grant_fn_t fn,
875 void *data)
876{
877 unsigned int goffset;
878 unsigned int glen;
879 unsigned long xen_pfn;
880
881 len = min_t(unsigned int, PAGE_SIZE - offset, len);
882 goffset = xen_offset_in_page(offset);
883
884 xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset);
885
886 while (len) {
887 glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len);
888 fn(pfn_to_gfn(xen_pfn), goffset, glen, data);
889
890 goffset = 0;
891 xen_pfn++;
892 len -= glen;
893 }
894}
895EXPORT_SYMBOL_GPL(gnttab_foreach_grant_in_range);
896
897void gnttab_foreach_grant(struct page **pages,
898 unsigned int nr_grefs,
899 xen_grant_fn_t fn,
900 void *data)
901{
902 unsigned int goffset = 0;
903 unsigned long xen_pfn = 0;
904 unsigned int i;
905
906 for (i = 0; i < nr_grefs; i++) {
907 if ((i % XEN_PFN_PER_PAGE) == 0) {
908 xen_pfn = page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);
909 goffset = 0;
910 }
911
912 fn(pfn_to_gfn(xen_pfn), goffset, XEN_PAGE_SIZE, data);
913
914 goffset += XEN_PAGE_SIZE;
915 xen_pfn++;
916 }
917}
918
919int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
920 struct gnttab_map_grant_ref *kmap_ops,
921 struct page **pages, unsigned int count)
922{
923 int i, ret;
924
925 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
926 if (ret)
927 return ret;
928
929 for (i = 0; i < count; i++) {
930 /* Retry eagain maps */
931 if (map_ops[i].status == GNTST_eagain)
932 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
933 &map_ops[i].status, __func__);
934
935 if (map_ops[i].status == GNTST_okay) {
936 struct xen_page_foreign *foreign;
937
938 SetPageForeign(pages[i]);
939 foreign = xen_page_foreign(pages[i]);
940 foreign->domid = map_ops[i].dom;
941 foreign->gref = map_ops[i].ref;
942 }
943 }
944
945 return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);
946}
947EXPORT_SYMBOL_GPL(gnttab_map_refs);
948
949int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
950 struct gnttab_unmap_grant_ref *kunmap_ops,
951 struct page **pages, unsigned int count)
952{
953 unsigned int i;
954 int ret;
955
956 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
957 if (ret)
958 return ret;
959
960 for (i = 0; i < count; i++)
961 ClearPageForeign(pages[i]);
962
963 return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count);
964}
965EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
966
967#define GNTTAB_UNMAP_REFS_DELAY 5
968
969static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
970
971static void gnttab_unmap_work(struct work_struct *work)
972{
973 struct gntab_unmap_queue_data
974 *unmap_data = container_of(work,
975 struct gntab_unmap_queue_data,
976 gnttab_work.work);
977 if (unmap_data->age != UINT_MAX)
978 unmap_data->age++;
979 __gnttab_unmap_refs_async(unmap_data);
980}
981
982static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
983{
984 int ret;
985 int pc;
986
987 for (pc = 0; pc < item->count; pc++) {
988 if (page_count(item->pages[pc]) > 1) {
989 unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);
990 schedule_delayed_work(&item->gnttab_work,
991 msecs_to_jiffies(delay));
992 return;
993 }
994 }
995
996 ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,
997 item->pages, item->count);
998 item->done(ret, item);
999}
1000
1001void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
1002{
1003 INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);
1004 item->age = 0;
1005
1006 __gnttab_unmap_refs_async(item);
1007}
1008EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async);
1009
1010static void unmap_refs_callback(int result,
1011 struct gntab_unmap_queue_data *data)
1012{
1013 struct unmap_refs_callback_data *d = data->data;
1014
1015 d->result = result;
1016 complete(&d->completion);
1017}
1018
1019int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item)
1020{
1021 struct unmap_refs_callback_data data;
1022
1023 init_completion(&data.completion);
1024 item->data = &data;
1025 item->done = &unmap_refs_callback;
1026 gnttab_unmap_refs_async(item);
1027 wait_for_completion(&data.completion);
1028
1029 return data.result;
1030}
1031EXPORT_SYMBOL_GPL(gnttab_unmap_refs_sync);
1032
1033static unsigned int nr_status_frames(unsigned int nr_grant_frames)
1034{
1035 BUG_ON(gnttab_interface == NULL);
1036 return gnttab_frames(nr_grant_frames, SPP);
1037}
1038
1039static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
1040{
1041 int rc;
1042
1043 rc = arch_gnttab_map_shared(frames, nr_gframes,
1044 gnttab_max_grant_frames(),
1045 &gnttab_shared.addr);
1046 BUG_ON(rc);
1047
1048 return 0;
1049}
1050
1051static void gnttab_unmap_frames_v1(void)
1052{
1053 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1054}
1055
1056static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
1057{
1058 uint64_t *sframes;
1059 unsigned int nr_sframes;
1060 struct gnttab_get_status_frames getframes;
1061 int rc;
1062
1063 nr_sframes = nr_status_frames(nr_gframes);
1064
1065 /* No need for kzalloc as it is initialized in following hypercall
1066 * GNTTABOP_get_status_frames.
1067 */
1068 sframes = kmalloc_array(nr_sframes, sizeof(uint64_t), GFP_ATOMIC);
1069 if (!sframes)
1070 return -ENOMEM;
1071
1072 getframes.dom = DOMID_SELF;
1073 getframes.nr_frames = nr_sframes;
1074 set_xen_guest_handle(getframes.frame_list, sframes);
1075
1076 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1077 &getframes, 1);
1078 if (rc == -ENOSYS) {
1079 kfree(sframes);
1080 return -ENOSYS;
1081 }
1082
1083 BUG_ON(rc || getframes.status);
1084
1085 rc = arch_gnttab_map_status(sframes, nr_sframes,
1086 nr_status_frames(gnttab_max_grant_frames()),
1087 &grstatus);
1088 BUG_ON(rc);
1089 kfree(sframes);
1090
1091 rc = arch_gnttab_map_shared(frames, nr_gframes,
1092 gnttab_max_grant_frames(),
1093 &gnttab_shared.addr);
1094 BUG_ON(rc);
1095
1096 return 0;
1097}
1098
1099static void gnttab_unmap_frames_v2(void)
1100{
1101 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1102 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
1103}
1104
1105static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1106{
1107 struct gnttab_setup_table setup;
1108 xen_pfn_t *frames;
1109 unsigned int nr_gframes = end_idx + 1;
1110 int rc;
1111
1112 if (xen_feature(XENFEAT_auto_translated_physmap)) {
1113 struct xen_add_to_physmap xatp;
1114 unsigned int i = end_idx;
1115 rc = 0;
1116 BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes);
1117 /*
1118 * Loop backwards, so that the first hypercall has the largest
1119 * index, ensuring that the table will grow only once.
1120 */
1121 do {
1122 xatp.domid = DOMID_SELF;
1123 xatp.idx = i;
1124 xatp.space = XENMAPSPACE_grant_table;
1125 xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i];
1126 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1127 if (rc != 0) {
1128 pr_warn("grant table add_to_physmap failed, err=%d\n",
1129 rc);
1130 break;
1131 }
1132 } while (i-- > start_idx);
1133
1134 return rc;
1135 }
1136
1137 /* No need for kzalloc as it is initialized in following hypercall
1138 * GNTTABOP_setup_table.
1139 */
1140 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1141 if (!frames)
1142 return -ENOMEM;
1143
1144 setup.dom = DOMID_SELF;
1145 setup.nr_frames = nr_gframes;
1146 set_xen_guest_handle(setup.frame_list, frames);
1147
1148 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1149 if (rc == -ENOSYS) {
1150 kfree(frames);
1151 return -ENOSYS;
1152 }
1153
1154 BUG_ON(rc || setup.status);
1155
1156 rc = gnttab_interface->map_frames(frames, nr_gframes);
1157
1158 kfree(frames);
1159
1160 return rc;
1161}
1162
1163static const struct gnttab_ops gnttab_v1_ops = {
1164 .version = 1,
1165 .grefs_per_grant_frame = XEN_PAGE_SIZE /
1166 sizeof(struct grant_entry_v1),
1167 .map_frames = gnttab_map_frames_v1,
1168 .unmap_frames = gnttab_unmap_frames_v1,
1169 .update_entry = gnttab_update_entry_v1,
1170 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1171 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1172 .query_foreign_access = gnttab_query_foreign_access_v1,
1173};
1174
1175static const struct gnttab_ops gnttab_v2_ops = {
1176 .version = 2,
1177 .grefs_per_grant_frame = XEN_PAGE_SIZE /
1178 sizeof(union grant_entry_v2),
1179 .map_frames = gnttab_map_frames_v2,
1180 .unmap_frames = gnttab_unmap_frames_v2,
1181 .update_entry = gnttab_update_entry_v2,
1182 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1183 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1184 .query_foreign_access = gnttab_query_foreign_access_v2,
1185};
1186
1187static bool gnttab_need_v2(void)
1188{
1189#ifdef CONFIG_X86
1190 uint32_t base, width;
1191
1192 if (xen_pv_domain()) {
1193 base = xen_cpuid_base();
1194 if (cpuid_eax(base) < 5)
1195 return false; /* Information not available, use V1. */
1196 width = cpuid_ebx(base + 5) &
1197 XEN_CPUID_MACHINE_ADDRESS_WIDTH_MASK;
1198 return width > 32 + PAGE_SHIFT;
1199 }
1200#endif
1201 return !!(max_possible_pfn >> 32);
1202}
1203
1204static void gnttab_request_version(void)
1205{
1206 long rc;
1207 struct gnttab_set_version gsv;
1208
1209 if (gnttab_need_v2())
1210 gsv.version = 2;
1211 else
1212 gsv.version = 1;
1213
1214 /* Boot parameter overrides automatic selection. */
1215 if (xen_gnttab_version >= 1 && xen_gnttab_version <= 2)
1216 gsv.version = xen_gnttab_version;
1217
1218 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1219 if (rc == 0 && gsv.version == 2)
1220 gnttab_interface = &gnttab_v2_ops;
1221 else
1222 gnttab_interface = &gnttab_v1_ops;
1223 pr_info("Grant tables using version %d layout\n",
1224 gnttab_interface->version);
1225}
1226
1227static int gnttab_setup(void)
1228{
1229 unsigned int max_nr_gframes;
1230
1231 max_nr_gframes = gnttab_max_grant_frames();
1232 if (max_nr_gframes < nr_grant_frames)
1233 return -ENOSYS;
1234
1235 if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {
1236 gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;
1237 if (gnttab_shared.addr == NULL) {
1238 pr_warn("gnttab share frames (addr=0x%08lx) is not mapped!\n",
1239 (unsigned long)xen_auto_xlat_grant_frames.vaddr);
1240 return -ENOMEM;
1241 }
1242 }
1243 return gnttab_map(0, nr_grant_frames - 1);
1244}
1245
1246int gnttab_resume(void)
1247{
1248 gnttab_request_version();
1249 return gnttab_setup();
1250}
1251
1252int gnttab_suspend(void)
1253{
1254 if (!xen_feature(XENFEAT_auto_translated_physmap))
1255 gnttab_interface->unmap_frames();
1256 return 0;
1257}
1258
1259static int gnttab_expand(unsigned int req_entries)
1260{
1261 int rc;
1262 unsigned int cur, extra;
1263
1264 BUG_ON(gnttab_interface == NULL);
1265 cur = nr_grant_frames;
1266 extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) /
1267 gnttab_interface->grefs_per_grant_frame);
1268 if (cur + extra > gnttab_max_grant_frames()) {
1269 pr_warn_ratelimited("xen/grant-table: max_grant_frames reached"
1270 " cur=%u extra=%u limit=%u"
1271 " gnttab_free_count=%u req_entries=%u\n",
1272 cur, extra, gnttab_max_grant_frames(),
1273 gnttab_free_count, req_entries);
1274 return -ENOSPC;
1275 }
1276
1277 rc = gnttab_map(cur, cur + extra - 1);
1278 if (rc == 0)
1279 rc = grow_gnttab_list(extra);
1280
1281 return rc;
1282}
1283
1284int gnttab_init(void)
1285{
1286 int i;
1287 unsigned long max_nr_grant_frames;
1288 unsigned int max_nr_glist_frames, nr_glist_frames;
1289 unsigned int nr_init_grefs;
1290 int ret;
1291
1292 gnttab_request_version();
1293 max_nr_grant_frames = gnttab_max_grant_frames();
1294 nr_grant_frames = 1;
1295
1296 /* Determine the maximum number of frames required for the
1297 * grant reference free list on the current hypervisor.
1298 */
1299 BUG_ON(gnttab_interface == NULL);
1300 max_nr_glist_frames = (max_nr_grant_frames *
1301 gnttab_interface->grefs_per_grant_frame / RPP);
1302
1303 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1304 GFP_KERNEL);
1305 if (gnttab_list == NULL)
1306 return -ENOMEM;
1307
1308 nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
1309 for (i = 0; i < nr_glist_frames; i++) {
1310 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1311 if (gnttab_list[i] == NULL) {
1312 ret = -ENOMEM;
1313 goto ini_nomem;
1314 }
1315 }
1316
1317 ret = arch_gnttab_init(max_nr_grant_frames,
1318 nr_status_frames(max_nr_grant_frames));
1319 if (ret < 0)
1320 goto ini_nomem;
1321
1322 if (gnttab_setup() < 0) {
1323 ret = -ENODEV;
1324 goto ini_nomem;
1325 }
1326
1327 nr_init_grefs = nr_grant_frames *
1328 gnttab_interface->grefs_per_grant_frame;
1329
1330 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1331 gnttab_entry(i) = i + 1;
1332
1333 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1334 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1335 gnttab_free_head = NR_RESERVED_ENTRIES;
1336
1337 printk("Grant table initialized\n");
1338 return 0;
1339
1340 ini_nomem:
1341 for (i--; i >= 0; i--)
1342 free_page((unsigned long)gnttab_list[i]);
1343 kfree(gnttab_list);
1344 return ret;
1345}
1346EXPORT_SYMBOL_GPL(gnttab_init);
1347
1348static int __gnttab_init(void)
1349{
1350 if (!xen_domain())
1351 return -ENODEV;
1352
1353 /* Delay grant-table initialization in the PV on HVM case */
1354 if (xen_hvm_domain() && !xen_pvh_domain())
1355 return 0;
1356
1357 return gnttab_init();
1358}
1359/* Starts after core_initcall so that xen_pvh_gnttab_setup can be called
1360 * beforehand to initialize xen_auto_xlat_grant_frames. */
1361core_initcall_sync(__gnttab_init);
1/******************************************************************************
2 * grant_table.c
3 *
4 * Granting foreign access to our memory reservation.
5 *
6 * Copyright (c) 2005-2006, Christopher Clark
7 * Copyright (c) 2004-2005, K A Fraser
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
35
36#include <linux/module.h>
37#include <linux/sched.h>
38#include <linux/mm.h>
39#include <linux/slab.h>
40#include <linux/vmalloc.h>
41#include <linux/uaccess.h>
42#include <linux/io.h>
43#include <linux/delay.h>
44#include <linux/hardirq.h>
45
46#include <xen/xen.h>
47#include <xen/interface/xen.h>
48#include <xen/page.h>
49#include <xen/grant_table.h>
50#include <xen/interface/memory.h>
51#include <xen/hvc-console.h>
52#include <xen/swiotlb-xen.h>
53#include <asm/xen/hypercall.h>
54#include <asm/xen/interface.h>
55
56#include <asm/pgtable.h>
57#include <asm/sync_bitops.h>
58
59/* External tools reserve first few grant table entries. */
60#define NR_RESERVED_ENTRIES 8
61#define GNTTAB_LIST_END 0xffffffff
62
63static grant_ref_t **gnttab_list;
64static unsigned int nr_grant_frames;
65static int gnttab_free_count;
66static grant_ref_t gnttab_free_head;
67static DEFINE_SPINLOCK(gnttab_list_lock);
68struct grant_frames xen_auto_xlat_grant_frames;
69
70static union {
71 struct grant_entry_v1 *v1;
72 union grant_entry_v2 *v2;
73 void *addr;
74} gnttab_shared;
75
76/*This is a structure of function pointers for grant table*/
77struct gnttab_ops {
78 /*
79 * Mapping a list of frames for storing grant entries. Frames parameter
80 * is used to store grant table address when grant table being setup,
81 * nr_gframes is the number of frames to map grant table. Returning
82 * GNTST_okay means success and negative value means failure.
83 */
84 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
85 /*
86 * Release a list of frames which are mapped in map_frames for grant
87 * entry status.
88 */
89 void (*unmap_frames)(void);
90 /*
91 * Introducing a valid entry into the grant table, granting the frame of
92 * this grant entry to domain for accessing or transfering. Ref
93 * parameter is reference of this introduced grant entry, domid is id of
94 * granted domain, frame is the page frame to be granted, and flags is
95 * status of the grant entry to be updated.
96 */
97 void (*update_entry)(grant_ref_t ref, domid_t domid,
98 unsigned long frame, unsigned flags);
99 /*
100 * Stop granting a grant entry to domain for accessing. Ref parameter is
101 * reference of a grant entry whose grant access will be stopped,
102 * readonly is not in use in this function. If the grant entry is
103 * currently mapped for reading or writing, just return failure(==0)
104 * directly and don't tear down the grant access. Otherwise, stop grant
105 * access for this entry and return success(==1).
106 */
107 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
108 /*
109 * Stop granting a grant entry to domain for transfer. Ref parameter is
110 * reference of a grant entry whose grant transfer will be stopped. If
111 * tranfer has not started, just reclaim the grant entry and return
112 * failure(==0). Otherwise, wait for the transfer to complete and then
113 * return the frame.
114 */
115 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
116 /*
117 * Query the status of a grant entry. Ref parameter is reference of
118 * queried grant entry, return value is the status of queried entry.
119 * Detailed status(writing/reading) can be gotten from the return value
120 * by bit operations.
121 */
122 int (*query_foreign_access)(grant_ref_t ref);
123 /*
124 * Grant a domain to access a range of bytes within the page referred by
125 * an available grant entry. Ref parameter is reference of a grant entry
126 * which will be sub-page accessed, domid is id of grantee domain, frame
127 * is frame address of subpage grant, flags is grant type and flag
128 * information, page_off is offset of the range of bytes, and length is
129 * length of bytes to be accessed.
130 */
131 void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
132 unsigned long frame, int flags,
133 unsigned page_off, unsigned length);
134 /*
135 * Redirect an available grant entry on domain A to another grant
136 * reference of domain B, then allow domain C to use grant reference
137 * of domain B transitively. Ref parameter is an available grant entry
138 * reference on domain A, domid is id of domain C which accesses grant
139 * entry transitively, flags is grant type and flag information,
140 * trans_domid is id of domain B whose grant entry is finally accessed
141 * transitively, trans_gref is grant entry transitive reference of
142 * domain B.
143 */
144 void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
145 domid_t trans_domid, grant_ref_t trans_gref);
146};
147
148static struct gnttab_ops *gnttab_interface;
149
150/*This reflects status of grant entries, so act as a global value*/
151static grant_status_t *grstatus;
152
153static int grant_table_version;
154static int grefs_per_grant_frame;
155
156static struct gnttab_free_callback *gnttab_free_callback_list;
157
158static int gnttab_expand(unsigned int req_entries);
159
160#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
161#define SPP (PAGE_SIZE / sizeof(grant_status_t))
162
163static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
164{
165 return &gnttab_list[(entry) / RPP][(entry) % RPP];
166}
167/* This can be used as an l-value */
168#define gnttab_entry(entry) (*__gnttab_entry(entry))
169
170static int get_free_entries(unsigned count)
171{
172 unsigned long flags;
173 int ref, rc = 0;
174 grant_ref_t head;
175
176 spin_lock_irqsave(&gnttab_list_lock, flags);
177
178 if ((gnttab_free_count < count) &&
179 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
180 spin_unlock_irqrestore(&gnttab_list_lock, flags);
181 return rc;
182 }
183
184 ref = head = gnttab_free_head;
185 gnttab_free_count -= count;
186 while (count-- > 1)
187 head = gnttab_entry(head);
188 gnttab_free_head = gnttab_entry(head);
189 gnttab_entry(head) = GNTTAB_LIST_END;
190
191 spin_unlock_irqrestore(&gnttab_list_lock, flags);
192
193 return ref;
194}
195
196static void do_free_callbacks(void)
197{
198 struct gnttab_free_callback *callback, *next;
199
200 callback = gnttab_free_callback_list;
201 gnttab_free_callback_list = NULL;
202
203 while (callback != NULL) {
204 next = callback->next;
205 if (gnttab_free_count >= callback->count) {
206 callback->next = NULL;
207 callback->fn(callback->arg);
208 } else {
209 callback->next = gnttab_free_callback_list;
210 gnttab_free_callback_list = callback;
211 }
212 callback = next;
213 }
214}
215
216static inline void check_free_callbacks(void)
217{
218 if (unlikely(gnttab_free_callback_list))
219 do_free_callbacks();
220}
221
222static void put_free_entry(grant_ref_t ref)
223{
224 unsigned long flags;
225 spin_lock_irqsave(&gnttab_list_lock, flags);
226 gnttab_entry(ref) = gnttab_free_head;
227 gnttab_free_head = ref;
228 gnttab_free_count++;
229 check_free_callbacks();
230 spin_unlock_irqrestore(&gnttab_list_lock, flags);
231}
232
233/*
234 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
235 * Introducing a valid entry into the grant table:
236 * 1. Write ent->domid.
237 * 2. Write ent->frame:
238 * GTF_permit_access: Frame to which access is permitted.
239 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
240 * frame, or zero if none.
241 * 3. Write memory barrier (WMB).
242 * 4. Write ent->flags, inc. valid type.
243 */
244static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
245 unsigned long frame, unsigned flags)
246{
247 gnttab_shared.v1[ref].domid = domid;
248 gnttab_shared.v1[ref].frame = frame;
249 wmb();
250 gnttab_shared.v1[ref].flags = flags;
251}
252
253static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
254 unsigned long frame, unsigned flags)
255{
256 gnttab_shared.v2[ref].hdr.domid = domid;
257 gnttab_shared.v2[ref].full_page.frame = frame;
258 wmb();
259 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
260}
261
262/*
263 * Public grant-issuing interface functions
264 */
265void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
266 unsigned long frame, int readonly)
267{
268 gnttab_interface->update_entry(ref, domid, frame,
269 GTF_permit_access | (readonly ? GTF_readonly : 0));
270}
271EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
272
273int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
274 int readonly)
275{
276 int ref;
277
278 ref = get_free_entries(1);
279 if (unlikely(ref < 0))
280 return -ENOSPC;
281
282 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
283
284 return ref;
285}
286EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
287
288static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
289 unsigned long frame, int flags,
290 unsigned page_off, unsigned length)
291{
292 gnttab_shared.v2[ref].sub_page.frame = frame;
293 gnttab_shared.v2[ref].sub_page.page_off = page_off;
294 gnttab_shared.v2[ref].sub_page.length = length;
295 gnttab_shared.v2[ref].hdr.domid = domid;
296 wmb();
297 gnttab_shared.v2[ref].hdr.flags =
298 GTF_permit_access | GTF_sub_page | flags;
299}
300
301int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
302 unsigned long frame, int flags,
303 unsigned page_off,
304 unsigned length)
305{
306 if (flags & (GTF_accept_transfer | GTF_reading |
307 GTF_writing | GTF_transitive))
308 return -EPERM;
309
310 if (gnttab_interface->update_subpage_entry == NULL)
311 return -ENOSYS;
312
313 gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
314 page_off, length);
315
316 return 0;
317}
318EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
319
320int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
321 int flags, unsigned page_off,
322 unsigned length)
323{
324 int ref, rc;
325
326 ref = get_free_entries(1);
327 if (unlikely(ref < 0))
328 return -ENOSPC;
329
330 rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
331 page_off, length);
332 if (rc < 0) {
333 put_free_entry(ref);
334 return rc;
335 }
336
337 return ref;
338}
339EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
340
341bool gnttab_subpage_grants_available(void)
342{
343 return gnttab_interface->update_subpage_entry != NULL;
344}
345EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
346
347static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
348 int flags, domid_t trans_domid,
349 grant_ref_t trans_gref)
350{
351 gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
352 gnttab_shared.v2[ref].transitive.gref = trans_gref;
353 gnttab_shared.v2[ref].hdr.domid = domid;
354 wmb();
355 gnttab_shared.v2[ref].hdr.flags =
356 GTF_permit_access | GTF_transitive | flags;
357}
358
359int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
360 int flags, domid_t trans_domid,
361 grant_ref_t trans_gref)
362{
363 if (flags & (GTF_accept_transfer | GTF_reading |
364 GTF_writing | GTF_sub_page))
365 return -EPERM;
366
367 if (gnttab_interface->update_trans_entry == NULL)
368 return -ENOSYS;
369
370 gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
371 trans_gref);
372
373 return 0;
374}
375EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
376
377int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
378 domid_t trans_domid,
379 grant_ref_t trans_gref)
380{
381 int ref, rc;
382
383 ref = get_free_entries(1);
384 if (unlikely(ref < 0))
385 return -ENOSPC;
386
387 rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
388 trans_domid, trans_gref);
389 if (rc < 0) {
390 put_free_entry(ref);
391 return rc;
392 }
393
394 return ref;
395}
396EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
397
398bool gnttab_trans_grants_available(void)
399{
400 return gnttab_interface->update_trans_entry != NULL;
401}
402EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
403
404static int gnttab_query_foreign_access_v1(grant_ref_t ref)
405{
406 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
407}
408
409static int gnttab_query_foreign_access_v2(grant_ref_t ref)
410{
411 return grstatus[ref] & (GTF_reading|GTF_writing);
412}
413
414int gnttab_query_foreign_access(grant_ref_t ref)
415{
416 return gnttab_interface->query_foreign_access(ref);
417}
418EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
419
420static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
421{
422 u16 flags, nflags;
423 u16 *pflags;
424
425 pflags = &gnttab_shared.v1[ref].flags;
426 nflags = *pflags;
427 do {
428 flags = nflags;
429 if (flags & (GTF_reading|GTF_writing))
430 return 0;
431 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
432
433 return 1;
434}
435
436static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
437{
438 gnttab_shared.v2[ref].hdr.flags = 0;
439 mb();
440 if (grstatus[ref] & (GTF_reading|GTF_writing)) {
441 return 0;
442 } else {
443 /* The read of grstatus needs to have acquire
444 semantics. On x86, reads already have
445 that, and we just need to protect against
446 compiler reorderings. On other
447 architectures we may need a full
448 barrier. */
449#ifdef CONFIG_X86
450 barrier();
451#else
452 mb();
453#endif
454 }
455
456 return 1;
457}
458
459static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
460{
461 return gnttab_interface->end_foreign_access_ref(ref, readonly);
462}
463
464int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
465{
466 if (_gnttab_end_foreign_access_ref(ref, readonly))
467 return 1;
468 pr_warn("WARNING: g.e. %#x still in use!\n", ref);
469 return 0;
470}
471EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
472
473struct deferred_entry {
474 struct list_head list;
475 grant_ref_t ref;
476 bool ro;
477 uint16_t warn_delay;
478 struct page *page;
479};
480static LIST_HEAD(deferred_list);
481static void gnttab_handle_deferred(unsigned long);
482static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
483
484static void gnttab_handle_deferred(unsigned long unused)
485{
486 unsigned int nr = 10;
487 struct deferred_entry *first = NULL;
488 unsigned long flags;
489
490 spin_lock_irqsave(&gnttab_list_lock, flags);
491 while (nr--) {
492 struct deferred_entry *entry
493 = list_first_entry(&deferred_list,
494 struct deferred_entry, list);
495
496 if (entry == first)
497 break;
498 list_del(&entry->list);
499 spin_unlock_irqrestore(&gnttab_list_lock, flags);
500 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
501 put_free_entry(entry->ref);
502 if (entry->page) {
503 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
504 entry->ref, page_to_pfn(entry->page));
505 __free_page(entry->page);
506 } else
507 pr_info("freeing g.e. %#x\n", entry->ref);
508 kfree(entry);
509 entry = NULL;
510 } else {
511 if (!--entry->warn_delay)
512 pr_info("g.e. %#x still pending\n", entry->ref);
513 if (!first)
514 first = entry;
515 }
516 spin_lock_irqsave(&gnttab_list_lock, flags);
517 if (entry)
518 list_add_tail(&entry->list, &deferred_list);
519 else if (list_empty(&deferred_list))
520 break;
521 }
522 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
523 deferred_timer.expires = jiffies + HZ;
524 add_timer(&deferred_timer);
525 }
526 spin_unlock_irqrestore(&gnttab_list_lock, flags);
527}
528
529static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
530 struct page *page)
531{
532 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
533 const char *what = KERN_WARNING "leaking";
534
535 if (entry) {
536 unsigned long flags;
537
538 entry->ref = ref;
539 entry->ro = readonly;
540 entry->page = page;
541 entry->warn_delay = 60;
542 spin_lock_irqsave(&gnttab_list_lock, flags);
543 list_add_tail(&entry->list, &deferred_list);
544 if (!timer_pending(&deferred_timer)) {
545 deferred_timer.expires = jiffies + HZ;
546 add_timer(&deferred_timer);
547 }
548 spin_unlock_irqrestore(&gnttab_list_lock, flags);
549 what = KERN_DEBUG "deferring";
550 }
551 printk("%s g.e. %#x (pfn %#lx)\n",
552 what, ref, page ? page_to_pfn(page) : -1);
553}
554
555void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
556 unsigned long page)
557{
558 if (gnttab_end_foreign_access_ref(ref, readonly)) {
559 put_free_entry(ref);
560 if (page != 0)
561 free_page(page);
562 } else
563 gnttab_add_deferred(ref, readonly,
564 page ? virt_to_page(page) : NULL);
565}
566EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
567
568int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
569{
570 int ref;
571
572 ref = get_free_entries(1);
573 if (unlikely(ref < 0))
574 return -ENOSPC;
575 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
576
577 return ref;
578}
579EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
580
581void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
582 unsigned long pfn)
583{
584 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
585}
586EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
587
588static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
589{
590 unsigned long frame;
591 u16 flags;
592 u16 *pflags;
593
594 pflags = &gnttab_shared.v1[ref].flags;
595
596 /*
597 * If a transfer is not even yet started, try to reclaim the grant
598 * reference and return failure (== 0).
599 */
600 while (!((flags = *pflags) & GTF_transfer_committed)) {
601 if (sync_cmpxchg(pflags, flags, 0) == flags)
602 return 0;
603 cpu_relax();
604 }
605
606 /* If a transfer is in progress then wait until it is completed. */
607 while (!(flags & GTF_transfer_completed)) {
608 flags = *pflags;
609 cpu_relax();
610 }
611
612 rmb(); /* Read the frame number /after/ reading completion status. */
613 frame = gnttab_shared.v1[ref].frame;
614 BUG_ON(frame == 0);
615
616 return frame;
617}
618
619static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
620{
621 unsigned long frame;
622 u16 flags;
623 u16 *pflags;
624
625 pflags = &gnttab_shared.v2[ref].hdr.flags;
626
627 /*
628 * If a transfer is not even yet started, try to reclaim the grant
629 * reference and return failure (== 0).
630 */
631 while (!((flags = *pflags) & GTF_transfer_committed)) {
632 if (sync_cmpxchg(pflags, flags, 0) == flags)
633 return 0;
634 cpu_relax();
635 }
636
637 /* If a transfer is in progress then wait until it is completed. */
638 while (!(flags & GTF_transfer_completed)) {
639 flags = *pflags;
640 cpu_relax();
641 }
642
643 rmb(); /* Read the frame number /after/ reading completion status. */
644 frame = gnttab_shared.v2[ref].full_page.frame;
645 BUG_ON(frame == 0);
646
647 return frame;
648}
649
650unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
651{
652 return gnttab_interface->end_foreign_transfer_ref(ref);
653}
654EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
655
656unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
657{
658 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
659 put_free_entry(ref);
660 return frame;
661}
662EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
663
664void gnttab_free_grant_reference(grant_ref_t ref)
665{
666 put_free_entry(ref);
667}
668EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
669
670void gnttab_free_grant_references(grant_ref_t head)
671{
672 grant_ref_t ref;
673 unsigned long flags;
674 int count = 1;
675 if (head == GNTTAB_LIST_END)
676 return;
677 spin_lock_irqsave(&gnttab_list_lock, flags);
678 ref = head;
679 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
680 ref = gnttab_entry(ref);
681 count++;
682 }
683 gnttab_entry(ref) = gnttab_free_head;
684 gnttab_free_head = head;
685 gnttab_free_count += count;
686 check_free_callbacks();
687 spin_unlock_irqrestore(&gnttab_list_lock, flags);
688}
689EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
690
691int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
692{
693 int h = get_free_entries(count);
694
695 if (h < 0)
696 return -ENOSPC;
697
698 *head = h;
699
700 return 0;
701}
702EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
703
704int gnttab_empty_grant_references(const grant_ref_t *private_head)
705{
706 return (*private_head == GNTTAB_LIST_END);
707}
708EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
709
710int gnttab_claim_grant_reference(grant_ref_t *private_head)
711{
712 grant_ref_t g = *private_head;
713 if (unlikely(g == GNTTAB_LIST_END))
714 return -ENOSPC;
715 *private_head = gnttab_entry(g);
716 return g;
717}
718EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
719
720void gnttab_release_grant_reference(grant_ref_t *private_head,
721 grant_ref_t release)
722{
723 gnttab_entry(release) = *private_head;
724 *private_head = release;
725}
726EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
727
728void gnttab_request_free_callback(struct gnttab_free_callback *callback,
729 void (*fn)(void *), void *arg, u16 count)
730{
731 unsigned long flags;
732 struct gnttab_free_callback *cb;
733
734 spin_lock_irqsave(&gnttab_list_lock, flags);
735
736 /* Check if the callback is already on the list */
737 cb = gnttab_free_callback_list;
738 while (cb) {
739 if (cb == callback)
740 goto out;
741 cb = cb->next;
742 }
743
744 callback->fn = fn;
745 callback->arg = arg;
746 callback->count = count;
747 callback->next = gnttab_free_callback_list;
748 gnttab_free_callback_list = callback;
749 check_free_callbacks();
750out:
751 spin_unlock_irqrestore(&gnttab_list_lock, flags);
752}
753EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
754
755void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
756{
757 struct gnttab_free_callback **pcb;
758 unsigned long flags;
759
760 spin_lock_irqsave(&gnttab_list_lock, flags);
761 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
762 if (*pcb == callback) {
763 *pcb = callback->next;
764 break;
765 }
766 }
767 spin_unlock_irqrestore(&gnttab_list_lock, flags);
768}
769EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
770
771static int grow_gnttab_list(unsigned int more_frames)
772{
773 unsigned int new_nr_grant_frames, extra_entries, i;
774 unsigned int nr_glist_frames, new_nr_glist_frames;
775
776 BUG_ON(grefs_per_grant_frame == 0);
777
778 new_nr_grant_frames = nr_grant_frames + more_frames;
779 extra_entries = more_frames * grefs_per_grant_frame;
780
781 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
782 new_nr_glist_frames =
783 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
784 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
785 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
786 if (!gnttab_list[i])
787 goto grow_nomem;
788 }
789
790
791 for (i = grefs_per_grant_frame * nr_grant_frames;
792 i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++)
793 gnttab_entry(i) = i + 1;
794
795 gnttab_entry(i) = gnttab_free_head;
796 gnttab_free_head = grefs_per_grant_frame * nr_grant_frames;
797 gnttab_free_count += extra_entries;
798
799 nr_grant_frames = new_nr_grant_frames;
800
801 check_free_callbacks();
802
803 return 0;
804
805grow_nomem:
806 for ( ; i >= nr_glist_frames; i--)
807 free_page((unsigned long) gnttab_list[i]);
808 return -ENOMEM;
809}
810
811static unsigned int __max_nr_grant_frames(void)
812{
813 struct gnttab_query_size query;
814 int rc;
815
816 query.dom = DOMID_SELF;
817
818 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
819 if ((rc < 0) || (query.status != GNTST_okay))
820 return 4; /* Legacy max supported number of frames */
821
822 return query.max_nr_frames;
823}
824
825unsigned int gnttab_max_grant_frames(void)
826{
827 unsigned int xen_max = __max_nr_grant_frames();
828 static unsigned int boot_max_nr_grant_frames;
829
830 /* First time, initialize it properly. */
831 if (!boot_max_nr_grant_frames)
832 boot_max_nr_grant_frames = __max_nr_grant_frames();
833
834 if (xen_max > boot_max_nr_grant_frames)
835 return boot_max_nr_grant_frames;
836 return xen_max;
837}
838EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
839
840int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
841{
842 xen_pfn_t *pfn;
843 unsigned int max_nr_gframes = __max_nr_grant_frames();
844 unsigned int i;
845 void *vaddr;
846
847 if (xen_auto_xlat_grant_frames.count)
848 return -EINVAL;
849
850 vaddr = xen_remap(addr, PAGE_SIZE * max_nr_gframes);
851 if (vaddr == NULL) {
852 pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",
853 &addr);
854 return -ENOMEM;
855 }
856 pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL);
857 if (!pfn) {
858 xen_unmap(vaddr);
859 return -ENOMEM;
860 }
861 for (i = 0; i < max_nr_gframes; i++)
862 pfn[i] = PFN_DOWN(addr) + i;
863
864 xen_auto_xlat_grant_frames.vaddr = vaddr;
865 xen_auto_xlat_grant_frames.pfn = pfn;
866 xen_auto_xlat_grant_frames.count = max_nr_gframes;
867
868 return 0;
869}
870EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames);
871
872void gnttab_free_auto_xlat_frames(void)
873{
874 if (!xen_auto_xlat_grant_frames.count)
875 return;
876 kfree(xen_auto_xlat_grant_frames.pfn);
877 xen_unmap(xen_auto_xlat_grant_frames.vaddr);
878
879 xen_auto_xlat_grant_frames.pfn = NULL;
880 xen_auto_xlat_grant_frames.count = 0;
881 xen_auto_xlat_grant_frames.vaddr = NULL;
882}
883EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
884
885/* Handling of paged out grant targets (GNTST_eagain) */
886#define MAX_DELAY 256
887static inline void
888gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
889 const char *func)
890{
891 unsigned delay = 1;
892
893 do {
894 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
895 if (*status == GNTST_eagain)
896 msleep(delay++);
897 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
898
899 if (delay >= MAX_DELAY) {
900 pr_err("%s: %s eagain grant\n", func, current->comm);
901 *status = GNTST_bad_page;
902 }
903}
904
905void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
906{
907 struct gnttab_map_grant_ref *op;
908
909 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
910 BUG();
911 for (op = batch; op < batch + count; op++)
912 if (op->status == GNTST_eagain)
913 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
914 &op->status, __func__);
915}
916EXPORT_SYMBOL_GPL(gnttab_batch_map);
917
918void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
919{
920 struct gnttab_copy *op;
921
922 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
923 BUG();
924 for (op = batch; op < batch + count; op++)
925 if (op->status == GNTST_eagain)
926 gnttab_retry_eagain_gop(GNTTABOP_copy, op,
927 &op->status, __func__);
928}
929EXPORT_SYMBOL_GPL(gnttab_batch_copy);
930
931int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
932 struct gnttab_map_grant_ref *kmap_ops,
933 struct page **pages, unsigned int count)
934{
935 int i, ret;
936
937 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
938 if (ret)
939 return ret;
940
941 /* Retry eagain maps */
942 for (i = 0; i < count; i++)
943 if (map_ops[i].status == GNTST_eagain)
944 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
945 &map_ops[i].status, __func__);
946
947 return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);
948}
949EXPORT_SYMBOL_GPL(gnttab_map_refs);
950
951int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
952 struct gnttab_map_grant_ref *kmap_ops,
953 struct page **pages, unsigned int count)
954{
955 int ret;
956
957 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
958 if (ret)
959 return ret;
960
961 return clear_foreign_p2m_mapping(unmap_ops, kmap_ops, pages, count);
962}
963EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
964
965static unsigned nr_status_frames(unsigned nr_grant_frames)
966{
967 BUG_ON(grefs_per_grant_frame == 0);
968 return (nr_grant_frames * grefs_per_grant_frame + SPP - 1) / SPP;
969}
970
971static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
972{
973 int rc;
974
975 rc = arch_gnttab_map_shared(frames, nr_gframes,
976 gnttab_max_grant_frames(),
977 &gnttab_shared.addr);
978 BUG_ON(rc);
979
980 return 0;
981}
982
983static void gnttab_unmap_frames_v1(void)
984{
985 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
986}
987
988static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
989{
990 uint64_t *sframes;
991 unsigned int nr_sframes;
992 struct gnttab_get_status_frames getframes;
993 int rc;
994
995 nr_sframes = nr_status_frames(nr_gframes);
996
997 /* No need for kzalloc as it is initialized in following hypercall
998 * GNTTABOP_get_status_frames.
999 */
1000 sframes = kmalloc(nr_sframes * sizeof(uint64_t), GFP_ATOMIC);
1001 if (!sframes)
1002 return -ENOMEM;
1003
1004 getframes.dom = DOMID_SELF;
1005 getframes.nr_frames = nr_sframes;
1006 set_xen_guest_handle(getframes.frame_list, sframes);
1007
1008 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
1009 &getframes, 1);
1010 if (rc == -ENOSYS) {
1011 kfree(sframes);
1012 return -ENOSYS;
1013 }
1014
1015 BUG_ON(rc || getframes.status);
1016
1017 rc = arch_gnttab_map_status(sframes, nr_sframes,
1018 nr_status_frames(gnttab_max_grant_frames()),
1019 &grstatus);
1020 BUG_ON(rc);
1021 kfree(sframes);
1022
1023 rc = arch_gnttab_map_shared(frames, nr_gframes,
1024 gnttab_max_grant_frames(),
1025 &gnttab_shared.addr);
1026 BUG_ON(rc);
1027
1028 return 0;
1029}
1030
1031static void gnttab_unmap_frames_v2(void)
1032{
1033 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
1034 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
1035}
1036
1037static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
1038{
1039 struct gnttab_setup_table setup;
1040 xen_pfn_t *frames;
1041 unsigned int nr_gframes = end_idx + 1;
1042 int rc;
1043
1044 if (xen_feature(XENFEAT_auto_translated_physmap)) {
1045 struct xen_add_to_physmap xatp;
1046 unsigned int i = end_idx;
1047 rc = 0;
1048 BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes);
1049 /*
1050 * Loop backwards, so that the first hypercall has the largest
1051 * index, ensuring that the table will grow only once.
1052 */
1053 do {
1054 xatp.domid = DOMID_SELF;
1055 xatp.idx = i;
1056 xatp.space = XENMAPSPACE_grant_table;
1057 xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i];
1058 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
1059 if (rc != 0) {
1060 pr_warn("grant table add_to_physmap failed, err=%d\n",
1061 rc);
1062 break;
1063 }
1064 } while (i-- > start_idx);
1065
1066 return rc;
1067 }
1068
1069 /* No need for kzalloc as it is initialized in following hypercall
1070 * GNTTABOP_setup_table.
1071 */
1072 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1073 if (!frames)
1074 return -ENOMEM;
1075
1076 setup.dom = DOMID_SELF;
1077 setup.nr_frames = nr_gframes;
1078 set_xen_guest_handle(setup.frame_list, frames);
1079
1080 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1081 if (rc == -ENOSYS) {
1082 kfree(frames);
1083 return -ENOSYS;
1084 }
1085
1086 BUG_ON(rc || setup.status);
1087
1088 rc = gnttab_interface->map_frames(frames, nr_gframes);
1089
1090 kfree(frames);
1091
1092 return rc;
1093}
1094
1095static struct gnttab_ops gnttab_v1_ops = {
1096 .map_frames = gnttab_map_frames_v1,
1097 .unmap_frames = gnttab_unmap_frames_v1,
1098 .update_entry = gnttab_update_entry_v1,
1099 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
1100 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
1101 .query_foreign_access = gnttab_query_foreign_access_v1,
1102};
1103
1104static struct gnttab_ops gnttab_v2_ops = {
1105 .map_frames = gnttab_map_frames_v2,
1106 .unmap_frames = gnttab_unmap_frames_v2,
1107 .update_entry = gnttab_update_entry_v2,
1108 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
1109 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
1110 .query_foreign_access = gnttab_query_foreign_access_v2,
1111 .update_subpage_entry = gnttab_update_subpage_entry_v2,
1112 .update_trans_entry = gnttab_update_trans_entry_v2,
1113};
1114
1115static void gnttab_request_version(void)
1116{
1117 int rc;
1118 struct gnttab_set_version gsv;
1119
1120 gsv.version = 1;
1121
1122 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1123 if (rc == 0 && gsv.version == 2) {
1124 grant_table_version = 2;
1125 grefs_per_grant_frame = PAGE_SIZE / sizeof(union grant_entry_v2);
1126 gnttab_interface = &gnttab_v2_ops;
1127 } else if (grant_table_version == 2) {
1128 /*
1129 * If we've already used version 2 features,
1130 * but then suddenly discover that they're not
1131 * available (e.g. migrating to an older
1132 * version of Xen), almost unbounded badness
1133 * can happen.
1134 */
1135 panic("we need grant tables version 2, but only version 1 is available");
1136 } else {
1137 grant_table_version = 1;
1138 grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1);
1139 gnttab_interface = &gnttab_v1_ops;
1140 }
1141 pr_info("Grant tables using version %d layout\n", grant_table_version);
1142}
1143
1144static int gnttab_setup(void)
1145{
1146 unsigned int max_nr_gframes;
1147
1148 max_nr_gframes = gnttab_max_grant_frames();
1149 if (max_nr_gframes < nr_grant_frames)
1150 return -ENOSYS;
1151
1152 if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {
1153 gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;
1154 if (gnttab_shared.addr == NULL) {
1155 pr_warn("gnttab share frames (addr=0x%08lx) is not mapped!\n",
1156 (unsigned long)xen_auto_xlat_grant_frames.vaddr);
1157 return -ENOMEM;
1158 }
1159 }
1160 return gnttab_map(0, nr_grant_frames - 1);
1161}
1162
1163int gnttab_resume(void)
1164{
1165 gnttab_request_version();
1166 return gnttab_setup();
1167}
1168
1169int gnttab_suspend(void)
1170{
1171 gnttab_interface->unmap_frames();
1172 return 0;
1173}
1174
1175static int gnttab_expand(unsigned int req_entries)
1176{
1177 int rc;
1178 unsigned int cur, extra;
1179
1180 BUG_ON(grefs_per_grant_frame == 0);
1181 cur = nr_grant_frames;
1182 extra = ((req_entries + (grefs_per_grant_frame-1)) /
1183 grefs_per_grant_frame);
1184 if (cur + extra > gnttab_max_grant_frames())
1185 return -ENOSPC;
1186
1187 rc = gnttab_map(cur, cur + extra - 1);
1188 if (rc == 0)
1189 rc = grow_gnttab_list(extra);
1190
1191 return rc;
1192}
1193
1194int gnttab_init(void)
1195{
1196 int i;
1197 unsigned int max_nr_glist_frames, nr_glist_frames;
1198 unsigned int nr_init_grefs;
1199 int ret;
1200
1201 gnttab_request_version();
1202 nr_grant_frames = 1;
1203
1204 /* Determine the maximum number of frames required for the
1205 * grant reference free list on the current hypervisor.
1206 */
1207 BUG_ON(grefs_per_grant_frame == 0);
1208 max_nr_glist_frames = (gnttab_max_grant_frames() *
1209 grefs_per_grant_frame / RPP);
1210
1211 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1212 GFP_KERNEL);
1213 if (gnttab_list == NULL)
1214 return -ENOMEM;
1215
1216 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP;
1217 for (i = 0; i < nr_glist_frames; i++) {
1218 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1219 if (gnttab_list[i] == NULL) {
1220 ret = -ENOMEM;
1221 goto ini_nomem;
1222 }
1223 }
1224
1225 if (gnttab_setup() < 0) {
1226 ret = -ENODEV;
1227 goto ini_nomem;
1228 }
1229
1230 nr_init_grefs = nr_grant_frames * grefs_per_grant_frame;
1231
1232 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1233 gnttab_entry(i) = i + 1;
1234
1235 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1236 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1237 gnttab_free_head = NR_RESERVED_ENTRIES;
1238
1239 printk("Grant table initialized\n");
1240 return 0;
1241
1242 ini_nomem:
1243 for (i--; i >= 0; i--)
1244 free_page((unsigned long)gnttab_list[i]);
1245 kfree(gnttab_list);
1246 return ret;
1247}
1248EXPORT_SYMBOL_GPL(gnttab_init);
1249
1250static int __gnttab_init(void)
1251{
1252 /* Delay grant-table initialization in the PV on HVM case */
1253 if (xen_hvm_domain())
1254 return 0;
1255
1256 if (!xen_pv_domain())
1257 return -ENODEV;
1258
1259 return gnttab_init();
1260}
1261/* Starts after core_initcall so that xen_pvh_gnttab_setup can be called
1262 * beforehand to initialize xen_auto_xlat_grant_frames. */
1263core_initcall_sync(__gnttab_init);