Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright 2013 Red Hat Inc.
4 *
5 * Authors: Jérôme Glisse <jglisse@redhat.com>
6 *
7 * See Documentation/vm/hmm.rst for reasons and overview of what HMM is.
8 */
9#ifndef LINUX_HMM_H
10#define LINUX_HMM_H
11
12#include <linux/kconfig.h>
13#include <linux/pgtable.h>
14
15#include <linux/device.h>
16#include <linux/migrate.h>
17#include <linux/memremap.h>
18#include <linux/completion.h>
19#include <linux/mmu_notifier.h>
20
21/*
22 * On output:
23 * 0 - The page is faultable and a future call with
24 * HMM_PFN_REQ_FAULT could succeed.
25 * HMM_PFN_VALID - the pfn field points to a valid PFN. This PFN is at
26 * least readable. If dev_private_owner is !NULL then this could
27 * point at a DEVICE_PRIVATE page.
28 * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
29 * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
30 * fail. ie poisoned memory, special pages, no vma, etc
31 *
32 * On input:
33 * 0 - Return the current state of the page, do not fault it.
34 * HMM_PFN_REQ_FAULT - The output must have HMM_PFN_VALID or hmm_range_fault()
35 * will fail
36 * HMM_PFN_REQ_WRITE - The output must have HMM_PFN_WRITE or hmm_range_fault()
37 * will fail. Must be combined with HMM_PFN_REQ_FAULT.
38 */
39enum hmm_pfn_flags {
40 /* Output fields and flags */
41 HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
42 HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
43 HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
44 HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
45
46 /* Input flags */
47 HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
48 HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
49
50 HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
51};
52
53/*
54 * hmm_pfn_to_page() - return struct page pointed to by a device entry
55 *
56 * This must be called under the caller 'user_lock' after a successful
57 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
58 * already.
59 */
60static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn)
61{
62 return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);
63}
64
65/*
66 * hmm_pfn_to_map_order() - return the CPU mapping size order
67 *
68 * This is optionally useful to optimize processing of the pfn result
69 * array. It indicates that the page starts at the order aligned VA and is
70 * 1<<order bytes long. Every pfn within an high order page will have the
71 * same pfn flags, both access protections and the map_order. The caller must
72 * be careful with edge cases as the start and end VA of the given page may
73 * extend past the range used with hmm_range_fault().
74 *
75 * This must be called under the caller 'user_lock' after a successful
76 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
77 * already.
78 */
79static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)
80{
81 return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;
82}
83
84/*
85 * struct hmm_range - track invalidation lock on virtual address range
86 *
87 * @notifier: a mmu_interval_notifier that includes the start/end
88 * @notifier_seq: result of mmu_interval_read_begin()
89 * @start: range virtual start address (inclusive)
90 * @end: range virtual end address (exclusive)
91 * @hmm_pfns: array of pfns (big enough for the range)
92 * @default_flags: default flags for the range (write, read, ... see hmm doc)
93 * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter
94 * @dev_private_owner: owner of device private pages
95 */
96struct hmm_range {
97 struct mmu_interval_notifier *notifier;
98 unsigned long notifier_seq;
99 unsigned long start;
100 unsigned long end;
101 unsigned long *hmm_pfns;
102 unsigned long default_flags;
103 unsigned long pfn_flags_mask;
104 void *dev_private_owner;
105};
106
107/*
108 * Please see Documentation/vm/hmm.rst for how to use the range API.
109 */
110int hmm_range_fault(struct hmm_range *range);
111
112/*
113 * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
114 *
115 * When waiting for mmu notifiers we need some kind of time out otherwise we
116 * could potentially wait for ever, 1000ms ie 1s sounds like a long time to
117 * wait already.
118 */
119#define HMM_RANGE_DEFAULT_TIMEOUT 1000
120
121#endif /* LINUX_HMM_H */
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright 2013 Red Hat Inc.
4 *
5 * Authors: Jérôme Glisse <jglisse@redhat.com>
6 *
7 * See Documentation/mm/hmm.rst for reasons and overview of what HMM is.
8 */
9#ifndef LINUX_HMM_H
10#define LINUX_HMM_H
11
12#include <linux/mm.h>
13
14struct mmu_interval_notifier;
15
16/*
17 * On output:
18 * 0 - The page is faultable and a future call with
19 * HMM_PFN_REQ_FAULT could succeed.
20 * HMM_PFN_VALID - the pfn field points to a valid PFN. This PFN is at
21 * least readable. If dev_private_owner is !NULL then this could
22 * point at a DEVICE_PRIVATE page.
23 * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
24 * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
25 * fail. ie poisoned memory, special pages, no vma, etc
26 *
27 * On input:
28 * 0 - Return the current state of the page, do not fault it.
29 * HMM_PFN_REQ_FAULT - The output must have HMM_PFN_VALID or hmm_range_fault()
30 * will fail
31 * HMM_PFN_REQ_WRITE - The output must have HMM_PFN_WRITE or hmm_range_fault()
32 * will fail. Must be combined with HMM_PFN_REQ_FAULT.
33 */
34enum hmm_pfn_flags {
35 /* Output fields and flags */
36 HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
37 HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
38 HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
39 HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
40
41 /* Input flags */
42 HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
43 HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
44
45 HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
46};
47
48/*
49 * hmm_pfn_to_page() - return struct page pointed to by a device entry
50 *
51 * This must be called under the caller 'user_lock' after a successful
52 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
53 * already.
54 */
55static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn)
56{
57 return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);
58}
59
60/*
61 * hmm_pfn_to_map_order() - return the CPU mapping size order
62 *
63 * This is optionally useful to optimize processing of the pfn result
64 * array. It indicates that the page starts at the order aligned VA and is
65 * 1<<order bytes long. Every pfn within an high order page will have the
66 * same pfn flags, both access protections and the map_order. The caller must
67 * be careful with edge cases as the start and end VA of the given page may
68 * extend past the range used with hmm_range_fault().
69 *
70 * This must be called under the caller 'user_lock' after a successful
71 * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
72 * already.
73 */
74static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)
75{
76 return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;
77}
78
79/*
80 * struct hmm_range - track invalidation lock on virtual address range
81 *
82 * @notifier: a mmu_interval_notifier that includes the start/end
83 * @notifier_seq: result of mmu_interval_read_begin()
84 * @start: range virtual start address (inclusive)
85 * @end: range virtual end address (exclusive)
86 * @hmm_pfns: array of pfns (big enough for the range)
87 * @default_flags: default flags for the range (write, read, ... see hmm doc)
88 * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter
89 * @dev_private_owner: owner of device private pages
90 */
91struct hmm_range {
92 struct mmu_interval_notifier *notifier;
93 unsigned long notifier_seq;
94 unsigned long start;
95 unsigned long end;
96 unsigned long *hmm_pfns;
97 unsigned long default_flags;
98 unsigned long pfn_flags_mask;
99 void *dev_private_owner;
100};
101
102/*
103 * Please see Documentation/mm/hmm.rst for how to use the range API.
104 */
105int hmm_range_fault(struct hmm_range *range);
106
107/*
108 * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
109 *
110 * When waiting for mmu notifiers we need some kind of time out otherwise we
111 * could potentially wait for ever, 1000ms ie 1s sounds like a long time to
112 * wait already.
113 */
114#define HMM_RANGE_DEFAULT_TIMEOUT 1000
115
116#endif /* LINUX_HMM_H */