Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2013 Red Hat Inc.
4 *
5 * Authors: Jérôme Glisse <jglisse@redhat.com>
6 */
7/*
8 * Refer to include/linux/hmm.h for information about heterogeneous memory
9 * management or HMM for short.
10 */
11#include <linux/pagewalk.h>
12#include <linux/hmm.h>
13#include <linux/init.h>
14#include <linux/rmap.h>
15#include <linux/swap.h>
16#include <linux/slab.h>
17#include <linux/sched.h>
18#include <linux/mmzone.h>
19#include <linux/pagemap.h>
20#include <linux/swapops.h>
21#include <linux/hugetlb.h>
22#include <linux/memremap.h>
23#include <linux/sched/mm.h>
24#include <linux/jump_label.h>
25#include <linux/dma-mapping.h>
26#include <linux/mmu_notifier.h>
27#include <linux/memory_hotplug.h>
28
29#include "internal.h"
30
31struct hmm_vma_walk {
32 struct hmm_range *range;
33 unsigned long last;
34};
35
36enum {
37 HMM_NEED_FAULT = 1 << 0,
38 HMM_NEED_WRITE_FAULT = 1 << 1,
39 HMM_NEED_ALL_BITS = HMM_NEED_FAULT | HMM_NEED_WRITE_FAULT,
40};
41
42static int hmm_pfns_fill(unsigned long addr, unsigned long end,
43 struct hmm_range *range, unsigned long cpu_flags)
44{
45 unsigned long i = (addr - range->start) >> PAGE_SHIFT;
46
47 for (; addr < end; addr += PAGE_SIZE, i++)
48 range->hmm_pfns[i] = cpu_flags;
49 return 0;
50}
51
52/*
53 * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s)
54 * @addr: range virtual start address (inclusive)
55 * @end: range virtual end address (exclusive)
56 * @required_fault: HMM_NEED_* flags
57 * @walk: mm_walk structure
58 * Return: -EBUSY after page fault, or page fault error
59 *
60 * This function will be called whenever pmd_none() or pte_none() returns true,
61 * or whenever there is no page directory covering the virtual address range.
62 */
63static int hmm_vma_fault(unsigned long addr, unsigned long end,
64 unsigned int required_fault, struct mm_walk *walk)
65{
66 struct hmm_vma_walk *hmm_vma_walk = walk->private;
67 struct vm_area_struct *vma = walk->vma;
68 unsigned int fault_flags = FAULT_FLAG_REMOTE;
69
70 WARN_ON_ONCE(!required_fault);
71 hmm_vma_walk->last = addr;
72
73 if (required_fault & HMM_NEED_WRITE_FAULT) {
74 if (!(vma->vm_flags & VM_WRITE))
75 return -EPERM;
76 fault_flags |= FAULT_FLAG_WRITE;
77 }
78
79 for (; addr < end; addr += PAGE_SIZE)
80 if (handle_mm_fault(vma, addr, fault_flags, NULL) &
81 VM_FAULT_ERROR)
82 return -EFAULT;
83 return -EBUSY;
84}
85
86static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
87 unsigned long pfn_req_flags,
88 unsigned long cpu_flags)
89{
90 struct hmm_range *range = hmm_vma_walk->range;
91
92 /*
93 * So we not only consider the individual per page request we also
94 * consider the default flags requested for the range. The API can
95 * be used 2 ways. The first one where the HMM user coalesces
96 * multiple page faults into one request and sets flags per pfn for
97 * those faults. The second one where the HMM user wants to pre-
98 * fault a range with specific flags. For the latter one it is a
99 * waste to have the user pre-fill the pfn arrays with a default
100 * flags value.
101 */
102 pfn_req_flags &= range->pfn_flags_mask;
103 pfn_req_flags |= range->default_flags;
104
105 /* We aren't ask to do anything ... */
106 if (!(pfn_req_flags & HMM_PFN_REQ_FAULT))
107 return 0;
108
109 /* Need to write fault ? */
110 if ((pfn_req_flags & HMM_PFN_REQ_WRITE) &&
111 !(cpu_flags & HMM_PFN_WRITE))
112 return HMM_NEED_FAULT | HMM_NEED_WRITE_FAULT;
113
114 /* If CPU page table is not valid then we need to fault */
115 if (!(cpu_flags & HMM_PFN_VALID))
116 return HMM_NEED_FAULT;
117 return 0;
118}
119
120static unsigned int
121hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
122 const unsigned long hmm_pfns[], unsigned long npages,
123 unsigned long cpu_flags)
124{
125 struct hmm_range *range = hmm_vma_walk->range;
126 unsigned int required_fault = 0;
127 unsigned long i;
128
129 /*
130 * If the default flags do not request to fault pages, and the mask does
131 * not allow for individual pages to be faulted, then
132 * hmm_pte_need_fault() will always return 0.
133 */
134 if (!((range->default_flags | range->pfn_flags_mask) &
135 HMM_PFN_REQ_FAULT))
136 return 0;
137
138 for (i = 0; i < npages; ++i) {
139 required_fault |= hmm_pte_need_fault(hmm_vma_walk, hmm_pfns[i],
140 cpu_flags);
141 if (required_fault == HMM_NEED_ALL_BITS)
142 return required_fault;
143 }
144 return required_fault;
145}
146
147static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
148 __always_unused int depth, struct mm_walk *walk)
149{
150 struct hmm_vma_walk *hmm_vma_walk = walk->private;
151 struct hmm_range *range = hmm_vma_walk->range;
152 unsigned int required_fault;
153 unsigned long i, npages;
154 unsigned long *hmm_pfns;
155
156 i = (addr - range->start) >> PAGE_SHIFT;
157 npages = (end - addr) >> PAGE_SHIFT;
158 hmm_pfns = &range->hmm_pfns[i];
159 required_fault =
160 hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0);
161 if (!walk->vma) {
162 if (required_fault)
163 return -EFAULT;
164 return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR);
165 }
166 if (required_fault)
167 return hmm_vma_fault(addr, end, required_fault, walk);
168 return hmm_pfns_fill(addr, end, range, 0);
169}
170
171static inline unsigned long hmm_pfn_flags_order(unsigned long order)
172{
173 return order << HMM_PFN_ORDER_SHIFT;
174}
175
176static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,
177 pmd_t pmd)
178{
179 if (pmd_protnone(pmd))
180 return 0;
181 return (pmd_write(pmd) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :
182 HMM_PFN_VALID) |
183 hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT);
184}
185
186#ifdef CONFIG_TRANSPARENT_HUGEPAGE
187static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
188 unsigned long end, unsigned long hmm_pfns[],
189 pmd_t pmd)
190{
191 struct hmm_vma_walk *hmm_vma_walk = walk->private;
192 struct hmm_range *range = hmm_vma_walk->range;
193 unsigned long pfn, npages, i;
194 unsigned int required_fault;
195 unsigned long cpu_flags;
196
197 npages = (end - addr) >> PAGE_SHIFT;
198 cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
199 required_fault =
200 hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags);
201 if (required_fault)
202 return hmm_vma_fault(addr, end, required_fault, walk);
203
204 pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
205 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
206 hmm_pfns[i] = pfn | cpu_flags;
207 return 0;
208}
209#else /* CONFIG_TRANSPARENT_HUGEPAGE */
210/* stub to allow the code below to compile */
211int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
212 unsigned long end, unsigned long hmm_pfns[], pmd_t pmd);
213#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
214
215static inline unsigned long pte_to_hmm_pfn_flags(struct hmm_range *range,
216 pte_t pte)
217{
218 if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte))
219 return 0;
220 return pte_write(pte) ? (HMM_PFN_VALID | HMM_PFN_WRITE) : HMM_PFN_VALID;
221}
222
223static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
224 unsigned long end, pmd_t *pmdp, pte_t *ptep,
225 unsigned long *hmm_pfn)
226{
227 struct hmm_vma_walk *hmm_vma_walk = walk->private;
228 struct hmm_range *range = hmm_vma_walk->range;
229 unsigned int required_fault;
230 unsigned long cpu_flags;
231 pte_t pte = ptep_get(ptep);
232 uint64_t pfn_req_flags = *hmm_pfn;
233
234 if (pte_none_mostly(pte)) {
235 required_fault =
236 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
237 if (required_fault)
238 goto fault;
239 *hmm_pfn = 0;
240 return 0;
241 }
242
243 if (!pte_present(pte)) {
244 swp_entry_t entry = pte_to_swp_entry(pte);
245
246 /*
247 * Don't fault in device private pages owned by the caller,
248 * just report the PFN.
249 */
250 if (is_device_private_entry(entry) &&
251 pfn_swap_entry_to_page(entry)->pgmap->owner ==
252 range->dev_private_owner) {
253 cpu_flags = HMM_PFN_VALID;
254 if (is_writable_device_private_entry(entry))
255 cpu_flags |= HMM_PFN_WRITE;
256 *hmm_pfn = swp_offset_pfn(entry) | cpu_flags;
257 return 0;
258 }
259
260 required_fault =
261 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
262 if (!required_fault) {
263 *hmm_pfn = 0;
264 return 0;
265 }
266
267 if (!non_swap_entry(entry))
268 goto fault;
269
270 if (is_device_private_entry(entry))
271 goto fault;
272
273 if (is_device_exclusive_entry(entry))
274 goto fault;
275
276 if (is_migration_entry(entry)) {
277 pte_unmap(ptep);
278 hmm_vma_walk->last = addr;
279 migration_entry_wait(walk->mm, pmdp, addr);
280 return -EBUSY;
281 }
282
283 /* Report error for everything else */
284 pte_unmap(ptep);
285 return -EFAULT;
286 }
287
288 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
289 required_fault =
290 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);
291 if (required_fault)
292 goto fault;
293
294 /*
295 * Bypass devmap pte such as DAX page when all pfn requested
296 * flags(pfn_req_flags) are fulfilled.
297 * Since each architecture defines a struct page for the zero page, just
298 * fall through and treat it like a normal page.
299 */
300 if (!vm_normal_page(walk->vma, addr, pte) &&
301 !pte_devmap(pte) &&
302 !is_zero_pfn(pte_pfn(pte))) {
303 if (hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0)) {
304 pte_unmap(ptep);
305 return -EFAULT;
306 }
307 *hmm_pfn = HMM_PFN_ERROR;
308 return 0;
309 }
310
311 *hmm_pfn = pte_pfn(pte) | cpu_flags;
312 return 0;
313
314fault:
315 pte_unmap(ptep);
316 /* Fault any virtual address we were asked to fault */
317 return hmm_vma_fault(addr, end, required_fault, walk);
318}
319
320static int hmm_vma_walk_pmd(pmd_t *pmdp,
321 unsigned long start,
322 unsigned long end,
323 struct mm_walk *walk)
324{
325 struct hmm_vma_walk *hmm_vma_walk = walk->private;
326 struct hmm_range *range = hmm_vma_walk->range;
327 unsigned long *hmm_pfns =
328 &range->hmm_pfns[(start - range->start) >> PAGE_SHIFT];
329 unsigned long npages = (end - start) >> PAGE_SHIFT;
330 unsigned long addr = start;
331 pte_t *ptep;
332 pmd_t pmd;
333
334again:
335 pmd = pmdp_get_lockless(pmdp);
336 if (pmd_none(pmd))
337 return hmm_vma_walk_hole(start, end, -1, walk);
338
339 if (thp_migration_supported() && is_pmd_migration_entry(pmd)) {
340 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0)) {
341 hmm_vma_walk->last = addr;
342 pmd_migration_entry_wait(walk->mm, pmdp);
343 return -EBUSY;
344 }
345 return hmm_pfns_fill(start, end, range, 0);
346 }
347
348 if (!pmd_present(pmd)) {
349 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0))
350 return -EFAULT;
351 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
352 }
353
354 if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) {
355 /*
356 * No need to take pmd_lock here, even if some other thread
357 * is splitting the huge pmd we will get that event through
358 * mmu_notifier callback.
359 *
360 * So just read pmd value and check again it's a transparent
361 * huge or device mapping one and compute corresponding pfn
362 * values.
363 */
364 pmd = pmdp_get_lockless(pmdp);
365 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
366 goto again;
367
368 return hmm_vma_handle_pmd(walk, addr, end, hmm_pfns, pmd);
369 }
370
371 /*
372 * We have handled all the valid cases above ie either none, migration,
373 * huge or transparent huge. At this point either it is a valid pmd
374 * entry pointing to pte directory or it is a bad pmd that will not
375 * recover.
376 */
377 if (pmd_bad(pmd)) {
378 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0))
379 return -EFAULT;
380 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
381 }
382
383 ptep = pte_offset_map(pmdp, addr);
384 if (!ptep)
385 goto again;
386 for (; addr < end; addr += PAGE_SIZE, ptep++, hmm_pfns++) {
387 int r;
388
389 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, hmm_pfns);
390 if (r) {
391 /* hmm_vma_handle_pte() did pte_unmap() */
392 return r;
393 }
394 }
395 pte_unmap(ptep - 1);
396 return 0;
397}
398
399#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \
400 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
401static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range *range,
402 pud_t pud)
403{
404 if (!pud_present(pud))
405 return 0;
406 return (pud_write(pud) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :
407 HMM_PFN_VALID) |
408 hmm_pfn_flags_order(PUD_SHIFT - PAGE_SHIFT);
409}
410
411static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
412 struct mm_walk *walk)
413{
414 struct hmm_vma_walk *hmm_vma_walk = walk->private;
415 struct hmm_range *range = hmm_vma_walk->range;
416 unsigned long addr = start;
417 pud_t pud;
418 spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma);
419
420 if (!ptl)
421 return 0;
422
423 /* Normally we don't want to split the huge page */
424 walk->action = ACTION_CONTINUE;
425
426 pud = READ_ONCE(*pudp);
427 if (pud_none(pud)) {
428 spin_unlock(ptl);
429 return hmm_vma_walk_hole(start, end, -1, walk);
430 }
431
432 if (pud_huge(pud) && pud_devmap(pud)) {
433 unsigned long i, npages, pfn;
434 unsigned int required_fault;
435 unsigned long *hmm_pfns;
436 unsigned long cpu_flags;
437
438 if (!pud_present(pud)) {
439 spin_unlock(ptl);
440 return hmm_vma_walk_hole(start, end, -1, walk);
441 }
442
443 i = (addr - range->start) >> PAGE_SHIFT;
444 npages = (end - addr) >> PAGE_SHIFT;
445 hmm_pfns = &range->hmm_pfns[i];
446
447 cpu_flags = pud_to_hmm_pfn_flags(range, pud);
448 required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns,
449 npages, cpu_flags);
450 if (required_fault) {
451 spin_unlock(ptl);
452 return hmm_vma_fault(addr, end, required_fault, walk);
453 }
454
455 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
456 for (i = 0; i < npages; ++i, ++pfn)
457 hmm_pfns[i] = pfn | cpu_flags;
458 goto out_unlock;
459 }
460
461 /* Ask for the PUD to be split */
462 walk->action = ACTION_SUBTREE;
463
464out_unlock:
465 spin_unlock(ptl);
466 return 0;
467}
468#else
469#define hmm_vma_walk_pud NULL
470#endif
471
472#ifdef CONFIG_HUGETLB_PAGE
473static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
474 unsigned long start, unsigned long end,
475 struct mm_walk *walk)
476{
477 unsigned long addr = start, i, pfn;
478 struct hmm_vma_walk *hmm_vma_walk = walk->private;
479 struct hmm_range *range = hmm_vma_walk->range;
480 struct vm_area_struct *vma = walk->vma;
481 unsigned int required_fault;
482 unsigned long pfn_req_flags;
483 unsigned long cpu_flags;
484 spinlock_t *ptl;
485 pte_t entry;
486
487 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);
488 entry = huge_ptep_get(pte);
489
490 i = (start - range->start) >> PAGE_SHIFT;
491 pfn_req_flags = range->hmm_pfns[i];
492 cpu_flags = pte_to_hmm_pfn_flags(range, entry) |
493 hmm_pfn_flags_order(huge_page_order(hstate_vma(vma)));
494 required_fault =
495 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);
496 if (required_fault) {
497 int ret;
498
499 spin_unlock(ptl);
500 hugetlb_vma_unlock_read(vma);
501 /*
502 * Avoid deadlock: drop the vma lock before calling
503 * hmm_vma_fault(), which will itself potentially take and
504 * drop the vma lock. This is also correct from a
505 * protection point of view, because there is no further
506 * use here of either pte or ptl after dropping the vma
507 * lock.
508 */
509 ret = hmm_vma_fault(addr, end, required_fault, walk);
510 hugetlb_vma_lock_read(vma);
511 return ret;
512 }
513
514 pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
515 for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
516 range->hmm_pfns[i] = pfn | cpu_flags;
517
518 spin_unlock(ptl);
519 return 0;
520}
521#else
522#define hmm_vma_walk_hugetlb_entry NULL
523#endif /* CONFIG_HUGETLB_PAGE */
524
525static int hmm_vma_walk_test(unsigned long start, unsigned long end,
526 struct mm_walk *walk)
527{
528 struct hmm_vma_walk *hmm_vma_walk = walk->private;
529 struct hmm_range *range = hmm_vma_walk->range;
530 struct vm_area_struct *vma = walk->vma;
531
532 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)) &&
533 vma->vm_flags & VM_READ)
534 return 0;
535
536 /*
537 * vma ranges that don't have struct page backing them or map I/O
538 * devices directly cannot be handled by hmm_range_fault().
539 *
540 * If the vma does not allow read access, then assume that it does not
541 * allow write access either. HMM does not support architectures that
542 * allow write without read.
543 *
544 * If a fault is requested for an unsupported range then it is a hard
545 * failure.
546 */
547 if (hmm_range_need_fault(hmm_vma_walk,
548 range->hmm_pfns +
549 ((start - range->start) >> PAGE_SHIFT),
550 (end - start) >> PAGE_SHIFT, 0))
551 return -EFAULT;
552
553 hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
554
555 /* Skip this vma and continue processing the next vma. */
556 return 1;
557}
558
559static const struct mm_walk_ops hmm_walk_ops = {
560 .pud_entry = hmm_vma_walk_pud,
561 .pmd_entry = hmm_vma_walk_pmd,
562 .pte_hole = hmm_vma_walk_hole,
563 .hugetlb_entry = hmm_vma_walk_hugetlb_entry,
564 .test_walk = hmm_vma_walk_test,
565 .walk_lock = PGWALK_RDLOCK,
566};
567
568/**
569 * hmm_range_fault - try to fault some address in a virtual address range
570 * @range: argument structure
571 *
572 * Returns 0 on success or one of the following error codes:
573 *
574 * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma
575 * (e.g., device file vma).
576 * -ENOMEM: Out of memory.
577 * -EPERM: Invalid permission (e.g., asking for write and range is read
578 * only).
579 * -EBUSY: The range has been invalidated and the caller needs to wait for
580 * the invalidation to finish.
581 * -EFAULT: A page was requested to be valid and could not be made valid
582 * ie it has no backing VMA or it is illegal to access
583 *
584 * This is similar to get_user_pages(), except that it can read the page tables
585 * without mutating them (ie causing faults).
586 */
587int hmm_range_fault(struct hmm_range *range)
588{
589 struct hmm_vma_walk hmm_vma_walk = {
590 .range = range,
591 .last = range->start,
592 };
593 struct mm_struct *mm = range->notifier->mm;
594 int ret;
595
596 mmap_assert_locked(mm);
597
598 do {
599 /* If range is no longer valid force retry. */
600 if (mmu_interval_check_retry(range->notifier,
601 range->notifier_seq))
602 return -EBUSY;
603 ret = walk_page_range(mm, hmm_vma_walk.last, range->end,
604 &hmm_walk_ops, &hmm_vma_walk);
605 /*
606 * When -EBUSY is returned the loop restarts with
607 * hmm_vma_walk.last set to an address that has not been stored
608 * in pfns. All entries < last in the pfn array are set to their
609 * output, and all >= are still at their input values.
610 */
611 } while (ret == -EBUSY);
612 return ret;
613}
614EXPORT_SYMBOL(hmm_range_fault);
1/*
2 * Copyright 2013 Red Hat Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Authors: Jérôme Glisse <jglisse@redhat.com>
15 */
16/*
17 * Refer to include/linux/hmm.h for information about heterogeneous memory
18 * management or HMM for short.
19 */
20#include <linux/mm.h>
21#include <linux/hmm.h>
22#include <linux/init.h>
23#include <linux/rmap.h>
24#include <linux/swap.h>
25#include <linux/slab.h>
26#include <linux/sched.h>
27#include <linux/mmzone.h>
28#include <linux/pagemap.h>
29#include <linux/swapops.h>
30#include <linux/hugetlb.h>
31#include <linux/memremap.h>
32#include <linux/jump_label.h>
33#include <linux/mmu_notifier.h>
34#include <linux/memory_hotplug.h>
35
36#define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
37
38#if defined(CONFIG_DEVICE_PRIVATE) || defined(CONFIG_DEVICE_PUBLIC)
39/*
40 * Device private memory see HMM (Documentation/vm/hmm.txt) or hmm.h
41 */
42DEFINE_STATIC_KEY_FALSE(device_private_key);
43EXPORT_SYMBOL(device_private_key);
44#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */
45
46
47#if IS_ENABLED(CONFIG_HMM_MIRROR)
48static const struct mmu_notifier_ops hmm_mmu_notifier_ops;
49
50/*
51 * struct hmm - HMM per mm struct
52 *
53 * @mm: mm struct this HMM struct is bound to
54 * @lock: lock protecting ranges list
55 * @sequence: we track updates to the CPU page table with a sequence number
56 * @ranges: list of range being snapshotted
57 * @mirrors: list of mirrors for this mm
58 * @mmu_notifier: mmu notifier to track updates to CPU page table
59 * @mirrors_sem: read/write semaphore protecting the mirrors list
60 */
61struct hmm {
62 struct mm_struct *mm;
63 spinlock_t lock;
64 atomic_t sequence;
65 struct list_head ranges;
66 struct list_head mirrors;
67 struct mmu_notifier mmu_notifier;
68 struct rw_semaphore mirrors_sem;
69};
70
71/*
72 * hmm_register - register HMM against an mm (HMM internal)
73 *
74 * @mm: mm struct to attach to
75 *
76 * This is not intended to be used directly by device drivers. It allocates an
77 * HMM struct if mm does not have one, and initializes it.
78 */
79static struct hmm *hmm_register(struct mm_struct *mm)
80{
81 struct hmm *hmm = READ_ONCE(mm->hmm);
82 bool cleanup = false;
83
84 /*
85 * The hmm struct can only be freed once the mm_struct goes away,
86 * hence we should always have pre-allocated an new hmm struct
87 * above.
88 */
89 if (hmm)
90 return hmm;
91
92 hmm = kmalloc(sizeof(*hmm), GFP_KERNEL);
93 if (!hmm)
94 return NULL;
95 INIT_LIST_HEAD(&hmm->mirrors);
96 init_rwsem(&hmm->mirrors_sem);
97 atomic_set(&hmm->sequence, 0);
98 hmm->mmu_notifier.ops = NULL;
99 INIT_LIST_HEAD(&hmm->ranges);
100 spin_lock_init(&hmm->lock);
101 hmm->mm = mm;
102
103 /*
104 * We should only get here if hold the mmap_sem in write mode ie on
105 * registration of first mirror through hmm_mirror_register()
106 */
107 hmm->mmu_notifier.ops = &hmm_mmu_notifier_ops;
108 if (__mmu_notifier_register(&hmm->mmu_notifier, mm)) {
109 kfree(hmm);
110 return NULL;
111 }
112
113 spin_lock(&mm->page_table_lock);
114 if (!mm->hmm)
115 mm->hmm = hmm;
116 else
117 cleanup = true;
118 spin_unlock(&mm->page_table_lock);
119
120 if (cleanup) {
121 mmu_notifier_unregister(&hmm->mmu_notifier, mm);
122 kfree(hmm);
123 }
124
125 return mm->hmm;
126}
127
128void hmm_mm_destroy(struct mm_struct *mm)
129{
130 kfree(mm->hmm);
131}
132
133static void hmm_invalidate_range(struct hmm *hmm,
134 enum hmm_update_type action,
135 unsigned long start,
136 unsigned long end)
137{
138 struct hmm_mirror *mirror;
139 struct hmm_range *range;
140
141 spin_lock(&hmm->lock);
142 list_for_each_entry(range, &hmm->ranges, list) {
143 unsigned long addr, idx, npages;
144
145 if (end < range->start || start >= range->end)
146 continue;
147
148 range->valid = false;
149 addr = max(start, range->start);
150 idx = (addr - range->start) >> PAGE_SHIFT;
151 npages = (min(range->end, end) - addr) >> PAGE_SHIFT;
152 memset(&range->pfns[idx], 0, sizeof(*range->pfns) * npages);
153 }
154 spin_unlock(&hmm->lock);
155
156 down_read(&hmm->mirrors_sem);
157 list_for_each_entry(mirror, &hmm->mirrors, list)
158 mirror->ops->sync_cpu_device_pagetables(mirror, action,
159 start, end);
160 up_read(&hmm->mirrors_sem);
161}
162
163static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm)
164{
165 struct hmm_mirror *mirror;
166 struct hmm *hmm = mm->hmm;
167
168 down_write(&hmm->mirrors_sem);
169 mirror = list_first_entry_or_null(&hmm->mirrors, struct hmm_mirror,
170 list);
171 while (mirror) {
172 list_del_init(&mirror->list);
173 if (mirror->ops->release) {
174 /*
175 * Drop mirrors_sem so callback can wait on any pending
176 * work that might itself trigger mmu_notifier callback
177 * and thus would deadlock with us.
178 */
179 up_write(&hmm->mirrors_sem);
180 mirror->ops->release(mirror);
181 down_write(&hmm->mirrors_sem);
182 }
183 mirror = list_first_entry_or_null(&hmm->mirrors,
184 struct hmm_mirror, list);
185 }
186 up_write(&hmm->mirrors_sem);
187}
188
189static void hmm_invalidate_range_start(struct mmu_notifier *mn,
190 struct mm_struct *mm,
191 unsigned long start,
192 unsigned long end)
193{
194 struct hmm *hmm = mm->hmm;
195
196 VM_BUG_ON(!hmm);
197
198 atomic_inc(&hmm->sequence);
199}
200
201static void hmm_invalidate_range_end(struct mmu_notifier *mn,
202 struct mm_struct *mm,
203 unsigned long start,
204 unsigned long end)
205{
206 struct hmm *hmm = mm->hmm;
207
208 VM_BUG_ON(!hmm);
209
210 hmm_invalidate_range(mm->hmm, HMM_UPDATE_INVALIDATE, start, end);
211}
212
213static const struct mmu_notifier_ops hmm_mmu_notifier_ops = {
214 .release = hmm_release,
215 .invalidate_range_start = hmm_invalidate_range_start,
216 .invalidate_range_end = hmm_invalidate_range_end,
217};
218
219/*
220 * hmm_mirror_register() - register a mirror against an mm
221 *
222 * @mirror: new mirror struct to register
223 * @mm: mm to register against
224 *
225 * To start mirroring a process address space, the device driver must register
226 * an HMM mirror struct.
227 *
228 * THE mm->mmap_sem MUST BE HELD IN WRITE MODE !
229 */
230int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm)
231{
232 /* Sanity check */
233 if (!mm || !mirror || !mirror->ops)
234 return -EINVAL;
235
236again:
237 mirror->hmm = hmm_register(mm);
238 if (!mirror->hmm)
239 return -ENOMEM;
240
241 down_write(&mirror->hmm->mirrors_sem);
242 if (mirror->hmm->mm == NULL) {
243 /*
244 * A racing hmm_mirror_unregister() is about to destroy the hmm
245 * struct. Try again to allocate a new one.
246 */
247 up_write(&mirror->hmm->mirrors_sem);
248 mirror->hmm = NULL;
249 goto again;
250 } else {
251 list_add(&mirror->list, &mirror->hmm->mirrors);
252 up_write(&mirror->hmm->mirrors_sem);
253 }
254
255 return 0;
256}
257EXPORT_SYMBOL(hmm_mirror_register);
258
259/*
260 * hmm_mirror_unregister() - unregister a mirror
261 *
262 * @mirror: new mirror struct to register
263 *
264 * Stop mirroring a process address space, and cleanup.
265 */
266void hmm_mirror_unregister(struct hmm_mirror *mirror)
267{
268 bool should_unregister = false;
269 struct mm_struct *mm;
270 struct hmm *hmm;
271
272 if (mirror->hmm == NULL)
273 return;
274
275 hmm = mirror->hmm;
276 down_write(&hmm->mirrors_sem);
277 list_del_init(&mirror->list);
278 should_unregister = list_empty(&hmm->mirrors);
279 mirror->hmm = NULL;
280 mm = hmm->mm;
281 hmm->mm = NULL;
282 up_write(&hmm->mirrors_sem);
283
284 if (!should_unregister || mm == NULL)
285 return;
286
287 spin_lock(&mm->page_table_lock);
288 if (mm->hmm == hmm)
289 mm->hmm = NULL;
290 spin_unlock(&mm->page_table_lock);
291
292 mmu_notifier_unregister_no_release(&hmm->mmu_notifier, mm);
293 kfree(hmm);
294}
295EXPORT_SYMBOL(hmm_mirror_unregister);
296
297struct hmm_vma_walk {
298 struct hmm_range *range;
299 unsigned long last;
300 bool fault;
301 bool block;
302};
303
304static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,
305 bool write_fault, uint64_t *pfn)
306{
307 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_REMOTE;
308 struct hmm_vma_walk *hmm_vma_walk = walk->private;
309 struct hmm_range *range = hmm_vma_walk->range;
310 struct vm_area_struct *vma = walk->vma;
311 int r;
312
313 flags |= hmm_vma_walk->block ? 0 : FAULT_FLAG_ALLOW_RETRY;
314 flags |= write_fault ? FAULT_FLAG_WRITE : 0;
315 r = handle_mm_fault(vma, addr, flags);
316 if (r & VM_FAULT_RETRY)
317 return -EBUSY;
318 if (r & VM_FAULT_ERROR) {
319 *pfn = range->values[HMM_PFN_ERROR];
320 return -EFAULT;
321 }
322
323 return -EAGAIN;
324}
325
326static int hmm_pfns_bad(unsigned long addr,
327 unsigned long end,
328 struct mm_walk *walk)
329{
330 struct hmm_vma_walk *hmm_vma_walk = walk->private;
331 struct hmm_range *range = hmm_vma_walk->range;
332 uint64_t *pfns = range->pfns;
333 unsigned long i;
334
335 i = (addr - range->start) >> PAGE_SHIFT;
336 for (; addr < end; addr += PAGE_SIZE, i++)
337 pfns[i] = range->values[HMM_PFN_ERROR];
338
339 return 0;
340}
341
342/*
343 * hmm_vma_walk_hole() - handle a range lacking valid pmd or pte(s)
344 * @start: range virtual start address (inclusive)
345 * @end: range virtual end address (exclusive)
346 * @fault: should we fault or not ?
347 * @write_fault: write fault ?
348 * @walk: mm_walk structure
349 * Returns: 0 on success, -EAGAIN after page fault, or page fault error
350 *
351 * This function will be called whenever pmd_none() or pte_none() returns true,
352 * or whenever there is no page directory covering the virtual address range.
353 */
354static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end,
355 bool fault, bool write_fault,
356 struct mm_walk *walk)
357{
358 struct hmm_vma_walk *hmm_vma_walk = walk->private;
359 struct hmm_range *range = hmm_vma_walk->range;
360 uint64_t *pfns = range->pfns;
361 unsigned long i;
362
363 hmm_vma_walk->last = addr;
364 i = (addr - range->start) >> PAGE_SHIFT;
365 for (; addr < end; addr += PAGE_SIZE, i++) {
366 pfns[i] = range->values[HMM_PFN_NONE];
367 if (fault || write_fault) {
368 int ret;
369
370 ret = hmm_vma_do_fault(walk, addr, write_fault,
371 &pfns[i]);
372 if (ret != -EAGAIN)
373 return ret;
374 }
375 }
376
377 return (fault || write_fault) ? -EAGAIN : 0;
378}
379
380static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
381 uint64_t pfns, uint64_t cpu_flags,
382 bool *fault, bool *write_fault)
383{
384 struct hmm_range *range = hmm_vma_walk->range;
385
386 *fault = *write_fault = false;
387 if (!hmm_vma_walk->fault)
388 return;
389
390 /* We aren't ask to do anything ... */
391 if (!(pfns & range->flags[HMM_PFN_VALID]))
392 return;
393 /* If this is device memory than only fault if explicitly requested */
394 if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
395 /* Do we fault on device memory ? */
396 if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) {
397 *write_fault = pfns & range->flags[HMM_PFN_WRITE];
398 *fault = true;
399 }
400 return;
401 }
402
403 /* If CPU page table is not valid then we need to fault */
404 *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);
405 /* Need to write fault ? */
406 if ((pfns & range->flags[HMM_PFN_WRITE]) &&
407 !(cpu_flags & range->flags[HMM_PFN_WRITE])) {
408 *write_fault = true;
409 *fault = true;
410 }
411}
412
413static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
414 const uint64_t *pfns, unsigned long npages,
415 uint64_t cpu_flags, bool *fault,
416 bool *write_fault)
417{
418 unsigned long i;
419
420 if (!hmm_vma_walk->fault) {
421 *fault = *write_fault = false;
422 return;
423 }
424
425 for (i = 0; i < npages; ++i) {
426 hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags,
427 fault, write_fault);
428 if ((*fault) || (*write_fault))
429 return;
430 }
431}
432
433static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
434 struct mm_walk *walk)
435{
436 struct hmm_vma_walk *hmm_vma_walk = walk->private;
437 struct hmm_range *range = hmm_vma_walk->range;
438 bool fault, write_fault;
439 unsigned long i, npages;
440 uint64_t *pfns;
441
442 i = (addr - range->start) >> PAGE_SHIFT;
443 npages = (end - addr) >> PAGE_SHIFT;
444 pfns = &range->pfns[i];
445 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
446 0, &fault, &write_fault);
447 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
448}
449
450static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)
451{
452 if (pmd_protnone(pmd))
453 return 0;
454 return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |
455 range->flags[HMM_PFN_WRITE] :
456 range->flags[HMM_PFN_VALID];
457}
458
459static int hmm_vma_handle_pmd(struct mm_walk *walk,
460 unsigned long addr,
461 unsigned long end,
462 uint64_t *pfns,
463 pmd_t pmd)
464{
465 struct hmm_vma_walk *hmm_vma_walk = walk->private;
466 struct hmm_range *range = hmm_vma_walk->range;
467 unsigned long pfn, npages, i;
468 bool fault, write_fault;
469 uint64_t cpu_flags;
470
471 npages = (end - addr) >> PAGE_SHIFT;
472 cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
473 hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags,
474 &fault, &write_fault);
475
476 if (pmd_protnone(pmd) || fault || write_fault)
477 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
478
479 pfn = pmd_pfn(pmd) + pte_index(addr);
480 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
481 pfns[i] = hmm_pfn_from_pfn(range, pfn) | cpu_flags;
482 hmm_vma_walk->last = end;
483 return 0;
484}
485
486static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)
487{
488 if (pte_none(pte) || !pte_present(pte))
489 return 0;
490 return pte_write(pte) ? range->flags[HMM_PFN_VALID] |
491 range->flags[HMM_PFN_WRITE] :
492 range->flags[HMM_PFN_VALID];
493}
494
495static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
496 unsigned long end, pmd_t *pmdp, pte_t *ptep,
497 uint64_t *pfn)
498{
499 struct hmm_vma_walk *hmm_vma_walk = walk->private;
500 struct hmm_range *range = hmm_vma_walk->range;
501 struct vm_area_struct *vma = walk->vma;
502 bool fault, write_fault;
503 uint64_t cpu_flags;
504 pte_t pte = *ptep;
505 uint64_t orig_pfn = *pfn;
506
507 *pfn = range->values[HMM_PFN_NONE];
508 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
509 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
510 &fault, &write_fault);
511
512 if (pte_none(pte)) {
513 if (fault || write_fault)
514 goto fault;
515 return 0;
516 }
517
518 if (!pte_present(pte)) {
519 swp_entry_t entry = pte_to_swp_entry(pte);
520
521 if (!non_swap_entry(entry)) {
522 if (fault || write_fault)
523 goto fault;
524 return 0;
525 }
526
527 /*
528 * This is a special swap entry, ignore migration, use
529 * device and report anything else as error.
530 */
531 if (is_device_private_entry(entry)) {
532 cpu_flags = range->flags[HMM_PFN_VALID] |
533 range->flags[HMM_PFN_DEVICE_PRIVATE];
534 cpu_flags |= is_write_device_private_entry(entry) ?
535 range->flags[HMM_PFN_WRITE] : 0;
536 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
537 &fault, &write_fault);
538 if (fault || write_fault)
539 goto fault;
540 *pfn = hmm_pfn_from_pfn(range, swp_offset(entry));
541 *pfn |= cpu_flags;
542 return 0;
543 }
544
545 if (is_migration_entry(entry)) {
546 if (fault || write_fault) {
547 pte_unmap(ptep);
548 hmm_vma_walk->last = addr;
549 migration_entry_wait(vma->vm_mm,
550 pmdp, addr);
551 return -EAGAIN;
552 }
553 return 0;
554 }
555
556 /* Report error for everything else */
557 *pfn = range->values[HMM_PFN_ERROR];
558 return -EFAULT;
559 }
560
561 if (fault || write_fault)
562 goto fault;
563
564 *pfn = hmm_pfn_from_pfn(range, pte_pfn(pte)) | cpu_flags;
565 return 0;
566
567fault:
568 pte_unmap(ptep);
569 /* Fault any virtual address we were asked to fault */
570 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
571}
572
573static int hmm_vma_walk_pmd(pmd_t *pmdp,
574 unsigned long start,
575 unsigned long end,
576 struct mm_walk *walk)
577{
578 struct hmm_vma_walk *hmm_vma_walk = walk->private;
579 struct hmm_range *range = hmm_vma_walk->range;
580 uint64_t *pfns = range->pfns;
581 unsigned long addr = start, i;
582 pte_t *ptep;
583
584 i = (addr - range->start) >> PAGE_SHIFT;
585
586again:
587 if (pmd_none(*pmdp))
588 return hmm_vma_walk_hole(start, end, walk);
589
590 if (pmd_huge(*pmdp) && (range->vma->vm_flags & VM_HUGETLB))
591 return hmm_pfns_bad(start, end, walk);
592
593 if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
594 pmd_t pmd;
595
596 /*
597 * No need to take pmd_lock here, even if some other threads
598 * is splitting the huge pmd we will get that event through
599 * mmu_notifier callback.
600 *
601 * So just read pmd value and check again its a transparent
602 * huge or device mapping one and compute corresponding pfn
603 * values.
604 */
605 pmd = pmd_read_atomic(pmdp);
606 barrier();
607 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
608 goto again;
609
610 return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
611 }
612
613 if (pmd_bad(*pmdp))
614 return hmm_pfns_bad(start, end, walk);
615
616 ptep = pte_offset_map(pmdp, addr);
617 for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
618 int r;
619
620 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
621 if (r) {
622 /* hmm_vma_handle_pte() did unmap pte directory */
623 hmm_vma_walk->last = addr;
624 return r;
625 }
626 }
627 pte_unmap(ptep - 1);
628
629 hmm_vma_walk->last = addr;
630 return 0;
631}
632
633static void hmm_pfns_clear(struct hmm_range *range,
634 uint64_t *pfns,
635 unsigned long addr,
636 unsigned long end)
637{
638 for (; addr < end; addr += PAGE_SIZE, pfns++)
639 *pfns = range->values[HMM_PFN_NONE];
640}
641
642static void hmm_pfns_special(struct hmm_range *range)
643{
644 unsigned long addr = range->start, i = 0;
645
646 for (; addr < range->end; addr += PAGE_SIZE, i++)
647 range->pfns[i] = range->values[HMM_PFN_SPECIAL];
648}
649
650/*
651 * hmm_vma_get_pfns() - snapshot CPU page table for a range of virtual addresses
652 * @range: range being snapshotted
653 * Returns: -EINVAL if invalid argument, -ENOMEM out of memory, -EPERM invalid
654 * vma permission, 0 success
655 *
656 * This snapshots the CPU page table for a range of virtual addresses. Snapshot
657 * validity is tracked by range struct. See hmm_vma_range_done() for further
658 * information.
659 *
660 * The range struct is initialized here. It tracks the CPU page table, but only
661 * if the function returns success (0), in which case the caller must then call
662 * hmm_vma_range_done() to stop CPU page table update tracking on this range.
663 *
664 * NOT CALLING hmm_vma_range_done() IF FUNCTION RETURNS 0 WILL LEAD TO SERIOUS
665 * MEMORY CORRUPTION ! YOU HAVE BEEN WARNED !
666 */
667int hmm_vma_get_pfns(struct hmm_range *range)
668{
669 struct vm_area_struct *vma = range->vma;
670 struct hmm_vma_walk hmm_vma_walk;
671 struct mm_walk mm_walk;
672 struct hmm *hmm;
673
674 /* Sanity check, this really should not happen ! */
675 if (range->start < vma->vm_start || range->start >= vma->vm_end)
676 return -EINVAL;
677 if (range->end < vma->vm_start || range->end > vma->vm_end)
678 return -EINVAL;
679
680 hmm = hmm_register(vma->vm_mm);
681 if (!hmm)
682 return -ENOMEM;
683 /* Caller must have registered a mirror, via hmm_mirror_register() ! */
684 if (!hmm->mmu_notifier.ops)
685 return -EINVAL;
686
687 /* FIXME support hugetlb fs */
688 if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) {
689 hmm_pfns_special(range);
690 return -EINVAL;
691 }
692
693 if (!(vma->vm_flags & VM_READ)) {
694 /*
695 * If vma do not allow read access, then assume that it does
696 * not allow write access, either. Architecture that allow
697 * write without read access are not supported by HMM, because
698 * operations such has atomic access would not work.
699 */
700 hmm_pfns_clear(range, range->pfns, range->start, range->end);
701 return -EPERM;
702 }
703
704 /* Initialize range to track CPU page table update */
705 spin_lock(&hmm->lock);
706 range->valid = true;
707 list_add_rcu(&range->list, &hmm->ranges);
708 spin_unlock(&hmm->lock);
709
710 hmm_vma_walk.fault = false;
711 hmm_vma_walk.range = range;
712 mm_walk.private = &hmm_vma_walk;
713
714 mm_walk.vma = vma;
715 mm_walk.mm = vma->vm_mm;
716 mm_walk.pte_entry = NULL;
717 mm_walk.test_walk = NULL;
718 mm_walk.hugetlb_entry = NULL;
719 mm_walk.pmd_entry = hmm_vma_walk_pmd;
720 mm_walk.pte_hole = hmm_vma_walk_hole;
721
722 walk_page_range(range->start, range->end, &mm_walk);
723 return 0;
724}
725EXPORT_SYMBOL(hmm_vma_get_pfns);
726
727/*
728 * hmm_vma_range_done() - stop tracking change to CPU page table over a range
729 * @range: range being tracked
730 * Returns: false if range data has been invalidated, true otherwise
731 *
732 * Range struct is used to track updates to the CPU page table after a call to
733 * either hmm_vma_get_pfns() or hmm_vma_fault(). Once the device driver is done
734 * using the data, or wants to lock updates to the data it got from those
735 * functions, it must call the hmm_vma_range_done() function, which will then
736 * stop tracking CPU page table updates.
737 *
738 * Note that device driver must still implement general CPU page table update
739 * tracking either by using hmm_mirror (see hmm_mirror_register()) or by using
740 * the mmu_notifier API directly.
741 *
742 * CPU page table update tracking done through hmm_range is only temporary and
743 * to be used while trying to duplicate CPU page table contents for a range of
744 * virtual addresses.
745 *
746 * There are two ways to use this :
747 * again:
748 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
749 * trans = device_build_page_table_update_transaction(pfns);
750 * device_page_table_lock();
751 * if (!hmm_vma_range_done(range)) {
752 * device_page_table_unlock();
753 * goto again;
754 * }
755 * device_commit_transaction(trans);
756 * device_page_table_unlock();
757 *
758 * Or:
759 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
760 * device_page_table_lock();
761 * hmm_vma_range_done(range);
762 * device_update_page_table(range->pfns);
763 * device_page_table_unlock();
764 */
765bool hmm_vma_range_done(struct hmm_range *range)
766{
767 unsigned long npages = (range->end - range->start) >> PAGE_SHIFT;
768 struct hmm *hmm;
769
770 if (range->end <= range->start) {
771 BUG();
772 return false;
773 }
774
775 hmm = hmm_register(range->vma->vm_mm);
776 if (!hmm) {
777 memset(range->pfns, 0, sizeof(*range->pfns) * npages);
778 return false;
779 }
780
781 spin_lock(&hmm->lock);
782 list_del_rcu(&range->list);
783 spin_unlock(&hmm->lock);
784
785 return range->valid;
786}
787EXPORT_SYMBOL(hmm_vma_range_done);
788
789/*
790 * hmm_vma_fault() - try to fault some address in a virtual address range
791 * @range: range being faulted
792 * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem)
793 * Returns: 0 success, error otherwise (-EAGAIN means mmap_sem have been drop)
794 *
795 * This is similar to a regular CPU page fault except that it will not trigger
796 * any memory migration if the memory being faulted is not accessible by CPUs.
797 *
798 * On error, for one virtual address in the range, the function will mark the
799 * corresponding HMM pfn entry with an error flag.
800 *
801 * Expected use pattern:
802 * retry:
803 * down_read(&mm->mmap_sem);
804 * // Find vma and address device wants to fault, initialize hmm_pfn_t
805 * // array accordingly
806 * ret = hmm_vma_fault(range, write, block);
807 * switch (ret) {
808 * case -EAGAIN:
809 * hmm_vma_range_done(range);
810 * // You might want to rate limit or yield to play nicely, you may
811 * // also commit any valid pfn in the array assuming that you are
812 * // getting true from hmm_vma_range_monitor_end()
813 * goto retry;
814 * case 0:
815 * break;
816 * case -ENOMEM:
817 * case -EINVAL:
818 * case -EPERM:
819 * default:
820 * // Handle error !
821 * up_read(&mm->mmap_sem)
822 * return;
823 * }
824 * // Take device driver lock that serialize device page table update
825 * driver_lock_device_page_table_update();
826 * hmm_vma_range_done(range);
827 * // Commit pfns we got from hmm_vma_fault()
828 * driver_unlock_device_page_table_update();
829 * up_read(&mm->mmap_sem)
830 *
831 * YOU MUST CALL hmm_vma_range_done() AFTER THIS FUNCTION RETURN SUCCESS (0)
832 * BEFORE FREEING THE range struct OR YOU WILL HAVE SERIOUS MEMORY CORRUPTION !
833 *
834 * YOU HAVE BEEN WARNED !
835 */
836int hmm_vma_fault(struct hmm_range *range, bool block)
837{
838 struct vm_area_struct *vma = range->vma;
839 unsigned long start = range->start;
840 struct hmm_vma_walk hmm_vma_walk;
841 struct mm_walk mm_walk;
842 struct hmm *hmm;
843 int ret;
844
845 /* Sanity check, this really should not happen ! */
846 if (range->start < vma->vm_start || range->start >= vma->vm_end)
847 return -EINVAL;
848 if (range->end < vma->vm_start || range->end > vma->vm_end)
849 return -EINVAL;
850
851 hmm = hmm_register(vma->vm_mm);
852 if (!hmm) {
853 hmm_pfns_clear(range, range->pfns, range->start, range->end);
854 return -ENOMEM;
855 }
856 /* Caller must have registered a mirror using hmm_mirror_register() */
857 if (!hmm->mmu_notifier.ops)
858 return -EINVAL;
859
860 /* FIXME support hugetlb fs */
861 if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) {
862 hmm_pfns_special(range);
863 return -EINVAL;
864 }
865
866 if (!(vma->vm_flags & VM_READ)) {
867 /*
868 * If vma do not allow read access, then assume that it does
869 * not allow write access, either. Architecture that allow
870 * write without read access are not supported by HMM, because
871 * operations such has atomic access would not work.
872 */
873 hmm_pfns_clear(range, range->pfns, range->start, range->end);
874 return -EPERM;
875 }
876
877 /* Initialize range to track CPU page table update */
878 spin_lock(&hmm->lock);
879 range->valid = true;
880 list_add_rcu(&range->list, &hmm->ranges);
881 spin_unlock(&hmm->lock);
882
883 hmm_vma_walk.fault = true;
884 hmm_vma_walk.block = block;
885 hmm_vma_walk.range = range;
886 mm_walk.private = &hmm_vma_walk;
887 hmm_vma_walk.last = range->start;
888
889 mm_walk.vma = vma;
890 mm_walk.mm = vma->vm_mm;
891 mm_walk.pte_entry = NULL;
892 mm_walk.test_walk = NULL;
893 mm_walk.hugetlb_entry = NULL;
894 mm_walk.pmd_entry = hmm_vma_walk_pmd;
895 mm_walk.pte_hole = hmm_vma_walk_hole;
896
897 do {
898 ret = walk_page_range(start, range->end, &mm_walk);
899 start = hmm_vma_walk.last;
900 } while (ret == -EAGAIN);
901
902 if (ret) {
903 unsigned long i;
904
905 i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
906 hmm_pfns_clear(range, &range->pfns[i], hmm_vma_walk.last,
907 range->end);
908 hmm_vma_range_done(range);
909 }
910 return ret;
911}
912EXPORT_SYMBOL(hmm_vma_fault);
913#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
914
915
916#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
917struct page *hmm_vma_alloc_locked_page(struct vm_area_struct *vma,
918 unsigned long addr)
919{
920 struct page *page;
921
922 page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
923 if (!page)
924 return NULL;
925 lock_page(page);
926 return page;
927}
928EXPORT_SYMBOL(hmm_vma_alloc_locked_page);
929
930
931static void hmm_devmem_ref_release(struct percpu_ref *ref)
932{
933 struct hmm_devmem *devmem;
934
935 devmem = container_of(ref, struct hmm_devmem, ref);
936 complete(&devmem->completion);
937}
938
939static void hmm_devmem_ref_exit(void *data)
940{
941 struct percpu_ref *ref = data;
942 struct hmm_devmem *devmem;
943
944 devmem = container_of(ref, struct hmm_devmem, ref);
945 percpu_ref_exit(ref);
946 devm_remove_action(devmem->device, &hmm_devmem_ref_exit, data);
947}
948
949static void hmm_devmem_ref_kill(void *data)
950{
951 struct percpu_ref *ref = data;
952 struct hmm_devmem *devmem;
953
954 devmem = container_of(ref, struct hmm_devmem, ref);
955 percpu_ref_kill(ref);
956 wait_for_completion(&devmem->completion);
957 devm_remove_action(devmem->device, &hmm_devmem_ref_kill, data);
958}
959
960static int hmm_devmem_fault(struct vm_area_struct *vma,
961 unsigned long addr,
962 const struct page *page,
963 unsigned int flags,
964 pmd_t *pmdp)
965{
966 struct hmm_devmem *devmem = page->pgmap->data;
967
968 return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);
969}
970
971static void hmm_devmem_free(struct page *page, void *data)
972{
973 struct hmm_devmem *devmem = data;
974
975 devmem->ops->free(devmem, page);
976}
977
978static DEFINE_MUTEX(hmm_devmem_lock);
979static RADIX_TREE(hmm_devmem_radix, GFP_KERNEL);
980
981static void hmm_devmem_radix_release(struct resource *resource)
982{
983 resource_size_t key, align_start, align_size;
984
985 align_start = resource->start & ~(PA_SECTION_SIZE - 1);
986 align_size = ALIGN(resource_size(resource), PA_SECTION_SIZE);
987
988 mutex_lock(&hmm_devmem_lock);
989 for (key = resource->start;
990 key <= resource->end;
991 key += PA_SECTION_SIZE)
992 radix_tree_delete(&hmm_devmem_radix, key >> PA_SECTION_SHIFT);
993 mutex_unlock(&hmm_devmem_lock);
994}
995
996static void hmm_devmem_release(struct device *dev, void *data)
997{
998 struct hmm_devmem *devmem = data;
999 struct resource *resource = devmem->resource;
1000 unsigned long start_pfn, npages;
1001 struct zone *zone;
1002 struct page *page;
1003
1004 if (percpu_ref_tryget_live(&devmem->ref)) {
1005 dev_WARN(dev, "%s: page mapping is still live!\n", __func__);
1006 percpu_ref_put(&devmem->ref);
1007 }
1008
1009 /* pages are dead and unused, undo the arch mapping */
1010 start_pfn = (resource->start & ~(PA_SECTION_SIZE - 1)) >> PAGE_SHIFT;
1011 npages = ALIGN(resource_size(resource), PA_SECTION_SIZE) >> PAGE_SHIFT;
1012
1013 page = pfn_to_page(start_pfn);
1014 zone = page_zone(page);
1015
1016 mem_hotplug_begin();
1017 if (resource->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY)
1018 __remove_pages(zone, start_pfn, npages, NULL);
1019 else
1020 arch_remove_memory(start_pfn << PAGE_SHIFT,
1021 npages << PAGE_SHIFT, NULL);
1022 mem_hotplug_done();
1023
1024 hmm_devmem_radix_release(resource);
1025}
1026
1027static int hmm_devmem_pages_create(struct hmm_devmem *devmem)
1028{
1029 resource_size_t key, align_start, align_size, align_end;
1030 struct device *device = devmem->device;
1031 int ret, nid, is_ram;
1032 unsigned long pfn;
1033
1034 align_start = devmem->resource->start & ~(PA_SECTION_SIZE - 1);
1035 align_size = ALIGN(devmem->resource->start +
1036 resource_size(devmem->resource),
1037 PA_SECTION_SIZE) - align_start;
1038
1039 is_ram = region_intersects(align_start, align_size,
1040 IORESOURCE_SYSTEM_RAM,
1041 IORES_DESC_NONE);
1042 if (is_ram == REGION_MIXED) {
1043 WARN_ONCE(1, "%s attempted on mixed region %pr\n",
1044 __func__, devmem->resource);
1045 return -ENXIO;
1046 }
1047 if (is_ram == REGION_INTERSECTS)
1048 return -ENXIO;
1049
1050 if (devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY)
1051 devmem->pagemap.type = MEMORY_DEVICE_PUBLIC;
1052 else
1053 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
1054
1055 devmem->pagemap.res = *devmem->resource;
1056 devmem->pagemap.page_fault = hmm_devmem_fault;
1057 devmem->pagemap.page_free = hmm_devmem_free;
1058 devmem->pagemap.dev = devmem->device;
1059 devmem->pagemap.ref = &devmem->ref;
1060 devmem->pagemap.data = devmem;
1061
1062 mutex_lock(&hmm_devmem_lock);
1063 align_end = align_start + align_size - 1;
1064 for (key = align_start; key <= align_end; key += PA_SECTION_SIZE) {
1065 struct hmm_devmem *dup;
1066
1067 dup = radix_tree_lookup(&hmm_devmem_radix,
1068 key >> PA_SECTION_SHIFT);
1069 if (dup) {
1070 dev_err(device, "%s: collides with mapping for %s\n",
1071 __func__, dev_name(dup->device));
1072 mutex_unlock(&hmm_devmem_lock);
1073 ret = -EBUSY;
1074 goto error;
1075 }
1076 ret = radix_tree_insert(&hmm_devmem_radix,
1077 key >> PA_SECTION_SHIFT,
1078 devmem);
1079 if (ret) {
1080 dev_err(device, "%s: failed: %d\n", __func__, ret);
1081 mutex_unlock(&hmm_devmem_lock);
1082 goto error_radix;
1083 }
1084 }
1085 mutex_unlock(&hmm_devmem_lock);
1086
1087 nid = dev_to_node(device);
1088 if (nid < 0)
1089 nid = numa_mem_id();
1090
1091 mem_hotplug_begin();
1092 /*
1093 * For device private memory we call add_pages() as we only need to
1094 * allocate and initialize struct page for the device memory. More-
1095 * over the device memory is un-accessible thus we do not want to
1096 * create a linear mapping for the memory like arch_add_memory()
1097 * would do.
1098 *
1099 * For device public memory, which is accesible by the CPU, we do
1100 * want the linear mapping and thus use arch_add_memory().
1101 */
1102 if (devmem->pagemap.type == MEMORY_DEVICE_PUBLIC)
1103 ret = arch_add_memory(nid, align_start, align_size, NULL,
1104 false);
1105 else
1106 ret = add_pages(nid, align_start >> PAGE_SHIFT,
1107 align_size >> PAGE_SHIFT, NULL, false);
1108 if (ret) {
1109 mem_hotplug_done();
1110 goto error_add_memory;
1111 }
1112 move_pfn_range_to_zone(&NODE_DATA(nid)->node_zones[ZONE_DEVICE],
1113 align_start >> PAGE_SHIFT,
1114 align_size >> PAGE_SHIFT, NULL);
1115 mem_hotplug_done();
1116
1117 for (pfn = devmem->pfn_first; pfn < devmem->pfn_last; pfn++) {
1118 struct page *page = pfn_to_page(pfn);
1119
1120 page->pgmap = &devmem->pagemap;
1121 }
1122 return 0;
1123
1124error_add_memory:
1125 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
1126error_radix:
1127 hmm_devmem_radix_release(devmem->resource);
1128error:
1129 return ret;
1130}
1131
1132static int hmm_devmem_match(struct device *dev, void *data, void *match_data)
1133{
1134 struct hmm_devmem *devmem = data;
1135
1136 return devmem->resource == match_data;
1137}
1138
1139static void hmm_devmem_pages_remove(struct hmm_devmem *devmem)
1140{
1141 devres_release(devmem->device, &hmm_devmem_release,
1142 &hmm_devmem_match, devmem->resource);
1143}
1144
1145/*
1146 * hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory
1147 *
1148 * @ops: memory event device driver callback (see struct hmm_devmem_ops)
1149 * @device: device struct to bind the resource too
1150 * @size: size in bytes of the device memory to add
1151 * Returns: pointer to new hmm_devmem struct ERR_PTR otherwise
1152 *
1153 * This function first finds an empty range of physical address big enough to
1154 * contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which
1155 * in turn allocates struct pages. It does not do anything beyond that; all
1156 * events affecting the memory will go through the various callbacks provided
1157 * by hmm_devmem_ops struct.
1158 *
1159 * Device driver should call this function during device initialization and
1160 * is then responsible of memory management. HMM only provides helpers.
1161 */
1162struct hmm_devmem *hmm_devmem_add(const struct hmm_devmem_ops *ops,
1163 struct device *device,
1164 unsigned long size)
1165{
1166 struct hmm_devmem *devmem;
1167 resource_size_t addr;
1168 int ret;
1169
1170 static_branch_enable(&device_private_key);
1171
1172 devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1173 GFP_KERNEL, dev_to_node(device));
1174 if (!devmem)
1175 return ERR_PTR(-ENOMEM);
1176
1177 init_completion(&devmem->completion);
1178 devmem->pfn_first = -1UL;
1179 devmem->pfn_last = -1UL;
1180 devmem->resource = NULL;
1181 devmem->device = device;
1182 devmem->ops = ops;
1183
1184 ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1185 0, GFP_KERNEL);
1186 if (ret)
1187 goto error_percpu_ref;
1188
1189 ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1190 if (ret)
1191 goto error_devm_add_action;
1192
1193 size = ALIGN(size, PA_SECTION_SIZE);
1194 addr = min((unsigned long)iomem_resource.end,
1195 (1UL << MAX_PHYSMEM_BITS) - 1);
1196 addr = addr - size + 1UL;
1197
1198 /*
1199 * FIXME add a new helper to quickly walk resource tree and find free
1200 * range
1201 *
1202 * FIXME what about ioport_resource resource ?
1203 */
1204 for (; addr > size && addr >= iomem_resource.start; addr -= size) {
1205 ret = region_intersects(addr, size, 0, IORES_DESC_NONE);
1206 if (ret != REGION_DISJOINT)
1207 continue;
1208
1209 devmem->resource = devm_request_mem_region(device, addr, size,
1210 dev_name(device));
1211 if (!devmem->resource) {
1212 ret = -ENOMEM;
1213 goto error_no_resource;
1214 }
1215 break;
1216 }
1217 if (!devmem->resource) {
1218 ret = -ERANGE;
1219 goto error_no_resource;
1220 }
1221
1222 devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
1223 devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1224 devmem->pfn_last = devmem->pfn_first +
1225 (resource_size(devmem->resource) >> PAGE_SHIFT);
1226
1227 ret = hmm_devmem_pages_create(devmem);
1228 if (ret)
1229 goto error_pages;
1230
1231 devres_add(device, devmem);
1232
1233 ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1234 if (ret) {
1235 hmm_devmem_remove(devmem);
1236 return ERR_PTR(ret);
1237 }
1238
1239 return devmem;
1240
1241error_pages:
1242 devm_release_mem_region(device, devmem->resource->start,
1243 resource_size(devmem->resource));
1244error_no_resource:
1245error_devm_add_action:
1246 hmm_devmem_ref_kill(&devmem->ref);
1247 hmm_devmem_ref_exit(&devmem->ref);
1248error_percpu_ref:
1249 devres_free(devmem);
1250 return ERR_PTR(ret);
1251}
1252EXPORT_SYMBOL(hmm_devmem_add);
1253
1254struct hmm_devmem *hmm_devmem_add_resource(const struct hmm_devmem_ops *ops,
1255 struct device *device,
1256 struct resource *res)
1257{
1258 struct hmm_devmem *devmem;
1259 int ret;
1260
1261 if (res->desc != IORES_DESC_DEVICE_PUBLIC_MEMORY)
1262 return ERR_PTR(-EINVAL);
1263
1264 static_branch_enable(&device_private_key);
1265
1266 devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1267 GFP_KERNEL, dev_to_node(device));
1268 if (!devmem)
1269 return ERR_PTR(-ENOMEM);
1270
1271 init_completion(&devmem->completion);
1272 devmem->pfn_first = -1UL;
1273 devmem->pfn_last = -1UL;
1274 devmem->resource = res;
1275 devmem->device = device;
1276 devmem->ops = ops;
1277
1278 ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1279 0, GFP_KERNEL);
1280 if (ret)
1281 goto error_percpu_ref;
1282
1283 ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1284 if (ret)
1285 goto error_devm_add_action;
1286
1287
1288 devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1289 devmem->pfn_last = devmem->pfn_first +
1290 (resource_size(devmem->resource) >> PAGE_SHIFT);
1291
1292 ret = hmm_devmem_pages_create(devmem);
1293 if (ret)
1294 goto error_devm_add_action;
1295
1296 devres_add(device, devmem);
1297
1298 ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1299 if (ret) {
1300 hmm_devmem_remove(devmem);
1301 return ERR_PTR(ret);
1302 }
1303
1304 return devmem;
1305
1306error_devm_add_action:
1307 hmm_devmem_ref_kill(&devmem->ref);
1308 hmm_devmem_ref_exit(&devmem->ref);
1309error_percpu_ref:
1310 devres_free(devmem);
1311 return ERR_PTR(ret);
1312}
1313EXPORT_SYMBOL(hmm_devmem_add_resource);
1314
1315/*
1316 * hmm_devmem_remove() - remove device memory (kill and free ZONE_DEVICE)
1317 *
1318 * @devmem: hmm_devmem struct use to track and manage the ZONE_DEVICE memory
1319 *
1320 * This will hot-unplug memory that was hotplugged by hmm_devmem_add on behalf
1321 * of the device driver. It will free struct page and remove the resource that
1322 * reserved the physical address range for this device memory.
1323 */
1324void hmm_devmem_remove(struct hmm_devmem *devmem)
1325{
1326 resource_size_t start, size;
1327 struct device *device;
1328 bool cdm = false;
1329
1330 if (!devmem)
1331 return;
1332
1333 device = devmem->device;
1334 start = devmem->resource->start;
1335 size = resource_size(devmem->resource);
1336
1337 cdm = devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY;
1338 hmm_devmem_ref_kill(&devmem->ref);
1339 hmm_devmem_ref_exit(&devmem->ref);
1340 hmm_devmem_pages_remove(devmem);
1341
1342 if (!cdm)
1343 devm_release_mem_region(device, start, size);
1344}
1345EXPORT_SYMBOL(hmm_devmem_remove);
1346
1347/*
1348 * A device driver that wants to handle multiple devices memory through a
1349 * single fake device can use hmm_device to do so. This is purely a helper
1350 * and it is not needed to make use of any HMM functionality.
1351 */
1352#define HMM_DEVICE_MAX 256
1353
1354static DECLARE_BITMAP(hmm_device_mask, HMM_DEVICE_MAX);
1355static DEFINE_SPINLOCK(hmm_device_lock);
1356static struct class *hmm_device_class;
1357static dev_t hmm_device_devt;
1358
1359static void hmm_device_release(struct device *device)
1360{
1361 struct hmm_device *hmm_device;
1362
1363 hmm_device = container_of(device, struct hmm_device, device);
1364 spin_lock(&hmm_device_lock);
1365 clear_bit(hmm_device->minor, hmm_device_mask);
1366 spin_unlock(&hmm_device_lock);
1367
1368 kfree(hmm_device);
1369}
1370
1371struct hmm_device *hmm_device_new(void *drvdata)
1372{
1373 struct hmm_device *hmm_device;
1374
1375 hmm_device = kzalloc(sizeof(*hmm_device), GFP_KERNEL);
1376 if (!hmm_device)
1377 return ERR_PTR(-ENOMEM);
1378
1379 spin_lock(&hmm_device_lock);
1380 hmm_device->minor = find_first_zero_bit(hmm_device_mask, HMM_DEVICE_MAX);
1381 if (hmm_device->minor >= HMM_DEVICE_MAX) {
1382 spin_unlock(&hmm_device_lock);
1383 kfree(hmm_device);
1384 return ERR_PTR(-EBUSY);
1385 }
1386 set_bit(hmm_device->minor, hmm_device_mask);
1387 spin_unlock(&hmm_device_lock);
1388
1389 dev_set_name(&hmm_device->device, "hmm_device%d", hmm_device->minor);
1390 hmm_device->device.devt = MKDEV(MAJOR(hmm_device_devt),
1391 hmm_device->minor);
1392 hmm_device->device.release = hmm_device_release;
1393 dev_set_drvdata(&hmm_device->device, drvdata);
1394 hmm_device->device.class = hmm_device_class;
1395 device_initialize(&hmm_device->device);
1396
1397 return hmm_device;
1398}
1399EXPORT_SYMBOL(hmm_device_new);
1400
1401void hmm_device_put(struct hmm_device *hmm_device)
1402{
1403 put_device(&hmm_device->device);
1404}
1405EXPORT_SYMBOL(hmm_device_put);
1406
1407static int __init hmm_init(void)
1408{
1409 int ret;
1410
1411 ret = alloc_chrdev_region(&hmm_device_devt, 0,
1412 HMM_DEVICE_MAX,
1413 "hmm_device");
1414 if (ret)
1415 return ret;
1416
1417 hmm_device_class = class_create(THIS_MODULE, "hmm_device");
1418 if (IS_ERR(hmm_device_class)) {
1419 unregister_chrdev_region(hmm_device_devt, HMM_DEVICE_MAX);
1420 return PTR_ERR(hmm_device_class);
1421 }
1422 return 0;
1423}
1424
1425device_initcall(hmm_init);
1426#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */