Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2013 Red Hat Inc.
4 *
5 * Authors: Jérôme Glisse <jglisse@redhat.com>
6 */
7/*
8 * Refer to include/linux/hmm.h for information about heterogeneous memory
9 * management or HMM for short.
10 */
11#include <linux/pagewalk.h>
12#include <linux/hmm.h>
13#include <linux/init.h>
14#include <linux/rmap.h>
15#include <linux/swap.h>
16#include <linux/slab.h>
17#include <linux/sched.h>
18#include <linux/mmzone.h>
19#include <linux/pagemap.h>
20#include <linux/swapops.h>
21#include <linux/hugetlb.h>
22#include <linux/memremap.h>
23#include <linux/sched/mm.h>
24#include <linux/jump_label.h>
25#include <linux/dma-mapping.h>
26#include <linux/mmu_notifier.h>
27#include <linux/memory_hotplug.h>
28
29static struct mmu_notifier *hmm_alloc_notifier(struct mm_struct *mm)
30{
31 struct hmm *hmm;
32
33 hmm = kzalloc(sizeof(*hmm), GFP_KERNEL);
34 if (!hmm)
35 return ERR_PTR(-ENOMEM);
36
37 init_waitqueue_head(&hmm->wq);
38 INIT_LIST_HEAD(&hmm->mirrors);
39 init_rwsem(&hmm->mirrors_sem);
40 INIT_LIST_HEAD(&hmm->ranges);
41 spin_lock_init(&hmm->ranges_lock);
42 hmm->notifiers = 0;
43 return &hmm->mmu_notifier;
44}
45
46static void hmm_free_notifier(struct mmu_notifier *mn)
47{
48 struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);
49
50 WARN_ON(!list_empty(&hmm->ranges));
51 WARN_ON(!list_empty(&hmm->mirrors));
52 kfree(hmm);
53}
54
55static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm)
56{
57 struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);
58 struct hmm_mirror *mirror;
59
60 /*
61 * Since hmm_range_register() holds the mmget() lock hmm_release() is
62 * prevented as long as a range exists.
63 */
64 WARN_ON(!list_empty_careful(&hmm->ranges));
65
66 down_read(&hmm->mirrors_sem);
67 list_for_each_entry(mirror, &hmm->mirrors, list) {
68 /*
69 * Note: The driver is not allowed to trigger
70 * hmm_mirror_unregister() from this thread.
71 */
72 if (mirror->ops->release)
73 mirror->ops->release(mirror);
74 }
75 up_read(&hmm->mirrors_sem);
76}
77
78static void notifiers_decrement(struct hmm *hmm)
79{
80 unsigned long flags;
81
82 spin_lock_irqsave(&hmm->ranges_lock, flags);
83 hmm->notifiers--;
84 if (!hmm->notifiers) {
85 struct hmm_range *range;
86
87 list_for_each_entry(range, &hmm->ranges, list) {
88 if (range->valid)
89 continue;
90 range->valid = true;
91 }
92 wake_up_all(&hmm->wq);
93 }
94 spin_unlock_irqrestore(&hmm->ranges_lock, flags);
95}
96
97static int hmm_invalidate_range_start(struct mmu_notifier *mn,
98 const struct mmu_notifier_range *nrange)
99{
100 struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);
101 struct hmm_mirror *mirror;
102 struct hmm_range *range;
103 unsigned long flags;
104 int ret = 0;
105
106 spin_lock_irqsave(&hmm->ranges_lock, flags);
107 hmm->notifiers++;
108 list_for_each_entry(range, &hmm->ranges, list) {
109 if (nrange->end < range->start || nrange->start >= range->end)
110 continue;
111
112 range->valid = false;
113 }
114 spin_unlock_irqrestore(&hmm->ranges_lock, flags);
115
116 if (mmu_notifier_range_blockable(nrange))
117 down_read(&hmm->mirrors_sem);
118 else if (!down_read_trylock(&hmm->mirrors_sem)) {
119 ret = -EAGAIN;
120 goto out;
121 }
122
123 list_for_each_entry(mirror, &hmm->mirrors, list) {
124 int rc;
125
126 rc = mirror->ops->sync_cpu_device_pagetables(mirror, nrange);
127 if (rc) {
128 if (WARN_ON(mmu_notifier_range_blockable(nrange) ||
129 rc != -EAGAIN))
130 continue;
131 ret = -EAGAIN;
132 break;
133 }
134 }
135 up_read(&hmm->mirrors_sem);
136
137out:
138 if (ret)
139 notifiers_decrement(hmm);
140 return ret;
141}
142
143static void hmm_invalidate_range_end(struct mmu_notifier *mn,
144 const struct mmu_notifier_range *nrange)
145{
146 struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier);
147
148 notifiers_decrement(hmm);
149}
150
151static const struct mmu_notifier_ops hmm_mmu_notifier_ops = {
152 .release = hmm_release,
153 .invalidate_range_start = hmm_invalidate_range_start,
154 .invalidate_range_end = hmm_invalidate_range_end,
155 .alloc_notifier = hmm_alloc_notifier,
156 .free_notifier = hmm_free_notifier,
157};
158
159/*
160 * hmm_mirror_register() - register a mirror against an mm
161 *
162 * @mirror: new mirror struct to register
163 * @mm: mm to register against
164 * Return: 0 on success, -ENOMEM if no memory, -EINVAL if invalid arguments
165 *
166 * To start mirroring a process address space, the device driver must register
167 * an HMM mirror struct.
168 *
169 * The caller cannot unregister the hmm_mirror while any ranges are
170 * registered.
171 *
172 * Callers using this function must put a call to mmu_notifier_synchronize()
173 * in their module exit functions.
174 */
175int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm)
176{
177 struct mmu_notifier *mn;
178
179 lockdep_assert_held_write(&mm->mmap_sem);
180
181 /* Sanity check */
182 if (!mm || !mirror || !mirror->ops)
183 return -EINVAL;
184
185 mn = mmu_notifier_get_locked(&hmm_mmu_notifier_ops, mm);
186 if (IS_ERR(mn))
187 return PTR_ERR(mn);
188 mirror->hmm = container_of(mn, struct hmm, mmu_notifier);
189
190 down_write(&mirror->hmm->mirrors_sem);
191 list_add(&mirror->list, &mirror->hmm->mirrors);
192 up_write(&mirror->hmm->mirrors_sem);
193
194 return 0;
195}
196EXPORT_SYMBOL(hmm_mirror_register);
197
198/*
199 * hmm_mirror_unregister() - unregister a mirror
200 *
201 * @mirror: mirror struct to unregister
202 *
203 * Stop mirroring a process address space, and cleanup.
204 */
205void hmm_mirror_unregister(struct hmm_mirror *mirror)
206{
207 struct hmm *hmm = mirror->hmm;
208
209 down_write(&hmm->mirrors_sem);
210 list_del(&mirror->list);
211 up_write(&hmm->mirrors_sem);
212 mmu_notifier_put(&hmm->mmu_notifier);
213}
214EXPORT_SYMBOL(hmm_mirror_unregister);
215
216struct hmm_vma_walk {
217 struct hmm_range *range;
218 struct dev_pagemap *pgmap;
219 unsigned long last;
220 unsigned int flags;
221};
222
223static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,
224 bool write_fault, uint64_t *pfn)
225{
226 unsigned int flags = FAULT_FLAG_REMOTE;
227 struct hmm_vma_walk *hmm_vma_walk = walk->private;
228 struct hmm_range *range = hmm_vma_walk->range;
229 struct vm_area_struct *vma = walk->vma;
230 vm_fault_t ret;
231
232 if (!vma)
233 goto err;
234
235 if (hmm_vma_walk->flags & HMM_FAULT_ALLOW_RETRY)
236 flags |= FAULT_FLAG_ALLOW_RETRY;
237 if (write_fault)
238 flags |= FAULT_FLAG_WRITE;
239
240 ret = handle_mm_fault(vma, addr, flags);
241 if (ret & VM_FAULT_RETRY) {
242 /* Note, handle_mm_fault did up_read(&mm->mmap_sem)) */
243 return -EAGAIN;
244 }
245 if (ret & VM_FAULT_ERROR)
246 goto err;
247
248 return -EBUSY;
249
250err:
251 *pfn = range->values[HMM_PFN_ERROR];
252 return -EFAULT;
253}
254
255static int hmm_pfns_bad(unsigned long addr,
256 unsigned long end,
257 struct mm_walk *walk)
258{
259 struct hmm_vma_walk *hmm_vma_walk = walk->private;
260 struct hmm_range *range = hmm_vma_walk->range;
261 uint64_t *pfns = range->pfns;
262 unsigned long i;
263
264 i = (addr - range->start) >> PAGE_SHIFT;
265 for (; addr < end; addr += PAGE_SIZE, i++)
266 pfns[i] = range->values[HMM_PFN_ERROR];
267
268 return 0;
269}
270
271/*
272 * hmm_vma_walk_hole_() - handle a range lacking valid pmd or pte(s)
273 * @addr: range virtual start address (inclusive)
274 * @end: range virtual end address (exclusive)
275 * @fault: should we fault or not ?
276 * @write_fault: write fault ?
277 * @walk: mm_walk structure
278 * Return: 0 on success, -EBUSY after page fault, or page fault error
279 *
280 * This function will be called whenever pmd_none() or pte_none() returns true,
281 * or whenever there is no page directory covering the virtual address range.
282 */
283static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end,
284 bool fault, bool write_fault,
285 struct mm_walk *walk)
286{
287 struct hmm_vma_walk *hmm_vma_walk = walk->private;
288 struct hmm_range *range = hmm_vma_walk->range;
289 uint64_t *pfns = range->pfns;
290 unsigned long i;
291
292 hmm_vma_walk->last = addr;
293 i = (addr - range->start) >> PAGE_SHIFT;
294
295 if (write_fault && walk->vma && !(walk->vma->vm_flags & VM_WRITE))
296 return -EPERM;
297
298 for (; addr < end; addr += PAGE_SIZE, i++) {
299 pfns[i] = range->values[HMM_PFN_NONE];
300 if (fault || write_fault) {
301 int ret;
302
303 ret = hmm_vma_do_fault(walk, addr, write_fault,
304 &pfns[i]);
305 if (ret != -EBUSY)
306 return ret;
307 }
308 }
309
310 return (fault || write_fault) ? -EBUSY : 0;
311}
312
313static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
314 uint64_t pfns, uint64_t cpu_flags,
315 bool *fault, bool *write_fault)
316{
317 struct hmm_range *range = hmm_vma_walk->range;
318
319 if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT)
320 return;
321
322 /*
323 * So we not only consider the individual per page request we also
324 * consider the default flags requested for the range. The API can
325 * be used 2 ways. The first one where the HMM user coalesces
326 * multiple page faults into one request and sets flags per pfn for
327 * those faults. The second one where the HMM user wants to pre-
328 * fault a range with specific flags. For the latter one it is a
329 * waste to have the user pre-fill the pfn arrays with a default
330 * flags value.
331 */
332 pfns = (pfns & range->pfn_flags_mask) | range->default_flags;
333
334 /* We aren't ask to do anything ... */
335 if (!(pfns & range->flags[HMM_PFN_VALID]))
336 return;
337 /* If this is device memory then only fault if explicitly requested */
338 if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
339 /* Do we fault on device memory ? */
340 if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) {
341 *write_fault = pfns & range->flags[HMM_PFN_WRITE];
342 *fault = true;
343 }
344 return;
345 }
346
347 /* If CPU page table is not valid then we need to fault */
348 *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);
349 /* Need to write fault ? */
350 if ((pfns & range->flags[HMM_PFN_WRITE]) &&
351 !(cpu_flags & range->flags[HMM_PFN_WRITE])) {
352 *write_fault = true;
353 *fault = true;
354 }
355}
356
357static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
358 const uint64_t *pfns, unsigned long npages,
359 uint64_t cpu_flags, bool *fault,
360 bool *write_fault)
361{
362 unsigned long i;
363
364 if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) {
365 *fault = *write_fault = false;
366 return;
367 }
368
369 *fault = *write_fault = false;
370 for (i = 0; i < npages; ++i) {
371 hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags,
372 fault, write_fault);
373 if ((*write_fault))
374 return;
375 }
376}
377
378static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
379 struct mm_walk *walk)
380{
381 struct hmm_vma_walk *hmm_vma_walk = walk->private;
382 struct hmm_range *range = hmm_vma_walk->range;
383 bool fault, write_fault;
384 unsigned long i, npages;
385 uint64_t *pfns;
386
387 i = (addr - range->start) >> PAGE_SHIFT;
388 npages = (end - addr) >> PAGE_SHIFT;
389 pfns = &range->pfns[i];
390 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
391 0, &fault, &write_fault);
392 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
393}
394
395static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)
396{
397 if (pmd_protnone(pmd))
398 return 0;
399 return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |
400 range->flags[HMM_PFN_WRITE] :
401 range->flags[HMM_PFN_VALID];
402}
403
404#ifdef CONFIG_TRANSPARENT_HUGEPAGE
405static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
406 unsigned long end, uint64_t *pfns, pmd_t pmd)
407{
408 struct hmm_vma_walk *hmm_vma_walk = walk->private;
409 struct hmm_range *range = hmm_vma_walk->range;
410 unsigned long pfn, npages, i;
411 bool fault, write_fault;
412 uint64_t cpu_flags;
413
414 npages = (end - addr) >> PAGE_SHIFT;
415 cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
416 hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags,
417 &fault, &write_fault);
418
419 if (pmd_protnone(pmd) || fault || write_fault)
420 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
421
422 pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
423 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
424 if (pmd_devmap(pmd)) {
425 hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
426 hmm_vma_walk->pgmap);
427 if (unlikely(!hmm_vma_walk->pgmap))
428 return -EBUSY;
429 }
430 pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags;
431 }
432 if (hmm_vma_walk->pgmap) {
433 put_dev_pagemap(hmm_vma_walk->pgmap);
434 hmm_vma_walk->pgmap = NULL;
435 }
436 hmm_vma_walk->last = end;
437 return 0;
438}
439#else /* CONFIG_TRANSPARENT_HUGEPAGE */
440/* stub to allow the code below to compile */
441int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
442 unsigned long end, uint64_t *pfns, pmd_t pmd);
443#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
444
445static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)
446{
447 if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte))
448 return 0;
449 return pte_write(pte) ? range->flags[HMM_PFN_VALID] |
450 range->flags[HMM_PFN_WRITE] :
451 range->flags[HMM_PFN_VALID];
452}
453
454static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
455 unsigned long end, pmd_t *pmdp, pte_t *ptep,
456 uint64_t *pfn)
457{
458 struct hmm_vma_walk *hmm_vma_walk = walk->private;
459 struct hmm_range *range = hmm_vma_walk->range;
460 bool fault, write_fault;
461 uint64_t cpu_flags;
462 pte_t pte = *ptep;
463 uint64_t orig_pfn = *pfn;
464
465 *pfn = range->values[HMM_PFN_NONE];
466 fault = write_fault = false;
467
468 if (pte_none(pte)) {
469 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0,
470 &fault, &write_fault);
471 if (fault || write_fault)
472 goto fault;
473 return 0;
474 }
475
476 if (!pte_present(pte)) {
477 swp_entry_t entry = pte_to_swp_entry(pte);
478
479 if (!non_swap_entry(entry)) {
480 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
481 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
482 &fault, &write_fault);
483 if (fault || write_fault)
484 goto fault;
485 return 0;
486 }
487
488 /*
489 * This is a special swap entry, ignore migration, use
490 * device and report anything else as error.
491 */
492 if (is_device_private_entry(entry)) {
493 cpu_flags = range->flags[HMM_PFN_VALID] |
494 range->flags[HMM_PFN_DEVICE_PRIVATE];
495 cpu_flags |= is_write_device_private_entry(entry) ?
496 range->flags[HMM_PFN_WRITE] : 0;
497 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
498 &fault, &write_fault);
499 if (fault || write_fault)
500 goto fault;
501 *pfn = hmm_device_entry_from_pfn(range,
502 swp_offset(entry));
503 *pfn |= cpu_flags;
504 return 0;
505 }
506
507 if (is_migration_entry(entry)) {
508 if (fault || write_fault) {
509 pte_unmap(ptep);
510 hmm_vma_walk->last = addr;
511 migration_entry_wait(walk->mm, pmdp, addr);
512 return -EBUSY;
513 }
514 return 0;
515 }
516
517 /* Report error for everything else */
518 *pfn = range->values[HMM_PFN_ERROR];
519 return -EFAULT;
520 } else {
521 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
522 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
523 &fault, &write_fault);
524 }
525
526 if (fault || write_fault)
527 goto fault;
528
529 if (pte_devmap(pte)) {
530 hmm_vma_walk->pgmap = get_dev_pagemap(pte_pfn(pte),
531 hmm_vma_walk->pgmap);
532 if (unlikely(!hmm_vma_walk->pgmap))
533 return -EBUSY;
534 } else if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && pte_special(pte)) {
535 *pfn = range->values[HMM_PFN_SPECIAL];
536 return -EFAULT;
537 }
538
539 *pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags;
540 return 0;
541
542fault:
543 if (hmm_vma_walk->pgmap) {
544 put_dev_pagemap(hmm_vma_walk->pgmap);
545 hmm_vma_walk->pgmap = NULL;
546 }
547 pte_unmap(ptep);
548 /* Fault any virtual address we were asked to fault */
549 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
550}
551
552static int hmm_vma_walk_pmd(pmd_t *pmdp,
553 unsigned long start,
554 unsigned long end,
555 struct mm_walk *walk)
556{
557 struct hmm_vma_walk *hmm_vma_walk = walk->private;
558 struct hmm_range *range = hmm_vma_walk->range;
559 uint64_t *pfns = range->pfns;
560 unsigned long addr = start, i;
561 pte_t *ptep;
562 pmd_t pmd;
563
564again:
565 pmd = READ_ONCE(*pmdp);
566 if (pmd_none(pmd))
567 return hmm_vma_walk_hole(start, end, walk);
568
569 if (thp_migration_supported() && is_pmd_migration_entry(pmd)) {
570 bool fault, write_fault;
571 unsigned long npages;
572 uint64_t *pfns;
573
574 i = (addr - range->start) >> PAGE_SHIFT;
575 npages = (end - addr) >> PAGE_SHIFT;
576 pfns = &range->pfns[i];
577
578 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
579 0, &fault, &write_fault);
580 if (fault || write_fault) {
581 hmm_vma_walk->last = addr;
582 pmd_migration_entry_wait(walk->mm, pmdp);
583 return -EBUSY;
584 }
585 return 0;
586 } else if (!pmd_present(pmd))
587 return hmm_pfns_bad(start, end, walk);
588
589 if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) {
590 /*
591 * No need to take pmd_lock here, even if some other thread
592 * is splitting the huge pmd we will get that event through
593 * mmu_notifier callback.
594 *
595 * So just read pmd value and check again it's a transparent
596 * huge or device mapping one and compute corresponding pfn
597 * values.
598 */
599 pmd = pmd_read_atomic(pmdp);
600 barrier();
601 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
602 goto again;
603
604 i = (addr - range->start) >> PAGE_SHIFT;
605 return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
606 }
607
608 /*
609 * We have handled all the valid cases above ie either none, migration,
610 * huge or transparent huge. At this point either it is a valid pmd
611 * entry pointing to pte directory or it is a bad pmd that will not
612 * recover.
613 */
614 if (pmd_bad(pmd))
615 return hmm_pfns_bad(start, end, walk);
616
617 ptep = pte_offset_map(pmdp, addr);
618 i = (addr - range->start) >> PAGE_SHIFT;
619 for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
620 int r;
621
622 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
623 if (r) {
624 /* hmm_vma_handle_pte() did unmap pte directory */
625 hmm_vma_walk->last = addr;
626 return r;
627 }
628 }
629 if (hmm_vma_walk->pgmap) {
630 /*
631 * We do put_dev_pagemap() here and not in hmm_vma_handle_pte()
632 * so that we can leverage get_dev_pagemap() optimization which
633 * will not re-take a reference on a pgmap if we already have
634 * one.
635 */
636 put_dev_pagemap(hmm_vma_walk->pgmap);
637 hmm_vma_walk->pgmap = NULL;
638 }
639 pte_unmap(ptep - 1);
640
641 hmm_vma_walk->last = addr;
642 return 0;
643}
644
645#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \
646 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
647static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud)
648{
649 if (!pud_present(pud))
650 return 0;
651 return pud_write(pud) ? range->flags[HMM_PFN_VALID] |
652 range->flags[HMM_PFN_WRITE] :
653 range->flags[HMM_PFN_VALID];
654}
655
656static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
657 struct mm_walk *walk)
658{
659 struct hmm_vma_walk *hmm_vma_walk = walk->private;
660 struct hmm_range *range = hmm_vma_walk->range;
661 unsigned long addr = start, next;
662 pmd_t *pmdp;
663 pud_t pud;
664 int ret;
665
666again:
667 pud = READ_ONCE(*pudp);
668 if (pud_none(pud))
669 return hmm_vma_walk_hole(start, end, walk);
670
671 if (pud_huge(pud) && pud_devmap(pud)) {
672 unsigned long i, npages, pfn;
673 uint64_t *pfns, cpu_flags;
674 bool fault, write_fault;
675
676 if (!pud_present(pud))
677 return hmm_vma_walk_hole(start, end, walk);
678
679 i = (addr - range->start) >> PAGE_SHIFT;
680 npages = (end - addr) >> PAGE_SHIFT;
681 pfns = &range->pfns[i];
682
683 cpu_flags = pud_to_hmm_pfn_flags(range, pud);
684 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
685 cpu_flags, &fault, &write_fault);
686 if (fault || write_fault)
687 return hmm_vma_walk_hole_(addr, end, fault,
688 write_fault, walk);
689
690 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
691 for (i = 0; i < npages; ++i, ++pfn) {
692 hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
693 hmm_vma_walk->pgmap);
694 if (unlikely(!hmm_vma_walk->pgmap))
695 return -EBUSY;
696 pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
697 cpu_flags;
698 }
699 if (hmm_vma_walk->pgmap) {
700 put_dev_pagemap(hmm_vma_walk->pgmap);
701 hmm_vma_walk->pgmap = NULL;
702 }
703 hmm_vma_walk->last = end;
704 return 0;
705 }
706
707 split_huge_pud(walk->vma, pudp, addr);
708 if (pud_none(*pudp))
709 goto again;
710
711 pmdp = pmd_offset(pudp, addr);
712 do {
713 next = pmd_addr_end(addr, end);
714 ret = hmm_vma_walk_pmd(pmdp, addr, next, walk);
715 if (ret)
716 return ret;
717 } while (pmdp++, addr = next, addr != end);
718
719 return 0;
720}
721#else
722#define hmm_vma_walk_pud NULL
723#endif
724
725#ifdef CONFIG_HUGETLB_PAGE
726static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
727 unsigned long start, unsigned long end,
728 struct mm_walk *walk)
729{
730 unsigned long addr = start, i, pfn;
731 struct hmm_vma_walk *hmm_vma_walk = walk->private;
732 struct hmm_range *range = hmm_vma_walk->range;
733 struct vm_area_struct *vma = walk->vma;
734 uint64_t orig_pfn, cpu_flags;
735 bool fault, write_fault;
736 spinlock_t *ptl;
737 pte_t entry;
738 int ret = 0;
739
740 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);
741 entry = huge_ptep_get(pte);
742
743 i = (start - range->start) >> PAGE_SHIFT;
744 orig_pfn = range->pfns[i];
745 range->pfns[i] = range->values[HMM_PFN_NONE];
746 cpu_flags = pte_to_hmm_pfn_flags(range, entry);
747 fault = write_fault = false;
748 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
749 &fault, &write_fault);
750 if (fault || write_fault) {
751 ret = -ENOENT;
752 goto unlock;
753 }
754
755 pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
756 for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
757 range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
758 cpu_flags;
759 hmm_vma_walk->last = end;
760
761unlock:
762 spin_unlock(ptl);
763
764 if (ret == -ENOENT)
765 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
766
767 return ret;
768}
769#else
770#define hmm_vma_walk_hugetlb_entry NULL
771#endif /* CONFIG_HUGETLB_PAGE */
772
773static void hmm_pfns_clear(struct hmm_range *range,
774 uint64_t *pfns,
775 unsigned long addr,
776 unsigned long end)
777{
778 for (; addr < end; addr += PAGE_SIZE, pfns++)
779 *pfns = range->values[HMM_PFN_NONE];
780}
781
782/*
783 * hmm_range_register() - start tracking change to CPU page table over a range
784 * @range: range
785 * @mm: the mm struct for the range of virtual address
786 *
787 * Return: 0 on success, -EFAULT if the address space is no longer valid
788 *
789 * Track updates to the CPU page table see include/linux/hmm.h
790 */
791int hmm_range_register(struct hmm_range *range, struct hmm_mirror *mirror)
792{
793 struct hmm *hmm = mirror->hmm;
794 unsigned long flags;
795
796 range->valid = false;
797 range->hmm = NULL;
798
799 if ((range->start & (PAGE_SIZE - 1)) || (range->end & (PAGE_SIZE - 1)))
800 return -EINVAL;
801 if (range->start >= range->end)
802 return -EINVAL;
803
804 /* Prevent hmm_release() from running while the range is valid */
805 if (!mmget_not_zero(hmm->mmu_notifier.mm))
806 return -EFAULT;
807
808 /* Initialize range to track CPU page table updates. */
809 spin_lock_irqsave(&hmm->ranges_lock, flags);
810
811 range->hmm = hmm;
812 list_add(&range->list, &hmm->ranges);
813
814 /*
815 * If there are any concurrent notifiers we have to wait for them for
816 * the range to be valid (see hmm_range_wait_until_valid()).
817 */
818 if (!hmm->notifiers)
819 range->valid = true;
820 spin_unlock_irqrestore(&hmm->ranges_lock, flags);
821
822 return 0;
823}
824EXPORT_SYMBOL(hmm_range_register);
825
826/*
827 * hmm_range_unregister() - stop tracking change to CPU page table over a range
828 * @range: range
829 *
830 * Range struct is used to track updates to the CPU page table after a call to
831 * hmm_range_register(). See include/linux/hmm.h for how to use it.
832 */
833void hmm_range_unregister(struct hmm_range *range)
834{
835 struct hmm *hmm = range->hmm;
836 unsigned long flags;
837
838 spin_lock_irqsave(&hmm->ranges_lock, flags);
839 list_del_init(&range->list);
840 spin_unlock_irqrestore(&hmm->ranges_lock, flags);
841
842 /* Drop reference taken by hmm_range_register() */
843 mmput(hmm->mmu_notifier.mm);
844
845 /*
846 * The range is now invalid and the ref on the hmm is dropped, so
847 * poison the pointer. Leave other fields in place, for the caller's
848 * use.
849 */
850 range->valid = false;
851 memset(&range->hmm, POISON_INUSE, sizeof(range->hmm));
852}
853EXPORT_SYMBOL(hmm_range_unregister);
854
855static const struct mm_walk_ops hmm_walk_ops = {
856 .pud_entry = hmm_vma_walk_pud,
857 .pmd_entry = hmm_vma_walk_pmd,
858 .pte_hole = hmm_vma_walk_hole,
859 .hugetlb_entry = hmm_vma_walk_hugetlb_entry,
860};
861
862/**
863 * hmm_range_fault - try to fault some address in a virtual address range
864 * @range: range being faulted
865 * @flags: HMM_FAULT_* flags
866 *
867 * Return: the number of valid pages in range->pfns[] (from range start
868 * address), which may be zero. On error one of the following status codes
869 * can be returned:
870 *
871 * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma
872 * (e.g., device file vma).
873 * -ENOMEM: Out of memory.
874 * -EPERM: Invalid permission (e.g., asking for write and range is read
875 * only).
876 * -EAGAIN: A page fault needs to be retried and mmap_sem was dropped.
877 * -EBUSY: The range has been invalidated and the caller needs to wait for
878 * the invalidation to finish.
879 * -EFAULT: Invalid (i.e., either no valid vma or it is illegal to access
880 * that range) number of valid pages in range->pfns[] (from
881 * range start address).
882 *
883 * This is similar to a regular CPU page fault except that it will not trigger
884 * any memory migration if the memory being faulted is not accessible by CPUs
885 * and caller does not ask for migration.
886 *
887 * On error, for one virtual address in the range, the function will mark the
888 * corresponding HMM pfn entry with an error flag.
889 */
890long hmm_range_fault(struct hmm_range *range, unsigned int flags)
891{
892 const unsigned long device_vma = VM_IO | VM_PFNMAP | VM_MIXEDMAP;
893 unsigned long start = range->start, end;
894 struct hmm_vma_walk hmm_vma_walk;
895 struct hmm *hmm = range->hmm;
896 struct vm_area_struct *vma;
897 int ret;
898
899 lockdep_assert_held(&hmm->mmu_notifier.mm->mmap_sem);
900
901 do {
902 /* If range is no longer valid force retry. */
903 if (!range->valid)
904 return -EBUSY;
905
906 vma = find_vma(hmm->mmu_notifier.mm, start);
907 if (vma == NULL || (vma->vm_flags & device_vma))
908 return -EFAULT;
909
910 if (!(vma->vm_flags & VM_READ)) {
911 /*
912 * If vma do not allow read access, then assume that it
913 * does not allow write access, either. HMM does not
914 * support architecture that allow write without read.
915 */
916 hmm_pfns_clear(range, range->pfns,
917 range->start, range->end);
918 return -EPERM;
919 }
920
921 hmm_vma_walk.pgmap = NULL;
922 hmm_vma_walk.last = start;
923 hmm_vma_walk.flags = flags;
924 hmm_vma_walk.range = range;
925 end = min(range->end, vma->vm_end);
926
927 walk_page_range(vma->vm_mm, start, end, &hmm_walk_ops,
928 &hmm_vma_walk);
929
930 do {
931 ret = walk_page_range(vma->vm_mm, start, end,
932 &hmm_walk_ops, &hmm_vma_walk);
933 start = hmm_vma_walk.last;
934
935 /* Keep trying while the range is valid. */
936 } while (ret == -EBUSY && range->valid);
937
938 if (ret) {
939 unsigned long i;
940
941 i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
942 hmm_pfns_clear(range, &range->pfns[i],
943 hmm_vma_walk.last, range->end);
944 return ret;
945 }
946 start = end;
947
948 } while (start < range->end);
949
950 return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
951}
952EXPORT_SYMBOL(hmm_range_fault);
953
954/**
955 * hmm_range_dma_map - hmm_range_fault() and dma map page all in one.
956 * @range: range being faulted
957 * @device: device to map page to
958 * @daddrs: array of dma addresses for the mapped pages
959 * @flags: HMM_FAULT_*
960 *
961 * Return: the number of pages mapped on success (including zero), or any
962 * status return from hmm_range_fault() otherwise.
963 */
964long hmm_range_dma_map(struct hmm_range *range, struct device *device,
965 dma_addr_t *daddrs, unsigned int flags)
966{
967 unsigned long i, npages, mapped;
968 long ret;
969
970 ret = hmm_range_fault(range, flags);
971 if (ret <= 0)
972 return ret ? ret : -EBUSY;
973
974 npages = (range->end - range->start) >> PAGE_SHIFT;
975 for (i = 0, mapped = 0; i < npages; ++i) {
976 enum dma_data_direction dir = DMA_TO_DEVICE;
977 struct page *page;
978
979 /*
980 * FIXME need to update DMA API to provide invalid DMA address
981 * value instead of a function to test dma address value. This
982 * would remove lot of dumb code duplicated accross many arch.
983 *
984 * For now setting it to 0 here is good enough as the pfns[]
985 * value is what is use to check what is valid and what isn't.
986 */
987 daddrs[i] = 0;
988
989 page = hmm_device_entry_to_page(range, range->pfns[i]);
990 if (page == NULL)
991 continue;
992
993 /* Check if range is being invalidated */
994 if (!range->valid) {
995 ret = -EBUSY;
996 goto unmap;
997 }
998
999 /* If it is read and write than map bi-directional. */
1000 if (range->pfns[i] & range->flags[HMM_PFN_WRITE])
1001 dir = DMA_BIDIRECTIONAL;
1002
1003 daddrs[i] = dma_map_page(device, page, 0, PAGE_SIZE, dir);
1004 if (dma_mapping_error(device, daddrs[i])) {
1005 ret = -EFAULT;
1006 goto unmap;
1007 }
1008
1009 mapped++;
1010 }
1011
1012 return mapped;
1013
1014unmap:
1015 for (npages = i, i = 0; (i < npages) && mapped; ++i) {
1016 enum dma_data_direction dir = DMA_TO_DEVICE;
1017 struct page *page;
1018
1019 page = hmm_device_entry_to_page(range, range->pfns[i]);
1020 if (page == NULL)
1021 continue;
1022
1023 if (dma_mapping_error(device, daddrs[i]))
1024 continue;
1025
1026 /* If it is read and write than map bi-directional. */
1027 if (range->pfns[i] & range->flags[HMM_PFN_WRITE])
1028 dir = DMA_BIDIRECTIONAL;
1029
1030 dma_unmap_page(device, daddrs[i], PAGE_SIZE, dir);
1031 mapped--;
1032 }
1033
1034 return ret;
1035}
1036EXPORT_SYMBOL(hmm_range_dma_map);
1037
1038/**
1039 * hmm_range_dma_unmap() - unmap range of that was map with hmm_range_dma_map()
1040 * @range: range being unmapped
1041 * @device: device against which dma map was done
1042 * @daddrs: dma address of mapped pages
1043 * @dirty: dirty page if it had the write flag set
1044 * Return: number of page unmapped on success, -EINVAL otherwise
1045 *
1046 * Note that caller MUST abide by mmu notifier or use HMM mirror and abide
1047 * to the sync_cpu_device_pagetables() callback so that it is safe here to
1048 * call set_page_dirty(). Caller must also take appropriate locks to avoid
1049 * concurrent mmu notifier or sync_cpu_device_pagetables() to make progress.
1050 */
1051long hmm_range_dma_unmap(struct hmm_range *range,
1052 struct device *device,
1053 dma_addr_t *daddrs,
1054 bool dirty)
1055{
1056 unsigned long i, npages;
1057 long cpages = 0;
1058
1059 /* Sanity check. */
1060 if (range->end <= range->start)
1061 return -EINVAL;
1062 if (!daddrs)
1063 return -EINVAL;
1064 if (!range->pfns)
1065 return -EINVAL;
1066
1067 npages = (range->end - range->start) >> PAGE_SHIFT;
1068 for (i = 0; i < npages; ++i) {
1069 enum dma_data_direction dir = DMA_TO_DEVICE;
1070 struct page *page;
1071
1072 page = hmm_device_entry_to_page(range, range->pfns[i]);
1073 if (page == NULL)
1074 continue;
1075
1076 /* If it is read and write than map bi-directional. */
1077 if (range->pfns[i] & range->flags[HMM_PFN_WRITE]) {
1078 dir = DMA_BIDIRECTIONAL;
1079
1080 /*
1081 * See comments in function description on why it is
1082 * safe here to call set_page_dirty()
1083 */
1084 if (dirty)
1085 set_page_dirty(page);
1086 }
1087
1088 /* Unmap and clear pfns/dma address */
1089 dma_unmap_page(device, daddrs[i], PAGE_SIZE, dir);
1090 range->pfns[i] = range->values[HMM_PFN_NONE];
1091 /* FIXME see comments in hmm_vma_dma_map() */
1092 daddrs[i] = 0;
1093 cpages++;
1094 }
1095
1096 return cpages;
1097}
1098EXPORT_SYMBOL(hmm_range_dma_unmap);
1/*
2 * Copyright 2013 Red Hat Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Authors: Jérôme Glisse <jglisse@redhat.com>
15 */
16/*
17 * Refer to include/linux/hmm.h for information about heterogeneous memory
18 * management or HMM for short.
19 */
20#include <linux/mm.h>
21#include <linux/hmm.h>
22#include <linux/init.h>
23#include <linux/rmap.h>
24#include <linux/swap.h>
25#include <linux/slab.h>
26#include <linux/sched.h>
27#include <linux/mmzone.h>
28#include <linux/pagemap.h>
29#include <linux/swapops.h>
30#include <linux/hugetlb.h>
31#include <linux/memremap.h>
32#include <linux/jump_label.h>
33#include <linux/mmu_notifier.h>
34#include <linux/memory_hotplug.h>
35
36#define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
37
38#if defined(CONFIG_DEVICE_PRIVATE) || defined(CONFIG_DEVICE_PUBLIC)
39/*
40 * Device private memory see HMM (Documentation/vm/hmm.txt) or hmm.h
41 */
42DEFINE_STATIC_KEY_FALSE(device_private_key);
43EXPORT_SYMBOL(device_private_key);
44#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */
45
46
47#if IS_ENABLED(CONFIG_HMM_MIRROR)
48static const struct mmu_notifier_ops hmm_mmu_notifier_ops;
49
50/*
51 * struct hmm - HMM per mm struct
52 *
53 * @mm: mm struct this HMM struct is bound to
54 * @lock: lock protecting ranges list
55 * @sequence: we track updates to the CPU page table with a sequence number
56 * @ranges: list of range being snapshotted
57 * @mirrors: list of mirrors for this mm
58 * @mmu_notifier: mmu notifier to track updates to CPU page table
59 * @mirrors_sem: read/write semaphore protecting the mirrors list
60 */
61struct hmm {
62 struct mm_struct *mm;
63 spinlock_t lock;
64 atomic_t sequence;
65 struct list_head ranges;
66 struct list_head mirrors;
67 struct mmu_notifier mmu_notifier;
68 struct rw_semaphore mirrors_sem;
69};
70
71/*
72 * hmm_register - register HMM against an mm (HMM internal)
73 *
74 * @mm: mm struct to attach to
75 *
76 * This is not intended to be used directly by device drivers. It allocates an
77 * HMM struct if mm does not have one, and initializes it.
78 */
79static struct hmm *hmm_register(struct mm_struct *mm)
80{
81 struct hmm *hmm = READ_ONCE(mm->hmm);
82 bool cleanup = false;
83
84 /*
85 * The hmm struct can only be freed once the mm_struct goes away,
86 * hence we should always have pre-allocated an new hmm struct
87 * above.
88 */
89 if (hmm)
90 return hmm;
91
92 hmm = kmalloc(sizeof(*hmm), GFP_KERNEL);
93 if (!hmm)
94 return NULL;
95 INIT_LIST_HEAD(&hmm->mirrors);
96 init_rwsem(&hmm->mirrors_sem);
97 atomic_set(&hmm->sequence, 0);
98 hmm->mmu_notifier.ops = NULL;
99 INIT_LIST_HEAD(&hmm->ranges);
100 spin_lock_init(&hmm->lock);
101 hmm->mm = mm;
102
103 /*
104 * We should only get here if hold the mmap_sem in write mode ie on
105 * registration of first mirror through hmm_mirror_register()
106 */
107 hmm->mmu_notifier.ops = &hmm_mmu_notifier_ops;
108 if (__mmu_notifier_register(&hmm->mmu_notifier, mm)) {
109 kfree(hmm);
110 return NULL;
111 }
112
113 spin_lock(&mm->page_table_lock);
114 if (!mm->hmm)
115 mm->hmm = hmm;
116 else
117 cleanup = true;
118 spin_unlock(&mm->page_table_lock);
119
120 if (cleanup) {
121 mmu_notifier_unregister(&hmm->mmu_notifier, mm);
122 kfree(hmm);
123 }
124
125 return mm->hmm;
126}
127
128void hmm_mm_destroy(struct mm_struct *mm)
129{
130 kfree(mm->hmm);
131}
132
133static void hmm_invalidate_range(struct hmm *hmm,
134 enum hmm_update_type action,
135 unsigned long start,
136 unsigned long end)
137{
138 struct hmm_mirror *mirror;
139 struct hmm_range *range;
140
141 spin_lock(&hmm->lock);
142 list_for_each_entry(range, &hmm->ranges, list) {
143 unsigned long addr, idx, npages;
144
145 if (end < range->start || start >= range->end)
146 continue;
147
148 range->valid = false;
149 addr = max(start, range->start);
150 idx = (addr - range->start) >> PAGE_SHIFT;
151 npages = (min(range->end, end) - addr) >> PAGE_SHIFT;
152 memset(&range->pfns[idx], 0, sizeof(*range->pfns) * npages);
153 }
154 spin_unlock(&hmm->lock);
155
156 down_read(&hmm->mirrors_sem);
157 list_for_each_entry(mirror, &hmm->mirrors, list)
158 mirror->ops->sync_cpu_device_pagetables(mirror, action,
159 start, end);
160 up_read(&hmm->mirrors_sem);
161}
162
163static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm)
164{
165 struct hmm_mirror *mirror;
166 struct hmm *hmm = mm->hmm;
167
168 down_write(&hmm->mirrors_sem);
169 mirror = list_first_entry_or_null(&hmm->mirrors, struct hmm_mirror,
170 list);
171 while (mirror) {
172 list_del_init(&mirror->list);
173 if (mirror->ops->release) {
174 /*
175 * Drop mirrors_sem so callback can wait on any pending
176 * work that might itself trigger mmu_notifier callback
177 * and thus would deadlock with us.
178 */
179 up_write(&hmm->mirrors_sem);
180 mirror->ops->release(mirror);
181 down_write(&hmm->mirrors_sem);
182 }
183 mirror = list_first_entry_or_null(&hmm->mirrors,
184 struct hmm_mirror, list);
185 }
186 up_write(&hmm->mirrors_sem);
187}
188
189static void hmm_invalidate_range_start(struct mmu_notifier *mn,
190 struct mm_struct *mm,
191 unsigned long start,
192 unsigned long end)
193{
194 struct hmm *hmm = mm->hmm;
195
196 VM_BUG_ON(!hmm);
197
198 atomic_inc(&hmm->sequence);
199}
200
201static void hmm_invalidate_range_end(struct mmu_notifier *mn,
202 struct mm_struct *mm,
203 unsigned long start,
204 unsigned long end)
205{
206 struct hmm *hmm = mm->hmm;
207
208 VM_BUG_ON(!hmm);
209
210 hmm_invalidate_range(mm->hmm, HMM_UPDATE_INVALIDATE, start, end);
211}
212
213static const struct mmu_notifier_ops hmm_mmu_notifier_ops = {
214 .release = hmm_release,
215 .invalidate_range_start = hmm_invalidate_range_start,
216 .invalidate_range_end = hmm_invalidate_range_end,
217};
218
219/*
220 * hmm_mirror_register() - register a mirror against an mm
221 *
222 * @mirror: new mirror struct to register
223 * @mm: mm to register against
224 *
225 * To start mirroring a process address space, the device driver must register
226 * an HMM mirror struct.
227 *
228 * THE mm->mmap_sem MUST BE HELD IN WRITE MODE !
229 */
230int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm)
231{
232 /* Sanity check */
233 if (!mm || !mirror || !mirror->ops)
234 return -EINVAL;
235
236again:
237 mirror->hmm = hmm_register(mm);
238 if (!mirror->hmm)
239 return -ENOMEM;
240
241 down_write(&mirror->hmm->mirrors_sem);
242 if (mirror->hmm->mm == NULL) {
243 /*
244 * A racing hmm_mirror_unregister() is about to destroy the hmm
245 * struct. Try again to allocate a new one.
246 */
247 up_write(&mirror->hmm->mirrors_sem);
248 mirror->hmm = NULL;
249 goto again;
250 } else {
251 list_add(&mirror->list, &mirror->hmm->mirrors);
252 up_write(&mirror->hmm->mirrors_sem);
253 }
254
255 return 0;
256}
257EXPORT_SYMBOL(hmm_mirror_register);
258
259/*
260 * hmm_mirror_unregister() - unregister a mirror
261 *
262 * @mirror: new mirror struct to register
263 *
264 * Stop mirroring a process address space, and cleanup.
265 */
266void hmm_mirror_unregister(struct hmm_mirror *mirror)
267{
268 bool should_unregister = false;
269 struct mm_struct *mm;
270 struct hmm *hmm;
271
272 if (mirror->hmm == NULL)
273 return;
274
275 hmm = mirror->hmm;
276 down_write(&hmm->mirrors_sem);
277 list_del_init(&mirror->list);
278 should_unregister = list_empty(&hmm->mirrors);
279 mirror->hmm = NULL;
280 mm = hmm->mm;
281 hmm->mm = NULL;
282 up_write(&hmm->mirrors_sem);
283
284 if (!should_unregister || mm == NULL)
285 return;
286
287 spin_lock(&mm->page_table_lock);
288 if (mm->hmm == hmm)
289 mm->hmm = NULL;
290 spin_unlock(&mm->page_table_lock);
291
292 mmu_notifier_unregister_no_release(&hmm->mmu_notifier, mm);
293 kfree(hmm);
294}
295EXPORT_SYMBOL(hmm_mirror_unregister);
296
297struct hmm_vma_walk {
298 struct hmm_range *range;
299 unsigned long last;
300 bool fault;
301 bool block;
302};
303
304static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,
305 bool write_fault, uint64_t *pfn)
306{
307 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_REMOTE;
308 struct hmm_vma_walk *hmm_vma_walk = walk->private;
309 struct hmm_range *range = hmm_vma_walk->range;
310 struct vm_area_struct *vma = walk->vma;
311 int r;
312
313 flags |= hmm_vma_walk->block ? 0 : FAULT_FLAG_ALLOW_RETRY;
314 flags |= write_fault ? FAULT_FLAG_WRITE : 0;
315 r = handle_mm_fault(vma, addr, flags);
316 if (r & VM_FAULT_RETRY)
317 return -EBUSY;
318 if (r & VM_FAULT_ERROR) {
319 *pfn = range->values[HMM_PFN_ERROR];
320 return -EFAULT;
321 }
322
323 return -EAGAIN;
324}
325
326static int hmm_pfns_bad(unsigned long addr,
327 unsigned long end,
328 struct mm_walk *walk)
329{
330 struct hmm_vma_walk *hmm_vma_walk = walk->private;
331 struct hmm_range *range = hmm_vma_walk->range;
332 uint64_t *pfns = range->pfns;
333 unsigned long i;
334
335 i = (addr - range->start) >> PAGE_SHIFT;
336 for (; addr < end; addr += PAGE_SIZE, i++)
337 pfns[i] = range->values[HMM_PFN_ERROR];
338
339 return 0;
340}
341
342/*
343 * hmm_vma_walk_hole() - handle a range lacking valid pmd or pte(s)
344 * @start: range virtual start address (inclusive)
345 * @end: range virtual end address (exclusive)
346 * @fault: should we fault or not ?
347 * @write_fault: write fault ?
348 * @walk: mm_walk structure
349 * Returns: 0 on success, -EAGAIN after page fault, or page fault error
350 *
351 * This function will be called whenever pmd_none() or pte_none() returns true,
352 * or whenever there is no page directory covering the virtual address range.
353 */
354static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end,
355 bool fault, bool write_fault,
356 struct mm_walk *walk)
357{
358 struct hmm_vma_walk *hmm_vma_walk = walk->private;
359 struct hmm_range *range = hmm_vma_walk->range;
360 uint64_t *pfns = range->pfns;
361 unsigned long i;
362
363 hmm_vma_walk->last = addr;
364 i = (addr - range->start) >> PAGE_SHIFT;
365 for (; addr < end; addr += PAGE_SIZE, i++) {
366 pfns[i] = range->values[HMM_PFN_NONE];
367 if (fault || write_fault) {
368 int ret;
369
370 ret = hmm_vma_do_fault(walk, addr, write_fault,
371 &pfns[i]);
372 if (ret != -EAGAIN)
373 return ret;
374 }
375 }
376
377 return (fault || write_fault) ? -EAGAIN : 0;
378}
379
380static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
381 uint64_t pfns, uint64_t cpu_flags,
382 bool *fault, bool *write_fault)
383{
384 struct hmm_range *range = hmm_vma_walk->range;
385
386 *fault = *write_fault = false;
387 if (!hmm_vma_walk->fault)
388 return;
389
390 /* We aren't ask to do anything ... */
391 if (!(pfns & range->flags[HMM_PFN_VALID]))
392 return;
393 /* If this is device memory than only fault if explicitly requested */
394 if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
395 /* Do we fault on device memory ? */
396 if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) {
397 *write_fault = pfns & range->flags[HMM_PFN_WRITE];
398 *fault = true;
399 }
400 return;
401 }
402
403 /* If CPU page table is not valid then we need to fault */
404 *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);
405 /* Need to write fault ? */
406 if ((pfns & range->flags[HMM_PFN_WRITE]) &&
407 !(cpu_flags & range->flags[HMM_PFN_WRITE])) {
408 *write_fault = true;
409 *fault = true;
410 }
411}
412
413static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
414 const uint64_t *pfns, unsigned long npages,
415 uint64_t cpu_flags, bool *fault,
416 bool *write_fault)
417{
418 unsigned long i;
419
420 if (!hmm_vma_walk->fault) {
421 *fault = *write_fault = false;
422 return;
423 }
424
425 for (i = 0; i < npages; ++i) {
426 hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags,
427 fault, write_fault);
428 if ((*fault) || (*write_fault))
429 return;
430 }
431}
432
433static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
434 struct mm_walk *walk)
435{
436 struct hmm_vma_walk *hmm_vma_walk = walk->private;
437 struct hmm_range *range = hmm_vma_walk->range;
438 bool fault, write_fault;
439 unsigned long i, npages;
440 uint64_t *pfns;
441
442 i = (addr - range->start) >> PAGE_SHIFT;
443 npages = (end - addr) >> PAGE_SHIFT;
444 pfns = &range->pfns[i];
445 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
446 0, &fault, &write_fault);
447 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
448}
449
450static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)
451{
452 if (pmd_protnone(pmd))
453 return 0;
454 return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |
455 range->flags[HMM_PFN_WRITE] :
456 range->flags[HMM_PFN_VALID];
457}
458
459static int hmm_vma_handle_pmd(struct mm_walk *walk,
460 unsigned long addr,
461 unsigned long end,
462 uint64_t *pfns,
463 pmd_t pmd)
464{
465 struct hmm_vma_walk *hmm_vma_walk = walk->private;
466 struct hmm_range *range = hmm_vma_walk->range;
467 unsigned long pfn, npages, i;
468 bool fault, write_fault;
469 uint64_t cpu_flags;
470
471 npages = (end - addr) >> PAGE_SHIFT;
472 cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
473 hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags,
474 &fault, &write_fault);
475
476 if (pmd_protnone(pmd) || fault || write_fault)
477 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
478
479 pfn = pmd_pfn(pmd) + pte_index(addr);
480 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
481 pfns[i] = hmm_pfn_from_pfn(range, pfn) | cpu_flags;
482 hmm_vma_walk->last = end;
483 return 0;
484}
485
486static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)
487{
488 if (pte_none(pte) || !pte_present(pte))
489 return 0;
490 return pte_write(pte) ? range->flags[HMM_PFN_VALID] |
491 range->flags[HMM_PFN_WRITE] :
492 range->flags[HMM_PFN_VALID];
493}
494
495static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
496 unsigned long end, pmd_t *pmdp, pte_t *ptep,
497 uint64_t *pfn)
498{
499 struct hmm_vma_walk *hmm_vma_walk = walk->private;
500 struct hmm_range *range = hmm_vma_walk->range;
501 struct vm_area_struct *vma = walk->vma;
502 bool fault, write_fault;
503 uint64_t cpu_flags;
504 pte_t pte = *ptep;
505 uint64_t orig_pfn = *pfn;
506
507 *pfn = range->values[HMM_PFN_NONE];
508 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
509 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
510 &fault, &write_fault);
511
512 if (pte_none(pte)) {
513 if (fault || write_fault)
514 goto fault;
515 return 0;
516 }
517
518 if (!pte_present(pte)) {
519 swp_entry_t entry = pte_to_swp_entry(pte);
520
521 if (!non_swap_entry(entry)) {
522 if (fault || write_fault)
523 goto fault;
524 return 0;
525 }
526
527 /*
528 * This is a special swap entry, ignore migration, use
529 * device and report anything else as error.
530 */
531 if (is_device_private_entry(entry)) {
532 cpu_flags = range->flags[HMM_PFN_VALID] |
533 range->flags[HMM_PFN_DEVICE_PRIVATE];
534 cpu_flags |= is_write_device_private_entry(entry) ?
535 range->flags[HMM_PFN_WRITE] : 0;
536 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
537 &fault, &write_fault);
538 if (fault || write_fault)
539 goto fault;
540 *pfn = hmm_pfn_from_pfn(range, swp_offset(entry));
541 *pfn |= cpu_flags;
542 return 0;
543 }
544
545 if (is_migration_entry(entry)) {
546 if (fault || write_fault) {
547 pte_unmap(ptep);
548 hmm_vma_walk->last = addr;
549 migration_entry_wait(vma->vm_mm,
550 pmdp, addr);
551 return -EAGAIN;
552 }
553 return 0;
554 }
555
556 /* Report error for everything else */
557 *pfn = range->values[HMM_PFN_ERROR];
558 return -EFAULT;
559 }
560
561 if (fault || write_fault)
562 goto fault;
563
564 *pfn = hmm_pfn_from_pfn(range, pte_pfn(pte)) | cpu_flags;
565 return 0;
566
567fault:
568 pte_unmap(ptep);
569 /* Fault any virtual address we were asked to fault */
570 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
571}
572
573static int hmm_vma_walk_pmd(pmd_t *pmdp,
574 unsigned long start,
575 unsigned long end,
576 struct mm_walk *walk)
577{
578 struct hmm_vma_walk *hmm_vma_walk = walk->private;
579 struct hmm_range *range = hmm_vma_walk->range;
580 uint64_t *pfns = range->pfns;
581 unsigned long addr = start, i;
582 pte_t *ptep;
583
584 i = (addr - range->start) >> PAGE_SHIFT;
585
586again:
587 if (pmd_none(*pmdp))
588 return hmm_vma_walk_hole(start, end, walk);
589
590 if (pmd_huge(*pmdp) && (range->vma->vm_flags & VM_HUGETLB))
591 return hmm_pfns_bad(start, end, walk);
592
593 if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
594 pmd_t pmd;
595
596 /*
597 * No need to take pmd_lock here, even if some other threads
598 * is splitting the huge pmd we will get that event through
599 * mmu_notifier callback.
600 *
601 * So just read pmd value and check again its a transparent
602 * huge or device mapping one and compute corresponding pfn
603 * values.
604 */
605 pmd = pmd_read_atomic(pmdp);
606 barrier();
607 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
608 goto again;
609
610 return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
611 }
612
613 if (pmd_bad(*pmdp))
614 return hmm_pfns_bad(start, end, walk);
615
616 ptep = pte_offset_map(pmdp, addr);
617 for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
618 int r;
619
620 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
621 if (r) {
622 /* hmm_vma_handle_pte() did unmap pte directory */
623 hmm_vma_walk->last = addr;
624 return r;
625 }
626 }
627 pte_unmap(ptep - 1);
628
629 hmm_vma_walk->last = addr;
630 return 0;
631}
632
633static void hmm_pfns_clear(struct hmm_range *range,
634 uint64_t *pfns,
635 unsigned long addr,
636 unsigned long end)
637{
638 for (; addr < end; addr += PAGE_SIZE, pfns++)
639 *pfns = range->values[HMM_PFN_NONE];
640}
641
642static void hmm_pfns_special(struct hmm_range *range)
643{
644 unsigned long addr = range->start, i = 0;
645
646 for (; addr < range->end; addr += PAGE_SIZE, i++)
647 range->pfns[i] = range->values[HMM_PFN_SPECIAL];
648}
649
650/*
651 * hmm_vma_get_pfns() - snapshot CPU page table for a range of virtual addresses
652 * @range: range being snapshotted
653 * Returns: -EINVAL if invalid argument, -ENOMEM out of memory, -EPERM invalid
654 * vma permission, 0 success
655 *
656 * This snapshots the CPU page table for a range of virtual addresses. Snapshot
657 * validity is tracked by range struct. See hmm_vma_range_done() for further
658 * information.
659 *
660 * The range struct is initialized here. It tracks the CPU page table, but only
661 * if the function returns success (0), in which case the caller must then call
662 * hmm_vma_range_done() to stop CPU page table update tracking on this range.
663 *
664 * NOT CALLING hmm_vma_range_done() IF FUNCTION RETURNS 0 WILL LEAD TO SERIOUS
665 * MEMORY CORRUPTION ! YOU HAVE BEEN WARNED !
666 */
667int hmm_vma_get_pfns(struct hmm_range *range)
668{
669 struct vm_area_struct *vma = range->vma;
670 struct hmm_vma_walk hmm_vma_walk;
671 struct mm_walk mm_walk;
672 struct hmm *hmm;
673
674 /* Sanity check, this really should not happen ! */
675 if (range->start < vma->vm_start || range->start >= vma->vm_end)
676 return -EINVAL;
677 if (range->end < vma->vm_start || range->end > vma->vm_end)
678 return -EINVAL;
679
680 hmm = hmm_register(vma->vm_mm);
681 if (!hmm)
682 return -ENOMEM;
683 /* Caller must have registered a mirror, via hmm_mirror_register() ! */
684 if (!hmm->mmu_notifier.ops)
685 return -EINVAL;
686
687 /* FIXME support hugetlb fs */
688 if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) {
689 hmm_pfns_special(range);
690 return -EINVAL;
691 }
692
693 if (!(vma->vm_flags & VM_READ)) {
694 /*
695 * If vma do not allow read access, then assume that it does
696 * not allow write access, either. Architecture that allow
697 * write without read access are not supported by HMM, because
698 * operations such has atomic access would not work.
699 */
700 hmm_pfns_clear(range, range->pfns, range->start, range->end);
701 return -EPERM;
702 }
703
704 /* Initialize range to track CPU page table update */
705 spin_lock(&hmm->lock);
706 range->valid = true;
707 list_add_rcu(&range->list, &hmm->ranges);
708 spin_unlock(&hmm->lock);
709
710 hmm_vma_walk.fault = false;
711 hmm_vma_walk.range = range;
712 mm_walk.private = &hmm_vma_walk;
713
714 mm_walk.vma = vma;
715 mm_walk.mm = vma->vm_mm;
716 mm_walk.pte_entry = NULL;
717 mm_walk.test_walk = NULL;
718 mm_walk.hugetlb_entry = NULL;
719 mm_walk.pmd_entry = hmm_vma_walk_pmd;
720 mm_walk.pte_hole = hmm_vma_walk_hole;
721
722 walk_page_range(range->start, range->end, &mm_walk);
723 return 0;
724}
725EXPORT_SYMBOL(hmm_vma_get_pfns);
726
727/*
728 * hmm_vma_range_done() - stop tracking change to CPU page table over a range
729 * @range: range being tracked
730 * Returns: false if range data has been invalidated, true otherwise
731 *
732 * Range struct is used to track updates to the CPU page table after a call to
733 * either hmm_vma_get_pfns() or hmm_vma_fault(). Once the device driver is done
734 * using the data, or wants to lock updates to the data it got from those
735 * functions, it must call the hmm_vma_range_done() function, which will then
736 * stop tracking CPU page table updates.
737 *
738 * Note that device driver must still implement general CPU page table update
739 * tracking either by using hmm_mirror (see hmm_mirror_register()) or by using
740 * the mmu_notifier API directly.
741 *
742 * CPU page table update tracking done through hmm_range is only temporary and
743 * to be used while trying to duplicate CPU page table contents for a range of
744 * virtual addresses.
745 *
746 * There are two ways to use this :
747 * again:
748 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
749 * trans = device_build_page_table_update_transaction(pfns);
750 * device_page_table_lock();
751 * if (!hmm_vma_range_done(range)) {
752 * device_page_table_unlock();
753 * goto again;
754 * }
755 * device_commit_transaction(trans);
756 * device_page_table_unlock();
757 *
758 * Or:
759 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
760 * device_page_table_lock();
761 * hmm_vma_range_done(range);
762 * device_update_page_table(range->pfns);
763 * device_page_table_unlock();
764 */
765bool hmm_vma_range_done(struct hmm_range *range)
766{
767 unsigned long npages = (range->end - range->start) >> PAGE_SHIFT;
768 struct hmm *hmm;
769
770 if (range->end <= range->start) {
771 BUG();
772 return false;
773 }
774
775 hmm = hmm_register(range->vma->vm_mm);
776 if (!hmm) {
777 memset(range->pfns, 0, sizeof(*range->pfns) * npages);
778 return false;
779 }
780
781 spin_lock(&hmm->lock);
782 list_del_rcu(&range->list);
783 spin_unlock(&hmm->lock);
784
785 return range->valid;
786}
787EXPORT_SYMBOL(hmm_vma_range_done);
788
789/*
790 * hmm_vma_fault() - try to fault some address in a virtual address range
791 * @range: range being faulted
792 * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem)
793 * Returns: 0 success, error otherwise (-EAGAIN means mmap_sem have been drop)
794 *
795 * This is similar to a regular CPU page fault except that it will not trigger
796 * any memory migration if the memory being faulted is not accessible by CPUs.
797 *
798 * On error, for one virtual address in the range, the function will mark the
799 * corresponding HMM pfn entry with an error flag.
800 *
801 * Expected use pattern:
802 * retry:
803 * down_read(&mm->mmap_sem);
804 * // Find vma and address device wants to fault, initialize hmm_pfn_t
805 * // array accordingly
806 * ret = hmm_vma_fault(range, write, block);
807 * switch (ret) {
808 * case -EAGAIN:
809 * hmm_vma_range_done(range);
810 * // You might want to rate limit or yield to play nicely, you may
811 * // also commit any valid pfn in the array assuming that you are
812 * // getting true from hmm_vma_range_monitor_end()
813 * goto retry;
814 * case 0:
815 * break;
816 * case -ENOMEM:
817 * case -EINVAL:
818 * case -EPERM:
819 * default:
820 * // Handle error !
821 * up_read(&mm->mmap_sem)
822 * return;
823 * }
824 * // Take device driver lock that serialize device page table update
825 * driver_lock_device_page_table_update();
826 * hmm_vma_range_done(range);
827 * // Commit pfns we got from hmm_vma_fault()
828 * driver_unlock_device_page_table_update();
829 * up_read(&mm->mmap_sem)
830 *
831 * YOU MUST CALL hmm_vma_range_done() AFTER THIS FUNCTION RETURN SUCCESS (0)
832 * BEFORE FREEING THE range struct OR YOU WILL HAVE SERIOUS MEMORY CORRUPTION !
833 *
834 * YOU HAVE BEEN WARNED !
835 */
836int hmm_vma_fault(struct hmm_range *range, bool block)
837{
838 struct vm_area_struct *vma = range->vma;
839 unsigned long start = range->start;
840 struct hmm_vma_walk hmm_vma_walk;
841 struct mm_walk mm_walk;
842 struct hmm *hmm;
843 int ret;
844
845 /* Sanity check, this really should not happen ! */
846 if (range->start < vma->vm_start || range->start >= vma->vm_end)
847 return -EINVAL;
848 if (range->end < vma->vm_start || range->end > vma->vm_end)
849 return -EINVAL;
850
851 hmm = hmm_register(vma->vm_mm);
852 if (!hmm) {
853 hmm_pfns_clear(range, range->pfns, range->start, range->end);
854 return -ENOMEM;
855 }
856 /* Caller must have registered a mirror using hmm_mirror_register() */
857 if (!hmm->mmu_notifier.ops)
858 return -EINVAL;
859
860 /* FIXME support hugetlb fs */
861 if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) {
862 hmm_pfns_special(range);
863 return -EINVAL;
864 }
865
866 if (!(vma->vm_flags & VM_READ)) {
867 /*
868 * If vma do not allow read access, then assume that it does
869 * not allow write access, either. Architecture that allow
870 * write without read access are not supported by HMM, because
871 * operations such has atomic access would not work.
872 */
873 hmm_pfns_clear(range, range->pfns, range->start, range->end);
874 return -EPERM;
875 }
876
877 /* Initialize range to track CPU page table update */
878 spin_lock(&hmm->lock);
879 range->valid = true;
880 list_add_rcu(&range->list, &hmm->ranges);
881 spin_unlock(&hmm->lock);
882
883 hmm_vma_walk.fault = true;
884 hmm_vma_walk.block = block;
885 hmm_vma_walk.range = range;
886 mm_walk.private = &hmm_vma_walk;
887 hmm_vma_walk.last = range->start;
888
889 mm_walk.vma = vma;
890 mm_walk.mm = vma->vm_mm;
891 mm_walk.pte_entry = NULL;
892 mm_walk.test_walk = NULL;
893 mm_walk.hugetlb_entry = NULL;
894 mm_walk.pmd_entry = hmm_vma_walk_pmd;
895 mm_walk.pte_hole = hmm_vma_walk_hole;
896
897 do {
898 ret = walk_page_range(start, range->end, &mm_walk);
899 start = hmm_vma_walk.last;
900 } while (ret == -EAGAIN);
901
902 if (ret) {
903 unsigned long i;
904
905 i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
906 hmm_pfns_clear(range, &range->pfns[i], hmm_vma_walk.last,
907 range->end);
908 hmm_vma_range_done(range);
909 }
910 return ret;
911}
912EXPORT_SYMBOL(hmm_vma_fault);
913#endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
914
915
916#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
917struct page *hmm_vma_alloc_locked_page(struct vm_area_struct *vma,
918 unsigned long addr)
919{
920 struct page *page;
921
922 page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
923 if (!page)
924 return NULL;
925 lock_page(page);
926 return page;
927}
928EXPORT_SYMBOL(hmm_vma_alloc_locked_page);
929
930
931static void hmm_devmem_ref_release(struct percpu_ref *ref)
932{
933 struct hmm_devmem *devmem;
934
935 devmem = container_of(ref, struct hmm_devmem, ref);
936 complete(&devmem->completion);
937}
938
939static void hmm_devmem_ref_exit(void *data)
940{
941 struct percpu_ref *ref = data;
942 struct hmm_devmem *devmem;
943
944 devmem = container_of(ref, struct hmm_devmem, ref);
945 percpu_ref_exit(ref);
946 devm_remove_action(devmem->device, &hmm_devmem_ref_exit, data);
947}
948
949static void hmm_devmem_ref_kill(void *data)
950{
951 struct percpu_ref *ref = data;
952 struct hmm_devmem *devmem;
953
954 devmem = container_of(ref, struct hmm_devmem, ref);
955 percpu_ref_kill(ref);
956 wait_for_completion(&devmem->completion);
957 devm_remove_action(devmem->device, &hmm_devmem_ref_kill, data);
958}
959
960static int hmm_devmem_fault(struct vm_area_struct *vma,
961 unsigned long addr,
962 const struct page *page,
963 unsigned int flags,
964 pmd_t *pmdp)
965{
966 struct hmm_devmem *devmem = page->pgmap->data;
967
968 return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);
969}
970
971static void hmm_devmem_free(struct page *page, void *data)
972{
973 struct hmm_devmem *devmem = data;
974
975 devmem->ops->free(devmem, page);
976}
977
978static DEFINE_MUTEX(hmm_devmem_lock);
979static RADIX_TREE(hmm_devmem_radix, GFP_KERNEL);
980
981static void hmm_devmem_radix_release(struct resource *resource)
982{
983 resource_size_t key, align_start, align_size;
984
985 align_start = resource->start & ~(PA_SECTION_SIZE - 1);
986 align_size = ALIGN(resource_size(resource), PA_SECTION_SIZE);
987
988 mutex_lock(&hmm_devmem_lock);
989 for (key = resource->start;
990 key <= resource->end;
991 key += PA_SECTION_SIZE)
992 radix_tree_delete(&hmm_devmem_radix, key >> PA_SECTION_SHIFT);
993 mutex_unlock(&hmm_devmem_lock);
994}
995
996static void hmm_devmem_release(struct device *dev, void *data)
997{
998 struct hmm_devmem *devmem = data;
999 struct resource *resource = devmem->resource;
1000 unsigned long start_pfn, npages;
1001 struct zone *zone;
1002 struct page *page;
1003
1004 if (percpu_ref_tryget_live(&devmem->ref)) {
1005 dev_WARN(dev, "%s: page mapping is still live!\n", __func__);
1006 percpu_ref_put(&devmem->ref);
1007 }
1008
1009 /* pages are dead and unused, undo the arch mapping */
1010 start_pfn = (resource->start & ~(PA_SECTION_SIZE - 1)) >> PAGE_SHIFT;
1011 npages = ALIGN(resource_size(resource), PA_SECTION_SIZE) >> PAGE_SHIFT;
1012
1013 page = pfn_to_page(start_pfn);
1014 zone = page_zone(page);
1015
1016 mem_hotplug_begin();
1017 if (resource->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY)
1018 __remove_pages(zone, start_pfn, npages, NULL);
1019 else
1020 arch_remove_memory(start_pfn << PAGE_SHIFT,
1021 npages << PAGE_SHIFT, NULL);
1022 mem_hotplug_done();
1023
1024 hmm_devmem_radix_release(resource);
1025}
1026
1027static int hmm_devmem_pages_create(struct hmm_devmem *devmem)
1028{
1029 resource_size_t key, align_start, align_size, align_end;
1030 struct device *device = devmem->device;
1031 int ret, nid, is_ram;
1032 unsigned long pfn;
1033
1034 align_start = devmem->resource->start & ~(PA_SECTION_SIZE - 1);
1035 align_size = ALIGN(devmem->resource->start +
1036 resource_size(devmem->resource),
1037 PA_SECTION_SIZE) - align_start;
1038
1039 is_ram = region_intersects(align_start, align_size,
1040 IORESOURCE_SYSTEM_RAM,
1041 IORES_DESC_NONE);
1042 if (is_ram == REGION_MIXED) {
1043 WARN_ONCE(1, "%s attempted on mixed region %pr\n",
1044 __func__, devmem->resource);
1045 return -ENXIO;
1046 }
1047 if (is_ram == REGION_INTERSECTS)
1048 return -ENXIO;
1049
1050 if (devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY)
1051 devmem->pagemap.type = MEMORY_DEVICE_PUBLIC;
1052 else
1053 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
1054
1055 devmem->pagemap.res = *devmem->resource;
1056 devmem->pagemap.page_fault = hmm_devmem_fault;
1057 devmem->pagemap.page_free = hmm_devmem_free;
1058 devmem->pagemap.dev = devmem->device;
1059 devmem->pagemap.ref = &devmem->ref;
1060 devmem->pagemap.data = devmem;
1061
1062 mutex_lock(&hmm_devmem_lock);
1063 align_end = align_start + align_size - 1;
1064 for (key = align_start; key <= align_end; key += PA_SECTION_SIZE) {
1065 struct hmm_devmem *dup;
1066
1067 dup = radix_tree_lookup(&hmm_devmem_radix,
1068 key >> PA_SECTION_SHIFT);
1069 if (dup) {
1070 dev_err(device, "%s: collides with mapping for %s\n",
1071 __func__, dev_name(dup->device));
1072 mutex_unlock(&hmm_devmem_lock);
1073 ret = -EBUSY;
1074 goto error;
1075 }
1076 ret = radix_tree_insert(&hmm_devmem_radix,
1077 key >> PA_SECTION_SHIFT,
1078 devmem);
1079 if (ret) {
1080 dev_err(device, "%s: failed: %d\n", __func__, ret);
1081 mutex_unlock(&hmm_devmem_lock);
1082 goto error_radix;
1083 }
1084 }
1085 mutex_unlock(&hmm_devmem_lock);
1086
1087 nid = dev_to_node(device);
1088 if (nid < 0)
1089 nid = numa_mem_id();
1090
1091 mem_hotplug_begin();
1092 /*
1093 * For device private memory we call add_pages() as we only need to
1094 * allocate and initialize struct page for the device memory. More-
1095 * over the device memory is un-accessible thus we do not want to
1096 * create a linear mapping for the memory like arch_add_memory()
1097 * would do.
1098 *
1099 * For device public memory, which is accesible by the CPU, we do
1100 * want the linear mapping and thus use arch_add_memory().
1101 */
1102 if (devmem->pagemap.type == MEMORY_DEVICE_PUBLIC)
1103 ret = arch_add_memory(nid, align_start, align_size, NULL,
1104 false);
1105 else
1106 ret = add_pages(nid, align_start >> PAGE_SHIFT,
1107 align_size >> PAGE_SHIFT, NULL, false);
1108 if (ret) {
1109 mem_hotplug_done();
1110 goto error_add_memory;
1111 }
1112 move_pfn_range_to_zone(&NODE_DATA(nid)->node_zones[ZONE_DEVICE],
1113 align_start >> PAGE_SHIFT,
1114 align_size >> PAGE_SHIFT, NULL);
1115 mem_hotplug_done();
1116
1117 for (pfn = devmem->pfn_first; pfn < devmem->pfn_last; pfn++) {
1118 struct page *page = pfn_to_page(pfn);
1119
1120 page->pgmap = &devmem->pagemap;
1121 }
1122 return 0;
1123
1124error_add_memory:
1125 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
1126error_radix:
1127 hmm_devmem_radix_release(devmem->resource);
1128error:
1129 return ret;
1130}
1131
1132static int hmm_devmem_match(struct device *dev, void *data, void *match_data)
1133{
1134 struct hmm_devmem *devmem = data;
1135
1136 return devmem->resource == match_data;
1137}
1138
1139static void hmm_devmem_pages_remove(struct hmm_devmem *devmem)
1140{
1141 devres_release(devmem->device, &hmm_devmem_release,
1142 &hmm_devmem_match, devmem->resource);
1143}
1144
1145/*
1146 * hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory
1147 *
1148 * @ops: memory event device driver callback (see struct hmm_devmem_ops)
1149 * @device: device struct to bind the resource too
1150 * @size: size in bytes of the device memory to add
1151 * Returns: pointer to new hmm_devmem struct ERR_PTR otherwise
1152 *
1153 * This function first finds an empty range of physical address big enough to
1154 * contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which
1155 * in turn allocates struct pages. It does not do anything beyond that; all
1156 * events affecting the memory will go through the various callbacks provided
1157 * by hmm_devmem_ops struct.
1158 *
1159 * Device driver should call this function during device initialization and
1160 * is then responsible of memory management. HMM only provides helpers.
1161 */
1162struct hmm_devmem *hmm_devmem_add(const struct hmm_devmem_ops *ops,
1163 struct device *device,
1164 unsigned long size)
1165{
1166 struct hmm_devmem *devmem;
1167 resource_size_t addr;
1168 int ret;
1169
1170 static_branch_enable(&device_private_key);
1171
1172 devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1173 GFP_KERNEL, dev_to_node(device));
1174 if (!devmem)
1175 return ERR_PTR(-ENOMEM);
1176
1177 init_completion(&devmem->completion);
1178 devmem->pfn_first = -1UL;
1179 devmem->pfn_last = -1UL;
1180 devmem->resource = NULL;
1181 devmem->device = device;
1182 devmem->ops = ops;
1183
1184 ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1185 0, GFP_KERNEL);
1186 if (ret)
1187 goto error_percpu_ref;
1188
1189 ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1190 if (ret)
1191 goto error_devm_add_action;
1192
1193 size = ALIGN(size, PA_SECTION_SIZE);
1194 addr = min((unsigned long)iomem_resource.end,
1195 (1UL << MAX_PHYSMEM_BITS) - 1);
1196 addr = addr - size + 1UL;
1197
1198 /*
1199 * FIXME add a new helper to quickly walk resource tree and find free
1200 * range
1201 *
1202 * FIXME what about ioport_resource resource ?
1203 */
1204 for (; addr > size && addr >= iomem_resource.start; addr -= size) {
1205 ret = region_intersects(addr, size, 0, IORES_DESC_NONE);
1206 if (ret != REGION_DISJOINT)
1207 continue;
1208
1209 devmem->resource = devm_request_mem_region(device, addr, size,
1210 dev_name(device));
1211 if (!devmem->resource) {
1212 ret = -ENOMEM;
1213 goto error_no_resource;
1214 }
1215 break;
1216 }
1217 if (!devmem->resource) {
1218 ret = -ERANGE;
1219 goto error_no_resource;
1220 }
1221
1222 devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
1223 devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1224 devmem->pfn_last = devmem->pfn_first +
1225 (resource_size(devmem->resource) >> PAGE_SHIFT);
1226
1227 ret = hmm_devmem_pages_create(devmem);
1228 if (ret)
1229 goto error_pages;
1230
1231 devres_add(device, devmem);
1232
1233 ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1234 if (ret) {
1235 hmm_devmem_remove(devmem);
1236 return ERR_PTR(ret);
1237 }
1238
1239 return devmem;
1240
1241error_pages:
1242 devm_release_mem_region(device, devmem->resource->start,
1243 resource_size(devmem->resource));
1244error_no_resource:
1245error_devm_add_action:
1246 hmm_devmem_ref_kill(&devmem->ref);
1247 hmm_devmem_ref_exit(&devmem->ref);
1248error_percpu_ref:
1249 devres_free(devmem);
1250 return ERR_PTR(ret);
1251}
1252EXPORT_SYMBOL(hmm_devmem_add);
1253
1254struct hmm_devmem *hmm_devmem_add_resource(const struct hmm_devmem_ops *ops,
1255 struct device *device,
1256 struct resource *res)
1257{
1258 struct hmm_devmem *devmem;
1259 int ret;
1260
1261 if (res->desc != IORES_DESC_DEVICE_PUBLIC_MEMORY)
1262 return ERR_PTR(-EINVAL);
1263
1264 static_branch_enable(&device_private_key);
1265
1266 devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1267 GFP_KERNEL, dev_to_node(device));
1268 if (!devmem)
1269 return ERR_PTR(-ENOMEM);
1270
1271 init_completion(&devmem->completion);
1272 devmem->pfn_first = -1UL;
1273 devmem->pfn_last = -1UL;
1274 devmem->resource = res;
1275 devmem->device = device;
1276 devmem->ops = ops;
1277
1278 ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1279 0, GFP_KERNEL);
1280 if (ret)
1281 goto error_percpu_ref;
1282
1283 ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1284 if (ret)
1285 goto error_devm_add_action;
1286
1287
1288 devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1289 devmem->pfn_last = devmem->pfn_first +
1290 (resource_size(devmem->resource) >> PAGE_SHIFT);
1291
1292 ret = hmm_devmem_pages_create(devmem);
1293 if (ret)
1294 goto error_devm_add_action;
1295
1296 devres_add(device, devmem);
1297
1298 ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1299 if (ret) {
1300 hmm_devmem_remove(devmem);
1301 return ERR_PTR(ret);
1302 }
1303
1304 return devmem;
1305
1306error_devm_add_action:
1307 hmm_devmem_ref_kill(&devmem->ref);
1308 hmm_devmem_ref_exit(&devmem->ref);
1309error_percpu_ref:
1310 devres_free(devmem);
1311 return ERR_PTR(ret);
1312}
1313EXPORT_SYMBOL(hmm_devmem_add_resource);
1314
1315/*
1316 * hmm_devmem_remove() - remove device memory (kill and free ZONE_DEVICE)
1317 *
1318 * @devmem: hmm_devmem struct use to track and manage the ZONE_DEVICE memory
1319 *
1320 * This will hot-unplug memory that was hotplugged by hmm_devmem_add on behalf
1321 * of the device driver. It will free struct page and remove the resource that
1322 * reserved the physical address range for this device memory.
1323 */
1324void hmm_devmem_remove(struct hmm_devmem *devmem)
1325{
1326 resource_size_t start, size;
1327 struct device *device;
1328 bool cdm = false;
1329
1330 if (!devmem)
1331 return;
1332
1333 device = devmem->device;
1334 start = devmem->resource->start;
1335 size = resource_size(devmem->resource);
1336
1337 cdm = devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY;
1338 hmm_devmem_ref_kill(&devmem->ref);
1339 hmm_devmem_ref_exit(&devmem->ref);
1340 hmm_devmem_pages_remove(devmem);
1341
1342 if (!cdm)
1343 devm_release_mem_region(device, start, size);
1344}
1345EXPORT_SYMBOL(hmm_devmem_remove);
1346
1347/*
1348 * A device driver that wants to handle multiple devices memory through a
1349 * single fake device can use hmm_device to do so. This is purely a helper
1350 * and it is not needed to make use of any HMM functionality.
1351 */
1352#define HMM_DEVICE_MAX 256
1353
1354static DECLARE_BITMAP(hmm_device_mask, HMM_DEVICE_MAX);
1355static DEFINE_SPINLOCK(hmm_device_lock);
1356static struct class *hmm_device_class;
1357static dev_t hmm_device_devt;
1358
1359static void hmm_device_release(struct device *device)
1360{
1361 struct hmm_device *hmm_device;
1362
1363 hmm_device = container_of(device, struct hmm_device, device);
1364 spin_lock(&hmm_device_lock);
1365 clear_bit(hmm_device->minor, hmm_device_mask);
1366 spin_unlock(&hmm_device_lock);
1367
1368 kfree(hmm_device);
1369}
1370
1371struct hmm_device *hmm_device_new(void *drvdata)
1372{
1373 struct hmm_device *hmm_device;
1374
1375 hmm_device = kzalloc(sizeof(*hmm_device), GFP_KERNEL);
1376 if (!hmm_device)
1377 return ERR_PTR(-ENOMEM);
1378
1379 spin_lock(&hmm_device_lock);
1380 hmm_device->minor = find_first_zero_bit(hmm_device_mask, HMM_DEVICE_MAX);
1381 if (hmm_device->minor >= HMM_DEVICE_MAX) {
1382 spin_unlock(&hmm_device_lock);
1383 kfree(hmm_device);
1384 return ERR_PTR(-EBUSY);
1385 }
1386 set_bit(hmm_device->minor, hmm_device_mask);
1387 spin_unlock(&hmm_device_lock);
1388
1389 dev_set_name(&hmm_device->device, "hmm_device%d", hmm_device->minor);
1390 hmm_device->device.devt = MKDEV(MAJOR(hmm_device_devt),
1391 hmm_device->minor);
1392 hmm_device->device.release = hmm_device_release;
1393 dev_set_drvdata(&hmm_device->device, drvdata);
1394 hmm_device->device.class = hmm_device_class;
1395 device_initialize(&hmm_device->device);
1396
1397 return hmm_device;
1398}
1399EXPORT_SYMBOL(hmm_device_new);
1400
1401void hmm_device_put(struct hmm_device *hmm_device)
1402{
1403 put_device(&hmm_device->device);
1404}
1405EXPORT_SYMBOL(hmm_device_put);
1406
1407static int __init hmm_init(void)
1408{
1409 int ret;
1410
1411 ret = alloc_chrdev_region(&hmm_device_devt, 0,
1412 HMM_DEVICE_MAX,
1413 "hmm_device");
1414 if (ret)
1415 return ret;
1416
1417 hmm_device_class = class_create(THIS_MODULE, "hmm_device");
1418 if (IS_ERR(hmm_device_class)) {
1419 unregister_chrdev_region(hmm_device_devt, HMM_DEVICE_MAX);
1420 return PTR_ERR(hmm_device_class);
1421 }
1422 return 0;
1423}
1424
1425device_initcall(hmm_init);
1426#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */