Linux Audio

Check our new training course

Loading...
v5.14.15
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * This is a module to test the HMM (Heterogeneous Memory Management)
   4 * mirror and zone device private memory migration APIs of the kernel.
   5 * Userspace programs can register with the driver to mirror their own address
   6 * space and can use the device to read/write any valid virtual address.
   7 */
   8#include <linux/init.h>
   9#include <linux/fs.h>
  10#include <linux/mm.h>
  11#include <linux/module.h>
  12#include <linux/kernel.h>
  13#include <linux/cdev.h>
  14#include <linux/device.h>
 
  15#include <linux/mutex.h>
  16#include <linux/rwsem.h>
  17#include <linux/sched.h>
  18#include <linux/slab.h>
  19#include <linux/highmem.h>
  20#include <linux/delay.h>
  21#include <linux/pagemap.h>
  22#include <linux/hmm.h>
  23#include <linux/vmalloc.h>
  24#include <linux/swap.h>
  25#include <linux/swapops.h>
  26#include <linux/sched/mm.h>
  27#include <linux/platform_device.h>
  28#include <linux/rmap.h>
 
 
  29
  30#include "test_hmm_uapi.h"
  31
  32#define DMIRROR_NDEVICES		2
  33#define DMIRROR_RANGE_FAULT_TIMEOUT	1000
  34#define DEVMEM_CHUNK_SIZE		(256 * 1024 * 1024U)
  35#define DEVMEM_CHUNKS_RESERVE		16
  36
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  37static const struct dev_pagemap_ops dmirror_devmem_ops;
  38static const struct mmu_interval_notifier_ops dmirror_min_ops;
  39static dev_t dmirror_dev;
  40
  41struct dmirror_device;
  42
  43struct dmirror_bounce {
  44	void			*ptr;
  45	unsigned long		size;
  46	unsigned long		addr;
  47	unsigned long		cpages;
  48};
  49
  50#define DPT_XA_TAG_ATOMIC 1UL
  51#define DPT_XA_TAG_WRITE 3UL
  52
  53/*
  54 * Data structure to track address ranges and register for mmu interval
  55 * notifier updates.
  56 */
  57struct dmirror_interval {
  58	struct mmu_interval_notifier	notifier;
  59	struct dmirror			*dmirror;
  60};
  61
  62/*
  63 * Data attached to the open device file.
  64 * Note that it might be shared after a fork().
  65 */
  66struct dmirror {
  67	struct dmirror_device		*mdevice;
  68	struct xarray			pt;
  69	struct mmu_interval_notifier	notifier;
  70	struct mutex			mutex;
  71};
  72
  73/*
  74 * ZONE_DEVICE pages for migration and simulating device memory.
  75 */
  76struct dmirror_chunk {
  77	struct dev_pagemap	pagemap;
  78	struct dmirror_device	*mdevice;
 
  79};
  80
  81/*
  82 * Per device data.
  83 */
  84struct dmirror_device {
  85	struct cdev		cdevice;
  86	struct hmm_devmem	*devmem;
 
  87
  88	unsigned int		devmem_capacity;
  89	unsigned int		devmem_count;
  90	struct dmirror_chunk	**devmem_chunks;
  91	struct mutex		devmem_lock;	/* protects the above */
  92
  93	unsigned long		calloc;
  94	unsigned long		cfree;
  95	struct page		*free_pages;
  96	spinlock_t		lock;		/* protects the above */
  97};
  98
  99static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
 100
 101static int dmirror_bounce_init(struct dmirror_bounce *bounce,
 102			       unsigned long addr,
 103			       unsigned long size)
 104{
 105	bounce->addr = addr;
 106	bounce->size = size;
 107	bounce->cpages = 0;
 108	bounce->ptr = vmalloc(size);
 109	if (!bounce->ptr)
 110		return -ENOMEM;
 111	return 0;
 112}
 113
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 114static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
 115{
 116	vfree(bounce->ptr);
 117}
 118
 119static int dmirror_fops_open(struct inode *inode, struct file *filp)
 120{
 121	struct cdev *cdev = inode->i_cdev;
 122	struct dmirror *dmirror;
 123	int ret;
 124
 125	/* Mirror this process address space */
 126	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
 127	if (dmirror == NULL)
 128		return -ENOMEM;
 129
 130	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
 131	mutex_init(&dmirror->mutex);
 132	xa_init(&dmirror->pt);
 133
 134	ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
 135				0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
 136	if (ret) {
 137		kfree(dmirror);
 138		return ret;
 139	}
 140
 141	filp->private_data = dmirror;
 142	return 0;
 143}
 144
 145static int dmirror_fops_release(struct inode *inode, struct file *filp)
 146{
 147	struct dmirror *dmirror = filp->private_data;
 148
 149	mmu_interval_notifier_remove(&dmirror->notifier);
 150	xa_destroy(&dmirror->pt);
 151	kfree(dmirror);
 152	return 0;
 153}
 154
 
 
 
 
 
 155static struct dmirror_device *dmirror_page_to_device(struct page *page)
 156
 157{
 158	return container_of(page->pgmap, struct dmirror_chunk,
 159			    pagemap)->mdevice;
 160}
 161
 162static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
 163{
 164	unsigned long *pfns = range->hmm_pfns;
 165	unsigned long pfn;
 166
 167	for (pfn = (range->start >> PAGE_SHIFT);
 168	     pfn < (range->end >> PAGE_SHIFT);
 169	     pfn++, pfns++) {
 170		struct page *page;
 171		void *entry;
 172
 173		/*
 174		 * Since we asked for hmm_range_fault() to populate pages,
 175		 * it shouldn't return an error entry on success.
 176		 */
 177		WARN_ON(*pfns & HMM_PFN_ERROR);
 178		WARN_ON(!(*pfns & HMM_PFN_VALID));
 179
 180		page = hmm_pfn_to_page(*pfns);
 181		WARN_ON(!page);
 182
 183		entry = page;
 184		if (*pfns & HMM_PFN_WRITE)
 185			entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
 186		else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))
 187			return -EFAULT;
 188		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 189		if (xa_is_err(entry))
 190			return xa_err(entry);
 191	}
 192
 193	return 0;
 194}
 195
 196static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
 197			      unsigned long end)
 198{
 199	unsigned long pfn;
 200	void *entry;
 201
 202	/*
 203	 * The XArray doesn't hold references to pages since it relies on
 204	 * the mmu notifier to clear page pointers when they become stale.
 205	 * Therefore, it is OK to just clear the entry.
 206	 */
 207	xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
 208			  end >> PAGE_SHIFT)
 209		xa_erase(&dmirror->pt, pfn);
 210}
 211
 212static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,
 213				const struct mmu_notifier_range *range,
 214				unsigned long cur_seq)
 215{
 216	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
 217
 218	/*
 219	 * Ignore invalidation callbacks for device private pages since
 220	 * the invalidation is handled as part of the migration process.
 221	 */
 222	if (range->event == MMU_NOTIFY_MIGRATE &&
 223	    range->owner == dmirror->mdevice)
 224		return true;
 225
 226	if (mmu_notifier_range_blockable(range))
 227		mutex_lock(&dmirror->mutex);
 228	else if (!mutex_trylock(&dmirror->mutex))
 229		return false;
 230
 231	mmu_interval_set_seq(mni, cur_seq);
 232	dmirror_do_update(dmirror, range->start, range->end);
 233
 234	mutex_unlock(&dmirror->mutex);
 235	return true;
 236}
 237
 238static const struct mmu_interval_notifier_ops dmirror_min_ops = {
 239	.invalidate = dmirror_interval_invalidate,
 240};
 241
 242static int dmirror_range_fault(struct dmirror *dmirror,
 243				struct hmm_range *range)
 244{
 245	struct mm_struct *mm = dmirror->notifier.mm;
 246	unsigned long timeout =
 247		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 248	int ret;
 249
 250	while (true) {
 251		if (time_after(jiffies, timeout)) {
 252			ret = -EBUSY;
 253			goto out;
 254		}
 255
 256		range->notifier_seq = mmu_interval_read_begin(range->notifier);
 257		mmap_read_lock(mm);
 258		ret = hmm_range_fault(range);
 259		mmap_read_unlock(mm);
 260		if (ret) {
 261			if (ret == -EBUSY)
 262				continue;
 263			goto out;
 264		}
 265
 266		mutex_lock(&dmirror->mutex);
 267		if (mmu_interval_read_retry(range->notifier,
 268					    range->notifier_seq)) {
 269			mutex_unlock(&dmirror->mutex);
 270			continue;
 271		}
 272		break;
 273	}
 274
 275	ret = dmirror_do_fault(dmirror, range);
 276
 277	mutex_unlock(&dmirror->mutex);
 278out:
 279	return ret;
 280}
 281
 282static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
 283			 unsigned long end, bool write)
 284{
 285	struct mm_struct *mm = dmirror->notifier.mm;
 286	unsigned long addr;
 287	unsigned long pfns[64];
 288	struct hmm_range range = {
 289		.notifier = &dmirror->notifier,
 290		.hmm_pfns = pfns,
 291		.pfn_flags_mask = 0,
 292		.default_flags =
 293			HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
 294		.dev_private_owner = dmirror->mdevice,
 295	};
 296	int ret = 0;
 297
 298	/* Since the mm is for the mirrored process, get a reference first. */
 299	if (!mmget_not_zero(mm))
 300		return 0;
 301
 302	for (addr = start; addr < end; addr = range.end) {
 303		range.start = addr;
 304		range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
 305
 306		ret = dmirror_range_fault(dmirror, &range);
 307		if (ret)
 308			break;
 309	}
 310
 311	mmput(mm);
 312	return ret;
 313}
 314
 315static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
 316			   unsigned long end, struct dmirror_bounce *bounce)
 317{
 318	unsigned long pfn;
 319	void *ptr;
 320
 321	ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
 322
 323	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 324		void *entry;
 325		struct page *page;
 326		void *tmp;
 327
 328		entry = xa_load(&dmirror->pt, pfn);
 329		page = xa_untag_pointer(entry);
 330		if (!page)
 331			return -ENOENT;
 332
 333		tmp = kmap(page);
 334		memcpy(ptr, tmp, PAGE_SIZE);
 335		kunmap(page);
 336
 337		ptr += PAGE_SIZE;
 338		bounce->cpages++;
 339	}
 340
 341	return 0;
 342}
 343
 344static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
 345{
 346	struct dmirror_bounce bounce;
 347	unsigned long start, end;
 348	unsigned long size = cmd->npages << PAGE_SHIFT;
 349	int ret;
 350
 351	start = cmd->addr;
 352	end = start + size;
 353	if (end < start)
 354		return -EINVAL;
 355
 356	ret = dmirror_bounce_init(&bounce, start, size);
 357	if (ret)
 358		return ret;
 359
 360	while (1) {
 361		mutex_lock(&dmirror->mutex);
 362		ret = dmirror_do_read(dmirror, start, end, &bounce);
 363		mutex_unlock(&dmirror->mutex);
 364		if (ret != -ENOENT)
 365			break;
 366
 367		start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
 368		ret = dmirror_fault(dmirror, start, end, false);
 369		if (ret)
 370			break;
 371		cmd->faults++;
 372	}
 373
 374	if (ret == 0) {
 375		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 376				 bounce.size))
 377			ret = -EFAULT;
 378	}
 379	cmd->cpages = bounce.cpages;
 380	dmirror_bounce_fini(&bounce);
 381	return ret;
 382}
 383
 384static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
 385			    unsigned long end, struct dmirror_bounce *bounce)
 386{
 387	unsigned long pfn;
 388	void *ptr;
 389
 390	ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
 391
 392	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 393		void *entry;
 394		struct page *page;
 395		void *tmp;
 396
 397		entry = xa_load(&dmirror->pt, pfn);
 398		page = xa_untag_pointer(entry);
 399		if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)
 400			return -ENOENT;
 401
 402		tmp = kmap(page);
 403		memcpy(tmp, ptr, PAGE_SIZE);
 404		kunmap(page);
 405
 406		ptr += PAGE_SIZE;
 407		bounce->cpages++;
 408	}
 409
 410	return 0;
 411}
 412
 413static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
 414{
 415	struct dmirror_bounce bounce;
 416	unsigned long start, end;
 417	unsigned long size = cmd->npages << PAGE_SHIFT;
 418	int ret;
 419
 420	start = cmd->addr;
 421	end = start + size;
 422	if (end < start)
 423		return -EINVAL;
 424
 425	ret = dmirror_bounce_init(&bounce, start, size);
 426	if (ret)
 427		return ret;
 428	if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),
 429			   bounce.size)) {
 430		ret = -EFAULT;
 431		goto fini;
 432	}
 433
 434	while (1) {
 435		mutex_lock(&dmirror->mutex);
 436		ret = dmirror_do_write(dmirror, start, end, &bounce);
 437		mutex_unlock(&dmirror->mutex);
 438		if (ret != -ENOENT)
 439			break;
 440
 441		start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
 442		ret = dmirror_fault(dmirror, start, end, true);
 443		if (ret)
 444			break;
 445		cmd->faults++;
 446	}
 447
 448fini:
 449	cmd->cpages = bounce.cpages;
 450	dmirror_bounce_fini(&bounce);
 451	return ret;
 452}
 453
 454static bool dmirror_allocate_chunk(struct dmirror_device *mdevice,
 455				   struct page **ppage)
 456{
 457	struct dmirror_chunk *devmem;
 458	struct resource *res;
 459	unsigned long pfn;
 460	unsigned long pfn_first;
 461	unsigned long pfn_last;
 462	void *ptr;
 
 463
 464	devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);
 465	if (!devmem)
 466		return false;
 467
 468	res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,
 469				      "hmm_dmirror");
 470	if (IS_ERR(res))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 471		goto err_devmem;
 
 472
 473	devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
 474	devmem->pagemap.range.start = res->start;
 475	devmem->pagemap.range.end = res->end;
 476	devmem->pagemap.nr_range = 1;
 477	devmem->pagemap.ops = &dmirror_devmem_ops;
 478	devmem->pagemap.owner = mdevice;
 479
 480	mutex_lock(&mdevice->devmem_lock);
 481
 482	if (mdevice->devmem_count == mdevice->devmem_capacity) {
 483		struct dmirror_chunk **new_chunks;
 484		unsigned int new_capacity;
 485
 486		new_capacity = mdevice->devmem_capacity +
 487				DEVMEM_CHUNKS_RESERVE;
 488		new_chunks = krealloc(mdevice->devmem_chunks,
 489				sizeof(new_chunks[0]) * new_capacity,
 490				GFP_KERNEL);
 491		if (!new_chunks)
 492			goto err_release;
 493		mdevice->devmem_capacity = new_capacity;
 494		mdevice->devmem_chunks = new_chunks;
 495	}
 496
 497	ptr = memremap_pages(&devmem->pagemap, numa_node_id());
 498	if (IS_ERR(ptr))
 
 
 
 
 499		goto err_release;
 
 500
 501	devmem->mdevice = mdevice;
 502	pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;
 503	pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);
 504	mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
 505
 506	mutex_unlock(&mdevice->devmem_lock);
 507
 508	pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
 509		DEVMEM_CHUNK_SIZE / (1024 * 1024),
 510		mdevice->devmem_count,
 511		mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),
 512		pfn_first, pfn_last);
 513
 514	spin_lock(&mdevice->lock);
 515	for (pfn = pfn_first; pfn < pfn_last; pfn++) {
 516		struct page *page = pfn_to_page(pfn);
 517
 518		page->zone_device_data = mdevice->free_pages;
 519		mdevice->free_pages = page;
 520	}
 521	if (ppage) {
 522		*ppage = mdevice->free_pages;
 523		mdevice->free_pages = (*ppage)->zone_device_data;
 524		mdevice->calloc++;
 525	}
 526	spin_unlock(&mdevice->lock);
 527
 528	return true;
 529
 530err_release:
 531	mutex_unlock(&mdevice->devmem_lock);
 532	release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));
 
 
 533err_devmem:
 534	kfree(devmem);
 535
 536	return false;
 537}
 538
 539static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
 540{
 541	struct page *dpage = NULL;
 542	struct page *rpage;
 543
 544	/*
 545	 * This is a fake device so we alloc real system memory to store
 546	 * our device memory.
 
 
 547	 */
 548	rpage = alloc_page(GFP_HIGHUSER);
 549	if (!rpage)
 550		return NULL;
 551
 
 552	spin_lock(&mdevice->lock);
 553
 554	if (mdevice->free_pages) {
 555		dpage = mdevice->free_pages;
 556		mdevice->free_pages = dpage->zone_device_data;
 557		mdevice->calloc++;
 558		spin_unlock(&mdevice->lock);
 559	} else {
 560		spin_unlock(&mdevice->lock);
 561		if (!dmirror_allocate_chunk(mdevice, &dpage))
 562			goto error;
 563	}
 564
 
 565	dpage->zone_device_data = rpage;
 566	get_page(dpage);
 567	lock_page(dpage);
 568	return dpage;
 569
 570error:
 571	__free_page(rpage);
 
 572	return NULL;
 573}
 574
 575static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
 576					   struct dmirror *dmirror)
 577{
 578	struct dmirror_device *mdevice = dmirror->mdevice;
 579	const unsigned long *src = args->src;
 580	unsigned long *dst = args->dst;
 581	unsigned long addr;
 582
 583	for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
 584						   src++, dst++) {
 585		struct page *spage;
 586		struct page *dpage;
 587		struct page *rpage;
 588
 589		if (!(*src & MIGRATE_PFN_MIGRATE))
 590			continue;
 591
 592		/*
 593		 * Note that spage might be NULL which is OK since it is an
 594		 * unallocated pte_none() or read-only zero page.
 595		 */
 596		spage = migrate_pfn_to_page(*src);
 
 
 
 
 597
 598		dpage = dmirror_devmem_alloc_page(mdevice);
 599		if (!dpage)
 600			continue;
 601
 602		rpage = dpage->zone_device_data;
 603		if (spage)
 604			copy_highpage(rpage, spage);
 605		else
 606			clear_highpage(rpage);
 607
 608		/*
 609		 * Normally, a device would use the page->zone_device_data to
 610		 * point to the mirror but here we use it to hold the page for
 611		 * the simulated device memory and that page holds the pointer
 612		 * to the mirror.
 613		 */
 614		rpage->zone_device_data = dmirror;
 615
 616		*dst = migrate_pfn(page_to_pfn(dpage)) |
 617			    MIGRATE_PFN_LOCKED;
 
 618		if ((*src & MIGRATE_PFN_WRITE) ||
 619		    (!spage && args->vma->vm_flags & VM_WRITE))
 620			*dst |= MIGRATE_PFN_WRITE;
 621	}
 622}
 623
 624static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,
 625			     unsigned long end)
 626{
 627	unsigned long pfn;
 628
 629	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 630		void *entry;
 631
 632		entry = xa_load(&dmirror->pt, pfn);
 633		if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC)
 634			return -EPERM;
 635	}
 636
 637	return 0;
 638}
 639
 640static int dmirror_atomic_map(unsigned long start, unsigned long end,
 641			      struct page **pages, struct dmirror *dmirror)
 642{
 643	unsigned long pfn, mapped = 0;
 644	int i;
 645
 646	/* Map the migrated pages into the device's page tables. */
 647	mutex_lock(&dmirror->mutex);
 648
 649	for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) {
 650		void *entry;
 651
 652		if (!pages[i])
 653			continue;
 654
 655		entry = pages[i];
 656		entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC);
 657		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 658		if (xa_is_err(entry)) {
 659			mutex_unlock(&dmirror->mutex);
 660			return xa_err(entry);
 661		}
 662
 663		mapped++;
 664	}
 665
 666	mutex_unlock(&dmirror->mutex);
 667	return mapped;
 668}
 669
 670static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
 671					    struct dmirror *dmirror)
 672{
 673	unsigned long start = args->start;
 674	unsigned long end = args->end;
 675	const unsigned long *src = args->src;
 676	const unsigned long *dst = args->dst;
 677	unsigned long pfn;
 678
 679	/* Map the migrated pages into the device's page tables. */
 680	mutex_lock(&dmirror->mutex);
 681
 682	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
 683								src++, dst++) {
 684		struct page *dpage;
 685		void *entry;
 686
 687		if (!(*src & MIGRATE_PFN_MIGRATE))
 688			continue;
 689
 690		dpage = migrate_pfn_to_page(*dst);
 691		if (!dpage)
 692			continue;
 693
 694		/*
 695		 * Store the page that holds the data so the page table
 696		 * doesn't have to deal with ZONE_DEVICE private pages.
 697		 */
 698		entry = dpage->zone_device_data;
 699		if (*dst & MIGRATE_PFN_WRITE)
 700			entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
 701		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 702		if (xa_is_err(entry)) {
 703			mutex_unlock(&dmirror->mutex);
 704			return xa_err(entry);
 705		}
 706	}
 707
 708	mutex_unlock(&dmirror->mutex);
 709	return 0;
 710}
 711
 712static int dmirror_exclusive(struct dmirror *dmirror,
 713			     struct hmm_dmirror_cmd *cmd)
 714{
 715	unsigned long start, end, addr;
 716	unsigned long size = cmd->npages << PAGE_SHIFT;
 717	struct mm_struct *mm = dmirror->notifier.mm;
 718	struct page *pages[64];
 719	struct dmirror_bounce bounce;
 720	unsigned long next;
 721	int ret;
 722
 723	start = cmd->addr;
 724	end = start + size;
 725	if (end < start)
 726		return -EINVAL;
 727
 728	/* Since the mm is for the mirrored process, get a reference first. */
 729	if (!mmget_not_zero(mm))
 730		return -EINVAL;
 731
 732	mmap_read_lock(mm);
 733	for (addr = start; addr < end; addr = next) {
 734		unsigned long mapped;
 735		int i;
 736
 737		if (end < addr + (ARRAY_SIZE(pages) << PAGE_SHIFT))
 738			next = end;
 739		else
 740			next = addr + (ARRAY_SIZE(pages) << PAGE_SHIFT);
 741
 742		ret = make_device_exclusive_range(mm, addr, next, pages, NULL);
 743		mapped = dmirror_atomic_map(addr, next, pages, dmirror);
 
 
 
 
 
 
 744		for (i = 0; i < ret; i++) {
 745			if (pages[i]) {
 746				unlock_page(pages[i]);
 747				put_page(pages[i]);
 748			}
 749		}
 750
 751		if (addr + (mapped << PAGE_SHIFT) < next) {
 752			mmap_read_unlock(mm);
 753			mmput(mm);
 754			return -EBUSY;
 755		}
 756	}
 757	mmap_read_unlock(mm);
 758	mmput(mm);
 759
 760	/* Return the migrated data for verification. */
 761	ret = dmirror_bounce_init(&bounce, start, size);
 762	if (ret)
 763		return ret;
 764	mutex_lock(&dmirror->mutex);
 765	ret = dmirror_do_read(dmirror, start, end, &bounce);
 766	mutex_unlock(&dmirror->mutex);
 767	if (ret == 0) {
 768		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 769				 bounce.size))
 770			ret = -EFAULT;
 771	}
 772
 773	cmd->cpages = bounce.cpages;
 774	dmirror_bounce_fini(&bounce);
 775	return ret;
 776}
 777
 778static int dmirror_migrate(struct dmirror *dmirror,
 779			   struct hmm_dmirror_cmd *cmd)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 780{
 781	unsigned long start, end, addr;
 782	unsigned long size = cmd->npages << PAGE_SHIFT;
 783	struct mm_struct *mm = dmirror->notifier.mm;
 784	struct vm_area_struct *vma;
 785	unsigned long src_pfns[64];
 786	unsigned long dst_pfns[64];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 787	struct dmirror_bounce bounce;
 788	struct migrate_vma args;
 789	unsigned long next;
 790	int ret;
 791
 792	start = cmd->addr;
 793	end = start + size;
 794	if (end < start)
 795		return -EINVAL;
 796
 797	/* Since the mm is for the mirrored process, get a reference first. */
 798	if (!mmget_not_zero(mm))
 799		return -EINVAL;
 800
 801	mmap_read_lock(mm);
 802	for (addr = start; addr < end; addr = next) {
 803		vma = vma_lookup(mm, addr);
 804		if (!vma || !(vma->vm_flags & VM_READ)) {
 805			ret = -EINVAL;
 806			goto out;
 807		}
 808		next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
 809		if (next > vma->vm_end)
 810			next = vma->vm_end;
 811
 812		args.vma = vma;
 813		args.src = src_pfns;
 814		args.dst = dst_pfns;
 815		args.start = addr;
 816		args.end = next;
 817		args.pgmap_owner = dmirror->mdevice;
 818		args.flags = MIGRATE_VMA_SELECT_SYSTEM;
 819		ret = migrate_vma_setup(&args);
 820		if (ret)
 821			goto out;
 822
 
 823		dmirror_migrate_alloc_and_copy(&args, dmirror);
 824		migrate_vma_pages(&args);
 825		dmirror_migrate_finalize_and_map(&args, dmirror);
 826		migrate_vma_finalize(&args);
 827	}
 828	mmap_read_unlock(mm);
 829	mmput(mm);
 830
 831	/* Return the migrated data for verification. */
 
 
 
 832	ret = dmirror_bounce_init(&bounce, start, size);
 833	if (ret)
 834		return ret;
 835	mutex_lock(&dmirror->mutex);
 836	ret = dmirror_do_read(dmirror, start, end, &bounce);
 837	mutex_unlock(&dmirror->mutex);
 838	if (ret == 0) {
 839		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 840				 bounce.size))
 841			ret = -EFAULT;
 842	}
 843	cmd->cpages = bounce.cpages;
 844	dmirror_bounce_fini(&bounce);
 845	return ret;
 846
 847out:
 848	mmap_read_unlock(mm);
 849	mmput(mm);
 850	return ret;
 851}
 852
 853static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
 854			    unsigned char *perm, unsigned long entry)
 855{
 856	struct page *page;
 857
 858	if (entry & HMM_PFN_ERROR) {
 859		*perm = HMM_DMIRROR_PROT_ERROR;
 860		return;
 861	}
 862	if (!(entry & HMM_PFN_VALID)) {
 863		*perm = HMM_DMIRROR_PROT_NONE;
 864		return;
 865	}
 866
 867	page = hmm_pfn_to_page(entry);
 868	if (is_device_private_page(page)) {
 869		/* Is the page migrated to this device or some other? */
 870		if (dmirror->mdevice == dmirror_page_to_device(page))
 871			*perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;
 872		else
 873			*perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;
 
 
 
 
 
 
 874	} else if (is_zero_pfn(page_to_pfn(page)))
 875		*perm = HMM_DMIRROR_PROT_ZERO;
 876	else
 877		*perm = HMM_DMIRROR_PROT_NONE;
 878	if (entry & HMM_PFN_WRITE)
 879		*perm |= HMM_DMIRROR_PROT_WRITE;
 880	else
 881		*perm |= HMM_DMIRROR_PROT_READ;
 882	if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
 883		*perm |= HMM_DMIRROR_PROT_PMD;
 884	else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
 885		*perm |= HMM_DMIRROR_PROT_PUD;
 886}
 887
 888static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
 889				const struct mmu_notifier_range *range,
 890				unsigned long cur_seq)
 891{
 892	struct dmirror_interval *dmi =
 893		container_of(mni, struct dmirror_interval, notifier);
 894	struct dmirror *dmirror = dmi->dmirror;
 895
 896	if (mmu_notifier_range_blockable(range))
 897		mutex_lock(&dmirror->mutex);
 898	else if (!mutex_trylock(&dmirror->mutex))
 899		return false;
 900
 901	/*
 902	 * Snapshots only need to set the sequence number since any
 903	 * invalidation in the interval invalidates the whole snapshot.
 904	 */
 905	mmu_interval_set_seq(mni, cur_seq);
 906
 907	mutex_unlock(&dmirror->mutex);
 908	return true;
 909}
 910
 911static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {
 912	.invalidate = dmirror_snapshot_invalidate,
 913};
 914
 915static int dmirror_range_snapshot(struct dmirror *dmirror,
 916				  struct hmm_range *range,
 917				  unsigned char *perm)
 918{
 919	struct mm_struct *mm = dmirror->notifier.mm;
 920	struct dmirror_interval notifier;
 921	unsigned long timeout =
 922		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 923	unsigned long i;
 924	unsigned long n;
 925	int ret = 0;
 926
 927	notifier.dmirror = dmirror;
 928	range->notifier = &notifier.notifier;
 929
 930	ret = mmu_interval_notifier_insert(range->notifier, mm,
 931			range->start, range->end - range->start,
 932			&dmirror_mrn_ops);
 933	if (ret)
 934		return ret;
 935
 936	while (true) {
 937		if (time_after(jiffies, timeout)) {
 938			ret = -EBUSY;
 939			goto out;
 940		}
 941
 942		range->notifier_seq = mmu_interval_read_begin(range->notifier);
 943
 944		mmap_read_lock(mm);
 945		ret = hmm_range_fault(range);
 946		mmap_read_unlock(mm);
 947		if (ret) {
 948			if (ret == -EBUSY)
 949				continue;
 950			goto out;
 951		}
 952
 953		mutex_lock(&dmirror->mutex);
 954		if (mmu_interval_read_retry(range->notifier,
 955					    range->notifier_seq)) {
 956			mutex_unlock(&dmirror->mutex);
 957			continue;
 958		}
 959		break;
 960	}
 961
 962	n = (range->end - range->start) >> PAGE_SHIFT;
 963	for (i = 0; i < n; i++)
 964		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
 965
 966	mutex_unlock(&dmirror->mutex);
 967out:
 968	mmu_interval_notifier_remove(range->notifier);
 969	return ret;
 970}
 971
 972static int dmirror_snapshot(struct dmirror *dmirror,
 973			    struct hmm_dmirror_cmd *cmd)
 974{
 975	struct mm_struct *mm = dmirror->notifier.mm;
 976	unsigned long start, end;
 977	unsigned long size = cmd->npages << PAGE_SHIFT;
 978	unsigned long addr;
 979	unsigned long next;
 980	unsigned long pfns[64];
 981	unsigned char perm[64];
 982	char __user *uptr;
 983	struct hmm_range range = {
 984		.hmm_pfns = pfns,
 985		.dev_private_owner = dmirror->mdevice,
 986	};
 987	int ret = 0;
 988
 989	start = cmd->addr;
 990	end = start + size;
 991	if (end < start)
 992		return -EINVAL;
 993
 994	/* Since the mm is for the mirrored process, get a reference first. */
 995	if (!mmget_not_zero(mm))
 996		return -EINVAL;
 997
 998	/*
 999	 * Register a temporary notifier to detect invalidations even if it
1000	 * overlaps with other mmu_interval_notifiers.
1001	 */
1002	uptr = u64_to_user_ptr(cmd->ptr);
1003	for (addr = start; addr < end; addr = next) {
1004		unsigned long n;
1005
1006		next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
1007		range.start = addr;
1008		range.end = next;
1009
1010		ret = dmirror_range_snapshot(dmirror, &range, perm);
1011		if (ret)
1012			break;
1013
1014		n = (range.end - range.start) >> PAGE_SHIFT;
1015		if (copy_to_user(uptr, perm, n)) {
1016			ret = -EFAULT;
1017			break;
1018		}
1019
1020		cmd->cpages += n;
1021		uptr += n;
1022	}
1023	mmput(mm);
1024
1025	return ret;
1026}
1027
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1028static long dmirror_fops_unlocked_ioctl(struct file *filp,
1029					unsigned int command,
1030					unsigned long arg)
1031{
1032	void __user *uarg = (void __user *)arg;
1033	struct hmm_dmirror_cmd cmd;
1034	struct dmirror *dmirror;
1035	int ret;
1036
1037	dmirror = filp->private_data;
1038	if (!dmirror)
1039		return -EINVAL;
1040
1041	if (copy_from_user(&cmd, uarg, sizeof(cmd)))
1042		return -EFAULT;
1043
1044	if (cmd.addr & ~PAGE_MASK)
1045		return -EINVAL;
1046	if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
1047		return -EINVAL;
1048
1049	cmd.cpages = 0;
1050	cmd.faults = 0;
1051
1052	switch (command) {
1053	case HMM_DMIRROR_READ:
1054		ret = dmirror_read(dmirror, &cmd);
1055		break;
1056
1057	case HMM_DMIRROR_WRITE:
1058		ret = dmirror_write(dmirror, &cmd);
1059		break;
1060
1061	case HMM_DMIRROR_MIGRATE:
1062		ret = dmirror_migrate(dmirror, &cmd);
 
 
 
 
1063		break;
1064
1065	case HMM_DMIRROR_EXCLUSIVE:
1066		ret = dmirror_exclusive(dmirror, &cmd);
1067		break;
1068
1069	case HMM_DMIRROR_CHECK_EXCLUSIVE:
1070		ret = dmirror_check_atomic(dmirror, cmd.addr,
1071					cmd.addr + (cmd.npages << PAGE_SHIFT));
1072		break;
1073
1074	case HMM_DMIRROR_SNAPSHOT:
1075		ret = dmirror_snapshot(dmirror, &cmd);
1076		break;
1077
 
 
 
 
 
1078	default:
1079		return -EINVAL;
1080	}
1081	if (ret)
1082		return ret;
1083
1084	if (copy_to_user(uarg, &cmd, sizeof(cmd)))
1085		return -EFAULT;
1086
1087	return 0;
1088}
1089
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1090static const struct file_operations dmirror_fops = {
1091	.open		= dmirror_fops_open,
1092	.release	= dmirror_fops_release,
 
1093	.unlocked_ioctl = dmirror_fops_unlocked_ioctl,
1094	.llseek		= default_llseek,
1095	.owner		= THIS_MODULE,
1096};
1097
1098static void dmirror_devmem_free(struct page *page)
1099{
1100	struct page *rpage = page->zone_device_data;
1101	struct dmirror_device *mdevice;
1102
1103	if (rpage)
1104		__free_page(rpage);
1105
1106	mdevice = dmirror_page_to_device(page);
1107
1108	spin_lock(&mdevice->lock);
1109	mdevice->cfree++;
1110	page->zone_device_data = mdevice->free_pages;
1111	mdevice->free_pages = page;
1112	spin_unlock(&mdevice->lock);
1113}
1114
1115static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
1116						      struct dmirror *dmirror)
1117{
1118	const unsigned long *src = args->src;
1119	unsigned long *dst = args->dst;
1120	unsigned long start = args->start;
1121	unsigned long end = args->end;
1122	unsigned long addr;
1123
1124	for (addr = start; addr < end; addr += PAGE_SIZE,
1125				       src++, dst++) {
1126		struct page *dpage, *spage;
1127
1128		spage = migrate_pfn_to_page(*src);
1129		if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
1130			continue;
1131		spage = spage->zone_device_data;
1132
1133		dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
1134		if (!dpage)
1135			continue;
1136
1137		lock_page(dpage);
1138		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1139		copy_highpage(dpage, spage);
1140		*dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
1141		if (*src & MIGRATE_PFN_WRITE)
1142			*dst |= MIGRATE_PFN_WRITE;
1143	}
1144	return 0;
1145}
1146
1147static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
1148{
1149	struct migrate_vma args;
1150	unsigned long src_pfns;
1151	unsigned long dst_pfns;
1152	struct page *rpage;
1153	struct dmirror *dmirror;
1154	vm_fault_t ret;
1155
1156	/*
1157	 * Normally, a device would use the page->zone_device_data to point to
1158	 * the mirror but here we use it to hold the page for the simulated
1159	 * device memory and that page holds the pointer to the mirror.
1160	 */
1161	rpage = vmf->page->zone_device_data;
1162	dmirror = rpage->zone_device_data;
1163
1164	/* FIXME demonstrate how we can adjust migrate range */
1165	args.vma = vmf->vma;
1166	args.start = vmf->address;
1167	args.end = args.start + PAGE_SIZE;
1168	args.src = &src_pfns;
1169	args.dst = &dst_pfns;
1170	args.pgmap_owner = dmirror->mdevice;
1171	args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
 
1172
1173	if (migrate_vma_setup(&args))
1174		return VM_FAULT_SIGBUS;
1175
1176	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
1177	if (ret)
1178		return ret;
1179	migrate_vma_pages(&args);
1180	/*
1181	 * No device finalize step is needed since
1182	 * dmirror_devmem_fault_alloc_and_copy() will have already
1183	 * invalidated the device page table.
1184	 */
1185	migrate_vma_finalize(&args);
1186	return 0;
1187}
1188
1189static const struct dev_pagemap_ops dmirror_devmem_ops = {
1190	.page_free	= dmirror_devmem_free,
1191	.migrate_to_ram	= dmirror_devmem_fault,
1192};
1193
1194static int dmirror_device_init(struct dmirror_device *mdevice, int id)
1195{
1196	dev_t dev;
1197	int ret;
1198
1199	dev = MKDEV(MAJOR(dmirror_dev), id);
1200	mutex_init(&mdevice->devmem_lock);
1201	spin_lock_init(&mdevice->lock);
1202
1203	cdev_init(&mdevice->cdevice, &dmirror_fops);
1204	mdevice->cdevice.owner = THIS_MODULE;
1205	ret = cdev_add(&mdevice->cdevice, dev, 1);
 
 
 
1206	if (ret)
1207		return ret;
1208
1209	/* Build a list of free ZONE_DEVICE private struct pages */
1210	dmirror_allocate_chunk(mdevice, NULL);
 
1211
1212	return 0;
 
1213}
1214
1215static void dmirror_device_remove(struct dmirror_device *mdevice)
1216{
1217	unsigned int i;
1218
1219	if (mdevice->devmem_chunks) {
1220		for (i = 0; i < mdevice->devmem_count; i++) {
1221			struct dmirror_chunk *devmem =
1222				mdevice->devmem_chunks[i];
1223
1224			memunmap_pages(&devmem->pagemap);
1225			release_mem_region(devmem->pagemap.range.start,
1226					   range_len(&devmem->pagemap.range));
1227			kfree(devmem);
1228		}
1229		kfree(mdevice->devmem_chunks);
1230	}
1231
1232	cdev_del(&mdevice->cdevice);
1233}
1234
1235static int __init hmm_dmirror_init(void)
1236{
1237	int ret;
1238	int id;
 
1239
1240	ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,
1241				  "HMM_DMIRROR");
1242	if (ret)
1243		goto err_unreg;
1244
1245	for (id = 0; id < DMIRROR_NDEVICES; id++) {
 
 
 
 
 
 
 
 
 
 
 
1246		ret = dmirror_device_init(dmirror_devices + id, id);
1247		if (ret)
1248			goto err_chrdev;
1249	}
1250
1251	pr_info("HMM test module loaded. This is only for testing HMM.\n");
1252	return 0;
1253
1254err_chrdev:
1255	while (--id >= 0)
1256		dmirror_device_remove(dmirror_devices + id);
1257	unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1258err_unreg:
1259	return ret;
1260}
1261
1262static void __exit hmm_dmirror_exit(void)
1263{
1264	int id;
1265
1266	for (id = 0; id < DMIRROR_NDEVICES; id++)
1267		dmirror_device_remove(dmirror_devices + id);
 
1268	unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1269}
1270
1271module_init(hmm_dmirror_init);
1272module_exit(hmm_dmirror_exit);
 
1273MODULE_LICENSE("GPL");
v6.13.7
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * This is a module to test the HMM (Heterogeneous Memory Management)
   4 * mirror and zone device private memory migration APIs of the kernel.
   5 * Userspace programs can register with the driver to mirror their own address
   6 * space and can use the device to read/write any valid virtual address.
   7 */
   8#include <linux/init.h>
   9#include <linux/fs.h>
  10#include <linux/mm.h>
  11#include <linux/module.h>
  12#include <linux/kernel.h>
  13#include <linux/cdev.h>
  14#include <linux/device.h>
  15#include <linux/memremap.h>
  16#include <linux/mutex.h>
  17#include <linux/rwsem.h>
  18#include <linux/sched.h>
  19#include <linux/slab.h>
  20#include <linux/highmem.h>
  21#include <linux/delay.h>
  22#include <linux/pagemap.h>
  23#include <linux/hmm.h>
  24#include <linux/vmalloc.h>
  25#include <linux/swap.h>
  26#include <linux/swapops.h>
  27#include <linux/sched/mm.h>
  28#include <linux/platform_device.h>
  29#include <linux/rmap.h>
  30#include <linux/mmu_notifier.h>
  31#include <linux/migrate.h>
  32
  33#include "test_hmm_uapi.h"
  34
  35#define DMIRROR_NDEVICES		4
  36#define DMIRROR_RANGE_FAULT_TIMEOUT	1000
  37#define DEVMEM_CHUNK_SIZE		(256 * 1024 * 1024U)
  38#define DEVMEM_CHUNKS_RESERVE		16
  39
  40/*
  41 * For device_private pages, dpage is just a dummy struct page
  42 * representing a piece of device memory. dmirror_devmem_alloc_page
  43 * allocates a real system memory page as backing storage to fake a
  44 * real device. zone_device_data points to that backing page. But
  45 * for device_coherent memory, the struct page represents real
  46 * physical CPU-accessible memory that we can use directly.
  47 */
  48#define BACKING_PAGE(page) (is_device_private_page((page)) ? \
  49			   (page)->zone_device_data : (page))
  50
  51static unsigned long spm_addr_dev0;
  52module_param(spm_addr_dev0, long, 0644);
  53MODULE_PARM_DESC(spm_addr_dev0,
  54		"Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
  55
  56static unsigned long spm_addr_dev1;
  57module_param(spm_addr_dev1, long, 0644);
  58MODULE_PARM_DESC(spm_addr_dev1,
  59		"Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
  60
  61static const struct dev_pagemap_ops dmirror_devmem_ops;
  62static const struct mmu_interval_notifier_ops dmirror_min_ops;
  63static dev_t dmirror_dev;
  64
  65struct dmirror_device;
  66
  67struct dmirror_bounce {
  68	void			*ptr;
  69	unsigned long		size;
  70	unsigned long		addr;
  71	unsigned long		cpages;
  72};
  73
  74#define DPT_XA_TAG_ATOMIC 1UL
  75#define DPT_XA_TAG_WRITE 3UL
  76
  77/*
  78 * Data structure to track address ranges and register for mmu interval
  79 * notifier updates.
  80 */
  81struct dmirror_interval {
  82	struct mmu_interval_notifier	notifier;
  83	struct dmirror			*dmirror;
  84};
  85
  86/*
  87 * Data attached to the open device file.
  88 * Note that it might be shared after a fork().
  89 */
  90struct dmirror {
  91	struct dmirror_device		*mdevice;
  92	struct xarray			pt;
  93	struct mmu_interval_notifier	notifier;
  94	struct mutex			mutex;
  95};
  96
  97/*
  98 * ZONE_DEVICE pages for migration and simulating device memory.
  99 */
 100struct dmirror_chunk {
 101	struct dev_pagemap	pagemap;
 102	struct dmirror_device	*mdevice;
 103	bool remove;
 104};
 105
 106/*
 107 * Per device data.
 108 */
 109struct dmirror_device {
 110	struct cdev		cdevice;
 111	unsigned int            zone_device_type;
 112	struct device		device;
 113
 114	unsigned int		devmem_capacity;
 115	unsigned int		devmem_count;
 116	struct dmirror_chunk	**devmem_chunks;
 117	struct mutex		devmem_lock;	/* protects the above */
 118
 119	unsigned long		calloc;
 120	unsigned long		cfree;
 121	struct page		*free_pages;
 122	spinlock_t		lock;		/* protects the above */
 123};
 124
 125static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
 126
 127static int dmirror_bounce_init(struct dmirror_bounce *bounce,
 128			       unsigned long addr,
 129			       unsigned long size)
 130{
 131	bounce->addr = addr;
 132	bounce->size = size;
 133	bounce->cpages = 0;
 134	bounce->ptr = vmalloc(size);
 135	if (!bounce->ptr)
 136		return -ENOMEM;
 137	return 0;
 138}
 139
 140static bool dmirror_is_private_zone(struct dmirror_device *mdevice)
 141{
 142	return (mdevice->zone_device_type ==
 143		HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? true : false;
 144}
 145
 146static enum migrate_vma_direction
 147dmirror_select_device(struct dmirror *dmirror)
 148{
 149	return (dmirror->mdevice->zone_device_type ==
 150		HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ?
 151		MIGRATE_VMA_SELECT_DEVICE_PRIVATE :
 152		MIGRATE_VMA_SELECT_DEVICE_COHERENT;
 153}
 154
 155static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
 156{
 157	vfree(bounce->ptr);
 158}
 159
 160static int dmirror_fops_open(struct inode *inode, struct file *filp)
 161{
 162	struct cdev *cdev = inode->i_cdev;
 163	struct dmirror *dmirror;
 164	int ret;
 165
 166	/* Mirror this process address space */
 167	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
 168	if (dmirror == NULL)
 169		return -ENOMEM;
 170
 171	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
 172	mutex_init(&dmirror->mutex);
 173	xa_init(&dmirror->pt);
 174
 175	ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
 176				0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
 177	if (ret) {
 178		kfree(dmirror);
 179		return ret;
 180	}
 181
 182	filp->private_data = dmirror;
 183	return 0;
 184}
 185
 186static int dmirror_fops_release(struct inode *inode, struct file *filp)
 187{
 188	struct dmirror *dmirror = filp->private_data;
 189
 190	mmu_interval_notifier_remove(&dmirror->notifier);
 191	xa_destroy(&dmirror->pt);
 192	kfree(dmirror);
 193	return 0;
 194}
 195
 196static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page)
 197{
 198	return container_of(page->pgmap, struct dmirror_chunk, pagemap);
 199}
 200
 201static struct dmirror_device *dmirror_page_to_device(struct page *page)
 202
 203{
 204	return dmirror_page_to_chunk(page)->mdevice;
 
 205}
 206
 207static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
 208{
 209	unsigned long *pfns = range->hmm_pfns;
 210	unsigned long pfn;
 211
 212	for (pfn = (range->start >> PAGE_SHIFT);
 213	     pfn < (range->end >> PAGE_SHIFT);
 214	     pfn++, pfns++) {
 215		struct page *page;
 216		void *entry;
 217
 218		/*
 219		 * Since we asked for hmm_range_fault() to populate pages,
 220		 * it shouldn't return an error entry on success.
 221		 */
 222		WARN_ON(*pfns & HMM_PFN_ERROR);
 223		WARN_ON(!(*pfns & HMM_PFN_VALID));
 224
 225		page = hmm_pfn_to_page(*pfns);
 226		WARN_ON(!page);
 227
 228		entry = page;
 229		if (*pfns & HMM_PFN_WRITE)
 230			entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
 231		else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))
 232			return -EFAULT;
 233		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 234		if (xa_is_err(entry))
 235			return xa_err(entry);
 236	}
 237
 238	return 0;
 239}
 240
 241static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
 242			      unsigned long end)
 243{
 244	unsigned long pfn;
 245	void *entry;
 246
 247	/*
 248	 * The XArray doesn't hold references to pages since it relies on
 249	 * the mmu notifier to clear page pointers when they become stale.
 250	 * Therefore, it is OK to just clear the entry.
 251	 */
 252	xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
 253			  end >> PAGE_SHIFT)
 254		xa_erase(&dmirror->pt, pfn);
 255}
 256
 257static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,
 258				const struct mmu_notifier_range *range,
 259				unsigned long cur_seq)
 260{
 261	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
 262
 263	/*
 264	 * Ignore invalidation callbacks for device private pages since
 265	 * the invalidation is handled as part of the migration process.
 266	 */
 267	if (range->event == MMU_NOTIFY_MIGRATE &&
 268	    range->owner == dmirror->mdevice)
 269		return true;
 270
 271	if (mmu_notifier_range_blockable(range))
 272		mutex_lock(&dmirror->mutex);
 273	else if (!mutex_trylock(&dmirror->mutex))
 274		return false;
 275
 276	mmu_interval_set_seq(mni, cur_seq);
 277	dmirror_do_update(dmirror, range->start, range->end);
 278
 279	mutex_unlock(&dmirror->mutex);
 280	return true;
 281}
 282
 283static const struct mmu_interval_notifier_ops dmirror_min_ops = {
 284	.invalidate = dmirror_interval_invalidate,
 285};
 286
 287static int dmirror_range_fault(struct dmirror *dmirror,
 288				struct hmm_range *range)
 289{
 290	struct mm_struct *mm = dmirror->notifier.mm;
 291	unsigned long timeout =
 292		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 293	int ret;
 294
 295	while (true) {
 296		if (time_after(jiffies, timeout)) {
 297			ret = -EBUSY;
 298			goto out;
 299		}
 300
 301		range->notifier_seq = mmu_interval_read_begin(range->notifier);
 302		mmap_read_lock(mm);
 303		ret = hmm_range_fault(range);
 304		mmap_read_unlock(mm);
 305		if (ret) {
 306			if (ret == -EBUSY)
 307				continue;
 308			goto out;
 309		}
 310
 311		mutex_lock(&dmirror->mutex);
 312		if (mmu_interval_read_retry(range->notifier,
 313					    range->notifier_seq)) {
 314			mutex_unlock(&dmirror->mutex);
 315			continue;
 316		}
 317		break;
 318	}
 319
 320	ret = dmirror_do_fault(dmirror, range);
 321
 322	mutex_unlock(&dmirror->mutex);
 323out:
 324	return ret;
 325}
 326
 327static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
 328			 unsigned long end, bool write)
 329{
 330	struct mm_struct *mm = dmirror->notifier.mm;
 331	unsigned long addr;
 332	unsigned long pfns[64];
 333	struct hmm_range range = {
 334		.notifier = &dmirror->notifier,
 335		.hmm_pfns = pfns,
 336		.pfn_flags_mask = 0,
 337		.default_flags =
 338			HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
 339		.dev_private_owner = dmirror->mdevice,
 340	};
 341	int ret = 0;
 342
 343	/* Since the mm is for the mirrored process, get a reference first. */
 344	if (!mmget_not_zero(mm))
 345		return 0;
 346
 347	for (addr = start; addr < end; addr = range.end) {
 348		range.start = addr;
 349		range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
 350
 351		ret = dmirror_range_fault(dmirror, &range);
 352		if (ret)
 353			break;
 354	}
 355
 356	mmput(mm);
 357	return ret;
 358}
 359
 360static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
 361			   unsigned long end, struct dmirror_bounce *bounce)
 362{
 363	unsigned long pfn;
 364	void *ptr;
 365
 366	ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
 367
 368	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 369		void *entry;
 370		struct page *page;
 
 371
 372		entry = xa_load(&dmirror->pt, pfn);
 373		page = xa_untag_pointer(entry);
 374		if (!page)
 375			return -ENOENT;
 376
 377		memcpy_from_page(ptr, page, 0, PAGE_SIZE);
 
 
 378
 379		ptr += PAGE_SIZE;
 380		bounce->cpages++;
 381	}
 382
 383	return 0;
 384}
 385
 386static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
 387{
 388	struct dmirror_bounce bounce;
 389	unsigned long start, end;
 390	unsigned long size = cmd->npages << PAGE_SHIFT;
 391	int ret;
 392
 393	start = cmd->addr;
 394	end = start + size;
 395	if (end < start)
 396		return -EINVAL;
 397
 398	ret = dmirror_bounce_init(&bounce, start, size);
 399	if (ret)
 400		return ret;
 401
 402	while (1) {
 403		mutex_lock(&dmirror->mutex);
 404		ret = dmirror_do_read(dmirror, start, end, &bounce);
 405		mutex_unlock(&dmirror->mutex);
 406		if (ret != -ENOENT)
 407			break;
 408
 409		start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
 410		ret = dmirror_fault(dmirror, start, end, false);
 411		if (ret)
 412			break;
 413		cmd->faults++;
 414	}
 415
 416	if (ret == 0) {
 417		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 418				 bounce.size))
 419			ret = -EFAULT;
 420	}
 421	cmd->cpages = bounce.cpages;
 422	dmirror_bounce_fini(&bounce);
 423	return ret;
 424}
 425
 426static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
 427			    unsigned long end, struct dmirror_bounce *bounce)
 428{
 429	unsigned long pfn;
 430	void *ptr;
 431
 432	ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
 433
 434	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 435		void *entry;
 436		struct page *page;
 
 437
 438		entry = xa_load(&dmirror->pt, pfn);
 439		page = xa_untag_pointer(entry);
 440		if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)
 441			return -ENOENT;
 442
 443		memcpy_to_page(page, 0, ptr, PAGE_SIZE);
 
 
 444
 445		ptr += PAGE_SIZE;
 446		bounce->cpages++;
 447	}
 448
 449	return 0;
 450}
 451
 452static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
 453{
 454	struct dmirror_bounce bounce;
 455	unsigned long start, end;
 456	unsigned long size = cmd->npages << PAGE_SHIFT;
 457	int ret;
 458
 459	start = cmd->addr;
 460	end = start + size;
 461	if (end < start)
 462		return -EINVAL;
 463
 464	ret = dmirror_bounce_init(&bounce, start, size);
 465	if (ret)
 466		return ret;
 467	if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),
 468			   bounce.size)) {
 469		ret = -EFAULT;
 470		goto fini;
 471	}
 472
 473	while (1) {
 474		mutex_lock(&dmirror->mutex);
 475		ret = dmirror_do_write(dmirror, start, end, &bounce);
 476		mutex_unlock(&dmirror->mutex);
 477		if (ret != -ENOENT)
 478			break;
 479
 480		start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
 481		ret = dmirror_fault(dmirror, start, end, true);
 482		if (ret)
 483			break;
 484		cmd->faults++;
 485	}
 486
 487fini:
 488	cmd->cpages = bounce.cpages;
 489	dmirror_bounce_fini(&bounce);
 490	return ret;
 491}
 492
 493static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
 494				   struct page **ppage)
 495{
 496	struct dmirror_chunk *devmem;
 497	struct resource *res = NULL;
 498	unsigned long pfn;
 499	unsigned long pfn_first;
 500	unsigned long pfn_last;
 501	void *ptr;
 502	int ret = -ENOMEM;
 503
 504	devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);
 505	if (!devmem)
 506		return ret;
 507
 508	switch (mdevice->zone_device_type) {
 509	case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE:
 510		res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,
 511					      "hmm_dmirror");
 512		if (IS_ERR_OR_NULL(res))
 513			goto err_devmem;
 514		devmem->pagemap.range.start = res->start;
 515		devmem->pagemap.range.end = res->end;
 516		devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
 517		break;
 518	case HMM_DMIRROR_MEMORY_DEVICE_COHERENT:
 519		devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ?
 520							spm_addr_dev0 :
 521							spm_addr_dev1;
 522		devmem->pagemap.range.end = devmem->pagemap.range.start +
 523					    DEVMEM_CHUNK_SIZE - 1;
 524		devmem->pagemap.type = MEMORY_DEVICE_COHERENT;
 525		break;
 526	default:
 527		ret = -EINVAL;
 528		goto err_devmem;
 529	}
 530
 
 
 
 531	devmem->pagemap.nr_range = 1;
 532	devmem->pagemap.ops = &dmirror_devmem_ops;
 533	devmem->pagemap.owner = mdevice;
 534
 535	mutex_lock(&mdevice->devmem_lock);
 536
 537	if (mdevice->devmem_count == mdevice->devmem_capacity) {
 538		struct dmirror_chunk **new_chunks;
 539		unsigned int new_capacity;
 540
 541		new_capacity = mdevice->devmem_capacity +
 542				DEVMEM_CHUNKS_RESERVE;
 543		new_chunks = krealloc(mdevice->devmem_chunks,
 544				sizeof(new_chunks[0]) * new_capacity,
 545				GFP_KERNEL);
 546		if (!new_chunks)
 547			goto err_release;
 548		mdevice->devmem_capacity = new_capacity;
 549		mdevice->devmem_chunks = new_chunks;
 550	}
 
 551	ptr = memremap_pages(&devmem->pagemap, numa_node_id());
 552	if (IS_ERR_OR_NULL(ptr)) {
 553		if (ptr)
 554			ret = PTR_ERR(ptr);
 555		else
 556			ret = -EFAULT;
 557		goto err_release;
 558	}
 559
 560	devmem->mdevice = mdevice;
 561	pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;
 562	pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);
 563	mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
 564
 565	mutex_unlock(&mdevice->devmem_lock);
 566
 567	pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
 568		DEVMEM_CHUNK_SIZE / (1024 * 1024),
 569		mdevice->devmem_count,
 570		mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),
 571		pfn_first, pfn_last);
 572
 573	spin_lock(&mdevice->lock);
 574	for (pfn = pfn_first; pfn < pfn_last; pfn++) {
 575		struct page *page = pfn_to_page(pfn);
 576
 577		page->zone_device_data = mdevice->free_pages;
 578		mdevice->free_pages = page;
 579	}
 580	if (ppage) {
 581		*ppage = mdevice->free_pages;
 582		mdevice->free_pages = (*ppage)->zone_device_data;
 583		mdevice->calloc++;
 584	}
 585	spin_unlock(&mdevice->lock);
 586
 587	return 0;
 588
 589err_release:
 590	mutex_unlock(&mdevice->devmem_lock);
 591	if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)
 592		release_mem_region(devmem->pagemap.range.start,
 593				   range_len(&devmem->pagemap.range));
 594err_devmem:
 595	kfree(devmem);
 596
 597	return ret;
 598}
 599
 600static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
 601{
 602	struct page *dpage = NULL;
 603	struct page *rpage = NULL;
 604
 605	/*
 606	 * For ZONE_DEVICE private type, this is a fake device so we allocate
 607	 * real system memory to store our device memory.
 608	 * For ZONE_DEVICE coherent type we use the actual dpage to store the
 609	 * data and ignore rpage.
 610	 */
 611	if (dmirror_is_private_zone(mdevice)) {
 612		rpage = alloc_page(GFP_HIGHUSER);
 613		if (!rpage)
 614			return NULL;
 615	}
 616	spin_lock(&mdevice->lock);
 617
 618	if (mdevice->free_pages) {
 619		dpage = mdevice->free_pages;
 620		mdevice->free_pages = dpage->zone_device_data;
 621		mdevice->calloc++;
 622		spin_unlock(&mdevice->lock);
 623	} else {
 624		spin_unlock(&mdevice->lock);
 625		if (dmirror_allocate_chunk(mdevice, &dpage))
 626			goto error;
 627	}
 628
 629	zone_device_page_init(dpage);
 630	dpage->zone_device_data = rpage;
 
 
 631	return dpage;
 632
 633error:
 634	if (rpage)
 635		__free_page(rpage);
 636	return NULL;
 637}
 638
 639static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
 640					   struct dmirror *dmirror)
 641{
 642	struct dmirror_device *mdevice = dmirror->mdevice;
 643	const unsigned long *src = args->src;
 644	unsigned long *dst = args->dst;
 645	unsigned long addr;
 646
 647	for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
 648						   src++, dst++) {
 649		struct page *spage;
 650		struct page *dpage;
 651		struct page *rpage;
 652
 653		if (!(*src & MIGRATE_PFN_MIGRATE))
 654			continue;
 655
 656		/*
 657		 * Note that spage might be NULL which is OK since it is an
 658		 * unallocated pte_none() or read-only zero page.
 659		 */
 660		spage = migrate_pfn_to_page(*src);
 661		if (WARN(spage && is_zone_device_page(spage),
 662		     "page already in device spage pfn: 0x%lx\n",
 663		     page_to_pfn(spage)))
 664			continue;
 665
 666		dpage = dmirror_devmem_alloc_page(mdevice);
 667		if (!dpage)
 668			continue;
 669
 670		rpage = BACKING_PAGE(dpage);
 671		if (spage)
 672			copy_highpage(rpage, spage);
 673		else
 674			clear_highpage(rpage);
 675
 676		/*
 677		 * Normally, a device would use the page->zone_device_data to
 678		 * point to the mirror but here we use it to hold the page for
 679		 * the simulated device memory and that page holds the pointer
 680		 * to the mirror.
 681		 */
 682		rpage->zone_device_data = dmirror;
 683
 684		pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
 685			 page_to_pfn(spage), page_to_pfn(dpage));
 686		*dst = migrate_pfn(page_to_pfn(dpage));
 687		if ((*src & MIGRATE_PFN_WRITE) ||
 688		    (!spage && args->vma->vm_flags & VM_WRITE))
 689			*dst |= MIGRATE_PFN_WRITE;
 690	}
 691}
 692
 693static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,
 694			     unsigned long end)
 695{
 696	unsigned long pfn;
 697
 698	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
 699		void *entry;
 700
 701		entry = xa_load(&dmirror->pt, pfn);
 702		if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC)
 703			return -EPERM;
 704	}
 705
 706	return 0;
 707}
 708
 709static int dmirror_atomic_map(unsigned long start, unsigned long end,
 710			      struct page **pages, struct dmirror *dmirror)
 711{
 712	unsigned long pfn, mapped = 0;
 713	int i;
 714
 715	/* Map the migrated pages into the device's page tables. */
 716	mutex_lock(&dmirror->mutex);
 717
 718	for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) {
 719		void *entry;
 720
 721		if (!pages[i])
 722			continue;
 723
 724		entry = pages[i];
 725		entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC);
 726		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 727		if (xa_is_err(entry)) {
 728			mutex_unlock(&dmirror->mutex);
 729			return xa_err(entry);
 730		}
 731
 732		mapped++;
 733	}
 734
 735	mutex_unlock(&dmirror->mutex);
 736	return mapped;
 737}
 738
 739static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
 740					    struct dmirror *dmirror)
 741{
 742	unsigned long start = args->start;
 743	unsigned long end = args->end;
 744	const unsigned long *src = args->src;
 745	const unsigned long *dst = args->dst;
 746	unsigned long pfn;
 747
 748	/* Map the migrated pages into the device's page tables. */
 749	mutex_lock(&dmirror->mutex);
 750
 751	for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
 752								src++, dst++) {
 753		struct page *dpage;
 754		void *entry;
 755
 756		if (!(*src & MIGRATE_PFN_MIGRATE))
 757			continue;
 758
 759		dpage = migrate_pfn_to_page(*dst);
 760		if (!dpage)
 761			continue;
 762
 763		entry = BACKING_PAGE(dpage);
 
 
 
 
 764		if (*dst & MIGRATE_PFN_WRITE)
 765			entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
 766		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
 767		if (xa_is_err(entry)) {
 768			mutex_unlock(&dmirror->mutex);
 769			return xa_err(entry);
 770		}
 771	}
 772
 773	mutex_unlock(&dmirror->mutex);
 774	return 0;
 775}
 776
 777static int dmirror_exclusive(struct dmirror *dmirror,
 778			     struct hmm_dmirror_cmd *cmd)
 779{
 780	unsigned long start, end, addr;
 781	unsigned long size = cmd->npages << PAGE_SHIFT;
 782	struct mm_struct *mm = dmirror->notifier.mm;
 783	struct page *pages[64];
 784	struct dmirror_bounce bounce;
 785	unsigned long next;
 786	int ret;
 787
 788	start = cmd->addr;
 789	end = start + size;
 790	if (end < start)
 791		return -EINVAL;
 792
 793	/* Since the mm is for the mirrored process, get a reference first. */
 794	if (!mmget_not_zero(mm))
 795		return -EINVAL;
 796
 797	mmap_read_lock(mm);
 798	for (addr = start; addr < end; addr = next) {
 799		unsigned long mapped = 0;
 800		int i;
 801
 802		next = min(end, addr + (ARRAY_SIZE(pages) << PAGE_SHIFT));
 
 
 
 803
 804		ret = make_device_exclusive_range(mm, addr, next, pages, NULL);
 805		/*
 806		 * Do dmirror_atomic_map() iff all pages are marked for
 807		 * exclusive access to avoid accessing uninitialized
 808		 * fields of pages.
 809		 */
 810		if (ret == (next - addr) >> PAGE_SHIFT)
 811			mapped = dmirror_atomic_map(addr, next, pages, dmirror);
 812		for (i = 0; i < ret; i++) {
 813			if (pages[i]) {
 814				unlock_page(pages[i]);
 815				put_page(pages[i]);
 816			}
 817		}
 818
 819		if (addr + (mapped << PAGE_SHIFT) < next) {
 820			mmap_read_unlock(mm);
 821			mmput(mm);
 822			return -EBUSY;
 823		}
 824	}
 825	mmap_read_unlock(mm);
 826	mmput(mm);
 827
 828	/* Return the migrated data for verification. */
 829	ret = dmirror_bounce_init(&bounce, start, size);
 830	if (ret)
 831		return ret;
 832	mutex_lock(&dmirror->mutex);
 833	ret = dmirror_do_read(dmirror, start, end, &bounce);
 834	mutex_unlock(&dmirror->mutex);
 835	if (ret == 0) {
 836		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
 837				 bounce.size))
 838			ret = -EFAULT;
 839	}
 840
 841	cmd->cpages = bounce.cpages;
 842	dmirror_bounce_fini(&bounce);
 843	return ret;
 844}
 845
 846static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
 847						      struct dmirror *dmirror)
 848{
 849	const unsigned long *src = args->src;
 850	unsigned long *dst = args->dst;
 851	unsigned long start = args->start;
 852	unsigned long end = args->end;
 853	unsigned long addr;
 854
 855	for (addr = start; addr < end; addr += PAGE_SIZE,
 856				       src++, dst++) {
 857		struct page *dpage, *spage;
 858
 859		spage = migrate_pfn_to_page(*src);
 860		if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
 861			continue;
 862
 863		if (WARN_ON(!is_device_private_page(spage) &&
 864			    !is_device_coherent_page(spage)))
 865			continue;
 866		spage = BACKING_PAGE(spage);
 867		dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
 868		if (!dpage)
 869			continue;
 870		pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n",
 871			 page_to_pfn(spage), page_to_pfn(dpage));
 872
 873		lock_page(dpage);
 874		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
 875		copy_highpage(dpage, spage);
 876		*dst = migrate_pfn(page_to_pfn(dpage));
 877		if (*src & MIGRATE_PFN_WRITE)
 878			*dst |= MIGRATE_PFN_WRITE;
 879	}
 880	return 0;
 881}
 882
 883static unsigned long
 884dmirror_successful_migrated_pages(struct migrate_vma *migrate)
 885{
 886	unsigned long cpages = 0;
 887	unsigned long i;
 888
 889	for (i = 0; i < migrate->npages; i++) {
 890		if (migrate->src[i] & MIGRATE_PFN_VALID &&
 891		    migrate->src[i] & MIGRATE_PFN_MIGRATE)
 892			cpages++;
 893	}
 894	return cpages;
 895}
 896
 897static int dmirror_migrate_to_system(struct dmirror *dmirror,
 898				     struct hmm_dmirror_cmd *cmd)
 899{
 900	unsigned long start, end, addr;
 901	unsigned long size = cmd->npages << PAGE_SHIFT;
 902	struct mm_struct *mm = dmirror->notifier.mm;
 903	struct vm_area_struct *vma;
 904	unsigned long src_pfns[64] = { 0 };
 905	unsigned long dst_pfns[64] = { 0 };
 906	struct migrate_vma args = { 0 };
 907	unsigned long next;
 908	int ret;
 909
 910	start = cmd->addr;
 911	end = start + size;
 912	if (end < start)
 913		return -EINVAL;
 914
 915	/* Since the mm is for the mirrored process, get a reference first. */
 916	if (!mmget_not_zero(mm))
 917		return -EINVAL;
 918
 919	cmd->cpages = 0;
 920	mmap_read_lock(mm);
 921	for (addr = start; addr < end; addr = next) {
 922		vma = vma_lookup(mm, addr);
 923		if (!vma || !(vma->vm_flags & VM_READ)) {
 924			ret = -EINVAL;
 925			goto out;
 926		}
 927		next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
 928		if (next > vma->vm_end)
 929			next = vma->vm_end;
 930
 931		args.vma = vma;
 932		args.src = src_pfns;
 933		args.dst = dst_pfns;
 934		args.start = addr;
 935		args.end = next;
 936		args.pgmap_owner = dmirror->mdevice;
 937		args.flags = dmirror_select_device(dmirror);
 938
 939		ret = migrate_vma_setup(&args);
 940		if (ret)
 941			goto out;
 942
 943		pr_debug("Migrating from device mem to sys mem\n");
 944		dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
 945
 946		migrate_vma_pages(&args);
 947		cmd->cpages += dmirror_successful_migrated_pages(&args);
 948		migrate_vma_finalize(&args);
 949	}
 950out:
 951	mmap_read_unlock(mm);
 952	mmput(mm);
 953
 954	return ret;
 955}
 956
 957static int dmirror_migrate_to_device(struct dmirror *dmirror,
 958				struct hmm_dmirror_cmd *cmd)
 959{
 960	unsigned long start, end, addr;
 961	unsigned long size = cmd->npages << PAGE_SHIFT;
 962	struct mm_struct *mm = dmirror->notifier.mm;
 963	struct vm_area_struct *vma;
 964	unsigned long src_pfns[64] = { 0 };
 965	unsigned long dst_pfns[64] = { 0 };
 966	struct dmirror_bounce bounce;
 967	struct migrate_vma args = { 0 };
 968	unsigned long next;
 969	int ret;
 970
 971	start = cmd->addr;
 972	end = start + size;
 973	if (end < start)
 974		return -EINVAL;
 975
 976	/* Since the mm is for the mirrored process, get a reference first. */
 977	if (!mmget_not_zero(mm))
 978		return -EINVAL;
 979
 980	mmap_read_lock(mm);
 981	for (addr = start; addr < end; addr = next) {
 982		vma = vma_lookup(mm, addr);
 983		if (!vma || !(vma->vm_flags & VM_READ)) {
 984			ret = -EINVAL;
 985			goto out;
 986		}
 987		next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
 988		if (next > vma->vm_end)
 989			next = vma->vm_end;
 990
 991		args.vma = vma;
 992		args.src = src_pfns;
 993		args.dst = dst_pfns;
 994		args.start = addr;
 995		args.end = next;
 996		args.pgmap_owner = dmirror->mdevice;
 997		args.flags = MIGRATE_VMA_SELECT_SYSTEM;
 998		ret = migrate_vma_setup(&args);
 999		if (ret)
1000			goto out;
1001
1002		pr_debug("Migrating from sys mem to device mem\n");
1003		dmirror_migrate_alloc_and_copy(&args, dmirror);
1004		migrate_vma_pages(&args);
1005		dmirror_migrate_finalize_and_map(&args, dmirror);
1006		migrate_vma_finalize(&args);
1007	}
1008	mmap_read_unlock(mm);
1009	mmput(mm);
1010
1011	/*
1012	 * Return the migrated data for verification.
1013	 * Only for pages in device zone
1014	 */
1015	ret = dmirror_bounce_init(&bounce, start, size);
1016	if (ret)
1017		return ret;
1018	mutex_lock(&dmirror->mutex);
1019	ret = dmirror_do_read(dmirror, start, end, &bounce);
1020	mutex_unlock(&dmirror->mutex);
1021	if (ret == 0) {
1022		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
1023				 bounce.size))
1024			ret = -EFAULT;
1025	}
1026	cmd->cpages = bounce.cpages;
1027	dmirror_bounce_fini(&bounce);
1028	return ret;
1029
1030out:
1031	mmap_read_unlock(mm);
1032	mmput(mm);
1033	return ret;
1034}
1035
1036static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
1037			    unsigned char *perm, unsigned long entry)
1038{
1039	struct page *page;
1040
1041	if (entry & HMM_PFN_ERROR) {
1042		*perm = HMM_DMIRROR_PROT_ERROR;
1043		return;
1044	}
1045	if (!(entry & HMM_PFN_VALID)) {
1046		*perm = HMM_DMIRROR_PROT_NONE;
1047		return;
1048	}
1049
1050	page = hmm_pfn_to_page(entry);
1051	if (is_device_private_page(page)) {
1052		/* Is the page migrated to this device or some other? */
1053		if (dmirror->mdevice == dmirror_page_to_device(page))
1054			*perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;
1055		else
1056			*perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;
1057	} else if (is_device_coherent_page(page)) {
1058		/* Is the page migrated to this device or some other? */
1059		if (dmirror->mdevice == dmirror_page_to_device(page))
1060			*perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL;
1061		else
1062			*perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE;
1063	} else if (is_zero_pfn(page_to_pfn(page)))
1064		*perm = HMM_DMIRROR_PROT_ZERO;
1065	else
1066		*perm = HMM_DMIRROR_PROT_NONE;
1067	if (entry & HMM_PFN_WRITE)
1068		*perm |= HMM_DMIRROR_PROT_WRITE;
1069	else
1070		*perm |= HMM_DMIRROR_PROT_READ;
1071	if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
1072		*perm |= HMM_DMIRROR_PROT_PMD;
1073	else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
1074		*perm |= HMM_DMIRROR_PROT_PUD;
1075}
1076
1077static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
1078				const struct mmu_notifier_range *range,
1079				unsigned long cur_seq)
1080{
1081	struct dmirror_interval *dmi =
1082		container_of(mni, struct dmirror_interval, notifier);
1083	struct dmirror *dmirror = dmi->dmirror;
1084
1085	if (mmu_notifier_range_blockable(range))
1086		mutex_lock(&dmirror->mutex);
1087	else if (!mutex_trylock(&dmirror->mutex))
1088		return false;
1089
1090	/*
1091	 * Snapshots only need to set the sequence number since any
1092	 * invalidation in the interval invalidates the whole snapshot.
1093	 */
1094	mmu_interval_set_seq(mni, cur_seq);
1095
1096	mutex_unlock(&dmirror->mutex);
1097	return true;
1098}
1099
1100static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {
1101	.invalidate = dmirror_snapshot_invalidate,
1102};
1103
1104static int dmirror_range_snapshot(struct dmirror *dmirror,
1105				  struct hmm_range *range,
1106				  unsigned char *perm)
1107{
1108	struct mm_struct *mm = dmirror->notifier.mm;
1109	struct dmirror_interval notifier;
1110	unsigned long timeout =
1111		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1112	unsigned long i;
1113	unsigned long n;
1114	int ret = 0;
1115
1116	notifier.dmirror = dmirror;
1117	range->notifier = &notifier.notifier;
1118
1119	ret = mmu_interval_notifier_insert(range->notifier, mm,
1120			range->start, range->end - range->start,
1121			&dmirror_mrn_ops);
1122	if (ret)
1123		return ret;
1124
1125	while (true) {
1126		if (time_after(jiffies, timeout)) {
1127			ret = -EBUSY;
1128			goto out;
1129		}
1130
1131		range->notifier_seq = mmu_interval_read_begin(range->notifier);
1132
1133		mmap_read_lock(mm);
1134		ret = hmm_range_fault(range);
1135		mmap_read_unlock(mm);
1136		if (ret) {
1137			if (ret == -EBUSY)
1138				continue;
1139			goto out;
1140		}
1141
1142		mutex_lock(&dmirror->mutex);
1143		if (mmu_interval_read_retry(range->notifier,
1144					    range->notifier_seq)) {
1145			mutex_unlock(&dmirror->mutex);
1146			continue;
1147		}
1148		break;
1149	}
1150
1151	n = (range->end - range->start) >> PAGE_SHIFT;
1152	for (i = 0; i < n; i++)
1153		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
1154
1155	mutex_unlock(&dmirror->mutex);
1156out:
1157	mmu_interval_notifier_remove(range->notifier);
1158	return ret;
1159}
1160
1161static int dmirror_snapshot(struct dmirror *dmirror,
1162			    struct hmm_dmirror_cmd *cmd)
1163{
1164	struct mm_struct *mm = dmirror->notifier.mm;
1165	unsigned long start, end;
1166	unsigned long size = cmd->npages << PAGE_SHIFT;
1167	unsigned long addr;
1168	unsigned long next;
1169	unsigned long pfns[64];
1170	unsigned char perm[64];
1171	char __user *uptr;
1172	struct hmm_range range = {
1173		.hmm_pfns = pfns,
1174		.dev_private_owner = dmirror->mdevice,
1175	};
1176	int ret = 0;
1177
1178	start = cmd->addr;
1179	end = start + size;
1180	if (end < start)
1181		return -EINVAL;
1182
1183	/* Since the mm is for the mirrored process, get a reference first. */
1184	if (!mmget_not_zero(mm))
1185		return -EINVAL;
1186
1187	/*
1188	 * Register a temporary notifier to detect invalidations even if it
1189	 * overlaps with other mmu_interval_notifiers.
1190	 */
1191	uptr = u64_to_user_ptr(cmd->ptr);
1192	for (addr = start; addr < end; addr = next) {
1193		unsigned long n;
1194
1195		next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
1196		range.start = addr;
1197		range.end = next;
1198
1199		ret = dmirror_range_snapshot(dmirror, &range, perm);
1200		if (ret)
1201			break;
1202
1203		n = (range.end - range.start) >> PAGE_SHIFT;
1204		if (copy_to_user(uptr, perm, n)) {
1205			ret = -EFAULT;
1206			break;
1207		}
1208
1209		cmd->cpages += n;
1210		uptr += n;
1211	}
1212	mmput(mm);
1213
1214	return ret;
1215}
1216
1217static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)
1218{
1219	unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT;
1220	unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT;
1221	unsigned long npages = end_pfn - start_pfn + 1;
1222	unsigned long i;
1223	unsigned long *src_pfns;
1224	unsigned long *dst_pfns;
1225
1226	src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
1227	dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
1228
1229	migrate_device_range(src_pfns, start_pfn, npages);
1230	for (i = 0; i < npages; i++) {
1231		struct page *dpage, *spage;
1232
1233		spage = migrate_pfn_to_page(src_pfns[i]);
1234		if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
1235			continue;
1236
1237		if (WARN_ON(!is_device_private_page(spage) &&
1238			    !is_device_coherent_page(spage)))
1239			continue;
1240		spage = BACKING_PAGE(spage);
1241		dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);
1242		lock_page(dpage);
1243		copy_highpage(dpage, spage);
1244		dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));
1245		if (src_pfns[i] & MIGRATE_PFN_WRITE)
1246			dst_pfns[i] |= MIGRATE_PFN_WRITE;
1247	}
1248	migrate_device_pages(src_pfns, dst_pfns, npages);
1249	migrate_device_finalize(src_pfns, dst_pfns, npages);
1250	kvfree(src_pfns);
1251	kvfree(dst_pfns);
1252}
1253
1254/* Removes free pages from the free list so they can't be re-allocated */
1255static void dmirror_remove_free_pages(struct dmirror_chunk *devmem)
1256{
1257	struct dmirror_device *mdevice = devmem->mdevice;
1258	struct page *page;
1259
1260	for (page = mdevice->free_pages; page; page = page->zone_device_data)
1261		if (dmirror_page_to_chunk(page) == devmem)
1262			mdevice->free_pages = page->zone_device_data;
1263}
1264
1265static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)
1266{
1267	unsigned int i;
1268
1269	mutex_lock(&mdevice->devmem_lock);
1270	if (mdevice->devmem_chunks) {
1271		for (i = 0; i < mdevice->devmem_count; i++) {
1272			struct dmirror_chunk *devmem =
1273				mdevice->devmem_chunks[i];
1274
1275			spin_lock(&mdevice->lock);
1276			devmem->remove = true;
1277			dmirror_remove_free_pages(devmem);
1278			spin_unlock(&mdevice->lock);
1279
1280			dmirror_device_evict_chunk(devmem);
1281			memunmap_pages(&devmem->pagemap);
1282			if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)
1283				release_mem_region(devmem->pagemap.range.start,
1284						   range_len(&devmem->pagemap.range));
1285			kfree(devmem);
1286		}
1287		mdevice->devmem_count = 0;
1288		mdevice->devmem_capacity = 0;
1289		mdevice->free_pages = NULL;
1290		kfree(mdevice->devmem_chunks);
1291		mdevice->devmem_chunks = NULL;
1292	}
1293	mutex_unlock(&mdevice->devmem_lock);
1294}
1295
1296static long dmirror_fops_unlocked_ioctl(struct file *filp,
1297					unsigned int command,
1298					unsigned long arg)
1299{
1300	void __user *uarg = (void __user *)arg;
1301	struct hmm_dmirror_cmd cmd;
1302	struct dmirror *dmirror;
1303	int ret;
1304
1305	dmirror = filp->private_data;
1306	if (!dmirror)
1307		return -EINVAL;
1308
1309	if (copy_from_user(&cmd, uarg, sizeof(cmd)))
1310		return -EFAULT;
1311
1312	if (cmd.addr & ~PAGE_MASK)
1313		return -EINVAL;
1314	if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
1315		return -EINVAL;
1316
1317	cmd.cpages = 0;
1318	cmd.faults = 0;
1319
1320	switch (command) {
1321	case HMM_DMIRROR_READ:
1322		ret = dmirror_read(dmirror, &cmd);
1323		break;
1324
1325	case HMM_DMIRROR_WRITE:
1326		ret = dmirror_write(dmirror, &cmd);
1327		break;
1328
1329	case HMM_DMIRROR_MIGRATE_TO_DEV:
1330		ret = dmirror_migrate_to_device(dmirror, &cmd);
1331		break;
1332
1333	case HMM_DMIRROR_MIGRATE_TO_SYS:
1334		ret = dmirror_migrate_to_system(dmirror, &cmd);
1335		break;
1336
1337	case HMM_DMIRROR_EXCLUSIVE:
1338		ret = dmirror_exclusive(dmirror, &cmd);
1339		break;
1340
1341	case HMM_DMIRROR_CHECK_EXCLUSIVE:
1342		ret = dmirror_check_atomic(dmirror, cmd.addr,
1343					cmd.addr + (cmd.npages << PAGE_SHIFT));
1344		break;
1345
1346	case HMM_DMIRROR_SNAPSHOT:
1347		ret = dmirror_snapshot(dmirror, &cmd);
1348		break;
1349
1350	case HMM_DMIRROR_RELEASE:
1351		dmirror_device_remove_chunks(dmirror->mdevice);
1352		ret = 0;
1353		break;
1354
1355	default:
1356		return -EINVAL;
1357	}
1358	if (ret)
1359		return ret;
1360
1361	if (copy_to_user(uarg, &cmd, sizeof(cmd)))
1362		return -EFAULT;
1363
1364	return 0;
1365}
1366
1367static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma)
1368{
1369	unsigned long addr;
1370
1371	for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
1372		struct page *page;
1373		int ret;
1374
1375		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1376		if (!page)
1377			return -ENOMEM;
1378
1379		ret = vm_insert_page(vma, addr, page);
1380		if (ret) {
1381			__free_page(page);
1382			return ret;
1383		}
1384		put_page(page);
1385	}
1386
1387	return 0;
1388}
1389
1390static const struct file_operations dmirror_fops = {
1391	.open		= dmirror_fops_open,
1392	.release	= dmirror_fops_release,
1393	.mmap		= dmirror_fops_mmap,
1394	.unlocked_ioctl = dmirror_fops_unlocked_ioctl,
1395	.llseek		= default_llseek,
1396	.owner		= THIS_MODULE,
1397};
1398
1399static void dmirror_devmem_free(struct page *page)
1400{
1401	struct page *rpage = BACKING_PAGE(page);
1402	struct dmirror_device *mdevice;
1403
1404	if (rpage != page)
1405		__free_page(rpage);
1406
1407	mdevice = dmirror_page_to_device(page);
 
1408	spin_lock(&mdevice->lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1409
1410	/* Return page to our allocator if not freeing the chunk */
1411	if (!dmirror_page_to_chunk(page)->remove) {
1412		mdevice->cfree++;
1413		page->zone_device_data = mdevice->free_pages;
1414		mdevice->free_pages = page;
 
1415	}
1416	spin_unlock(&mdevice->lock);
1417}
1418
1419static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
1420{
1421	struct migrate_vma args = { 0 };
1422	unsigned long src_pfns = 0;
1423	unsigned long dst_pfns = 0;
1424	struct page *rpage;
1425	struct dmirror *dmirror;
1426	vm_fault_t ret;
1427
1428	/*
1429	 * Normally, a device would use the page->zone_device_data to point to
1430	 * the mirror but here we use it to hold the page for the simulated
1431	 * device memory and that page holds the pointer to the mirror.
1432	 */
1433	rpage = vmf->page->zone_device_data;
1434	dmirror = rpage->zone_device_data;
1435
1436	/* FIXME demonstrate how we can adjust migrate range */
1437	args.vma = vmf->vma;
1438	args.start = vmf->address;
1439	args.end = args.start + PAGE_SIZE;
1440	args.src = &src_pfns;
1441	args.dst = &dst_pfns;
1442	args.pgmap_owner = dmirror->mdevice;
1443	args.flags = dmirror_select_device(dmirror);
1444	args.fault_page = vmf->page;
1445
1446	if (migrate_vma_setup(&args))
1447		return VM_FAULT_SIGBUS;
1448
1449	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
1450	if (ret)
1451		return ret;
1452	migrate_vma_pages(&args);
1453	/*
1454	 * No device finalize step is needed since
1455	 * dmirror_devmem_fault_alloc_and_copy() will have already
1456	 * invalidated the device page table.
1457	 */
1458	migrate_vma_finalize(&args);
1459	return 0;
1460}
1461
1462static const struct dev_pagemap_ops dmirror_devmem_ops = {
1463	.page_free	= dmirror_devmem_free,
1464	.migrate_to_ram	= dmirror_devmem_fault,
1465};
1466
1467static int dmirror_device_init(struct dmirror_device *mdevice, int id)
1468{
1469	dev_t dev;
1470	int ret;
1471
1472	dev = MKDEV(MAJOR(dmirror_dev), id);
1473	mutex_init(&mdevice->devmem_lock);
1474	spin_lock_init(&mdevice->lock);
1475
1476	cdev_init(&mdevice->cdevice, &dmirror_fops);
1477	mdevice->cdevice.owner = THIS_MODULE;
1478	device_initialize(&mdevice->device);
1479	mdevice->device.devt = dev;
1480
1481	ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);
1482	if (ret)
1483		return ret;
1484
1485	ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);
1486	if (ret)
1487		return ret;
1488
1489	/* Build a list of free ZONE_DEVICE struct pages */
1490	return dmirror_allocate_chunk(mdevice, NULL);
1491}
1492
1493static void dmirror_device_remove(struct dmirror_device *mdevice)
1494{
1495	dmirror_device_remove_chunks(mdevice);
1496	cdev_device_del(&mdevice->cdevice, &mdevice->device);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1497}
1498
1499static int __init hmm_dmirror_init(void)
1500{
1501	int ret;
1502	int id = 0;
1503	int ndevices = 0;
1504
1505	ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,
1506				  "HMM_DMIRROR");
1507	if (ret)
1508		goto err_unreg;
1509
1510	memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0]));
1511	dmirror_devices[ndevices++].zone_device_type =
1512				HMM_DMIRROR_MEMORY_DEVICE_PRIVATE;
1513	dmirror_devices[ndevices++].zone_device_type =
1514				HMM_DMIRROR_MEMORY_DEVICE_PRIVATE;
1515	if (spm_addr_dev0 && spm_addr_dev1) {
1516		dmirror_devices[ndevices++].zone_device_type =
1517					HMM_DMIRROR_MEMORY_DEVICE_COHERENT;
1518		dmirror_devices[ndevices++].zone_device_type =
1519					HMM_DMIRROR_MEMORY_DEVICE_COHERENT;
1520	}
1521	for (id = 0; id < ndevices; id++) {
1522		ret = dmirror_device_init(dmirror_devices + id, id);
1523		if (ret)
1524			goto err_chrdev;
1525	}
1526
1527	pr_info("HMM test module loaded. This is only for testing HMM.\n");
1528	return 0;
1529
1530err_chrdev:
1531	while (--id >= 0)
1532		dmirror_device_remove(dmirror_devices + id);
1533	unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1534err_unreg:
1535	return ret;
1536}
1537
1538static void __exit hmm_dmirror_exit(void)
1539{
1540	int id;
1541
1542	for (id = 0; id < DMIRROR_NDEVICES; id++)
1543		if (dmirror_devices[id].zone_device_type)
1544			dmirror_device_remove(dmirror_devices + id);
1545	unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1546}
1547
1548module_init(hmm_dmirror_init);
1549module_exit(hmm_dmirror_exit);
1550MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module");
1551MODULE_LICENSE("GPL");