1/*
   2 * linux/kernel/power/snapshot.c
   3 *
   4 * This file provides system snapshot/restore functionality for swsusp.
   5 *
   6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz>
   7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
   8 *
   9 * This file is released under the GPLv2.
  10 *
  11 */
  12
  13#include <linux/version.h>
  14#include <linux/module.h>
  15#include <linux/mm.h>
  16#include <linux/suspend.h>
  17#include <linux/delay.h>
  18#include <linux/bitops.h>
  19#include <linux/spinlock.h>
  20#include <linux/kernel.h>
  21#include <linux/pm.h>
  22#include <linux/device.h>
  23#include <linux/init.h>
  24#include <linux/bootmem.h>
  25#include <linux/syscalls.h>
  26#include <linux/console.h>
  27#include <linux/highmem.h>
  28#include <linux/list.h>
  29#include <linux/slab.h>
 
  30
  31#include <asm/uaccess.h>
  32#include <asm/mmu_context.h>
  33#include <asm/pgtable.h>
  34#include <asm/tlbflush.h>
  35#include <asm/io.h>
  36
  37#include "power.h"
  38
  39static int swsusp_page_is_free(struct page *);
  40static void swsusp_set_page_forbidden(struct page *);
  41static void swsusp_unset_page_forbidden(struct page *);
  42
  43/*
  44 * Number of bytes to reserve for memory allocations made by device drivers
  45 * from their ->freeze() and ->freeze_noirq() callbacks so that they don't
  46 * cause image creation to fail (tunable via /sys/power/reserved_size).
  47 */
  48unsigned long reserved_size;
  49
  50void __init hibernate_reserved_size_init(void)
  51{
  52	reserved_size = SPARE_PAGES * PAGE_SIZE;
  53}
  54
  55/*
  56 * Preferred image size in bytes (tunable via /sys/power/image_size).
  57 * When it is set to N, swsusp will do its best to ensure the image
  58 * size will not exceed N bytes, but if that is impossible, it will
  59 * try to create the smallest image possible.
  60 */
  61unsigned long image_size;
  62
  63void __init hibernate_image_size_init(void)
  64{
  65	image_size = ((totalram_pages * 2) / 5) * PAGE_SIZE;
  66}
  67
  68/* List of PBEs needed for restoring the pages that were allocated before
  69 * the suspend and included in the suspend image, but have also been
  70 * allocated by the "resume" kernel, so their contents cannot be written
  71 * directly to their "original" page frames.
  72 */
  73struct pbe *restore_pblist;
  74
  75/* Pointer to an auxiliary buffer (1 page) */
  76static void *buffer;
  77
  78/**
  79 *	@safe_needed - on resume, for storing the PBE list and the image,
  80 *	we can only use memory pages that do not conflict with the pages
  81 *	used before suspend.  The unsafe pages have PageNosaveFree set
  82 *	and we count them using unsafe_pages.
  83 *
  84 *	Each allocated image page is marked as PageNosave and PageNosaveFree
  85 *	so that swsusp_free() can release it.
  86 */
  87
  88#define PG_ANY		0
  89#define PG_SAFE		1
  90#define PG_UNSAFE_CLEAR	1
  91#define PG_UNSAFE_KEEP	0
  92
  93static unsigned int allocated_unsafe_pages;
  94
  95static void *get_image_page(gfp_t gfp_mask, int safe_needed)
  96{
  97	void *res;
  98
  99	res = (void *)get_zeroed_page(gfp_mask);
 100	if (safe_needed)
 101		while (res && swsusp_page_is_free(virt_to_page(res))) {
 102			/* The page is unsafe, mark it for swsusp_free() */
 103			swsusp_set_page_forbidden(virt_to_page(res));
 104			allocated_unsafe_pages++;
 105			res = (void *)get_zeroed_page(gfp_mask);
 106		}
 107	if (res) {
 108		swsusp_set_page_forbidden(virt_to_page(res));
 109		swsusp_set_page_free(virt_to_page(res));
 110	}
 111	return res;
 112}
 113
 114unsigned long get_safe_page(gfp_t gfp_mask)
 115{
 116	return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
 117}
 118
 119static struct page *alloc_image_page(gfp_t gfp_mask)
 120{
 121	struct page *page;
 122
 123	page = alloc_page(gfp_mask);
 124	if (page) {
 125		swsusp_set_page_forbidden(page);
 126		swsusp_set_page_free(page);
 127	}
 128	return page;
 129}
 130
 131/**
 132 *	free_image_page - free page represented by @addr, allocated with
 133 *	get_image_page (page flags set by it must be cleared)
 134 */
 135
 136static inline void free_image_page(void *addr, int clear_nosave_free)
 137{
 138	struct page *page;
 139
 140	BUG_ON(!virt_addr_valid(addr));
 141
 142	page = virt_to_page(addr);
 143
 144	swsusp_unset_page_forbidden(page);
 145	if (clear_nosave_free)
 146		swsusp_unset_page_free(page);
 147
 148	__free_page(page);
 149}
 150
 151/* struct linked_page is used to build chains of pages */
 152
 153#define LINKED_PAGE_DATA_SIZE	(PAGE_SIZE - sizeof(void *))
 154
 155struct linked_page {
 156	struct linked_page *next;
 157	char data[LINKED_PAGE_DATA_SIZE];
 158} __attribute__((packed));
 159
 160static inline void
 161free_list_of_pages(struct linked_page *list, int clear_page_nosave)
 162{
 163	while (list) {
 164		struct linked_page *lp = list->next;
 165
 166		free_image_page(list, clear_page_nosave);
 167		list = lp;
 168	}
 169}
 170
 171/**
 172  *	struct chain_allocator is used for allocating small objects out of
 173  *	a linked list of pages called 'the chain'.
 174  *
 175  *	The chain grows each time when there is no room for a new object in
 176  *	the current page.  The allocated objects cannot be freed individually.
 177  *	It is only possible to free them all at once, by freeing the entire
 178  *	chain.
 179  *
 180  *	NOTE: The chain allocator may be inefficient if the allocated objects
 181  *	are not much smaller than PAGE_SIZE.
 182  */
 183
 184struct chain_allocator {
 185	struct linked_page *chain;	/* the chain */
 186	unsigned int used_space;	/* total size of objects allocated out
 187					 * of the current page
 188					 */
 189	gfp_t gfp_mask;		/* mask for allocating pages */
 190	int safe_needed;	/* if set, only "safe" pages are allocated */
 191};
 192
 193static void
 194chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
 195{
 196	ca->chain = NULL;
 197	ca->used_space = LINKED_PAGE_DATA_SIZE;
 198	ca->gfp_mask = gfp_mask;
 199	ca->safe_needed = safe_needed;
 200}
 201
 202static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
 203{
 204	void *ret;
 205
 206	if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
 207		struct linked_page *lp;
 208
 209		lp = get_image_page(ca->gfp_mask, ca->safe_needed);
 210		if (!lp)
 211			return NULL;
 212
 213		lp->next = ca->chain;
 214		ca->chain = lp;
 215		ca->used_space = 0;
 216	}
 217	ret = ca->chain->data + ca->used_space;
 218	ca->used_space += size;
 219	return ret;
 220}
 221
 222/**
 223 *	Data types related to memory bitmaps.
 224 *
 225 *	Memory bitmap is a structure consiting of many linked lists of
 226 *	objects.  The main list's elements are of type struct zone_bitmap
 227 *	and each of them corresonds to one zone.  For each zone bitmap
 228 *	object there is a list of objects of type struct bm_block that
 229 *	represent each blocks of bitmap in which information is stored.
 230 *
 231 *	struct memory_bitmap contains a pointer to the main list of zone
 232 *	bitmap objects, a struct bm_position used for browsing the bitmap,
 233 *	and a pointer to the list of pages used for allocating all of the
 234 *	zone bitmap objects and bitmap block objects.
 235 *
 236 *	NOTE: It has to be possible to lay out the bitmap in memory
 237 *	using only allocations of order 0.  Additionally, the bitmap is
 238 *	designed to work with arbitrary number of zones (this is over the
 239 *	top for now, but let's avoid making unnecessary assumptions ;-).
 240 *
 241 *	struct zone_bitmap contains a pointer to a list of bitmap block
 242 *	objects and a pointer to the bitmap block object that has been
 243 *	most recently used for setting bits.  Additionally, it contains the
 244 *	pfns that correspond to the start and end of the represented zone.
 245 *
 246 *	struct bm_block contains a pointer to the memory page in which
 247 *	information is stored (in the form of a block of bitmap)
 248 *	It also contains the pfns that correspond to the start and end of
 249 *	the represented memory area.
 250 */
 251
 252#define BM_END_OF_MAP	(~0UL)
 253
 254#define BM_BITS_PER_BLOCK	(PAGE_SIZE * BITS_PER_BYTE)
 255
 256struct bm_block {
 257	struct list_head hook;	/* hook into a list of bitmap blocks */
 258	unsigned long start_pfn;	/* pfn represented by the first bit */
 259	unsigned long end_pfn;	/* pfn represented by the last bit plus 1 */
 260	unsigned long *data;	/* bitmap representing pages */
 261};
 262
 263static inline unsigned long bm_block_bits(struct bm_block *bb)
 264{
 265	return bb->end_pfn - bb->start_pfn;
 266}
 267
 268/* strcut bm_position is used for browsing memory bitmaps */
 269
 270struct bm_position {
 271	struct bm_block *block;
 272	int bit;
 273};
 274
 275struct memory_bitmap {
 276	struct list_head blocks;	/* list of bitmap blocks */
 277	struct linked_page *p_list;	/* list of pages used to store zone
 278					 * bitmap objects and bitmap block
 279					 * objects
 280					 */
 281	struct bm_position cur;	/* most recently used bit position */
 282};
 283
 284/* Functions that operate on memory bitmaps */
 285
 286static void memory_bm_position_reset(struct memory_bitmap *bm)
 287{
 288	bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook);
 289	bm->cur.bit = 0;
 290}
 291
 292static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
 293
 294/**
 295 *	create_bm_block_list - create a list of block bitmap objects
 296 *	@pages - number of pages to track
 297 *	@list - list to put the allocated blocks into
 298 *	@ca - chain allocator to be used for allocating memory
 299 */
 300static int create_bm_block_list(unsigned long pages,
 301				struct list_head *list,
 302				struct chain_allocator *ca)
 303{
 304	unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
 305
 306	while (nr_blocks-- > 0) {
 307		struct bm_block *bb;
 308
 309		bb = chain_alloc(ca, sizeof(struct bm_block));
 310		if (!bb)
 311			return -ENOMEM;
 312		list_add(&bb->hook, list);
 313	}
 314
 315	return 0;
 316}
 317
 318struct mem_extent {
 319	struct list_head hook;
 320	unsigned long start;
 321	unsigned long end;
 322};
 323
 324/**
 325 *	free_mem_extents - free a list of memory extents
 326 *	@list - list of extents to empty
 327 */
 328static void free_mem_extents(struct list_head *list)
 329{
 330	struct mem_extent *ext, *aux;
 331
 332	list_for_each_entry_safe(ext, aux, list, hook) {
 333		list_del(&ext->hook);
 334		kfree(ext);
 335	}
 336}
 337
 338/**
 339 *	create_mem_extents - create a list of memory extents representing
 340 *	                     contiguous ranges of PFNs
 341 *	@list - list to put the extents into
 342 *	@gfp_mask - mask to use for memory allocations
 343 */
 344static int create_mem_extents(struct list_head *list, gfp_t gfp_mask)
 345{
 346	struct zone *zone;
 347
 348	INIT_LIST_HEAD(list);
 349
 350	for_each_populated_zone(zone) {
 351		unsigned long zone_start, zone_end;
 352		struct mem_extent *ext, *cur, *aux;
 353
 354		zone_start = zone->zone_start_pfn;
 355		zone_end = zone->zone_start_pfn + zone->spanned_pages;
 356
 357		list_for_each_entry(ext, list, hook)
 358			if (zone_start <= ext->end)
 359				break;
 360
 361		if (&ext->hook == list || zone_end < ext->start) {
 362			/* New extent is necessary */
 363			struct mem_extent *new_ext;
 364
 365			new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask);
 366			if (!new_ext) {
 367				free_mem_extents(list);
 368				return -ENOMEM;
 369			}
 370			new_ext->start = zone_start;
 371			new_ext->end = zone_end;
 372			list_add_tail(&new_ext->hook, &ext->hook);
 373			continue;
 374		}
 375
 376		/* Merge this zone's range of PFNs with the existing one */
 377		if (zone_start < ext->start)
 378			ext->start = zone_start;
 379		if (zone_end > ext->end)
 380			ext->end = zone_end;
 381
 382		/* More merging may be possible */
 383		cur = ext;
 384		list_for_each_entry_safe_continue(cur, aux, list, hook) {
 385			if (zone_end < cur->start)
 386				break;
 387			if (zone_end < cur->end)
 388				ext->end = cur->end;
 389			list_del(&cur->hook);
 390			kfree(cur);
 391		}
 392	}
 393
 394	return 0;
 395}
 396
 397/**
 398  *	memory_bm_create - allocate memory for a memory bitmap
 399  */
 400static int
 401memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
 402{
 403	struct chain_allocator ca;
 404	struct list_head mem_extents;
 405	struct mem_extent *ext;
 406	int error;
 407
 408	chain_init(&ca, gfp_mask, safe_needed);
 409	INIT_LIST_HEAD(&bm->blocks);
 410
 411	error = create_mem_extents(&mem_extents, gfp_mask);
 412	if (error)
 413		return error;
 414
 415	list_for_each_entry(ext, &mem_extents, hook) {
 416		struct bm_block *bb;
 417		unsigned long pfn = ext->start;
 418		unsigned long pages = ext->end - ext->start;
 419
 420		bb = list_entry(bm->blocks.prev, struct bm_block, hook);
 421
 422		error = create_bm_block_list(pages, bm->blocks.prev, &ca);
 423		if (error)
 424			goto Error;
 425
 426		list_for_each_entry_continue(bb, &bm->blocks, hook) {
 427			bb->data = get_image_page(gfp_mask, safe_needed);
 428			if (!bb->data) {
 429				error = -ENOMEM;
 430				goto Error;
 431			}
 432
 433			bb->start_pfn = pfn;
 434			if (pages >= BM_BITS_PER_BLOCK) {
 435				pfn += BM_BITS_PER_BLOCK;
 436				pages -= BM_BITS_PER_BLOCK;
 437			} else {
 438				/* This is executed only once in the loop */
 439				pfn += pages;
 440			}
 441			bb->end_pfn = pfn;
 442		}
 443	}
 444
 445	bm->p_list = ca.chain;
 446	memory_bm_position_reset(bm);
 447 Exit:
 448	free_mem_extents(&mem_extents);
 449	return error;
 450
 451 Error:
 452	bm->p_list = ca.chain;
 453	memory_bm_free(bm, PG_UNSAFE_CLEAR);
 454	goto Exit;
 455}
 456
 457/**
 458  *	memory_bm_free - free memory occupied by the memory bitmap @bm
 459  */
 460static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
 461{
 462	struct bm_block *bb;
 463
 464	list_for_each_entry(bb, &bm->blocks, hook)
 465		if (bb->data)
 466			free_image_page(bb->data, clear_nosave_free);
 467
 468	free_list_of_pages(bm->p_list, clear_nosave_free);
 469
 470	INIT_LIST_HEAD(&bm->blocks);
 471}
 472
 473/**
 474 *	memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
 475 *	to given pfn.  The cur_zone_bm member of @bm and the cur_block member
 476 *	of @bm->cur_zone_bm are updated.
 477 */
 478static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
 479				void **addr, unsigned int *bit_nr)
 480{
 481	struct bm_block *bb;
 482
 483	/*
 484	 * Check if the pfn corresponds to the current bitmap block and find
 485	 * the block where it fits if this is not the case.
 486	 */
 487	bb = bm->cur.block;
 488	if (pfn < bb->start_pfn)
 489		list_for_each_entry_continue_reverse(bb, &bm->blocks, hook)
 490			if (pfn >= bb->start_pfn)
 491				break;
 492
 493	if (pfn >= bb->end_pfn)
 494		list_for_each_entry_continue(bb, &bm->blocks, hook)
 495			if (pfn >= bb->start_pfn && pfn < bb->end_pfn)
 496				break;
 497
 498	if (&bb->hook == &bm->blocks)
 499		return -EFAULT;
 500
 501	/* The block has been found */
 502	bm->cur.block = bb;
 503	pfn -= bb->start_pfn;
 504	bm->cur.bit = pfn + 1;
 505	*bit_nr = pfn;
 506	*addr = bb->data;
 507	return 0;
 508}
 509
 510static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
 511{
 512	void *addr;
 513	unsigned int bit;
 514	int error;
 515
 516	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 517	BUG_ON(error);
 518	set_bit(bit, addr);
 519}
 520
 521static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
 522{
 523	void *addr;
 524	unsigned int bit;
 525	int error;
 526
 527	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 528	if (!error)
 529		set_bit(bit, addr);
 530	return error;
 531}
 532
 533static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
 534{
 535	void *addr;
 536	unsigned int bit;
 537	int error;
 538
 539	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 540	BUG_ON(error);
 541	clear_bit(bit, addr);
 542}
 543
 544static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
 545{
 546	void *addr;
 547	unsigned int bit;
 548	int error;
 549
 550	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 551	BUG_ON(error);
 552	return test_bit(bit, addr);
 553}
 554
 555static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
 556{
 557	void *addr;
 558	unsigned int bit;
 559
 560	return !memory_bm_find_bit(bm, pfn, &addr, &bit);
 561}
 562
 563/**
 564 *	memory_bm_next_pfn - find the pfn that corresponds to the next set bit
 565 *	in the bitmap @bm.  If the pfn cannot be found, BM_END_OF_MAP is
 566 *	returned.
 567 *
 568 *	It is required to run memory_bm_position_reset() before the first call to
 569 *	this function.
 570 */
 571
 572static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
 573{
 574	struct bm_block *bb;
 575	int bit;
 576
 577	bb = bm->cur.block;
 578	do {
 579		bit = bm->cur.bit;
 580		bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
 581		if (bit < bm_block_bits(bb))
 582			goto Return_pfn;
 583
 584		bb = list_entry(bb->hook.next, struct bm_block, hook);
 585		bm->cur.block = bb;
 586		bm->cur.bit = 0;
 587	} while (&bb->hook != &bm->blocks);
 588
 589	memory_bm_position_reset(bm);
 590	return BM_END_OF_MAP;
 591
 592 Return_pfn:
 593	bm->cur.bit = bit + 1;
 594	return bb->start_pfn + bit;
 595}
 596
 597/**
 598 *	This structure represents a range of page frames the contents of which
 599 *	should not be saved during the suspend.
 600 */
 601
 602struct nosave_region {
 603	struct list_head list;
 604	unsigned long start_pfn;
 605	unsigned long end_pfn;
 606};
 607
 608static LIST_HEAD(nosave_regions);
 609
 610/**
 611 *	register_nosave_region - register a range of page frames the contents
 612 *	of which should not be saved during the suspend (to be used in the early
 613 *	initialization code)
 614 */
 615
 616void __init
 617__register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
 618			 int use_kmalloc)
 619{
 620	struct nosave_region *region;
 621
 622	if (start_pfn >= end_pfn)
 623		return;
 624
 625	if (!list_empty(&nosave_regions)) {
 626		/* Try to extend the previous region (they should be sorted) */
 627		region = list_entry(nosave_regions.prev,
 628					struct nosave_region, list);
 629		if (region->end_pfn == start_pfn) {
 630			region->end_pfn = end_pfn;
 631			goto Report;
 632		}
 633	}
 634	if (use_kmalloc) {
 635		/* during init, this shouldn't fail */
 636		region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
 637		BUG_ON(!region);
 638	} else
 639		/* This allocation cannot fail */
 640		region = alloc_bootmem(sizeof(struct nosave_region));
 641	region->start_pfn = start_pfn;
 642	region->end_pfn = end_pfn;
 643	list_add_tail(&region->list, &nosave_regions);
 644 Report:
 645	printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n",
 646		start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
 
 647}
 648
 649/*
 650 * Set bits in this map correspond to the page frames the contents of which
 651 * should not be saved during the suspend.
 652 */
 653static struct memory_bitmap *forbidden_pages_map;
 654
 655/* Set bits in this map correspond to free page frames. */
 656static struct memory_bitmap *free_pages_map;
 657
 658/*
 659 * Each page frame allocated for creating the image is marked by setting the
 660 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
 661 */
 662
 663void swsusp_set_page_free(struct page *page)
 664{
 665	if (free_pages_map)
 666		memory_bm_set_bit(free_pages_map, page_to_pfn(page));
 667}
 668
 669static int swsusp_page_is_free(struct page *page)
 670{
 671	return free_pages_map ?
 672		memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
 673}
 674
 675void swsusp_unset_page_free(struct page *page)
 676{
 677	if (free_pages_map)
 678		memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
 679}
 680
 681static void swsusp_set_page_forbidden(struct page *page)
 682{
 683	if (forbidden_pages_map)
 684		memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
 685}
 686
 687int swsusp_page_is_forbidden(struct page *page)
 688{
 689	return forbidden_pages_map ?
 690		memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
 691}
 692
 693static void swsusp_unset_page_forbidden(struct page *page)
 694{
 695	if (forbidden_pages_map)
 696		memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
 697}
 698
 699/**
 700 *	mark_nosave_pages - set bits corresponding to the page frames the
 701 *	contents of which should not be saved in a given bitmap.
 702 */
 703
 704static void mark_nosave_pages(struct memory_bitmap *bm)
 705{
 706	struct nosave_region *region;
 707
 708	if (list_empty(&nosave_regions))
 709		return;
 710
 711	list_for_each_entry(region, &nosave_regions, list) {
 712		unsigned long pfn;
 713
 714		pr_debug("PM: Marking nosave pages: %016lx - %016lx\n",
 715				region->start_pfn << PAGE_SHIFT,
 716				region->end_pfn << PAGE_SHIFT);
 
 717
 718		for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
 719			if (pfn_valid(pfn)) {
 720				/*
 721				 * It is safe to ignore the result of
 722				 * mem_bm_set_bit_check() here, since we won't
 723				 * touch the PFNs for which the error is
 724				 * returned anyway.
 725				 */
 726				mem_bm_set_bit_check(bm, pfn);
 727			}
 728	}
 729}
 730
 731/**
 732 *	create_basic_memory_bitmaps - create bitmaps needed for marking page
 733 *	frames that should not be saved and free page frames.  The pointers
 734 *	forbidden_pages_map and free_pages_map are only modified if everything
 735 *	goes well, because we don't want the bits to be used before both bitmaps
 736 *	are set up.
 737 */
 738
 739int create_basic_memory_bitmaps(void)
 740{
 741	struct memory_bitmap *bm1, *bm2;
 742	int error = 0;
 743
 744	BUG_ON(forbidden_pages_map || free_pages_map);
 
 
 
 745
 746	bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
 747	if (!bm1)
 748		return -ENOMEM;
 749
 750	error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
 751	if (error)
 752		goto Free_first_object;
 753
 754	bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
 755	if (!bm2)
 756		goto Free_first_bitmap;
 757
 758	error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
 759	if (error)
 760		goto Free_second_object;
 761
 762	forbidden_pages_map = bm1;
 763	free_pages_map = bm2;
 764	mark_nosave_pages(forbidden_pages_map);
 765
 766	pr_debug("PM: Basic memory bitmaps created\n");
 767
 768	return 0;
 769
 770 Free_second_object:
 771	kfree(bm2);
 772 Free_first_bitmap:
 773 	memory_bm_free(bm1, PG_UNSAFE_CLEAR);
 774 Free_first_object:
 775	kfree(bm1);
 776	return -ENOMEM;
 777}
 778
 779/**
 780 *	free_basic_memory_bitmaps - free memory bitmaps allocated by
 781 *	create_basic_memory_bitmaps().  The auxiliary pointers are necessary
 782 *	so that the bitmaps themselves are not referred to while they are being
 783 *	freed.
 784 */
 785
 786void free_basic_memory_bitmaps(void)
 787{
 788	struct memory_bitmap *bm1, *bm2;
 789
 790	BUG_ON(!(forbidden_pages_map && free_pages_map));
 
 791
 792	bm1 = forbidden_pages_map;
 793	bm2 = free_pages_map;
 794	forbidden_pages_map = NULL;
 795	free_pages_map = NULL;
 796	memory_bm_free(bm1, PG_UNSAFE_CLEAR);
 797	kfree(bm1);
 798	memory_bm_free(bm2, PG_UNSAFE_CLEAR);
 799	kfree(bm2);
 800
 801	pr_debug("PM: Basic memory bitmaps freed\n");
 802}
 803
 804/**
 805 *	snapshot_additional_pages - estimate the number of additional pages
 806 *	be needed for setting up the suspend image data structures for given
 807 *	zone (usually the returned value is greater than the exact number)
 808 */
 809
 810unsigned int snapshot_additional_pages(struct zone *zone)
 811{
 812	unsigned int res;
 813
 814	res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
 815	res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
 
 816	return 2 * res;
 817}
 818
 819#ifdef CONFIG_HIGHMEM
 820/**
 821 *	count_free_highmem_pages - compute the total number of free highmem
 822 *	pages, system-wide.
 823 */
 824
 825static unsigned int count_free_highmem_pages(void)
 826{
 827	struct zone *zone;
 828	unsigned int cnt = 0;
 829
 830	for_each_populated_zone(zone)
 831		if (is_highmem(zone))
 832			cnt += zone_page_state(zone, NR_FREE_PAGES);
 833
 834	return cnt;
 835}
 836
 837/**
 838 *	saveable_highmem_page - Determine whether a highmem page should be
 839 *	included in the suspend image.
 840 *
 841 *	We should save the page if it isn't Nosave or NosaveFree, or Reserved,
 842 *	and it isn't a part of a free chunk of pages.
 843 */
 844static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)
 845{
 846	struct page *page;
 847
 848	if (!pfn_valid(pfn))
 849		return NULL;
 850
 851	page = pfn_to_page(pfn);
 852	if (page_zone(page) != zone)
 853		return NULL;
 854
 855	BUG_ON(!PageHighMem(page));
 856
 857	if (swsusp_page_is_forbidden(page) ||  swsusp_page_is_free(page) ||
 858	    PageReserved(page))
 859		return NULL;
 860
 
 
 
 861	return page;
 862}
 863
 864/**
 865 *	count_highmem_pages - compute the total number of saveable highmem
 866 *	pages.
 867 */
 868
 869static unsigned int count_highmem_pages(void)
 870{
 871	struct zone *zone;
 872	unsigned int n = 0;
 873
 874	for_each_populated_zone(zone) {
 875		unsigned long pfn, max_zone_pfn;
 876
 877		if (!is_highmem(zone))
 878			continue;
 879
 880		mark_free_pages(zone);
 881		max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
 882		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
 883			if (saveable_highmem_page(zone, pfn))
 884				n++;
 885	}
 886	return n;
 887}
 888#else
 889static inline void *saveable_highmem_page(struct zone *z, unsigned long p)
 890{
 891	return NULL;
 892}
 893#endif /* CONFIG_HIGHMEM */
 894
 895/**
 896 *	saveable_page - Determine whether a non-highmem page should be included
 897 *	in the suspend image.
 898 *
 899 *	We should save the page if it isn't Nosave, and is not in the range
 900 *	of pages statically defined as 'unsaveable', and it isn't a part of
 901 *	a free chunk of pages.
 902 */
 903static struct page *saveable_page(struct zone *zone, unsigned long pfn)
 904{
 905	struct page *page;
 906
 907	if (!pfn_valid(pfn))
 908		return NULL;
 909
 910	page = pfn_to_page(pfn);
 911	if (page_zone(page) != zone)
 912		return NULL;
 913
 914	BUG_ON(PageHighMem(page));
 915
 916	if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
 917		return NULL;
 918
 919	if (PageReserved(page)
 920	    && (!kernel_page_present(page) || pfn_is_nosave(pfn)))
 921		return NULL;
 922
 
 
 
 923	return page;
 924}
 925
 926/**
 927 *	count_data_pages - compute the total number of saveable non-highmem
 928 *	pages.
 929 */
 930
 931static unsigned int count_data_pages(void)
 932{
 933	struct zone *zone;
 934	unsigned long pfn, max_zone_pfn;
 935	unsigned int n = 0;
 936
 937	for_each_populated_zone(zone) {
 938		if (is_highmem(zone))
 939			continue;
 940
 941		mark_free_pages(zone);
 942		max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
 943		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
 944			if (saveable_page(zone, pfn))
 945				n++;
 946	}
 947	return n;
 948}
 949
 950/* This is needed, because copy_page and memcpy are not usable for copying
 951 * task structs.
 952 */
 953static inline void do_copy_page(long *dst, long *src)
 954{
 955	int n;
 956
 957	for (n = PAGE_SIZE / sizeof(long); n; n--)
 958		*dst++ = *src++;
 959}
 960
 961
 962/**
 963 *	safe_copy_page - check if the page we are going to copy is marked as
 964 *		present in the kernel page tables (this always is the case if
 965 *		CONFIG_DEBUG_PAGEALLOC is not set and in that case
 966 *		kernel_page_present() always returns 'true').
 967 */
 968static void safe_copy_page(void *dst, struct page *s_page)
 969{
 970	if (kernel_page_present(s_page)) {
 971		do_copy_page(dst, page_address(s_page));
 972	} else {
 973		kernel_map_pages(s_page, 1, 1);
 974		do_copy_page(dst, page_address(s_page));
 975		kernel_map_pages(s_page, 1, 0);
 976	}
 977}
 978
 979
 980#ifdef CONFIG_HIGHMEM
 981static inline struct page *
 982page_is_saveable(struct zone *zone, unsigned long pfn)
 983{
 984	return is_highmem(zone) ?
 985		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
 986}
 987
 988static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
 989{
 990	struct page *s_page, *d_page;
 991	void *src, *dst;
 992
 993	s_page = pfn_to_page(src_pfn);
 994	d_page = pfn_to_page(dst_pfn);
 995	if (PageHighMem(s_page)) {
 996		src = kmap_atomic(s_page, KM_USER0);
 997		dst = kmap_atomic(d_page, KM_USER1);
 998		do_copy_page(dst, src);
 999		kunmap_atomic(dst, KM_USER1);
1000		kunmap_atomic(src, KM_USER0);
1001	} else {
1002		if (PageHighMem(d_page)) {
1003			/* Page pointed to by src may contain some kernel
1004			 * data modified by kmap_atomic()
1005			 */
1006			safe_copy_page(buffer, s_page);
1007			dst = kmap_atomic(d_page, KM_USER0);
1008			copy_page(dst, buffer);
1009			kunmap_atomic(dst, KM_USER0);
1010		} else {
1011			safe_copy_page(page_address(d_page), s_page);
1012		}
1013	}
1014}
1015#else
1016#define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)
1017
1018static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1019{
1020	safe_copy_page(page_address(pfn_to_page(dst_pfn)),
1021				pfn_to_page(src_pfn));
1022}
1023#endif /* CONFIG_HIGHMEM */
1024
1025static void
1026copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
1027{
1028	struct zone *zone;
1029	unsigned long pfn;
1030
1031	for_each_populated_zone(zone) {
1032		unsigned long max_zone_pfn;
1033
1034		mark_free_pages(zone);
1035		max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1036		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1037			if (page_is_saveable(zone, pfn))
1038				memory_bm_set_bit(orig_bm, pfn);
1039	}
1040	memory_bm_position_reset(orig_bm);
1041	memory_bm_position_reset(copy_bm);
1042	for(;;) {
1043		pfn = memory_bm_next_pfn(orig_bm);
1044		if (unlikely(pfn == BM_END_OF_MAP))
1045			break;
1046		copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1047	}
1048}
1049
1050/* Total number of image pages */
1051static unsigned int nr_copy_pages;
1052/* Number of pages needed for saving the original pfns of the image pages */
1053static unsigned int nr_meta_pages;
1054/*
1055 * Numbers of normal and highmem page frames allocated for hibernation image
1056 * before suspending devices.
1057 */
1058unsigned int alloc_normal, alloc_highmem;
1059/*
1060 * Memory bitmap used for marking saveable pages (during hibernation) or
1061 * hibernation image pages (during restore)
1062 */
1063static struct memory_bitmap orig_bm;
1064/*
1065 * Memory bitmap used during hibernation for marking allocated page frames that
1066 * will contain copies of saveable pages.  During restore it is initially used
1067 * for marking hibernation image pages, but then the set bits from it are
1068 * duplicated in @orig_bm and it is released.  On highmem systems it is next
1069 * used for marking "safe" highmem pages, but it has to be reinitialized for
1070 * this purpose.
1071 */
1072static struct memory_bitmap copy_bm;
1073
1074/**
1075 *	swsusp_free - free pages allocated for the suspend.
1076 *
1077 *	Suspend pages are alocated before the atomic copy is made, so we
1078 *	need to release them after the resume.
1079 */
1080
1081void swsusp_free(void)
1082{
1083	struct zone *zone;
1084	unsigned long pfn, max_zone_pfn;
1085
1086	for_each_populated_zone(zone) {
1087		max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1088		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1089			if (pfn_valid(pfn)) {
1090				struct page *page = pfn_to_page(pfn);
1091
1092				if (swsusp_page_is_forbidden(page) &&
1093				    swsusp_page_is_free(page)) {
1094					swsusp_unset_page_forbidden(page);
1095					swsusp_unset_page_free(page);
1096					__free_page(page);
1097				}
1098			}
1099	}
1100	nr_copy_pages = 0;
1101	nr_meta_pages = 0;
1102	restore_pblist = NULL;
1103	buffer = NULL;
1104	alloc_normal = 0;
1105	alloc_highmem = 0;
1106}
1107
1108/* Helper functions used for the shrinking of memory. */
1109
1110#define GFP_IMAGE	(GFP_KERNEL | __GFP_NOWARN)
1111
1112/**
1113 * preallocate_image_pages - Allocate a number of pages for hibernation image
1114 * @nr_pages: Number of page frames to allocate.
1115 * @mask: GFP flags to use for the allocation.
1116 *
1117 * Return value: Number of page frames actually allocated
1118 */
1119static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
1120{
1121	unsigned long nr_alloc = 0;
1122
1123	while (nr_pages > 0) {
1124		struct page *page;
1125
1126		page = alloc_image_page(mask);
1127		if (!page)
1128			break;
1129		memory_bm_set_bit(&copy_bm, page_to_pfn(page));
1130		if (PageHighMem(page))
1131			alloc_highmem++;
1132		else
1133			alloc_normal++;
1134		nr_pages--;
1135		nr_alloc++;
1136	}
1137
1138	return nr_alloc;
1139}
1140
1141static unsigned long preallocate_image_memory(unsigned long nr_pages,
1142					      unsigned long avail_normal)
1143{
1144	unsigned long alloc;
1145
1146	if (avail_normal <= alloc_normal)
1147		return 0;
1148
1149	alloc = avail_normal - alloc_normal;
1150	if (nr_pages < alloc)
1151		alloc = nr_pages;
1152
1153	return preallocate_image_pages(alloc, GFP_IMAGE);
1154}
1155
1156#ifdef CONFIG_HIGHMEM
1157static unsigned long preallocate_image_highmem(unsigned long nr_pages)
1158{
1159	return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
1160}
1161
1162/**
1163 *  __fraction - Compute (an approximation of) x * (multiplier / base)
1164 */
1165static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
1166{
1167	x *= multiplier;
1168	do_div(x, base);
1169	return (unsigned long)x;
1170}
1171
1172static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1173						unsigned long highmem,
1174						unsigned long total)
1175{
1176	unsigned long alloc = __fraction(nr_pages, highmem, total);
1177
1178	return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
1179}
1180#else /* CONFIG_HIGHMEM */
1181static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
1182{
1183	return 0;
1184}
1185
1186static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1187						unsigned long highmem,
1188						unsigned long total)
1189{
1190	return 0;
1191}
1192#endif /* CONFIG_HIGHMEM */
1193
1194/**
1195 * free_unnecessary_pages - Release preallocated pages not needed for the image
1196 */
1197static void free_unnecessary_pages(void)
1198{
1199	unsigned long save, to_free_normal, to_free_highmem;
1200
1201	save = count_data_pages();
1202	if (alloc_normal >= save) {
1203		to_free_normal = alloc_normal - save;
1204		save = 0;
1205	} else {
1206		to_free_normal = 0;
1207		save -= alloc_normal;
1208	}
1209	save += count_highmem_pages();
1210	if (alloc_highmem >= save) {
1211		to_free_highmem = alloc_highmem - save;
1212	} else {
1213		to_free_highmem = 0;
1214		save -= alloc_highmem;
1215		if (to_free_normal > save)
1216			to_free_normal -= save;
1217		else
1218			to_free_normal = 0;
1219	}
1220
1221	memory_bm_position_reset(&copy_bm);
1222
1223	while (to_free_normal > 0 || to_free_highmem > 0) {
1224		unsigned long pfn = memory_bm_next_pfn(&copy_bm);
1225		struct page *page = pfn_to_page(pfn);
1226
1227		if (PageHighMem(page)) {
1228			if (!to_free_highmem)
1229				continue;
1230			to_free_highmem--;
1231			alloc_highmem--;
1232		} else {
1233			if (!to_free_normal)
1234				continue;
1235			to_free_normal--;
1236			alloc_normal--;
1237		}
1238		memory_bm_clear_bit(&copy_bm, pfn);
1239		swsusp_unset_page_forbidden(page);
1240		swsusp_unset_page_free(page);
1241		__free_page(page);
1242	}
1243}
1244
1245/**
1246 * minimum_image_size - Estimate the minimum acceptable size of an image
1247 * @saveable: Number of saveable pages in the system.
1248 *
1249 * We want to avoid attempting to free too much memory too hard, so estimate the
1250 * minimum acceptable size of a hibernation image to use as the lower limit for
1251 * preallocating memory.
1252 *
1253 * We assume that the minimum image size should be proportional to
1254 *
1255 * [number of saveable pages] - [number of pages that can be freed in theory]
1256 *
1257 * where the second term is the sum of (1) reclaimable slab pages, (2) active
1258 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
1259 * minus mapped file pages.
1260 */
1261static unsigned long minimum_image_size(unsigned long saveable)
1262{
1263	unsigned long size;
1264
1265	size = global_page_state(NR_SLAB_RECLAIMABLE)
1266		+ global_page_state(NR_ACTIVE_ANON)
1267		+ global_page_state(NR_INACTIVE_ANON)
1268		+ global_page_state(NR_ACTIVE_FILE)
1269		+ global_page_state(NR_INACTIVE_FILE)
1270		- global_page_state(NR_FILE_MAPPED);
1271
1272	return saveable <= size ? 0 : saveable - size;
1273}
1274
1275/**
1276 * hibernate_preallocate_memory - Preallocate memory for hibernation image
1277 *
1278 * To create a hibernation image it is necessary to make a copy of every page
1279 * frame in use.  We also need a number of page frames to be free during
1280 * hibernation for allocations made while saving the image and for device
1281 * drivers, in case they need to allocate memory from their hibernation
1282 * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough
1283 * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through
1284 * /sys/power/reserved_size, respectively).  To make this happen, we compute the
1285 * total number of available page frames and allocate at least
1286 *
1287 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2
1288 *  + 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE)
1289 *
1290 * of them, which corresponds to the maximum size of a hibernation image.
1291 *
1292 * If image_size is set below the number following from the above formula,
1293 * the preallocation of memory is continued until the total number of saveable
1294 * pages in the system is below the requested image size or the minimum
1295 * acceptable image size returned by minimum_image_size(), whichever is greater.
1296 */
1297int hibernate_preallocate_memory(void)
1298{
1299	struct zone *zone;
1300	unsigned long saveable, size, max_size, count, highmem, pages = 0;
1301	unsigned long alloc, save_highmem, pages_highmem, avail_normal;
1302	struct timeval start, stop;
1303	int error;
1304
1305	printk(KERN_INFO "PM: Preallocating image memory... ");
1306	do_gettimeofday(&start);
1307
1308	error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
1309	if (error)
1310		goto err_out;
1311
1312	error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
1313	if (error)
1314		goto err_out;
1315
1316	alloc_normal = 0;
1317	alloc_highmem = 0;
1318
1319	/* Count the number of saveable data pages. */
1320	save_highmem = count_highmem_pages();
1321	saveable = count_data_pages();
1322
1323	/*
1324	 * Compute the total number of page frames we can use (count) and the
1325	 * number of pages needed for image metadata (size).
1326	 */
1327	count = saveable;
1328	saveable += save_highmem;
1329	highmem = save_highmem;
1330	size = 0;
1331	for_each_populated_zone(zone) {
1332		size += snapshot_additional_pages(zone);
1333		if (is_highmem(zone))
1334			highmem += zone_page_state(zone, NR_FREE_PAGES);
1335		else
1336			count += zone_page_state(zone, NR_FREE_PAGES);
1337	}
1338	avail_normal = count;
1339	count += highmem;
1340	count -= totalreserve_pages;
1341
 
 
 
1342	/* Compute the maximum number of saveable pages to leave in memory. */
1343	max_size = (count - (size + PAGES_FOR_IO)) / 2
1344			- 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE);
1345	/* Compute the desired number of image pages specified by image_size. */
1346	size = DIV_ROUND_UP(image_size, PAGE_SIZE);
1347	if (size > max_size)
1348		size = max_size;
1349	/*
1350	 * If the desired number of image pages is at least as large as the
1351	 * current number of saveable pages in memory, allocate page frames for
1352	 * the image and we're done.
1353	 */
1354	if (size >= saveable) {
1355		pages = preallocate_image_highmem(save_highmem);
1356		pages += preallocate_image_memory(saveable - pages, avail_normal);
1357		goto out;
1358	}
1359
1360	/* Estimate the minimum size of the image. */
1361	pages = minimum_image_size(saveable);
1362	/*
1363	 * To avoid excessive pressure on the normal zone, leave room in it to
1364	 * accommodate an image of the minimum size (unless it's already too
1365	 * small, in which case don't preallocate pages from it at all).
1366	 */
1367	if (avail_normal > pages)
1368		avail_normal -= pages;
1369	else
1370		avail_normal = 0;
1371	if (size < pages)
1372		size = min_t(unsigned long, pages, max_size);
1373
1374	/*
1375	 * Let the memory management subsystem know that we're going to need a
1376	 * large number of page frames to allocate and make it free some memory.
1377	 * NOTE: If this is not done, performance will be hurt badly in some
1378	 * test cases.
1379	 */
1380	shrink_all_memory(saveable - size);
1381
1382	/*
1383	 * The number of saveable pages in memory was too high, so apply some
1384	 * pressure to decrease it.  First, make room for the largest possible
1385	 * image and fail if that doesn't work.  Next, try to decrease the size
1386	 * of the image as much as indicated by 'size' using allocations from
1387	 * highmem and non-highmem zones separately.
1388	 */
1389	pages_highmem = preallocate_image_highmem(highmem / 2);
1390	alloc = (count - max_size) - pages_highmem;
 
 
 
 
1391	pages = preallocate_image_memory(alloc, avail_normal);
1392	if (pages < alloc) {
1393		/* We have exhausted non-highmem pages, try highmem. */
1394		alloc -= pages;
1395		pages += pages_highmem;
1396		pages_highmem = preallocate_image_highmem(alloc);
1397		if (pages_highmem < alloc)
1398			goto err_out;
1399		pages += pages_highmem;
1400		/*
1401		 * size is the desired number of saveable pages to leave in
1402		 * memory, so try to preallocate (all memory - size) pages.
1403		 */
1404		alloc = (count - pages) - size;
1405		pages += preallocate_image_highmem(alloc);
1406	} else {
1407		/*
1408		 * There are approximately max_size saveable pages at this point
1409		 * and we want to reduce this number down to size.
1410		 */
1411		alloc = max_size - size;
1412		size = preallocate_highmem_fraction(alloc, highmem, count);
1413		pages_highmem += size;
1414		alloc -= size;
1415		size = preallocate_image_memory(alloc, avail_normal);
1416		pages_highmem += preallocate_image_highmem(alloc - size);
1417		pages += pages_highmem + size;
1418	}
1419
1420	/*
1421	 * We only need as many page frames for the image as there are saveable
1422	 * pages in memory, but we have allocated more.  Release the excessive
1423	 * ones now.
1424	 */
1425	free_unnecessary_pages();
1426
1427 out:
1428	do_gettimeofday(&stop);
1429	printk(KERN_CONT "done (allocated %lu pages)\n", pages);
1430	swsusp_show_speed(&start, &stop, pages, "Allocated");
1431
1432	return 0;
1433
1434 err_out:
1435	printk(KERN_CONT "\n");
1436	swsusp_free();
1437	return -ENOMEM;
1438}
1439
1440#ifdef CONFIG_HIGHMEM
1441/**
1442  *	count_pages_for_highmem - compute the number of non-highmem pages
1443  *	that will be necessary for creating copies of highmem pages.
1444  */
1445
1446static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1447{
1448	unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
1449
1450	if (free_highmem >= nr_highmem)
1451		nr_highmem = 0;
1452	else
1453		nr_highmem -= free_highmem;
1454
1455	return nr_highmem;
1456}
1457#else
1458static unsigned int
1459count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1460#endif /* CONFIG_HIGHMEM */
1461
1462/**
1463 *	enough_free_mem - Make sure we have enough free memory for the
1464 *	snapshot image.
1465 */
1466
1467static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1468{
1469	struct zone *zone;
1470	unsigned int free = alloc_normal;
1471
1472	for_each_populated_zone(zone)
1473		if (!is_highmem(zone))
1474			free += zone_page_state(zone, NR_FREE_PAGES);
1475
1476	nr_pages += count_pages_for_highmem(nr_highmem);
1477	pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
1478		nr_pages, PAGES_FOR_IO, free);
1479
1480	return free > nr_pages + PAGES_FOR_IO;
1481}
1482
1483#ifdef CONFIG_HIGHMEM
1484/**
1485 *	get_highmem_buffer - if there are some highmem pages in the suspend
1486 *	image, we may need the buffer to copy them and/or load their data.
1487 */
1488
1489static inline int get_highmem_buffer(int safe_needed)
1490{
1491	buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1492	return buffer ? 0 : -ENOMEM;
1493}
1494
1495/**
1496 *	alloc_highmem_image_pages - allocate some highmem pages for the image.
1497 *	Try to allocate as many pages as needed, but if the number of free
1498 *	highmem pages is lesser than that, allocate them all.
1499 */
1500
1501static inline unsigned int
1502alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1503{
1504	unsigned int to_alloc = count_free_highmem_pages();
1505
1506	if (to_alloc > nr_highmem)
1507		to_alloc = nr_highmem;
1508
1509	nr_highmem -= to_alloc;
1510	while (to_alloc-- > 0) {
1511		struct page *page;
1512
1513		page = alloc_image_page(__GFP_HIGHMEM);
1514		memory_bm_set_bit(bm, page_to_pfn(page));
1515	}
1516	return nr_highmem;
1517}
1518#else
1519static inline int get_highmem_buffer(int safe_needed) { return 0; }
1520
1521static inline unsigned int
1522alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1523#endif /* CONFIG_HIGHMEM */
1524
1525/**
1526 *	swsusp_alloc - allocate memory for the suspend image
1527 *
1528 *	We first try to allocate as many highmem pages as there are
1529 *	saveable highmem pages in the system.  If that fails, we allocate
1530 *	non-highmem pages for the copies of the remaining highmem ones.
1531 *
1532 *	In this approach it is likely that the copies of highmem pages will
1533 *	also be located in the high memory, because of the way in which
1534 *	copy_data_pages() works.
1535 */
1536
1537static int
1538swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1539		unsigned int nr_pages, unsigned int nr_highmem)
1540{
1541	if (nr_highmem > 0) {
1542		if (get_highmem_buffer(PG_ANY))
1543			goto err_out;
1544		if (nr_highmem > alloc_highmem) {
1545			nr_highmem -= alloc_highmem;
1546			nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
1547		}
1548	}
1549	if (nr_pages > alloc_normal) {
1550		nr_pages -= alloc_normal;
1551		while (nr_pages-- > 0) {
1552			struct page *page;
1553
1554			page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1555			if (!page)
1556				goto err_out;
1557			memory_bm_set_bit(copy_bm, page_to_pfn(page));
1558		}
1559	}
1560
1561	return 0;
1562
1563 err_out:
1564	swsusp_free();
1565	return -ENOMEM;
1566}
1567
1568asmlinkage int swsusp_save(void)
1569{
1570	unsigned int nr_pages, nr_highmem;
1571
1572	printk(KERN_INFO "PM: Creating hibernation image:\n");
1573
1574	drain_local_pages(NULL);
1575	nr_pages = count_data_pages();
1576	nr_highmem = count_highmem_pages();
1577	printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem);
1578
1579	if (!enough_free_mem(nr_pages, nr_highmem)) {
1580		printk(KERN_ERR "PM: Not enough free memory\n");
1581		return -ENOMEM;
1582	}
1583
1584	if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1585		printk(KERN_ERR "PM: Memory allocation failed\n");
1586		return -ENOMEM;
1587	}
1588
1589	/* During allocating of suspend pagedir, new cold pages may appear.
1590	 * Kill them.
1591	 */
1592	drain_local_pages(NULL);
1593	copy_data_pages(&copy_bm, &orig_bm);
1594
1595	/*
1596	 * End of critical section. From now on, we can write to memory,
1597	 * but we should not touch disk. This specially means we must _not_
1598	 * touch swap space! Except we must write out our image of course.
1599	 */
1600
1601	nr_pages += nr_highmem;
1602	nr_copy_pages = nr_pages;
1603	nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1604
1605	printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n",
1606		nr_pages);
1607
1608	return 0;
1609}
1610
1611#ifndef CONFIG_ARCH_HIBERNATION_HEADER
1612static int init_header_complete(struct swsusp_info *info)
1613{
1614	memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1615	info->version_code = LINUX_VERSION_CODE;
1616	return 0;
1617}
1618
1619static char *check_image_kernel(struct swsusp_info *info)
1620{
1621	if (info->version_code != LINUX_VERSION_CODE)
1622		return "kernel version";
1623	if (strcmp(info->uts.sysname,init_utsname()->sysname))
1624		return "system type";
1625	if (strcmp(info->uts.release,init_utsname()->release))
1626		return "kernel release";
1627	if (strcmp(info->uts.version,init_utsname()->version))
1628		return "version";
1629	if (strcmp(info->uts.machine,init_utsname()->machine))
1630		return "machine";
1631	return NULL;
1632}
1633#endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1634
1635unsigned long snapshot_get_image_size(void)
1636{
1637	return nr_copy_pages + nr_meta_pages + 1;
1638}
1639
1640static int init_header(struct swsusp_info *info)
1641{
1642	memset(info, 0, sizeof(struct swsusp_info));
1643	info->num_physpages = num_physpages;
1644	info->image_pages = nr_copy_pages;
1645	info->pages = snapshot_get_image_size();
1646	info->size = info->pages;
1647	info->size <<= PAGE_SHIFT;
1648	return init_header_complete(info);
1649}
1650
1651/**
1652 *	pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1653 *	are stored in the array @buf[] (1 page at a time)
1654 */
1655
1656static inline void
1657pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1658{
1659	int j;
1660
1661	for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1662		buf[j] = memory_bm_next_pfn(bm);
1663		if (unlikely(buf[j] == BM_END_OF_MAP))
1664			break;
 
 
1665	}
1666}
1667
1668/**
1669 *	snapshot_read_next - used for reading the system memory snapshot.
1670 *
1671 *	On the first call to it @handle should point to a zeroed
1672 *	snapshot_handle structure.  The structure gets updated and a pointer
1673 *	to it should be passed to this function every next time.
1674 *
1675 *	On success the function returns a positive number.  Then, the caller
1676 *	is allowed to read up to the returned number of bytes from the memory
1677 *	location computed by the data_of() macro.
1678 *
1679 *	The function returns 0 to indicate the end of data stream condition,
1680 *	and a negative number is returned on error.  In such cases the
1681 *	structure pointed to by @handle is not updated and should not be used
1682 *	any more.
1683 */
1684
1685int snapshot_read_next(struct snapshot_handle *handle)
1686{
1687	if (handle->cur > nr_meta_pages + nr_copy_pages)
1688		return 0;
1689
1690	if (!buffer) {
1691		/* This makes the buffer be freed by swsusp_free() */
1692		buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1693		if (!buffer)
1694			return -ENOMEM;
1695	}
1696	if (!handle->cur) {
1697		int error;
1698
1699		error = init_header((struct swsusp_info *)buffer);
1700		if (error)
1701			return error;
1702		handle->buffer = buffer;
1703		memory_bm_position_reset(&orig_bm);
1704		memory_bm_position_reset(&copy_bm);
1705	} else if (handle->cur <= nr_meta_pages) {
1706		clear_page(buffer);
1707		pack_pfns(buffer, &orig_bm);
1708	} else {
1709		struct page *page;
1710
1711		page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1712		if (PageHighMem(page)) {
1713			/* Highmem pages are copied to the buffer,
1714			 * because we can't return with a kmapped
1715			 * highmem page (we may not be called again).
1716			 */
1717			void *kaddr;
1718
1719			kaddr = kmap_atomic(page, KM_USER0);
1720			copy_page(buffer, kaddr);
1721			kunmap_atomic(kaddr, KM_USER0);
1722			handle->buffer = buffer;
1723		} else {
1724			handle->buffer = page_address(page);
1725		}
1726	}
1727	handle->cur++;
1728	return PAGE_SIZE;
1729}
1730
1731/**
1732 *	mark_unsafe_pages - mark the pages that cannot be used for storing
1733 *	the image during resume, because they conflict with the pages that
1734 *	had been used before suspend
1735 */
1736
1737static int mark_unsafe_pages(struct memory_bitmap *bm)
1738{
1739	struct zone *zone;
1740	unsigned long pfn, max_zone_pfn;
1741
1742	/* Clear page flags */
1743	for_each_populated_zone(zone) {
1744		max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1745		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1746			if (pfn_valid(pfn))
1747				swsusp_unset_page_free(pfn_to_page(pfn));
1748	}
1749
1750	/* Mark pages that correspond to the "original" pfns as "unsafe" */
1751	memory_bm_position_reset(bm);
1752	do {
1753		pfn = memory_bm_next_pfn(bm);
1754		if (likely(pfn != BM_END_OF_MAP)) {
1755			if (likely(pfn_valid(pfn)))
1756				swsusp_set_page_free(pfn_to_page(pfn));
1757			else
1758				return -EFAULT;
1759		}
1760	} while (pfn != BM_END_OF_MAP);
1761
1762	allocated_unsafe_pages = 0;
1763
1764	return 0;
1765}
1766
1767static void
1768duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1769{
1770	unsigned long pfn;
1771
1772	memory_bm_position_reset(src);
1773	pfn = memory_bm_next_pfn(src);
1774	while (pfn != BM_END_OF_MAP) {
1775		memory_bm_set_bit(dst, pfn);
1776		pfn = memory_bm_next_pfn(src);
1777	}
1778}
1779
1780static int check_header(struct swsusp_info *info)
1781{
1782	char *reason;
1783
1784	reason = check_image_kernel(info);
1785	if (!reason && info->num_physpages != num_physpages)
1786		reason = "memory size";
1787	if (reason) {
1788		printk(KERN_ERR "PM: Image mismatch: %s\n", reason);
1789		return -EPERM;
1790	}
1791	return 0;
1792}
1793
1794/**
1795 *	load header - check the image header and copy data from it
1796 */
1797
1798static int
1799load_header(struct swsusp_info *info)
1800{
1801	int error;
1802
1803	restore_pblist = NULL;
1804	error = check_header(info);
1805	if (!error) {
1806		nr_copy_pages = info->image_pages;
1807		nr_meta_pages = info->pages - info->image_pages - 1;
1808	}
1809	return error;
1810}
1811
1812/**
1813 *	unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1814 *	the corresponding bit in the memory bitmap @bm
1815 */
1816static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1817{
1818	int j;
1819
1820	for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1821		if (unlikely(buf[j] == BM_END_OF_MAP))
1822			break;
1823
 
 
 
1824		if (memory_bm_pfn_present(bm, buf[j]))
1825			memory_bm_set_bit(bm, buf[j]);
1826		else
1827			return -EFAULT;
1828	}
1829
1830	return 0;
1831}
1832
1833/* List of "safe" pages that may be used to store data loaded from the suspend
1834 * image
1835 */
1836static struct linked_page *safe_pages_list;
1837
1838#ifdef CONFIG_HIGHMEM
1839/* struct highmem_pbe is used for creating the list of highmem pages that
1840 * should be restored atomically during the resume from disk, because the page
1841 * frames they have occupied before the suspend are in use.
1842 */
1843struct highmem_pbe {
1844	struct page *copy_page;	/* data is here now */
1845	struct page *orig_page;	/* data was here before the suspend */
1846	struct highmem_pbe *next;
1847};
1848
1849/* List of highmem PBEs needed for restoring the highmem pages that were
1850 * allocated before the suspend and included in the suspend image, but have
1851 * also been allocated by the "resume" kernel, so their contents cannot be
1852 * written directly to their "original" page frames.
1853 */
1854static struct highmem_pbe *highmem_pblist;
1855
1856/**
1857 *	count_highmem_image_pages - compute the number of highmem pages in the
1858 *	suspend image.  The bits in the memory bitmap @bm that correspond to the
1859 *	image pages are assumed to be set.
1860 */
1861
1862static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1863{
1864	unsigned long pfn;
1865	unsigned int cnt = 0;
1866
1867	memory_bm_position_reset(bm);
1868	pfn = memory_bm_next_pfn(bm);
1869	while (pfn != BM_END_OF_MAP) {
1870		if (PageHighMem(pfn_to_page(pfn)))
1871			cnt++;
1872
1873		pfn = memory_bm_next_pfn(bm);
1874	}
1875	return cnt;
1876}
1877
1878/**
1879 *	prepare_highmem_image - try to allocate as many highmem pages as
1880 *	there are highmem image pages (@nr_highmem_p points to the variable
1881 *	containing the number of highmem image pages).  The pages that are
1882 *	"safe" (ie. will not be overwritten when the suspend image is
1883 *	restored) have the corresponding bits set in @bm (it must be
1884 *	unitialized).
1885 *
1886 *	NOTE: This function should not be called if there are no highmem
1887 *	image pages.
1888 */
1889
1890static unsigned int safe_highmem_pages;
1891
1892static struct memory_bitmap *safe_highmem_bm;
1893
1894static int
1895prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1896{
1897	unsigned int to_alloc;
1898
1899	if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1900		return -ENOMEM;
1901
1902	if (get_highmem_buffer(PG_SAFE))
1903		return -ENOMEM;
1904
1905	to_alloc = count_free_highmem_pages();
1906	if (to_alloc > *nr_highmem_p)
1907		to_alloc = *nr_highmem_p;
1908	else
1909		*nr_highmem_p = to_alloc;
1910
1911	safe_highmem_pages = 0;
1912	while (to_alloc-- > 0) {
1913		struct page *page;
1914
1915		page = alloc_page(__GFP_HIGHMEM);
1916		if (!swsusp_page_is_free(page)) {
1917			/* The page is "safe", set its bit the bitmap */
1918			memory_bm_set_bit(bm, page_to_pfn(page));
1919			safe_highmem_pages++;
1920		}
1921		/* Mark the page as allocated */
1922		swsusp_set_page_forbidden(page);
1923		swsusp_set_page_free(page);
1924	}
1925	memory_bm_position_reset(bm);
1926	safe_highmem_bm = bm;
1927	return 0;
1928}
1929
1930/**
1931 *	get_highmem_page_buffer - for given highmem image page find the buffer
1932 *	that suspend_write_next() should set for its caller to write to.
1933 *
1934 *	If the page is to be saved to its "original" page frame or a copy of
1935 *	the page is to be made in the highmem, @buffer is returned.  Otherwise,
1936 *	the copy of the page is to be made in normal memory, so the address of
1937 *	the copy is returned.
1938 *
1939 *	If @buffer is returned, the caller of suspend_write_next() will write
1940 *	the page's contents to @buffer, so they will have to be copied to the
1941 *	right location on the next call to suspend_write_next() and it is done
1942 *	with the help of copy_last_highmem_page().  For this purpose, if
1943 *	@buffer is returned, @last_highmem page is set to the page to which
1944 *	the data will have to be copied from @buffer.
1945 */
1946
1947static struct page *last_highmem_page;
1948
1949static void *
1950get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1951{
1952	struct highmem_pbe *pbe;
1953	void *kaddr;
1954
1955	if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1956		/* We have allocated the "original" page frame and we can
1957		 * use it directly to store the loaded page.
1958		 */
1959		last_highmem_page = page;
1960		return buffer;
1961	}
1962	/* The "original" page frame has not been allocated and we have to
1963	 * use a "safe" page frame to store the loaded page.
1964	 */
1965	pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1966	if (!pbe) {
1967		swsusp_free();
1968		return ERR_PTR(-ENOMEM);
1969	}
1970	pbe->orig_page = page;
1971	if (safe_highmem_pages > 0) {
1972		struct page *tmp;
1973
1974		/* Copy of the page will be stored in high memory */
1975		kaddr = buffer;
1976		tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1977		safe_highmem_pages--;
1978		last_highmem_page = tmp;
1979		pbe->copy_page = tmp;
1980	} else {
1981		/* Copy of the page will be stored in normal memory */
1982		kaddr = safe_pages_list;
1983		safe_pages_list = safe_pages_list->next;
1984		pbe->copy_page = virt_to_page(kaddr);
1985	}
1986	pbe->next = highmem_pblist;
1987	highmem_pblist = pbe;
1988	return kaddr;
1989}
1990
1991/**
1992 *	copy_last_highmem_page - copy the contents of a highmem image from
1993 *	@buffer, where the caller of snapshot_write_next() has place them,
1994 *	to the right location represented by @last_highmem_page .
1995 */
1996
1997static void copy_last_highmem_page(void)
1998{
1999	if (last_highmem_page) {
2000		void *dst;
2001
2002		dst = kmap_atomic(last_highmem_page, KM_USER0);
2003		copy_page(dst, buffer);
2004		kunmap_atomic(dst, KM_USER0);
2005		last_highmem_page = NULL;
2006	}
2007}
2008
2009static inline int last_highmem_page_copied(void)
2010{
2011	return !last_highmem_page;
2012}
2013
2014static inline void free_highmem_data(void)
2015{
2016	if (safe_highmem_bm)
2017		memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
2018
2019	if (buffer)
2020		free_image_page(buffer, PG_UNSAFE_CLEAR);
2021}
2022#else
2023static inline int get_safe_write_buffer(void) { return 0; }
2024
2025static unsigned int
2026count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
2027
2028static inline int
2029prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
2030{
2031	return 0;
2032}
2033
2034static inline void *
2035get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
2036{
2037	return ERR_PTR(-EINVAL);
2038}
2039
2040static inline void copy_last_highmem_page(void) {}
2041static inline int last_highmem_page_copied(void) { return 1; }
2042static inline void free_highmem_data(void) {}
2043#endif /* CONFIG_HIGHMEM */
2044
2045/**
2046 *	prepare_image - use the memory bitmap @bm to mark the pages that will
2047 *	be overwritten in the process of restoring the system memory state
2048 *	from the suspend image ("unsafe" pages) and allocate memory for the
2049 *	image.
2050 *
2051 *	The idea is to allocate a new memory bitmap first and then allocate
2052 *	as many pages as needed for the image data, but not to assign these
2053 *	pages to specific tasks initially.  Instead, we just mark them as
2054 *	allocated and create a lists of "safe" pages that will be used
2055 *	later.  On systems with high memory a list of "safe" highmem pages is
2056 *	also created.
2057 */
2058
2059#define PBES_PER_LINKED_PAGE	(LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2060
2061static int
2062prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
2063{
2064	unsigned int nr_pages, nr_highmem;
2065	struct linked_page *sp_list, *lp;
2066	int error;
2067
2068	/* If there is no highmem, the buffer will not be necessary */
2069	free_image_page(buffer, PG_UNSAFE_CLEAR);
2070	buffer = NULL;
2071
2072	nr_highmem = count_highmem_image_pages(bm);
2073	error = mark_unsafe_pages(bm);
2074	if (error)
2075		goto Free;
2076
2077	error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
2078	if (error)
2079		goto Free;
2080
2081	duplicate_memory_bitmap(new_bm, bm);
2082	memory_bm_free(bm, PG_UNSAFE_KEEP);
2083	if (nr_highmem > 0) {
2084		error = prepare_highmem_image(bm, &nr_highmem);
2085		if (error)
2086			goto Free;
2087	}
2088	/* Reserve some safe pages for potential later use.
2089	 *
2090	 * NOTE: This way we make sure there will be enough safe pages for the
2091	 * chain_alloc() in get_buffer().  It is a bit wasteful, but
2092	 * nr_copy_pages cannot be greater than 50% of the memory anyway.
2093	 */
2094	sp_list = NULL;
2095	/* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
2096	nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2097	nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
2098	while (nr_pages > 0) {
2099		lp = get_image_page(GFP_ATOMIC, PG_SAFE);
2100		if (!lp) {
2101			error = -ENOMEM;
2102			goto Free;
2103		}
2104		lp->next = sp_list;
2105		sp_list = lp;
2106		nr_pages--;
2107	}
2108	/* Preallocate memory for the image */
2109	safe_pages_list = NULL;
2110	nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2111	while (nr_pages > 0) {
2112		lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
2113		if (!lp) {
2114			error = -ENOMEM;
2115			goto Free;
2116		}
2117		if (!swsusp_page_is_free(virt_to_page(lp))) {
2118			/* The page is "safe", add it to the list */
2119			lp->next = safe_pages_list;
2120			safe_pages_list = lp;
2121		}
2122		/* Mark the page as allocated */
2123		swsusp_set_page_forbidden(virt_to_page(lp));
2124		swsusp_set_page_free(virt_to_page(lp));
2125		nr_pages--;
2126	}
2127	/* Free the reserved safe pages so that chain_alloc() can use them */
2128	while (sp_list) {
2129		lp = sp_list->next;
2130		free_image_page(sp_list, PG_UNSAFE_CLEAR);
2131		sp_list = lp;
2132	}
2133	return 0;
2134
2135 Free:
2136	swsusp_free();
2137	return error;
2138}
2139
2140/**
2141 *	get_buffer - compute the address that snapshot_write_next() should
2142 *	set for its caller to write to.
2143 */
2144
2145static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
2146{
2147	struct pbe *pbe;
2148	struct page *page;
2149	unsigned long pfn = memory_bm_next_pfn(bm);
2150
2151	if (pfn == BM_END_OF_MAP)
2152		return ERR_PTR(-EFAULT);
2153
2154	page = pfn_to_page(pfn);
2155	if (PageHighMem(page))
2156		return get_highmem_page_buffer(page, ca);
2157
2158	if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
2159		/* We have allocated the "original" page frame and we can
2160		 * use it directly to store the loaded page.
2161		 */
2162		return page_address(page);
2163
2164	/* The "original" page frame has not been allocated and we have to
2165	 * use a "safe" page frame to store the loaded page.
2166	 */
2167	pbe = chain_alloc(ca, sizeof(struct pbe));
2168	if (!pbe) {
2169		swsusp_free();
2170		return ERR_PTR(-ENOMEM);
2171	}
2172	pbe->orig_address = page_address(page);
2173	pbe->address = safe_pages_list;
2174	safe_pages_list = safe_pages_list->next;
2175	pbe->next = restore_pblist;
2176	restore_pblist = pbe;
2177	return pbe->address;
2178}
2179
2180/**
2181 *	snapshot_write_next - used for writing the system memory snapshot.
2182 *
2183 *	On the first call to it @handle should point to a zeroed
2184 *	snapshot_handle structure.  The structure gets updated and a pointer
2185 *	to it should be passed to this function every next time.
2186 *
2187 *	On success the function returns a positive number.  Then, the caller
2188 *	is allowed to write up to the returned number of bytes to the memory
2189 *	location computed by the data_of() macro.
2190 *
2191 *	The function returns 0 to indicate the "end of file" condition,
2192 *	and a negative number is returned on error.  In such cases the
2193 *	structure pointed to by @handle is not updated and should not be used
2194 *	any more.
2195 */
2196
2197int snapshot_write_next(struct snapshot_handle *handle)
2198{
2199	static struct chain_allocator ca;
2200	int error = 0;
2201
2202	/* Check if we have already loaded the entire image */
2203	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages)
2204		return 0;
2205
2206	handle->sync_read = 1;
2207
2208	if (!handle->cur) {
2209		if (!buffer)
2210			/* This makes the buffer be freed by swsusp_free() */
2211			buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2212
2213		if (!buffer)
2214			return -ENOMEM;
2215
2216		handle->buffer = buffer;
2217	} else if (handle->cur == 1) {
2218		error = load_header(buffer);
2219		if (error)
2220			return error;
2221
2222		error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
2223		if (error)
2224			return error;
2225
 
 
 
 
 
2226	} else if (handle->cur <= nr_meta_pages + 1) {
2227		error = unpack_orig_pfns(buffer, &copy_bm);
2228		if (error)
2229			return error;
2230
2231		if (handle->cur == nr_meta_pages + 1) {
2232			error = prepare_image(&orig_bm, &copy_bm);
2233			if (error)
2234				return error;
2235
2236			chain_init(&ca, GFP_ATOMIC, PG_SAFE);
2237			memory_bm_position_reset(&orig_bm);
2238			restore_pblist = NULL;
2239			handle->buffer = get_buffer(&orig_bm, &ca);
2240			handle->sync_read = 0;
2241			if (IS_ERR(handle->buffer))
2242				return PTR_ERR(handle->buffer);
2243		}
2244	} else {
2245		copy_last_highmem_page();
 
 
2246		handle->buffer = get_buffer(&orig_bm, &ca);
2247		if (IS_ERR(handle->buffer))
2248			return PTR_ERR(handle->buffer);
2249		if (handle->buffer != buffer)
2250			handle->sync_read = 0;
2251	}
2252	handle->cur++;
2253	return PAGE_SIZE;
2254}
2255
2256/**
2257 *	snapshot_write_finalize - must be called after the last call to
2258 *	snapshot_write_next() in case the last page in the image happens
2259 *	to be a highmem page and its contents should be stored in the
2260 *	highmem.  Additionally, it releases the memory that will not be
2261 *	used any more.
2262 */
2263
2264void snapshot_write_finalize(struct snapshot_handle *handle)
2265{
2266	copy_last_highmem_page();
 
 
 
2267	/* Free only if we have loaded the image entirely */
2268	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) {
2269		memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
2270		free_highmem_data();
2271	}
2272}
2273
2274int snapshot_image_loaded(struct snapshot_handle *handle)
2275{
2276	return !(!nr_copy_pages || !last_highmem_page_copied() ||
2277			handle->cur <= nr_meta_pages + nr_copy_pages);
2278}
2279
2280#ifdef CONFIG_HIGHMEM
2281/* Assumes that @buf is ready and points to a "safe" page */
2282static inline void
2283swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
2284{
2285	void *kaddr1, *kaddr2;
2286
2287	kaddr1 = kmap_atomic(p1, KM_USER0);
2288	kaddr2 = kmap_atomic(p2, KM_USER1);
2289	copy_page(buf, kaddr1);
2290	copy_page(kaddr1, kaddr2);
2291	copy_page(kaddr2, buf);
2292	kunmap_atomic(kaddr2, KM_USER1);
2293	kunmap_atomic(kaddr1, KM_USER0);
2294}
2295
2296/**
2297 *	restore_highmem - for each highmem page that was allocated before
2298 *	the suspend and included in the suspend image, and also has been
2299 *	allocated by the "resume" kernel swap its current (ie. "before
2300 *	resume") contents with the previous (ie. "before suspend") one.
2301 *
2302 *	If the resume eventually fails, we can call this function once
2303 *	again and restore the "before resume" highmem state.
2304 */
2305
2306int restore_highmem(void)
2307{
2308	struct highmem_pbe *pbe = highmem_pblist;
2309	void *buf;
2310
2311	if (!pbe)
2312		return 0;
2313
2314	buf = get_image_page(GFP_ATOMIC, PG_SAFE);
2315	if (!buf)
2316		return -ENOMEM;
2317
2318	while (pbe) {
2319		swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
2320		pbe = pbe->next;
2321	}
2322	free_image_page(buf, PG_UNSAFE_CLEAR);
2323	return 0;
2324}
2325#endif /* CONFIG_HIGHMEM */