Hibernate: Do not oops on resume if image data are incorrect
[linux-2.6] / kernel / power / snapshot.c
1 /*
2  * linux/kernel/power/snapshot.c
3  *
4  * This file provides system snapshot/restore functionality for swsusp.
5  *
6  * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
7  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
8  *
9  * This file is released under the GPLv2.
10  *
11  */
12
13 #include <linux/version.h>
14 #include <linux/module.h>
15 #include <linux/mm.h>
16 #include <linux/suspend.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/spinlock.h>
20 #include <linux/kernel.h>
21 #include <linux/pm.h>
22 #include <linux/device.h>
23 #include <linux/init.h>
24 #include <linux/bootmem.h>
25 #include <linux/syscalls.h>
26 #include <linux/console.h>
27 #include <linux/highmem.h>
28
29 #include <asm/uaccess.h>
30 #include <asm/mmu_context.h>
31 #include <asm/pgtable.h>
32 #include <asm/tlbflush.h>
33 #include <asm/io.h>
34
35 #include "power.h"
36
37 static int swsusp_page_is_free(struct page *);
38 static void swsusp_set_page_forbidden(struct page *);
39 static void swsusp_unset_page_forbidden(struct page *);
40
41 /* List of PBEs needed for restoring the pages that were allocated before
42  * the suspend and included in the suspend image, but have also been
43  * allocated by the "resume" kernel, so their contents cannot be written
44  * directly to their "original" page frames.
45  */
46 struct pbe *restore_pblist;
47
48 /* Pointer to an auxiliary buffer (1 page) */
49 static void *buffer;
50
51 /**
52  *      @safe_needed - on resume, for storing the PBE list and the image,
53  *      we can only use memory pages that do not conflict with the pages
54  *      used before suspend.  The unsafe pages have PageNosaveFree set
55  *      and we count them using unsafe_pages.
56  *
57  *      Each allocated image page is marked as PageNosave and PageNosaveFree
58  *      so that swsusp_free() can release it.
59  */
60
61 #define PG_ANY          0
62 #define PG_SAFE         1
63 #define PG_UNSAFE_CLEAR 1
64 #define PG_UNSAFE_KEEP  0
65
66 static unsigned int allocated_unsafe_pages;
67
68 static void *get_image_page(gfp_t gfp_mask, int safe_needed)
69 {
70         void *res;
71
72         res = (void *)get_zeroed_page(gfp_mask);
73         if (safe_needed)
74                 while (res && swsusp_page_is_free(virt_to_page(res))) {
75                         /* The page is unsafe, mark it for swsusp_free() */
76                         swsusp_set_page_forbidden(virt_to_page(res));
77                         allocated_unsafe_pages++;
78                         res = (void *)get_zeroed_page(gfp_mask);
79                 }
80         if (res) {
81                 swsusp_set_page_forbidden(virt_to_page(res));
82                 swsusp_set_page_free(virt_to_page(res));
83         }
84         return res;
85 }
86
87 unsigned long get_safe_page(gfp_t gfp_mask)
88 {
89         return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
90 }
91
92 static struct page *alloc_image_page(gfp_t gfp_mask)
93 {
94         struct page *page;
95
96         page = alloc_page(gfp_mask);
97         if (page) {
98                 swsusp_set_page_forbidden(page);
99                 swsusp_set_page_free(page);
100         }
101         return page;
102 }
103
104 /**
105  *      free_image_page - free page represented by @addr, allocated with
106  *      get_image_page (page flags set by it must be cleared)
107  */
108
109 static inline void free_image_page(void *addr, int clear_nosave_free)
110 {
111         struct page *page;
112
113         BUG_ON(!virt_addr_valid(addr));
114
115         page = virt_to_page(addr);
116
117         swsusp_unset_page_forbidden(page);
118         if (clear_nosave_free)
119                 swsusp_unset_page_free(page);
120
121         __free_page(page);
122 }
123
124 /* struct linked_page is used to build chains of pages */
125
126 #define LINKED_PAGE_DATA_SIZE   (PAGE_SIZE - sizeof(void *))
127
128 struct linked_page {
129         struct linked_page *next;
130         char data[LINKED_PAGE_DATA_SIZE];
131 } __attribute__((packed));
132
133 static inline void
134 free_list_of_pages(struct linked_page *list, int clear_page_nosave)
135 {
136         while (list) {
137                 struct linked_page *lp = list->next;
138
139                 free_image_page(list, clear_page_nosave);
140                 list = lp;
141         }
142 }
143
144 /**
145   *     struct chain_allocator is used for allocating small objects out of
146   *     a linked list of pages called 'the chain'.
147   *
148   *     The chain grows each time when there is no room for a new object in
149   *     the current page.  The allocated objects cannot be freed individually.
150   *     It is only possible to free them all at once, by freeing the entire
151   *     chain.
152   *
153   *     NOTE: The chain allocator may be inefficient if the allocated objects
154   *     are not much smaller than PAGE_SIZE.
155   */
156
157 struct chain_allocator {
158         struct linked_page *chain;      /* the chain */
159         unsigned int used_space;        /* total size of objects allocated out
160                                          * of the current page
161                                          */
162         gfp_t gfp_mask;         /* mask for allocating pages */
163         int safe_needed;        /* if set, only "safe" pages are allocated */
164 };
165
166 static void
167 chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
168 {
169         ca->chain = NULL;
170         ca->used_space = LINKED_PAGE_DATA_SIZE;
171         ca->gfp_mask = gfp_mask;
172         ca->safe_needed = safe_needed;
173 }
174
175 static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
176 {
177         void *ret;
178
179         if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
180                 struct linked_page *lp;
181
182                 lp = get_image_page(ca->gfp_mask, ca->safe_needed);
183                 if (!lp)
184                         return NULL;
185
186                 lp->next = ca->chain;
187                 ca->chain = lp;
188                 ca->used_space = 0;
189         }
190         ret = ca->chain->data + ca->used_space;
191         ca->used_space += size;
192         return ret;
193 }
194
195 static void chain_free(struct chain_allocator *ca, int clear_page_nosave)
196 {
197         free_list_of_pages(ca->chain, clear_page_nosave);
198         memset(ca, 0, sizeof(struct chain_allocator));
199 }
200
201 /**
202  *      Data types related to memory bitmaps.
203  *
204  *      Memory bitmap is a structure consiting of many linked lists of
205  *      objects.  The main list's elements are of type struct zone_bitmap
206  *      and each of them corresonds to one zone.  For each zone bitmap
207  *      object there is a list of objects of type struct bm_block that
208  *      represent each blocks of bitmap in which information is stored.
209  *
210  *      struct memory_bitmap contains a pointer to the main list of zone
211  *      bitmap objects, a struct bm_position used for browsing the bitmap,
212  *      and a pointer to the list of pages used for allocating all of the
213  *      zone bitmap objects and bitmap block objects.
214  *
215  *      NOTE: It has to be possible to lay out the bitmap in memory
216  *      using only allocations of order 0.  Additionally, the bitmap is
217  *      designed to work with arbitrary number of zones (this is over the
218  *      top for now, but let's avoid making unnecessary assumptions ;-).
219  *
220  *      struct zone_bitmap contains a pointer to a list of bitmap block
221  *      objects and a pointer to the bitmap block object that has been
222  *      most recently used for setting bits.  Additionally, it contains the
223  *      pfns that correspond to the start and end of the represented zone.
224  *
225  *      struct bm_block contains a pointer to the memory page in which
226  *      information is stored (in the form of a block of bitmap)
227  *      It also contains the pfns that correspond to the start and end of
228  *      the represented memory area.
229  */
230
231 #define BM_END_OF_MAP   (~0UL)
232
233 #define BM_BITS_PER_BLOCK       (PAGE_SIZE << 3)
234
235 struct bm_block {
236         struct bm_block *next;          /* next element of the list */
237         unsigned long start_pfn;        /* pfn represented by the first bit */
238         unsigned long end_pfn;  /* pfn represented by the last bit plus 1 */
239         unsigned long *data;    /* bitmap representing pages */
240 };
241
242 static inline unsigned long bm_block_bits(struct bm_block *bb)
243 {
244         return bb->end_pfn - bb->start_pfn;
245 }
246
247 struct zone_bitmap {
248         struct zone_bitmap *next;       /* next element of the list */
249         unsigned long start_pfn;        /* minimal pfn in this zone */
250         unsigned long end_pfn;          /* maximal pfn in this zone plus 1 */
251         struct bm_block *bm_blocks;     /* list of bitmap blocks */
252         struct bm_block *cur_block;     /* recently used bitmap block */
253 };
254
255 /* strcut bm_position is used for browsing memory bitmaps */
256
257 struct bm_position {
258         struct zone_bitmap *zone_bm;
259         struct bm_block *block;
260         int bit;
261 };
262
263 struct memory_bitmap {
264         struct zone_bitmap *zone_bm_list;       /* list of zone bitmaps */
265         struct linked_page *p_list;     /* list of pages used to store zone
266                                          * bitmap objects and bitmap block
267                                          * objects
268                                          */
269         struct bm_position cur; /* most recently used bit position */
270 };
271
272 /* Functions that operate on memory bitmaps */
273
274 static void memory_bm_position_reset(struct memory_bitmap *bm)
275 {
276         struct zone_bitmap *zone_bm;
277
278         zone_bm = bm->zone_bm_list;
279         bm->cur.zone_bm = zone_bm;
280         bm->cur.block = zone_bm->bm_blocks;
281         bm->cur.bit = 0;
282 }
283
284 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
285
286 /**
287  *      create_bm_block_list - create a list of block bitmap objects
288  */
289
290 static inline struct bm_block *
291 create_bm_block_list(unsigned int nr_blocks, struct chain_allocator *ca)
292 {
293         struct bm_block *bblist = NULL;
294
295         while (nr_blocks-- > 0) {
296                 struct bm_block *bb;
297
298                 bb = chain_alloc(ca, sizeof(struct bm_block));
299                 if (!bb)
300                         return NULL;
301
302                 bb->next = bblist;
303                 bblist = bb;
304         }
305         return bblist;
306 }
307
308 /**
309  *      create_zone_bm_list - create a list of zone bitmap objects
310  */
311
312 static inline struct zone_bitmap *
313 create_zone_bm_list(unsigned int nr_zones, struct chain_allocator *ca)
314 {
315         struct zone_bitmap *zbmlist = NULL;
316
317         while (nr_zones-- > 0) {
318                 struct zone_bitmap *zbm;
319
320                 zbm = chain_alloc(ca, sizeof(struct zone_bitmap));
321                 if (!zbm)
322                         return NULL;
323
324                 zbm->next = zbmlist;
325                 zbmlist = zbm;
326         }
327         return zbmlist;
328 }
329
330 /**
331   *     memory_bm_create - allocate memory for a memory bitmap
332   */
333
334 static int
335 memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
336 {
337         struct chain_allocator ca;
338         struct zone *zone;
339         struct zone_bitmap *zone_bm;
340         struct bm_block *bb;
341         unsigned int nr;
342
343         chain_init(&ca, gfp_mask, safe_needed);
344
345         /* Compute the number of zones */
346         nr = 0;
347         for_each_zone(zone)
348                 if (populated_zone(zone))
349                         nr++;
350
351         /* Allocate the list of zones bitmap objects */
352         zone_bm = create_zone_bm_list(nr, &ca);
353         bm->zone_bm_list = zone_bm;
354         if (!zone_bm) {
355                 chain_free(&ca, PG_UNSAFE_CLEAR);
356                 return -ENOMEM;
357         }
358
359         /* Initialize the zone bitmap objects */
360         for_each_zone(zone) {
361                 unsigned long pfn;
362
363                 if (!populated_zone(zone))
364                         continue;
365
366                 zone_bm->start_pfn = zone->zone_start_pfn;
367                 zone_bm->end_pfn = zone->zone_start_pfn + zone->spanned_pages;
368                 /* Allocate the list of bitmap block objects */
369                 nr = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
370                 bb = create_bm_block_list(nr, &ca);
371                 zone_bm->bm_blocks = bb;
372                 zone_bm->cur_block = bb;
373                 if (!bb)
374                         goto Free;
375
376                 nr = zone->spanned_pages;
377                 pfn = zone->zone_start_pfn;
378                 /* Initialize the bitmap block objects */
379                 while (bb) {
380                         unsigned long *ptr;
381
382                         ptr = get_image_page(gfp_mask, safe_needed);
383                         bb->data = ptr;
384                         if (!ptr)
385                                 goto Free;
386
387                         bb->start_pfn = pfn;
388                         if (nr >= BM_BITS_PER_BLOCK) {
389                                 pfn += BM_BITS_PER_BLOCK;
390                                 nr -= BM_BITS_PER_BLOCK;
391                         } else {
392                                 /* This is executed only once in the loop */
393                                 pfn += nr;
394                         }
395                         bb->end_pfn = pfn;
396                         bb = bb->next;
397                 }
398                 zone_bm = zone_bm->next;
399         }
400         bm->p_list = ca.chain;
401         memory_bm_position_reset(bm);
402         return 0;
403
404  Free:
405         bm->p_list = ca.chain;
406         memory_bm_free(bm, PG_UNSAFE_CLEAR);
407         return -ENOMEM;
408 }
409
410 /**
411   *     memory_bm_free - free memory occupied by the memory bitmap @bm
412   */
413
414 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
415 {
416         struct zone_bitmap *zone_bm;
417
418         /* Free the list of bit blocks for each zone_bitmap object */
419         zone_bm = bm->zone_bm_list;
420         while (zone_bm) {
421                 struct bm_block *bb;
422
423                 bb = zone_bm->bm_blocks;
424                 while (bb) {
425                         if (bb->data)
426                                 free_image_page(bb->data, clear_nosave_free);
427                         bb = bb->next;
428                 }
429                 zone_bm = zone_bm->next;
430         }
431         free_list_of_pages(bm->p_list, clear_nosave_free);
432         bm->zone_bm_list = NULL;
433 }
434
435 /**
436  *      memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
437  *      to given pfn.  The cur_zone_bm member of @bm and the cur_block member
438  *      of @bm->cur_zone_bm are updated.
439  */
440
441 static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
442                                 void **addr, unsigned int *bit_nr)
443 {
444         struct zone_bitmap *zone_bm;
445         struct bm_block *bb;
446
447         /* Check if the pfn is from the current zone */
448         zone_bm = bm->cur.zone_bm;
449         if (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
450                 zone_bm = bm->zone_bm_list;
451                 /* We don't assume that the zones are sorted by pfns */
452                 while (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
453                         zone_bm = zone_bm->next;
454
455                         if (!zone_bm)
456                                 return -EFAULT;
457                 }
458                 bm->cur.zone_bm = zone_bm;
459         }
460         /* Check if the pfn corresponds to the current bitmap block */
461         bb = zone_bm->cur_block;
462         if (pfn < bb->start_pfn)
463                 bb = zone_bm->bm_blocks;
464
465         while (pfn >= bb->end_pfn) {
466                 bb = bb->next;
467
468                 BUG_ON(!bb);
469         }
470         zone_bm->cur_block = bb;
471         pfn -= bb->start_pfn;
472         *bit_nr = pfn;
473         *addr = bb->data;
474         return 0;
475 }
476
477 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
478 {
479         void *addr;
480         unsigned int bit;
481         int error;
482
483         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
484         BUG_ON(error);
485         set_bit(bit, addr);
486 }
487
488 static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
489 {
490         void *addr;
491         unsigned int bit;
492         int error;
493
494         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
495         if (!error)
496                 set_bit(bit, addr);
497         return error;
498 }
499
500 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
501 {
502         void *addr;
503         unsigned int bit;
504         int error;
505
506         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
507         BUG_ON(error);
508         clear_bit(bit, addr);
509 }
510
511 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
512 {
513         void *addr;
514         unsigned int bit;
515         int error;
516
517         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
518         BUG_ON(error);
519         return test_bit(bit, addr);
520 }
521
522 static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
523 {
524         void *addr;
525         unsigned int bit;
526
527         return !memory_bm_find_bit(bm, pfn, &addr, &bit);
528 }
529
530 /**
531  *      memory_bm_next_pfn - find the pfn that corresponds to the next set bit
532  *      in the bitmap @bm.  If the pfn cannot be found, BM_END_OF_MAP is
533  *      returned.
534  *
535  *      It is required to run memory_bm_position_reset() before the first call to
536  *      this function.
537  */
538
539 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
540 {
541         struct zone_bitmap *zone_bm;
542         struct bm_block *bb;
543         int bit;
544
545         do {
546                 bb = bm->cur.block;
547                 do {
548                         bit = bm->cur.bit;
549                         bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
550                         if (bit < bm_block_bits(bb))
551                                 goto Return_pfn;
552
553                         bb = bb->next;
554                         bm->cur.block = bb;
555                         bm->cur.bit = 0;
556                 } while (bb);
557                 zone_bm = bm->cur.zone_bm->next;
558                 if (zone_bm) {
559                         bm->cur.zone_bm = zone_bm;
560                         bm->cur.block = zone_bm->bm_blocks;
561                         bm->cur.bit = 0;
562                 }
563         } while (zone_bm);
564         memory_bm_position_reset(bm);
565         return BM_END_OF_MAP;
566
567  Return_pfn:
568         bm->cur.bit = bit + 1;
569         return bb->start_pfn + bit;
570 }
571
572 /**
573  *      This structure represents a range of page frames the contents of which
574  *      should not be saved during the suspend.
575  */
576
577 struct nosave_region {
578         struct list_head list;
579         unsigned long start_pfn;
580         unsigned long end_pfn;
581 };
582
583 static LIST_HEAD(nosave_regions);
584
585 /**
586  *      register_nosave_region - register a range of page frames the contents
587  *      of which should not be saved during the suspend (to be used in the early
588  *      initialization code)
589  */
590
591 void __init
592 __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
593                          int use_kmalloc)
594 {
595         struct nosave_region *region;
596
597         if (start_pfn >= end_pfn)
598                 return;
599
600         if (!list_empty(&nosave_regions)) {
601                 /* Try to extend the previous region (they should be sorted) */
602                 region = list_entry(nosave_regions.prev,
603                                         struct nosave_region, list);
604                 if (region->end_pfn == start_pfn) {
605                         region->end_pfn = end_pfn;
606                         goto Report;
607                 }
608         }
609         if (use_kmalloc) {
610                 /* during init, this shouldn't fail */
611                 region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
612                 BUG_ON(!region);
613         } else
614                 /* This allocation cannot fail */
615                 region = alloc_bootmem_low(sizeof(struct nosave_region));
616         region->start_pfn = start_pfn;
617         region->end_pfn = end_pfn;
618         list_add_tail(&region->list, &nosave_regions);
619  Report:
620         printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n",
621                 start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
622 }
623
624 /*
625  * Set bits in this map correspond to the page frames the contents of which
626  * should not be saved during the suspend.
627  */
628 static struct memory_bitmap *forbidden_pages_map;
629
630 /* Set bits in this map correspond to free page frames. */
631 static struct memory_bitmap *free_pages_map;
632
633 /*
634  * Each page frame allocated for creating the image is marked by setting the
635  * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
636  */
637
638 void swsusp_set_page_free(struct page *page)
639 {
640         if (free_pages_map)
641                 memory_bm_set_bit(free_pages_map, page_to_pfn(page));
642 }
643
644 static int swsusp_page_is_free(struct page *page)
645 {
646         return free_pages_map ?
647                 memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
648 }
649
650 void swsusp_unset_page_free(struct page *page)
651 {
652         if (free_pages_map)
653                 memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
654 }
655
656 static void swsusp_set_page_forbidden(struct page *page)
657 {
658         if (forbidden_pages_map)
659                 memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
660 }
661
662 int swsusp_page_is_forbidden(struct page *page)
663 {
664         return forbidden_pages_map ?
665                 memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
666 }
667
668 static void swsusp_unset_page_forbidden(struct page *page)
669 {
670         if (forbidden_pages_map)
671                 memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
672 }
673
674 /**
675  *      mark_nosave_pages - set bits corresponding to the page frames the
676  *      contents of which should not be saved in a given bitmap.
677  */
678
679 static void mark_nosave_pages(struct memory_bitmap *bm)
680 {
681         struct nosave_region *region;
682
683         if (list_empty(&nosave_regions))
684                 return;
685
686         list_for_each_entry(region, &nosave_regions, list) {
687                 unsigned long pfn;
688
689                 pr_debug("PM: Marking nosave pages: %016lx - %016lx\n",
690                                 region->start_pfn << PAGE_SHIFT,
691                                 region->end_pfn << PAGE_SHIFT);
692
693                 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
694                         if (pfn_valid(pfn)) {
695                                 /*
696                                  * It is safe to ignore the result of
697                                  * mem_bm_set_bit_check() here, since we won't
698                                  * touch the PFNs for which the error is
699                                  * returned anyway.
700                                  */
701                                 mem_bm_set_bit_check(bm, pfn);
702                         }
703         }
704 }
705
706 /**
707  *      create_basic_memory_bitmaps - create bitmaps needed for marking page
708  *      frames that should not be saved and free page frames.  The pointers
709  *      forbidden_pages_map and free_pages_map are only modified if everything
710  *      goes well, because we don't want the bits to be used before both bitmaps
711  *      are set up.
712  */
713
714 int create_basic_memory_bitmaps(void)
715 {
716         struct memory_bitmap *bm1, *bm2;
717         int error = 0;
718
719         BUG_ON(forbidden_pages_map || free_pages_map);
720
721         bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
722         if (!bm1)
723                 return -ENOMEM;
724
725         error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
726         if (error)
727                 goto Free_first_object;
728
729         bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
730         if (!bm2)
731                 goto Free_first_bitmap;
732
733         error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
734         if (error)
735                 goto Free_second_object;
736
737         forbidden_pages_map = bm1;
738         free_pages_map = bm2;
739         mark_nosave_pages(forbidden_pages_map);
740
741         pr_debug("PM: Basic memory bitmaps created\n");
742
743         return 0;
744
745  Free_second_object:
746         kfree(bm2);
747  Free_first_bitmap:
748         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
749  Free_first_object:
750         kfree(bm1);
751         return -ENOMEM;
752 }
753
754 /**
755  *      free_basic_memory_bitmaps - free memory bitmaps allocated by
756  *      create_basic_memory_bitmaps().  The auxiliary pointers are necessary
757  *      so that the bitmaps themselves are not referred to while they are being
758  *      freed.
759  */
760
761 void free_basic_memory_bitmaps(void)
762 {
763         struct memory_bitmap *bm1, *bm2;
764
765         BUG_ON(!(forbidden_pages_map && free_pages_map));
766
767         bm1 = forbidden_pages_map;
768         bm2 = free_pages_map;
769         forbidden_pages_map = NULL;
770         free_pages_map = NULL;
771         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
772         kfree(bm1);
773         memory_bm_free(bm2, PG_UNSAFE_CLEAR);
774         kfree(bm2);
775
776         pr_debug("PM: Basic memory bitmaps freed\n");
777 }
778
779 /**
780  *      snapshot_additional_pages - estimate the number of additional pages
781  *      be needed for setting up the suspend image data structures for given
782  *      zone (usually the returned value is greater than the exact number)
783  */
784
785 unsigned int snapshot_additional_pages(struct zone *zone)
786 {
787         unsigned int res;
788
789         res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
790         res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
791         return 2 * res;
792 }
793
794 #ifdef CONFIG_HIGHMEM
795 /**
796  *      count_free_highmem_pages - compute the total number of free highmem
797  *      pages, system-wide.
798  */
799
800 static unsigned int count_free_highmem_pages(void)
801 {
802         struct zone *zone;
803         unsigned int cnt = 0;
804
805         for_each_zone(zone)
806                 if (populated_zone(zone) && is_highmem(zone))
807                         cnt += zone_page_state(zone, NR_FREE_PAGES);
808
809         return cnt;
810 }
811
812 /**
813  *      saveable_highmem_page - Determine whether a highmem page should be
814  *      included in the suspend image.
815  *
816  *      We should save the page if it isn't Nosave or NosaveFree, or Reserved,
817  *      and it isn't a part of a free chunk of pages.
818  */
819
820 static struct page *saveable_highmem_page(unsigned long pfn)
821 {
822         struct page *page;
823
824         if (!pfn_valid(pfn))
825                 return NULL;
826
827         page = pfn_to_page(pfn);
828
829         BUG_ON(!PageHighMem(page));
830
831         if (swsusp_page_is_forbidden(page) ||  swsusp_page_is_free(page) ||
832             PageReserved(page))
833                 return NULL;
834
835         return page;
836 }
837
838 /**
839  *      count_highmem_pages - compute the total number of saveable highmem
840  *      pages.
841  */
842
843 unsigned int count_highmem_pages(void)
844 {
845         struct zone *zone;
846         unsigned int n = 0;
847
848         for_each_zone(zone) {
849                 unsigned long pfn, max_zone_pfn;
850
851                 if (!is_highmem(zone))
852                         continue;
853
854                 mark_free_pages(zone);
855                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
856                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
857                         if (saveable_highmem_page(pfn))
858                                 n++;
859         }
860         return n;
861 }
862 #else
863 static inline void *saveable_highmem_page(unsigned long pfn) { return NULL; }
864 #endif /* CONFIG_HIGHMEM */
865
866 /**
867  *      saveable_page - Determine whether a non-highmem page should be included
868  *      in the suspend image.
869  *
870  *      We should save the page if it isn't Nosave, and is not in the range
871  *      of pages statically defined as 'unsaveable', and it isn't a part of
872  *      a free chunk of pages.
873  */
874
875 static struct page *saveable_page(unsigned long pfn)
876 {
877         struct page *page;
878
879         if (!pfn_valid(pfn))
880                 return NULL;
881
882         page = pfn_to_page(pfn);
883
884         BUG_ON(PageHighMem(page));
885
886         if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
887                 return NULL;
888
889         if (PageReserved(page)
890             && (!kernel_page_present(page) || pfn_is_nosave(pfn)))
891                 return NULL;
892
893         return page;
894 }
895
896 /**
897  *      count_data_pages - compute the total number of saveable non-highmem
898  *      pages.
899  */
900
901 unsigned int count_data_pages(void)
902 {
903         struct zone *zone;
904         unsigned long pfn, max_zone_pfn;
905         unsigned int n = 0;
906
907         for_each_zone(zone) {
908                 if (is_highmem(zone))
909                         continue;
910
911                 mark_free_pages(zone);
912                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
913                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
914                         if(saveable_page(pfn))
915                                 n++;
916         }
917         return n;
918 }
919
920 /* This is needed, because copy_page and memcpy are not usable for copying
921  * task structs.
922  */
923 static inline void do_copy_page(long *dst, long *src)
924 {
925         int n;
926
927         for (n = PAGE_SIZE / sizeof(long); n; n--)
928                 *dst++ = *src++;
929 }
930
931
932 /**
933  *      safe_copy_page - check if the page we are going to copy is marked as
934  *              present in the kernel page tables (this always is the case if
935  *              CONFIG_DEBUG_PAGEALLOC is not set and in that case
936  *              kernel_page_present() always returns 'true').
937  */
938 static void safe_copy_page(void *dst, struct page *s_page)
939 {
940         if (kernel_page_present(s_page)) {
941                 do_copy_page(dst, page_address(s_page));
942         } else {
943                 kernel_map_pages(s_page, 1, 1);
944                 do_copy_page(dst, page_address(s_page));
945                 kernel_map_pages(s_page, 1, 0);
946         }
947 }
948
949
950 #ifdef CONFIG_HIGHMEM
951 static inline struct page *
952 page_is_saveable(struct zone *zone, unsigned long pfn)
953 {
954         return is_highmem(zone) ?
955                         saveable_highmem_page(pfn) : saveable_page(pfn);
956 }
957
958 static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
959 {
960         struct page *s_page, *d_page;
961         void *src, *dst;
962
963         s_page = pfn_to_page(src_pfn);
964         d_page = pfn_to_page(dst_pfn);
965         if (PageHighMem(s_page)) {
966                 src = kmap_atomic(s_page, KM_USER0);
967                 dst = kmap_atomic(d_page, KM_USER1);
968                 do_copy_page(dst, src);
969                 kunmap_atomic(src, KM_USER0);
970                 kunmap_atomic(dst, KM_USER1);
971         } else {
972                 if (PageHighMem(d_page)) {
973                         /* Page pointed to by src may contain some kernel
974                          * data modified by kmap_atomic()
975                          */
976                         safe_copy_page(buffer, s_page);
977                         dst = kmap_atomic(pfn_to_page(dst_pfn), KM_USER0);
978                         memcpy(dst, buffer, PAGE_SIZE);
979                         kunmap_atomic(dst, KM_USER0);
980                 } else {
981                         safe_copy_page(page_address(d_page), s_page);
982                 }
983         }
984 }
985 #else
986 #define page_is_saveable(zone, pfn)     saveable_page(pfn)
987
988 static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
989 {
990         safe_copy_page(page_address(pfn_to_page(dst_pfn)),
991                                 pfn_to_page(src_pfn));
992 }
993 #endif /* CONFIG_HIGHMEM */
994
995 static void
996 copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
997 {
998         struct zone *zone;
999         unsigned long pfn;
1000
1001         for_each_zone(zone) {
1002                 unsigned long max_zone_pfn;
1003
1004                 mark_free_pages(zone);
1005                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1006                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1007                         if (page_is_saveable(zone, pfn))
1008                                 memory_bm_set_bit(orig_bm, pfn);
1009         }
1010         memory_bm_position_reset(orig_bm);
1011         memory_bm_position_reset(copy_bm);
1012         for(;;) {
1013                 pfn = memory_bm_next_pfn(orig_bm);
1014                 if (unlikely(pfn == BM_END_OF_MAP))
1015                         break;
1016                 copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1017         }
1018 }
1019
1020 /* Total number of image pages */
1021 static unsigned int nr_copy_pages;
1022 /* Number of pages needed for saving the original pfns of the image pages */
1023 static unsigned int nr_meta_pages;
1024
1025 /**
1026  *      swsusp_free - free pages allocated for the suspend.
1027  *
1028  *      Suspend pages are alocated before the atomic copy is made, so we
1029  *      need to release them after the resume.
1030  */
1031
1032 void swsusp_free(void)
1033 {
1034         struct zone *zone;
1035         unsigned long pfn, max_zone_pfn;
1036
1037         for_each_zone(zone) {
1038                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1039                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1040                         if (pfn_valid(pfn)) {
1041                                 struct page *page = pfn_to_page(pfn);
1042
1043                                 if (swsusp_page_is_forbidden(page) &&
1044                                     swsusp_page_is_free(page)) {
1045                                         swsusp_unset_page_forbidden(page);
1046                                         swsusp_unset_page_free(page);
1047                                         __free_page(page);
1048                                 }
1049                         }
1050         }
1051         nr_copy_pages = 0;
1052         nr_meta_pages = 0;
1053         restore_pblist = NULL;
1054         buffer = NULL;
1055 }
1056
1057 #ifdef CONFIG_HIGHMEM
1058 /**
1059   *     count_pages_for_highmem - compute the number of non-highmem pages
1060   *     that will be necessary for creating copies of highmem pages.
1061   */
1062
1063 static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1064 {
1065         unsigned int free_highmem = count_free_highmem_pages();
1066
1067         if (free_highmem >= nr_highmem)
1068                 nr_highmem = 0;
1069         else
1070                 nr_highmem -= free_highmem;
1071
1072         return nr_highmem;
1073 }
1074 #else
1075 static unsigned int
1076 count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1077 #endif /* CONFIG_HIGHMEM */
1078
1079 /**
1080  *      enough_free_mem - Make sure we have enough free memory for the
1081  *      snapshot image.
1082  */
1083
1084 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1085 {
1086         struct zone *zone;
1087         unsigned int free = 0, meta = 0;
1088
1089         for_each_zone(zone) {
1090                 meta += snapshot_additional_pages(zone);
1091                 if (!is_highmem(zone))
1092                         free += zone_page_state(zone, NR_FREE_PAGES);
1093         }
1094
1095         nr_pages += count_pages_for_highmem(nr_highmem);
1096         pr_debug("PM: Normal pages needed: %u + %u + %u, available pages: %u\n",
1097                 nr_pages, PAGES_FOR_IO, meta, free);
1098
1099         return free > nr_pages + PAGES_FOR_IO + meta;
1100 }
1101
1102 #ifdef CONFIG_HIGHMEM
1103 /**
1104  *      get_highmem_buffer - if there are some highmem pages in the suspend
1105  *      image, we may need the buffer to copy them and/or load their data.
1106  */
1107
1108 static inline int get_highmem_buffer(int safe_needed)
1109 {
1110         buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1111         return buffer ? 0 : -ENOMEM;
1112 }
1113
1114 /**
1115  *      alloc_highmem_image_pages - allocate some highmem pages for the image.
1116  *      Try to allocate as many pages as needed, but if the number of free
1117  *      highmem pages is lesser than that, allocate them all.
1118  */
1119
1120 static inline unsigned int
1121 alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1122 {
1123         unsigned int to_alloc = count_free_highmem_pages();
1124
1125         if (to_alloc > nr_highmem)
1126                 to_alloc = nr_highmem;
1127
1128         nr_highmem -= to_alloc;
1129         while (to_alloc-- > 0) {
1130                 struct page *page;
1131
1132                 page = alloc_image_page(__GFP_HIGHMEM);
1133                 memory_bm_set_bit(bm, page_to_pfn(page));
1134         }
1135         return nr_highmem;
1136 }
1137 #else
1138 static inline int get_highmem_buffer(int safe_needed) { return 0; }
1139
1140 static inline unsigned int
1141 alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1142 #endif /* CONFIG_HIGHMEM */
1143
1144 /**
1145  *      swsusp_alloc - allocate memory for the suspend image
1146  *
1147  *      We first try to allocate as many highmem pages as there are
1148  *      saveable highmem pages in the system.  If that fails, we allocate
1149  *      non-highmem pages for the copies of the remaining highmem ones.
1150  *
1151  *      In this approach it is likely that the copies of highmem pages will
1152  *      also be located in the high memory, because of the way in which
1153  *      copy_data_pages() works.
1154  */
1155
1156 static int
1157 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1158                 unsigned int nr_pages, unsigned int nr_highmem)
1159 {
1160         int error;
1161
1162         error = memory_bm_create(orig_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
1163         if (error)
1164                 goto Free;
1165
1166         error = memory_bm_create(copy_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
1167         if (error)
1168                 goto Free;
1169
1170         if (nr_highmem > 0) {
1171                 error = get_highmem_buffer(PG_ANY);
1172                 if (error)
1173                         goto Free;
1174
1175                 nr_pages += alloc_highmem_image_pages(copy_bm, nr_highmem);
1176         }
1177         while (nr_pages-- > 0) {
1178                 struct page *page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1179
1180                 if (!page)
1181                         goto Free;
1182
1183                 memory_bm_set_bit(copy_bm, page_to_pfn(page));
1184         }
1185         return 0;
1186
1187  Free:
1188         swsusp_free();
1189         return -ENOMEM;
1190 }
1191
1192 /* Memory bitmap used for marking saveable pages (during suspend) or the
1193  * suspend image pages (during resume)
1194  */
1195 static struct memory_bitmap orig_bm;
1196 /* Memory bitmap used on suspend for marking allocated pages that will contain
1197  * the copies of saveable pages.  During resume it is initially used for
1198  * marking the suspend image pages, but then its set bits are duplicated in
1199  * @orig_bm and it is released.  Next, on systems with high memory, it may be
1200  * used for marking "safe" highmem pages, but it has to be reinitialized for
1201  * this purpose.
1202  */
1203 static struct memory_bitmap copy_bm;
1204
1205 asmlinkage int swsusp_save(void)
1206 {
1207         unsigned int nr_pages, nr_highmem;
1208
1209         printk(KERN_INFO "PM: Creating hibernation image: \n");
1210
1211         drain_local_pages(NULL);
1212         nr_pages = count_data_pages();
1213         nr_highmem = count_highmem_pages();
1214         printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem);
1215
1216         if (!enough_free_mem(nr_pages, nr_highmem)) {
1217                 printk(KERN_ERR "PM: Not enough free memory\n");
1218                 return -ENOMEM;
1219         }
1220
1221         if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1222                 printk(KERN_ERR "PM: Memory allocation failed\n");
1223                 return -ENOMEM;
1224         }
1225
1226         /* During allocating of suspend pagedir, new cold pages may appear.
1227          * Kill them.
1228          */
1229         drain_local_pages(NULL);
1230         copy_data_pages(&copy_bm, &orig_bm);
1231
1232         /*
1233          * End of critical section. From now on, we can write to memory,
1234          * but we should not touch disk. This specially means we must _not_
1235          * touch swap space! Except we must write out our image of course.
1236          */
1237
1238         nr_pages += nr_highmem;
1239         nr_copy_pages = nr_pages;
1240         nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1241
1242         printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n",
1243                 nr_pages);
1244
1245         return 0;
1246 }
1247
1248 #ifndef CONFIG_ARCH_HIBERNATION_HEADER
1249 static int init_header_complete(struct swsusp_info *info)
1250 {
1251         memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1252         info->version_code = LINUX_VERSION_CODE;
1253         return 0;
1254 }
1255
1256 static char *check_image_kernel(struct swsusp_info *info)
1257 {
1258         if (info->version_code != LINUX_VERSION_CODE)
1259                 return "kernel version";
1260         if (strcmp(info->uts.sysname,init_utsname()->sysname))
1261                 return "system type";
1262         if (strcmp(info->uts.release,init_utsname()->release))
1263                 return "kernel release";
1264         if (strcmp(info->uts.version,init_utsname()->version))
1265                 return "version";
1266         if (strcmp(info->uts.machine,init_utsname()->machine))
1267                 return "machine";
1268         return NULL;
1269 }
1270 #endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1271
1272 unsigned long snapshot_get_image_size(void)
1273 {
1274         return nr_copy_pages + nr_meta_pages + 1;
1275 }
1276
1277 static int init_header(struct swsusp_info *info)
1278 {
1279         memset(info, 0, sizeof(struct swsusp_info));
1280         info->num_physpages = num_physpages;
1281         info->image_pages = nr_copy_pages;
1282         info->pages = snapshot_get_image_size();
1283         info->size = info->pages;
1284         info->size <<= PAGE_SHIFT;
1285         return init_header_complete(info);
1286 }
1287
1288 /**
1289  *      pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1290  *      are stored in the array @buf[] (1 page at a time)
1291  */
1292
1293 static inline void
1294 pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1295 {
1296         int j;
1297
1298         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1299                 buf[j] = memory_bm_next_pfn(bm);
1300                 if (unlikely(buf[j] == BM_END_OF_MAP))
1301                         break;
1302         }
1303 }
1304
1305 /**
1306  *      snapshot_read_next - used for reading the system memory snapshot.
1307  *
1308  *      On the first call to it @handle should point to a zeroed
1309  *      snapshot_handle structure.  The structure gets updated and a pointer
1310  *      to it should be passed to this function every next time.
1311  *
1312  *      The @count parameter should contain the number of bytes the caller
1313  *      wants to read from the snapshot.  It must not be zero.
1314  *
1315  *      On success the function returns a positive number.  Then, the caller
1316  *      is allowed to read up to the returned number of bytes from the memory
1317  *      location computed by the data_of() macro.  The number returned
1318  *      may be smaller than @count, but this only happens if the read would
1319  *      cross a page boundary otherwise.
1320  *
1321  *      The function returns 0 to indicate the end of data stream condition,
1322  *      and a negative number is returned on error.  In such cases the
1323  *      structure pointed to by @handle is not updated and should not be used
1324  *      any more.
1325  */
1326
1327 int snapshot_read_next(struct snapshot_handle *handle, size_t count)
1328 {
1329         if (handle->cur > nr_meta_pages + nr_copy_pages)
1330                 return 0;
1331
1332         if (!buffer) {
1333                 /* This makes the buffer be freed by swsusp_free() */
1334                 buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1335                 if (!buffer)
1336                         return -ENOMEM;
1337         }
1338         if (!handle->offset) {
1339                 int error;
1340
1341                 error = init_header((struct swsusp_info *)buffer);
1342                 if (error)
1343                         return error;
1344                 handle->buffer = buffer;
1345                 memory_bm_position_reset(&orig_bm);
1346                 memory_bm_position_reset(&copy_bm);
1347         }
1348         if (handle->prev < handle->cur) {
1349                 if (handle->cur <= nr_meta_pages) {
1350                         memset(buffer, 0, PAGE_SIZE);
1351                         pack_pfns(buffer, &orig_bm);
1352                 } else {
1353                         struct page *page;
1354
1355                         page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1356                         if (PageHighMem(page)) {
1357                                 /* Highmem pages are copied to the buffer,
1358                                  * because we can't return with a kmapped
1359                                  * highmem page (we may not be called again).
1360                                  */
1361                                 void *kaddr;
1362
1363                                 kaddr = kmap_atomic(page, KM_USER0);
1364                                 memcpy(buffer, kaddr, PAGE_SIZE);
1365                                 kunmap_atomic(kaddr, KM_USER0);
1366                                 handle->buffer = buffer;
1367                         } else {
1368                                 handle->buffer = page_address(page);
1369                         }
1370                 }
1371                 handle->prev = handle->cur;
1372         }
1373         handle->buf_offset = handle->cur_offset;
1374         if (handle->cur_offset + count >= PAGE_SIZE) {
1375                 count = PAGE_SIZE - handle->cur_offset;
1376                 handle->cur_offset = 0;
1377                 handle->cur++;
1378         } else {
1379                 handle->cur_offset += count;
1380         }
1381         handle->offset += count;
1382         return count;
1383 }
1384
1385 /**
1386  *      mark_unsafe_pages - mark the pages that cannot be used for storing
1387  *      the image during resume, because they conflict with the pages that
1388  *      had been used before suspend
1389  */
1390
1391 static int mark_unsafe_pages(struct memory_bitmap *bm)
1392 {
1393         struct zone *zone;
1394         unsigned long pfn, max_zone_pfn;
1395
1396         /* Clear page flags */
1397         for_each_zone(zone) {
1398                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1399                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1400                         if (pfn_valid(pfn))
1401                                 swsusp_unset_page_free(pfn_to_page(pfn));
1402         }
1403
1404         /* Mark pages that correspond to the "original" pfns as "unsafe" */
1405         memory_bm_position_reset(bm);
1406         do {
1407                 pfn = memory_bm_next_pfn(bm);
1408                 if (likely(pfn != BM_END_OF_MAP)) {
1409                         if (likely(pfn_valid(pfn)))
1410                                 swsusp_set_page_free(pfn_to_page(pfn));
1411                         else
1412                                 return -EFAULT;
1413                 }
1414         } while (pfn != BM_END_OF_MAP);
1415
1416         allocated_unsafe_pages = 0;
1417
1418         return 0;
1419 }
1420
1421 static void
1422 duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1423 {
1424         unsigned long pfn;
1425
1426         memory_bm_position_reset(src);
1427         pfn = memory_bm_next_pfn(src);
1428         while (pfn != BM_END_OF_MAP) {
1429                 memory_bm_set_bit(dst, pfn);
1430                 pfn = memory_bm_next_pfn(src);
1431         }
1432 }
1433
1434 static int check_header(struct swsusp_info *info)
1435 {
1436         char *reason;
1437
1438         reason = check_image_kernel(info);
1439         if (!reason && info->num_physpages != num_physpages)
1440                 reason = "memory size";
1441         if (reason) {
1442                 printk(KERN_ERR "PM: Image mismatch: %s\n", reason);
1443                 return -EPERM;
1444         }
1445         return 0;
1446 }
1447
1448 /**
1449  *      load header - check the image header and copy data from it
1450  */
1451
1452 static int
1453 load_header(struct swsusp_info *info)
1454 {
1455         int error;
1456
1457         restore_pblist = NULL;
1458         error = check_header(info);
1459         if (!error) {
1460                 nr_copy_pages = info->image_pages;
1461                 nr_meta_pages = info->pages - info->image_pages - 1;
1462         }
1463         return error;
1464 }
1465
1466 /**
1467  *      unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1468  *      the corresponding bit in the memory bitmap @bm
1469  */
1470 static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1471 {
1472         int j;
1473
1474         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1475                 if (unlikely(buf[j] == BM_END_OF_MAP))
1476                         break;
1477
1478                 if (memory_bm_pfn_present(bm, buf[j]))
1479                         memory_bm_set_bit(bm, buf[j]);
1480                 else
1481                         return -EFAULT;
1482         }
1483
1484         return 0;
1485 }
1486
1487 /* List of "safe" pages that may be used to store data loaded from the suspend
1488  * image
1489  */
1490 static struct linked_page *safe_pages_list;
1491
1492 #ifdef CONFIG_HIGHMEM
1493 /* struct highmem_pbe is used for creating the list of highmem pages that
1494  * should be restored atomically during the resume from disk, because the page
1495  * frames they have occupied before the suspend are in use.
1496  */
1497 struct highmem_pbe {
1498         struct page *copy_page; /* data is here now */
1499         struct page *orig_page; /* data was here before the suspend */
1500         struct highmem_pbe *next;
1501 };
1502
1503 /* List of highmem PBEs needed for restoring the highmem pages that were
1504  * allocated before the suspend and included in the suspend image, but have
1505  * also been allocated by the "resume" kernel, so their contents cannot be
1506  * written directly to their "original" page frames.
1507  */
1508 static struct highmem_pbe *highmem_pblist;
1509
1510 /**
1511  *      count_highmem_image_pages - compute the number of highmem pages in the
1512  *      suspend image.  The bits in the memory bitmap @bm that correspond to the
1513  *      image pages are assumed to be set.
1514  */
1515
1516 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1517 {
1518         unsigned long pfn;
1519         unsigned int cnt = 0;
1520
1521         memory_bm_position_reset(bm);
1522         pfn = memory_bm_next_pfn(bm);
1523         while (pfn != BM_END_OF_MAP) {
1524                 if (PageHighMem(pfn_to_page(pfn)))
1525                         cnt++;
1526
1527                 pfn = memory_bm_next_pfn(bm);
1528         }
1529         return cnt;
1530 }
1531
1532 /**
1533  *      prepare_highmem_image - try to allocate as many highmem pages as
1534  *      there are highmem image pages (@nr_highmem_p points to the variable
1535  *      containing the number of highmem image pages).  The pages that are
1536  *      "safe" (ie. will not be overwritten when the suspend image is
1537  *      restored) have the corresponding bits set in @bm (it must be
1538  *      unitialized).
1539  *
1540  *      NOTE: This function should not be called if there are no highmem
1541  *      image pages.
1542  */
1543
1544 static unsigned int safe_highmem_pages;
1545
1546 static struct memory_bitmap *safe_highmem_bm;
1547
1548 static int
1549 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1550 {
1551         unsigned int to_alloc;
1552
1553         if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1554                 return -ENOMEM;
1555
1556         if (get_highmem_buffer(PG_SAFE))
1557                 return -ENOMEM;
1558
1559         to_alloc = count_free_highmem_pages();
1560         if (to_alloc > *nr_highmem_p)
1561                 to_alloc = *nr_highmem_p;
1562         else
1563                 *nr_highmem_p = to_alloc;
1564
1565         safe_highmem_pages = 0;
1566         while (to_alloc-- > 0) {
1567                 struct page *page;
1568
1569                 page = alloc_page(__GFP_HIGHMEM);
1570                 if (!swsusp_page_is_free(page)) {
1571                         /* The page is "safe", set its bit the bitmap */
1572                         memory_bm_set_bit(bm, page_to_pfn(page));
1573                         safe_highmem_pages++;
1574                 }
1575                 /* Mark the page as allocated */
1576                 swsusp_set_page_forbidden(page);
1577                 swsusp_set_page_free(page);
1578         }
1579         memory_bm_position_reset(bm);
1580         safe_highmem_bm = bm;
1581         return 0;
1582 }
1583
1584 /**
1585  *      get_highmem_page_buffer - for given highmem image page find the buffer
1586  *      that suspend_write_next() should set for its caller to write to.
1587  *
1588  *      If the page is to be saved to its "original" page frame or a copy of
1589  *      the page is to be made in the highmem, @buffer is returned.  Otherwise,
1590  *      the copy of the page is to be made in normal memory, so the address of
1591  *      the copy is returned.
1592  *
1593  *      If @buffer is returned, the caller of suspend_write_next() will write
1594  *      the page's contents to @buffer, so they will have to be copied to the
1595  *      right location on the next call to suspend_write_next() and it is done
1596  *      with the help of copy_last_highmem_page().  For this purpose, if
1597  *      @buffer is returned, @last_highmem page is set to the page to which
1598  *      the data will have to be copied from @buffer.
1599  */
1600
1601 static struct page *last_highmem_page;
1602
1603 static void *
1604 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1605 {
1606         struct highmem_pbe *pbe;
1607         void *kaddr;
1608
1609         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1610                 /* We have allocated the "original" page frame and we can
1611                  * use it directly to store the loaded page.
1612                  */
1613                 last_highmem_page = page;
1614                 return buffer;
1615         }
1616         /* The "original" page frame has not been allocated and we have to
1617          * use a "safe" page frame to store the loaded page.
1618          */
1619         pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1620         if (!pbe) {
1621                 swsusp_free();
1622                 return ERR_PTR(-ENOMEM);
1623         }
1624         pbe->orig_page = page;
1625         if (safe_highmem_pages > 0) {
1626                 struct page *tmp;
1627
1628                 /* Copy of the page will be stored in high memory */
1629                 kaddr = buffer;
1630                 tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1631                 safe_highmem_pages--;
1632                 last_highmem_page = tmp;
1633                 pbe->copy_page = tmp;
1634         } else {
1635                 /* Copy of the page will be stored in normal memory */
1636                 kaddr = safe_pages_list;
1637                 safe_pages_list = safe_pages_list->next;
1638                 pbe->copy_page = virt_to_page(kaddr);
1639         }
1640         pbe->next = highmem_pblist;
1641         highmem_pblist = pbe;
1642         return kaddr;
1643 }
1644
1645 /**
1646  *      copy_last_highmem_page - copy the contents of a highmem image from
1647  *      @buffer, where the caller of snapshot_write_next() has place them,
1648  *      to the right location represented by @last_highmem_page .
1649  */
1650
1651 static void copy_last_highmem_page(void)
1652 {
1653         if (last_highmem_page) {
1654                 void *dst;
1655
1656                 dst = kmap_atomic(last_highmem_page, KM_USER0);
1657                 memcpy(dst, buffer, PAGE_SIZE);
1658                 kunmap_atomic(dst, KM_USER0);
1659                 last_highmem_page = NULL;
1660         }
1661 }
1662
1663 static inline int last_highmem_page_copied(void)
1664 {
1665         return !last_highmem_page;
1666 }
1667
1668 static inline void free_highmem_data(void)
1669 {
1670         if (safe_highmem_bm)
1671                 memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
1672
1673         if (buffer)
1674                 free_image_page(buffer, PG_UNSAFE_CLEAR);
1675 }
1676 #else
1677 static inline int get_safe_write_buffer(void) { return 0; }
1678
1679 static unsigned int
1680 count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
1681
1682 static inline int
1683 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1684 {
1685         return 0;
1686 }
1687
1688 static inline void *
1689 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1690 {
1691         return ERR_PTR(-EINVAL);
1692 }
1693
1694 static inline void copy_last_highmem_page(void) {}
1695 static inline int last_highmem_page_copied(void) { return 1; }
1696 static inline void free_highmem_data(void) {}
1697 #endif /* CONFIG_HIGHMEM */
1698
1699 /**
1700  *      prepare_image - use the memory bitmap @bm to mark the pages that will
1701  *      be overwritten in the process of restoring the system memory state
1702  *      from the suspend image ("unsafe" pages) and allocate memory for the
1703  *      image.
1704  *
1705  *      The idea is to allocate a new memory bitmap first and then allocate
1706  *      as many pages as needed for the image data, but not to assign these
1707  *      pages to specific tasks initially.  Instead, we just mark them as
1708  *      allocated and create a lists of "safe" pages that will be used
1709  *      later.  On systems with high memory a list of "safe" highmem pages is
1710  *      also created.
1711  */
1712
1713 #define PBES_PER_LINKED_PAGE    (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
1714
1715 static int
1716 prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
1717 {
1718         unsigned int nr_pages, nr_highmem;
1719         struct linked_page *sp_list, *lp;
1720         int error;
1721
1722         /* If there is no highmem, the buffer will not be necessary */
1723         free_image_page(buffer, PG_UNSAFE_CLEAR);
1724         buffer = NULL;
1725
1726         nr_highmem = count_highmem_image_pages(bm);
1727         error = mark_unsafe_pages(bm);
1728         if (error)
1729                 goto Free;
1730
1731         error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
1732         if (error)
1733                 goto Free;
1734
1735         duplicate_memory_bitmap(new_bm, bm);
1736         memory_bm_free(bm, PG_UNSAFE_KEEP);
1737         if (nr_highmem > 0) {
1738                 error = prepare_highmem_image(bm, &nr_highmem);
1739                 if (error)
1740                         goto Free;
1741         }
1742         /* Reserve some safe pages for potential later use.
1743          *
1744          * NOTE: This way we make sure there will be enough safe pages for the
1745          * chain_alloc() in get_buffer().  It is a bit wasteful, but
1746          * nr_copy_pages cannot be greater than 50% of the memory anyway.
1747          */
1748         sp_list = NULL;
1749         /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
1750         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
1751         nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
1752         while (nr_pages > 0) {
1753                 lp = get_image_page(GFP_ATOMIC, PG_SAFE);
1754                 if (!lp) {
1755                         error = -ENOMEM;
1756                         goto Free;
1757                 }
1758                 lp->next = sp_list;
1759                 sp_list = lp;
1760                 nr_pages--;
1761         }
1762         /* Preallocate memory for the image */
1763         safe_pages_list = NULL;
1764         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
1765         while (nr_pages > 0) {
1766                 lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
1767                 if (!lp) {
1768                         error = -ENOMEM;
1769                         goto Free;
1770                 }
1771                 if (!swsusp_page_is_free(virt_to_page(lp))) {
1772                         /* The page is "safe", add it to the list */
1773                         lp->next = safe_pages_list;
1774                         safe_pages_list = lp;
1775                 }
1776                 /* Mark the page as allocated */
1777                 swsusp_set_page_forbidden(virt_to_page(lp));
1778                 swsusp_set_page_free(virt_to_page(lp));
1779                 nr_pages--;
1780         }
1781         /* Free the reserved safe pages so that chain_alloc() can use them */
1782         while (sp_list) {
1783                 lp = sp_list->next;
1784                 free_image_page(sp_list, PG_UNSAFE_CLEAR);
1785                 sp_list = lp;
1786         }
1787         return 0;
1788
1789  Free:
1790         swsusp_free();
1791         return error;
1792 }
1793
1794 /**
1795  *      get_buffer - compute the address that snapshot_write_next() should
1796  *      set for its caller to write to.
1797  */
1798
1799 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
1800 {
1801         struct pbe *pbe;
1802         struct page *page;
1803         unsigned long pfn = memory_bm_next_pfn(bm);
1804
1805         if (pfn == BM_END_OF_MAP)
1806                 return ERR_PTR(-EFAULT);
1807
1808         page = pfn_to_page(pfn);
1809         if (PageHighMem(page))
1810                 return get_highmem_page_buffer(page, ca);
1811
1812         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
1813                 /* We have allocated the "original" page frame and we can
1814                  * use it directly to store the loaded page.
1815                  */
1816                 return page_address(page);
1817
1818         /* The "original" page frame has not been allocated and we have to
1819          * use a "safe" page frame to store the loaded page.
1820          */
1821         pbe = chain_alloc(ca, sizeof(struct pbe));
1822         if (!pbe) {
1823                 swsusp_free();
1824                 return ERR_PTR(-ENOMEM);
1825         }
1826         pbe->orig_address = page_address(page);
1827         pbe->address = safe_pages_list;
1828         safe_pages_list = safe_pages_list->next;
1829         pbe->next = restore_pblist;
1830         restore_pblist = pbe;
1831         return pbe->address;
1832 }
1833
1834 /**
1835  *      snapshot_write_next - used for writing the system memory snapshot.
1836  *
1837  *      On the first call to it @handle should point to a zeroed
1838  *      snapshot_handle structure.  The structure gets updated and a pointer
1839  *      to it should be passed to this function every next time.
1840  *
1841  *      The @count parameter should contain the number of bytes the caller
1842  *      wants to write to the image.  It must not be zero.
1843  *
1844  *      On success the function returns a positive number.  Then, the caller
1845  *      is allowed to write up to the returned number of bytes to the memory
1846  *      location computed by the data_of() macro.  The number returned
1847  *      may be smaller than @count, but this only happens if the write would
1848  *      cross a page boundary otherwise.
1849  *
1850  *      The function returns 0 to indicate the "end of file" condition,
1851  *      and a negative number is returned on error.  In such cases the
1852  *      structure pointed to by @handle is not updated and should not be used
1853  *      any more.
1854  */
1855
1856 int snapshot_write_next(struct snapshot_handle *handle, size_t count)
1857 {
1858         static struct chain_allocator ca;
1859         int error = 0;
1860
1861         /* Check if we have already loaded the entire image */
1862         if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages)
1863                 return 0;
1864
1865         if (handle->offset == 0) {
1866                 if (!buffer)
1867                         /* This makes the buffer be freed by swsusp_free() */
1868                         buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1869
1870                 if (!buffer)
1871                         return -ENOMEM;
1872
1873                 handle->buffer = buffer;
1874         }
1875         handle->sync_read = 1;
1876         if (handle->prev < handle->cur) {
1877                 if (handle->prev == 0) {
1878                         error = load_header(buffer);
1879                         if (error)
1880                                 return error;
1881
1882                         error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
1883                         if (error)
1884                                 return error;
1885
1886                 } else if (handle->prev <= nr_meta_pages) {
1887                         error = unpack_orig_pfns(buffer, &copy_bm);
1888                         if (error)
1889                                 return error;
1890
1891                         if (handle->prev == nr_meta_pages) {
1892                                 error = prepare_image(&orig_bm, &copy_bm);
1893                                 if (error)
1894                                         return error;
1895
1896                                 chain_init(&ca, GFP_ATOMIC, PG_SAFE);
1897                                 memory_bm_position_reset(&orig_bm);
1898                                 restore_pblist = NULL;
1899                                 handle->buffer = get_buffer(&orig_bm, &ca);
1900                                 handle->sync_read = 0;
1901                                 if (IS_ERR(handle->buffer))
1902                                         return PTR_ERR(handle->buffer);
1903                         }
1904                 } else {
1905                         copy_last_highmem_page();
1906                         handle->buffer = get_buffer(&orig_bm, &ca);
1907                         if (IS_ERR(handle->buffer))
1908                                 return PTR_ERR(handle->buffer);
1909                         if (handle->buffer != buffer)
1910                                 handle->sync_read = 0;
1911                 }
1912                 handle->prev = handle->cur;
1913         }
1914         handle->buf_offset = handle->cur_offset;
1915         if (handle->cur_offset + count >= PAGE_SIZE) {
1916                 count = PAGE_SIZE - handle->cur_offset;
1917                 handle->cur_offset = 0;
1918                 handle->cur++;
1919         } else {
1920                 handle->cur_offset += count;
1921         }
1922         handle->offset += count;
1923         return count;
1924 }
1925
1926 /**
1927  *      snapshot_write_finalize - must be called after the last call to
1928  *      snapshot_write_next() in case the last page in the image happens
1929  *      to be a highmem page and its contents should be stored in the
1930  *      highmem.  Additionally, it releases the memory that will not be
1931  *      used any more.
1932  */
1933
1934 void snapshot_write_finalize(struct snapshot_handle *handle)
1935 {
1936         copy_last_highmem_page();
1937         /* Free only if we have loaded the image entirely */
1938         if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages) {
1939                 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
1940                 free_highmem_data();
1941         }
1942 }
1943
1944 int snapshot_image_loaded(struct snapshot_handle *handle)
1945 {
1946         return !(!nr_copy_pages || !last_highmem_page_copied() ||
1947                         handle->cur <= nr_meta_pages + nr_copy_pages);
1948 }
1949
1950 #ifdef CONFIG_HIGHMEM
1951 /* Assumes that @buf is ready and points to a "safe" page */
1952 static inline void
1953 swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
1954 {
1955         void *kaddr1, *kaddr2;
1956
1957         kaddr1 = kmap_atomic(p1, KM_USER0);
1958         kaddr2 = kmap_atomic(p2, KM_USER1);
1959         memcpy(buf, kaddr1, PAGE_SIZE);
1960         memcpy(kaddr1, kaddr2, PAGE_SIZE);
1961         memcpy(kaddr2, buf, PAGE_SIZE);
1962         kunmap_atomic(kaddr1, KM_USER0);
1963         kunmap_atomic(kaddr2, KM_USER1);
1964 }
1965
1966 /**
1967  *      restore_highmem - for each highmem page that was allocated before
1968  *      the suspend and included in the suspend image, and also has been
1969  *      allocated by the "resume" kernel swap its current (ie. "before
1970  *      resume") contents with the previous (ie. "before suspend") one.
1971  *
1972  *      If the resume eventually fails, we can call this function once
1973  *      again and restore the "before resume" highmem state.
1974  */
1975
1976 int restore_highmem(void)
1977 {
1978         struct highmem_pbe *pbe = highmem_pblist;
1979         void *buf;
1980
1981         if (!pbe)
1982                 return 0;
1983
1984         buf = get_image_page(GFP_ATOMIC, PG_SAFE);
1985         if (!buf)
1986                 return -ENOMEM;
1987
1988         while (pbe) {
1989                 swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
1990                 pbe = pbe->next;
1991         }
1992         free_image_page(buf, PG_UNSAFE_CLEAR);
1993         return 0;
1994 }
1995 #endif /* CONFIG_HIGHMEM */