Merge git://git.kernel.org/pub/scm/linux/kernel/git/holtmann/bluetooth-2.6
[linux-2.6] / kernel / power / snapshot.c
1 /*
2  * linux/kernel/power/snapshot.c
3  *
4  * This file provides system snapshot/restore functionality for swsusp.
5  *
6  * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
7  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
8  *
9  * This file is released under the GPLv2.
10  *
11  */
12
13 #include <linux/version.h>
14 #include <linux/module.h>
15 #include <linux/mm.h>
16 #include <linux/suspend.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/spinlock.h>
20 #include <linux/kernel.h>
21 #include <linux/pm.h>
22 #include <linux/device.h>
23 #include <linux/init.h>
24 #include <linux/bootmem.h>
25 #include <linux/syscalls.h>
26 #include <linux/console.h>
27 #include <linux/highmem.h>
28
29 #include <asm/uaccess.h>
30 #include <asm/mmu_context.h>
31 #include <asm/pgtable.h>
32 #include <asm/tlbflush.h>
33 #include <asm/io.h>
34
35 #include "power.h"
36
37 static int swsusp_page_is_free(struct page *);
38 static void swsusp_set_page_forbidden(struct page *);
39 static void swsusp_unset_page_forbidden(struct page *);
40
41 /* List of PBEs needed for restoring the pages that were allocated before
42  * the suspend and included in the suspend image, but have also been
43  * allocated by the "resume" kernel, so their contents cannot be written
44  * directly to their "original" page frames.
45  */
46 struct pbe *restore_pblist;
47
48 /* Pointer to an auxiliary buffer (1 page) */
49 static void *buffer;
50
51 /**
52  *      @safe_needed - on resume, for storing the PBE list and the image,
53  *      we can only use memory pages that do not conflict with the pages
54  *      used before suspend.  The unsafe pages have PageNosaveFree set
55  *      and we count them using unsafe_pages.
56  *
57  *      Each allocated image page is marked as PageNosave and PageNosaveFree
58  *      so that swsusp_free() can release it.
59  */
60
61 #define PG_ANY          0
62 #define PG_SAFE         1
63 #define PG_UNSAFE_CLEAR 1
64 #define PG_UNSAFE_KEEP  0
65
66 static unsigned int allocated_unsafe_pages;
67
68 static void *get_image_page(gfp_t gfp_mask, int safe_needed)
69 {
70         void *res;
71
72         res = (void *)get_zeroed_page(gfp_mask);
73         if (safe_needed)
74                 while (res && swsusp_page_is_free(virt_to_page(res))) {
75                         /* The page is unsafe, mark it for swsusp_free() */
76                         swsusp_set_page_forbidden(virt_to_page(res));
77                         allocated_unsafe_pages++;
78                         res = (void *)get_zeroed_page(gfp_mask);
79                 }
80         if (res) {
81                 swsusp_set_page_forbidden(virt_to_page(res));
82                 swsusp_set_page_free(virt_to_page(res));
83         }
84         return res;
85 }
86
87 unsigned long get_safe_page(gfp_t gfp_mask)
88 {
89         return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
90 }
91
92 static struct page *alloc_image_page(gfp_t gfp_mask)
93 {
94         struct page *page;
95
96         page = alloc_page(gfp_mask);
97         if (page) {
98                 swsusp_set_page_forbidden(page);
99                 swsusp_set_page_free(page);
100         }
101         return page;
102 }
103
104 /**
105  *      free_image_page - free page represented by @addr, allocated with
106  *      get_image_page (page flags set by it must be cleared)
107  */
108
109 static inline void free_image_page(void *addr, int clear_nosave_free)
110 {
111         struct page *page;
112
113         BUG_ON(!virt_addr_valid(addr));
114
115         page = virt_to_page(addr);
116
117         swsusp_unset_page_forbidden(page);
118         if (clear_nosave_free)
119                 swsusp_unset_page_free(page);
120
121         __free_page(page);
122 }
123
124 /* struct linked_page is used to build chains of pages */
125
126 #define LINKED_PAGE_DATA_SIZE   (PAGE_SIZE - sizeof(void *))
127
128 struct linked_page {
129         struct linked_page *next;
130         char data[LINKED_PAGE_DATA_SIZE];
131 } __attribute__((packed));
132
133 static inline void
134 free_list_of_pages(struct linked_page *list, int clear_page_nosave)
135 {
136         while (list) {
137                 struct linked_page *lp = list->next;
138
139                 free_image_page(list, clear_page_nosave);
140                 list = lp;
141         }
142 }
143
144 /**
145   *     struct chain_allocator is used for allocating small objects out of
146   *     a linked list of pages called 'the chain'.
147   *
148   *     The chain grows each time when there is no room for a new object in
149   *     the current page.  The allocated objects cannot be freed individually.
150   *     It is only possible to free them all at once, by freeing the entire
151   *     chain.
152   *
153   *     NOTE: The chain allocator may be inefficient if the allocated objects
154   *     are not much smaller than PAGE_SIZE.
155   */
156
157 struct chain_allocator {
158         struct linked_page *chain;      /* the chain */
159         unsigned int used_space;        /* total size of objects allocated out
160                                          * of the current page
161                                          */
162         gfp_t gfp_mask;         /* mask for allocating pages */
163         int safe_needed;        /* if set, only "safe" pages are allocated */
164 };
165
166 static void
167 chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
168 {
169         ca->chain = NULL;
170         ca->used_space = LINKED_PAGE_DATA_SIZE;
171         ca->gfp_mask = gfp_mask;
172         ca->safe_needed = safe_needed;
173 }
174
175 static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
176 {
177         void *ret;
178
179         if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
180                 struct linked_page *lp;
181
182                 lp = get_image_page(ca->gfp_mask, ca->safe_needed);
183                 if (!lp)
184                         return NULL;
185
186                 lp->next = ca->chain;
187                 ca->chain = lp;
188                 ca->used_space = 0;
189         }
190         ret = ca->chain->data + ca->used_space;
191         ca->used_space += size;
192         return ret;
193 }
194
195 static void chain_free(struct chain_allocator *ca, int clear_page_nosave)
196 {
197         free_list_of_pages(ca->chain, clear_page_nosave);
198         memset(ca, 0, sizeof(struct chain_allocator));
199 }
200
201 /**
202  *      Data types related to memory bitmaps.
203  *
204  *      Memory bitmap is a structure consiting of many linked lists of
205  *      objects.  The main list's elements are of type struct zone_bitmap
206  *      and each of them corresonds to one zone.  For each zone bitmap
207  *      object there is a list of objects of type struct bm_block that
208  *      represent each blocks of bit chunks in which information is
209  *      stored.
210  *
211  *      struct memory_bitmap contains a pointer to the main list of zone
212  *      bitmap objects, a struct bm_position used for browsing the bitmap,
213  *      and a pointer to the list of pages used for allocating all of the
214  *      zone bitmap objects and bitmap block objects.
215  *
216  *      NOTE: It has to be possible to lay out the bitmap in memory
217  *      using only allocations of order 0.  Additionally, the bitmap is
218  *      designed to work with arbitrary number of zones (this is over the
219  *      top for now, but let's avoid making unnecessary assumptions ;-).
220  *
221  *      struct zone_bitmap contains a pointer to a list of bitmap block
222  *      objects and a pointer to the bitmap block object that has been
223  *      most recently used for setting bits.  Additionally, it contains the
224  *      pfns that correspond to the start and end of the represented zone.
225  *
226  *      struct bm_block contains a pointer to the memory page in which
227  *      information is stored (in the form of a block of bit chunks
228  *      of type unsigned long each).  It also contains the pfns that
229  *      correspond to the start and end of the represented memory area and
230  *      the number of bit chunks in the block.
231  */
232
233 #define BM_END_OF_MAP   (~0UL)
234
235 #define BM_CHUNKS_PER_BLOCK     (PAGE_SIZE / sizeof(long))
236 #define BM_BITS_PER_CHUNK       (sizeof(long) << 3)
237 #define BM_BITS_PER_BLOCK       (PAGE_SIZE << 3)
238
239 struct bm_block {
240         struct bm_block *next;          /* next element of the list */
241         unsigned long start_pfn;        /* pfn represented by the first bit */
242         unsigned long end_pfn;  /* pfn represented by the last bit plus 1 */
243         unsigned int size;      /* number of bit chunks */
244         unsigned long *data;    /* chunks of bits representing pages */
245 };
246
247 struct zone_bitmap {
248         struct zone_bitmap *next;       /* next element of the list */
249         unsigned long start_pfn;        /* minimal pfn in this zone */
250         unsigned long end_pfn;          /* maximal pfn in this zone plus 1 */
251         struct bm_block *bm_blocks;     /* list of bitmap blocks */
252         struct bm_block *cur_block;     /* recently used bitmap block */
253 };
254
255 /* strcut bm_position is used for browsing memory bitmaps */
256
257 struct bm_position {
258         struct zone_bitmap *zone_bm;
259         struct bm_block *block;
260         int chunk;
261         int bit;
262 };
263
264 struct memory_bitmap {
265         struct zone_bitmap *zone_bm_list;       /* list of zone bitmaps */
266         struct linked_page *p_list;     /* list of pages used to store zone
267                                          * bitmap objects and bitmap block
268                                          * objects
269                                          */
270         struct bm_position cur; /* most recently used bit position */
271 };
272
273 /* Functions that operate on memory bitmaps */
274
275 static inline void memory_bm_reset_chunk(struct memory_bitmap *bm)
276 {
277         bm->cur.chunk = 0;
278         bm->cur.bit = -1;
279 }
280
281 static void memory_bm_position_reset(struct memory_bitmap *bm)
282 {
283         struct zone_bitmap *zone_bm;
284
285         zone_bm = bm->zone_bm_list;
286         bm->cur.zone_bm = zone_bm;
287         bm->cur.block = zone_bm->bm_blocks;
288         memory_bm_reset_chunk(bm);
289 }
290
291 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
292
293 /**
294  *      create_bm_block_list - create a list of block bitmap objects
295  */
296
297 static inline struct bm_block *
298 create_bm_block_list(unsigned int nr_blocks, struct chain_allocator *ca)
299 {
300         struct bm_block *bblist = NULL;
301
302         while (nr_blocks-- > 0) {
303                 struct bm_block *bb;
304
305                 bb = chain_alloc(ca, sizeof(struct bm_block));
306                 if (!bb)
307                         return NULL;
308
309                 bb->next = bblist;
310                 bblist = bb;
311         }
312         return bblist;
313 }
314
315 /**
316  *      create_zone_bm_list - create a list of zone bitmap objects
317  */
318
319 static inline struct zone_bitmap *
320 create_zone_bm_list(unsigned int nr_zones, struct chain_allocator *ca)
321 {
322         struct zone_bitmap *zbmlist = NULL;
323
324         while (nr_zones-- > 0) {
325                 struct zone_bitmap *zbm;
326
327                 zbm = chain_alloc(ca, sizeof(struct zone_bitmap));
328                 if (!zbm)
329                         return NULL;
330
331                 zbm->next = zbmlist;
332                 zbmlist = zbm;
333         }
334         return zbmlist;
335 }
336
337 /**
338   *     memory_bm_create - allocate memory for a memory bitmap
339   */
340
341 static int
342 memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
343 {
344         struct chain_allocator ca;
345         struct zone *zone;
346         struct zone_bitmap *zone_bm;
347         struct bm_block *bb;
348         unsigned int nr;
349
350         chain_init(&ca, gfp_mask, safe_needed);
351
352         /* Compute the number of zones */
353         nr = 0;
354         for_each_zone(zone)
355                 if (populated_zone(zone))
356                         nr++;
357
358         /* Allocate the list of zones bitmap objects */
359         zone_bm = create_zone_bm_list(nr, &ca);
360         bm->zone_bm_list = zone_bm;
361         if (!zone_bm) {
362                 chain_free(&ca, PG_UNSAFE_CLEAR);
363                 return -ENOMEM;
364         }
365
366         /* Initialize the zone bitmap objects */
367         for_each_zone(zone) {
368                 unsigned long pfn;
369
370                 if (!populated_zone(zone))
371                         continue;
372
373                 zone_bm->start_pfn = zone->zone_start_pfn;
374                 zone_bm->end_pfn = zone->zone_start_pfn + zone->spanned_pages;
375                 /* Allocate the list of bitmap block objects */
376                 nr = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
377                 bb = create_bm_block_list(nr, &ca);
378                 zone_bm->bm_blocks = bb;
379                 zone_bm->cur_block = bb;
380                 if (!bb)
381                         goto Free;
382
383                 nr = zone->spanned_pages;
384                 pfn = zone->zone_start_pfn;
385                 /* Initialize the bitmap block objects */
386                 while (bb) {
387                         unsigned long *ptr;
388
389                         ptr = get_image_page(gfp_mask, safe_needed);
390                         bb->data = ptr;
391                         if (!ptr)
392                                 goto Free;
393
394                         bb->start_pfn = pfn;
395                         if (nr >= BM_BITS_PER_BLOCK) {
396                                 pfn += BM_BITS_PER_BLOCK;
397                                 bb->size = BM_CHUNKS_PER_BLOCK;
398                                 nr -= BM_BITS_PER_BLOCK;
399                         } else {
400                                 /* This is executed only once in the loop */
401                                 pfn += nr;
402                                 bb->size = DIV_ROUND_UP(nr, BM_BITS_PER_CHUNK);
403                         }
404                         bb->end_pfn = pfn;
405                         bb = bb->next;
406                 }
407                 zone_bm = zone_bm->next;
408         }
409         bm->p_list = ca.chain;
410         memory_bm_position_reset(bm);
411         return 0;
412
413  Free:
414         bm->p_list = ca.chain;
415         memory_bm_free(bm, PG_UNSAFE_CLEAR);
416         return -ENOMEM;
417 }
418
419 /**
420   *     memory_bm_free - free memory occupied by the memory bitmap @bm
421   */
422
423 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
424 {
425         struct zone_bitmap *zone_bm;
426
427         /* Free the list of bit blocks for each zone_bitmap object */
428         zone_bm = bm->zone_bm_list;
429         while (zone_bm) {
430                 struct bm_block *bb;
431
432                 bb = zone_bm->bm_blocks;
433                 while (bb) {
434                         if (bb->data)
435                                 free_image_page(bb->data, clear_nosave_free);
436                         bb = bb->next;
437                 }
438                 zone_bm = zone_bm->next;
439         }
440         free_list_of_pages(bm->p_list, clear_nosave_free);
441         bm->zone_bm_list = NULL;
442 }
443
444 /**
445  *      memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
446  *      to given pfn.  The cur_zone_bm member of @bm and the cur_block member
447  *      of @bm->cur_zone_bm are updated.
448  */
449
450 static void memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
451                                 void **addr, unsigned int *bit_nr)
452 {
453         struct zone_bitmap *zone_bm;
454         struct bm_block *bb;
455
456         /* Check if the pfn is from the current zone */
457         zone_bm = bm->cur.zone_bm;
458         if (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
459                 zone_bm = bm->zone_bm_list;
460                 /* We don't assume that the zones are sorted by pfns */
461                 while (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
462                         zone_bm = zone_bm->next;
463
464                         BUG_ON(!zone_bm);
465                 }
466                 bm->cur.zone_bm = zone_bm;
467         }
468         /* Check if the pfn corresponds to the current bitmap block */
469         bb = zone_bm->cur_block;
470         if (pfn < bb->start_pfn)
471                 bb = zone_bm->bm_blocks;
472
473         while (pfn >= bb->end_pfn) {
474                 bb = bb->next;
475
476                 BUG_ON(!bb);
477         }
478         zone_bm->cur_block = bb;
479         pfn -= bb->start_pfn;
480         *bit_nr = pfn % BM_BITS_PER_CHUNK;
481         *addr = bb->data + pfn / BM_BITS_PER_CHUNK;
482 }
483
484 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
485 {
486         void *addr;
487         unsigned int bit;
488
489         memory_bm_find_bit(bm, pfn, &addr, &bit);
490         set_bit(bit, addr);
491 }
492
493 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
494 {
495         void *addr;
496         unsigned int bit;
497
498         memory_bm_find_bit(bm, pfn, &addr, &bit);
499         clear_bit(bit, addr);
500 }
501
502 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
503 {
504         void *addr;
505         unsigned int bit;
506
507         memory_bm_find_bit(bm, pfn, &addr, &bit);
508         return test_bit(bit, addr);
509 }
510
511 /* Two auxiliary functions for memory_bm_next_pfn */
512
513 /* Find the first set bit in the given chunk, if there is one */
514
515 static inline int next_bit_in_chunk(int bit, unsigned long *chunk_p)
516 {
517         bit++;
518         while (bit < BM_BITS_PER_CHUNK) {
519                 if (test_bit(bit, chunk_p))
520                         return bit;
521
522                 bit++;
523         }
524         return -1;
525 }
526
527 /* Find a chunk containing some bits set in given block of bits */
528
529 static inline int next_chunk_in_block(int n, struct bm_block *bb)
530 {
531         n++;
532         while (n < bb->size) {
533                 if (bb->data[n])
534                         return n;
535
536                 n++;
537         }
538         return -1;
539 }
540
541 /**
542  *      memory_bm_next_pfn - find the pfn that corresponds to the next set bit
543  *      in the bitmap @bm.  If the pfn cannot be found, BM_END_OF_MAP is
544  *      returned.
545  *
546  *      It is required to run memory_bm_position_reset() before the first call to
547  *      this function.
548  */
549
550 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
551 {
552         struct zone_bitmap *zone_bm;
553         struct bm_block *bb;
554         int chunk;
555         int bit;
556
557         do {
558                 bb = bm->cur.block;
559                 do {
560                         chunk = bm->cur.chunk;
561                         bit = bm->cur.bit;
562                         do {
563                                 bit = next_bit_in_chunk(bit, bb->data + chunk);
564                                 if (bit >= 0)
565                                         goto Return_pfn;
566
567                                 chunk = next_chunk_in_block(chunk, bb);
568                                 bit = -1;
569                         } while (chunk >= 0);
570                         bb = bb->next;
571                         bm->cur.block = bb;
572                         memory_bm_reset_chunk(bm);
573                 } while (bb);
574                 zone_bm = bm->cur.zone_bm->next;
575                 if (zone_bm) {
576                         bm->cur.zone_bm = zone_bm;
577                         bm->cur.block = zone_bm->bm_blocks;
578                         memory_bm_reset_chunk(bm);
579                 }
580         } while (zone_bm);
581         memory_bm_position_reset(bm);
582         return BM_END_OF_MAP;
583
584  Return_pfn:
585         bm->cur.chunk = chunk;
586         bm->cur.bit = bit;
587         return bb->start_pfn + chunk * BM_BITS_PER_CHUNK + bit;
588 }
589
590 /**
591  *      This structure represents a range of page frames the contents of which
592  *      should not be saved during the suspend.
593  */
594
595 struct nosave_region {
596         struct list_head list;
597         unsigned long start_pfn;
598         unsigned long end_pfn;
599 };
600
601 static LIST_HEAD(nosave_regions);
602
603 /**
604  *      register_nosave_region - register a range of page frames the contents
605  *      of which should not be saved during the suspend (to be used in the early
606  *      initialization code)
607  */
608
609 void __init
610 __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
611                          int use_kmalloc)
612 {
613         struct nosave_region *region;
614
615         if (start_pfn >= end_pfn)
616                 return;
617
618         if (!list_empty(&nosave_regions)) {
619                 /* Try to extend the previous region (they should be sorted) */
620                 region = list_entry(nosave_regions.prev,
621                                         struct nosave_region, list);
622                 if (region->end_pfn == start_pfn) {
623                         region->end_pfn = end_pfn;
624                         goto Report;
625                 }
626         }
627         if (use_kmalloc) {
628                 /* during init, this shouldn't fail */
629                 region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
630                 BUG_ON(!region);
631         } else
632                 /* This allocation cannot fail */
633                 region = alloc_bootmem_low(sizeof(struct nosave_region));
634         region->start_pfn = start_pfn;
635         region->end_pfn = end_pfn;
636         list_add_tail(&region->list, &nosave_regions);
637  Report:
638         printk("swsusp: Registered nosave memory region: %016lx - %016lx\n",
639                 start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
640 }
641
642 /*
643  * Set bits in this map correspond to the page frames the contents of which
644  * should not be saved during the suspend.
645  */
646 static struct memory_bitmap *forbidden_pages_map;
647
648 /* Set bits in this map correspond to free page frames. */
649 static struct memory_bitmap *free_pages_map;
650
651 /*
652  * Each page frame allocated for creating the image is marked by setting the
653  * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
654  */
655
656 void swsusp_set_page_free(struct page *page)
657 {
658         if (free_pages_map)
659                 memory_bm_set_bit(free_pages_map, page_to_pfn(page));
660 }
661
662 static int swsusp_page_is_free(struct page *page)
663 {
664         return free_pages_map ?
665                 memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
666 }
667
668 void swsusp_unset_page_free(struct page *page)
669 {
670         if (free_pages_map)
671                 memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
672 }
673
674 static void swsusp_set_page_forbidden(struct page *page)
675 {
676         if (forbidden_pages_map)
677                 memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
678 }
679
680 int swsusp_page_is_forbidden(struct page *page)
681 {
682         return forbidden_pages_map ?
683                 memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
684 }
685
686 static void swsusp_unset_page_forbidden(struct page *page)
687 {
688         if (forbidden_pages_map)
689                 memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
690 }
691
692 /**
693  *      mark_nosave_pages - set bits corresponding to the page frames the
694  *      contents of which should not be saved in a given bitmap.
695  */
696
697 static void mark_nosave_pages(struct memory_bitmap *bm)
698 {
699         struct nosave_region *region;
700
701         if (list_empty(&nosave_regions))
702                 return;
703
704         list_for_each_entry(region, &nosave_regions, list) {
705                 unsigned long pfn;
706
707                 printk("swsusp: Marking nosave pages: %016lx - %016lx\n",
708                                 region->start_pfn << PAGE_SHIFT,
709                                 region->end_pfn << PAGE_SHIFT);
710
711                 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
712                         memory_bm_set_bit(bm, pfn);
713         }
714 }
715
716 /**
717  *      create_basic_memory_bitmaps - create bitmaps needed for marking page
718  *      frames that should not be saved and free page frames.  The pointers
719  *      forbidden_pages_map and free_pages_map are only modified if everything
720  *      goes well, because we don't want the bits to be used before both bitmaps
721  *      are set up.
722  */
723
724 int create_basic_memory_bitmaps(void)
725 {
726         struct memory_bitmap *bm1, *bm2;
727         int error = 0;
728
729         BUG_ON(forbidden_pages_map || free_pages_map);
730
731         bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
732         if (!bm1)
733                 return -ENOMEM;
734
735         error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
736         if (error)
737                 goto Free_first_object;
738
739         bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
740         if (!bm2)
741                 goto Free_first_bitmap;
742
743         error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
744         if (error)
745                 goto Free_second_object;
746
747         forbidden_pages_map = bm1;
748         free_pages_map = bm2;
749         mark_nosave_pages(forbidden_pages_map);
750
751         printk("swsusp: Basic memory bitmaps created\n");
752
753         return 0;
754
755  Free_second_object:
756         kfree(bm2);
757  Free_first_bitmap:
758         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
759  Free_first_object:
760         kfree(bm1);
761         return -ENOMEM;
762 }
763
764 /**
765  *      free_basic_memory_bitmaps - free memory bitmaps allocated by
766  *      create_basic_memory_bitmaps().  The auxiliary pointers are necessary
767  *      so that the bitmaps themselves are not referred to while they are being
768  *      freed.
769  */
770
771 void free_basic_memory_bitmaps(void)
772 {
773         struct memory_bitmap *bm1, *bm2;
774
775         BUG_ON(!(forbidden_pages_map && free_pages_map));
776
777         bm1 = forbidden_pages_map;
778         bm2 = free_pages_map;
779         forbidden_pages_map = NULL;
780         free_pages_map = NULL;
781         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
782         kfree(bm1);
783         memory_bm_free(bm2, PG_UNSAFE_CLEAR);
784         kfree(bm2);
785
786         printk("swsusp: Basic memory bitmaps freed\n");
787 }
788
789 /**
790  *      snapshot_additional_pages - estimate the number of additional pages
791  *      be needed for setting up the suspend image data structures for given
792  *      zone (usually the returned value is greater than the exact number)
793  */
794
795 unsigned int snapshot_additional_pages(struct zone *zone)
796 {
797         unsigned int res;
798
799         res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
800         res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
801         return 2 * res;
802 }
803
804 #ifdef CONFIG_HIGHMEM
805 /**
806  *      count_free_highmem_pages - compute the total number of free highmem
807  *      pages, system-wide.
808  */
809
810 static unsigned int count_free_highmem_pages(void)
811 {
812         struct zone *zone;
813         unsigned int cnt = 0;
814
815         for_each_zone(zone)
816                 if (populated_zone(zone) && is_highmem(zone))
817                         cnt += zone_page_state(zone, NR_FREE_PAGES);
818
819         return cnt;
820 }
821
822 /**
823  *      saveable_highmem_page - Determine whether a highmem page should be
824  *      included in the suspend image.
825  *
826  *      We should save the page if it isn't Nosave or NosaveFree, or Reserved,
827  *      and it isn't a part of a free chunk of pages.
828  */
829
830 static struct page *saveable_highmem_page(unsigned long pfn)
831 {
832         struct page *page;
833
834         if (!pfn_valid(pfn))
835                 return NULL;
836
837         page = pfn_to_page(pfn);
838
839         BUG_ON(!PageHighMem(page));
840
841         if (swsusp_page_is_forbidden(page) ||  swsusp_page_is_free(page) ||
842             PageReserved(page))
843                 return NULL;
844
845         return page;
846 }
847
848 /**
849  *      count_highmem_pages - compute the total number of saveable highmem
850  *      pages.
851  */
852
853 unsigned int count_highmem_pages(void)
854 {
855         struct zone *zone;
856         unsigned int n = 0;
857
858         for_each_zone(zone) {
859                 unsigned long pfn, max_zone_pfn;
860
861                 if (!is_highmem(zone))
862                         continue;
863
864                 mark_free_pages(zone);
865                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
866                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
867                         if (saveable_highmem_page(pfn))
868                                 n++;
869         }
870         return n;
871 }
872 #else
873 static inline void *saveable_highmem_page(unsigned long pfn) { return NULL; }
874 static inline unsigned int count_highmem_pages(void) { return 0; }
875 #endif /* CONFIG_HIGHMEM */
876
877 /**
878  *      saveable - Determine whether a non-highmem page should be included in
879  *      the suspend image.
880  *
881  *      We should save the page if it isn't Nosave, and is not in the range
882  *      of pages statically defined as 'unsaveable', and it isn't a part of
883  *      a free chunk of pages.
884  */
885
886 static struct page *saveable_page(unsigned long pfn)
887 {
888         struct page *page;
889
890         if (!pfn_valid(pfn))
891                 return NULL;
892
893         page = pfn_to_page(pfn);
894
895         BUG_ON(PageHighMem(page));
896
897         if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
898                 return NULL;
899
900         if (PageReserved(page) && pfn_is_nosave(pfn))
901                 return NULL;
902
903         return page;
904 }
905
906 /**
907  *      count_data_pages - compute the total number of saveable non-highmem
908  *      pages.
909  */
910
911 unsigned int count_data_pages(void)
912 {
913         struct zone *zone;
914         unsigned long pfn, max_zone_pfn;
915         unsigned int n = 0;
916
917         for_each_zone(zone) {
918                 if (is_highmem(zone))
919                         continue;
920
921                 mark_free_pages(zone);
922                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
923                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
924                         if(saveable_page(pfn))
925                                 n++;
926         }
927         return n;
928 }
929
930 /* This is needed, because copy_page and memcpy are not usable for copying
931  * task structs.
932  */
933 static inline void do_copy_page(long *dst, long *src)
934 {
935         int n;
936
937         for (n = PAGE_SIZE / sizeof(long); n; n--)
938                 *dst++ = *src++;
939 }
940
941 #ifdef CONFIG_HIGHMEM
942 static inline struct page *
943 page_is_saveable(struct zone *zone, unsigned long pfn)
944 {
945         return is_highmem(zone) ?
946                         saveable_highmem_page(pfn) : saveable_page(pfn);
947 }
948
949 static inline void
950 copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
951 {
952         struct page *s_page, *d_page;
953         void *src, *dst;
954
955         s_page = pfn_to_page(src_pfn);
956         d_page = pfn_to_page(dst_pfn);
957         if (PageHighMem(s_page)) {
958                 src = kmap_atomic(s_page, KM_USER0);
959                 dst = kmap_atomic(d_page, KM_USER1);
960                 do_copy_page(dst, src);
961                 kunmap_atomic(src, KM_USER0);
962                 kunmap_atomic(dst, KM_USER1);
963         } else {
964                 src = page_address(s_page);
965                 if (PageHighMem(d_page)) {
966                         /* Page pointed to by src may contain some kernel
967                          * data modified by kmap_atomic()
968                          */
969                         do_copy_page(buffer, src);
970                         dst = kmap_atomic(pfn_to_page(dst_pfn), KM_USER0);
971                         memcpy(dst, buffer, PAGE_SIZE);
972                         kunmap_atomic(dst, KM_USER0);
973                 } else {
974                         dst = page_address(d_page);
975                         do_copy_page(dst, src);
976                 }
977         }
978 }
979 #else
980 #define page_is_saveable(zone, pfn)     saveable_page(pfn)
981
982 static inline void
983 copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
984 {
985         do_copy_page(page_address(pfn_to_page(dst_pfn)),
986                         page_address(pfn_to_page(src_pfn)));
987 }
988 #endif /* CONFIG_HIGHMEM */
989
990 static void
991 copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
992 {
993         struct zone *zone;
994         unsigned long pfn;
995
996         for_each_zone(zone) {
997                 unsigned long max_zone_pfn;
998
999                 mark_free_pages(zone);
1000                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1001                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1002                         if (page_is_saveable(zone, pfn))
1003                                 memory_bm_set_bit(orig_bm, pfn);
1004         }
1005         memory_bm_position_reset(orig_bm);
1006         memory_bm_position_reset(copy_bm);
1007         do {
1008                 pfn = memory_bm_next_pfn(orig_bm);
1009                 if (likely(pfn != BM_END_OF_MAP))
1010                         copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1011         } while (pfn != BM_END_OF_MAP);
1012 }
1013
1014 /* Total number of image pages */
1015 static unsigned int nr_copy_pages;
1016 /* Number of pages needed for saving the original pfns of the image pages */
1017 static unsigned int nr_meta_pages;
1018
1019 /**
1020  *      swsusp_free - free pages allocated for the suspend.
1021  *
1022  *      Suspend pages are alocated before the atomic copy is made, so we
1023  *      need to release them after the resume.
1024  */
1025
1026 void swsusp_free(void)
1027 {
1028         struct zone *zone;
1029         unsigned long pfn, max_zone_pfn;
1030
1031         for_each_zone(zone) {
1032                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1033                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1034                         if (pfn_valid(pfn)) {
1035                                 struct page *page = pfn_to_page(pfn);
1036
1037                                 if (swsusp_page_is_forbidden(page) &&
1038                                     swsusp_page_is_free(page)) {
1039                                         swsusp_unset_page_forbidden(page);
1040                                         swsusp_unset_page_free(page);
1041                                         __free_page(page);
1042                                 }
1043                         }
1044         }
1045         nr_copy_pages = 0;
1046         nr_meta_pages = 0;
1047         restore_pblist = NULL;
1048         buffer = NULL;
1049 }
1050
1051 #ifdef CONFIG_HIGHMEM
1052 /**
1053   *     count_pages_for_highmem - compute the number of non-highmem pages
1054   *     that will be necessary for creating copies of highmem pages.
1055   */
1056
1057 static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1058 {
1059         unsigned int free_highmem = count_free_highmem_pages();
1060
1061         if (free_highmem >= nr_highmem)
1062                 nr_highmem = 0;
1063         else
1064                 nr_highmem -= free_highmem;
1065
1066         return nr_highmem;
1067 }
1068 #else
1069 static unsigned int
1070 count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1071 #endif /* CONFIG_HIGHMEM */
1072
1073 /**
1074  *      enough_free_mem - Make sure we have enough free memory for the
1075  *      snapshot image.
1076  */
1077
1078 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1079 {
1080         struct zone *zone;
1081         unsigned int free = 0, meta = 0;
1082
1083         for_each_zone(zone) {
1084                 meta += snapshot_additional_pages(zone);
1085                 if (!is_highmem(zone))
1086                         free += zone_page_state(zone, NR_FREE_PAGES);
1087         }
1088
1089         nr_pages += count_pages_for_highmem(nr_highmem);
1090         pr_debug("swsusp: Normal pages needed: %u + %u + %u, available pages: %u\n",
1091                 nr_pages, PAGES_FOR_IO, meta, free);
1092
1093         return free > nr_pages + PAGES_FOR_IO + meta;
1094 }
1095
1096 #ifdef CONFIG_HIGHMEM
1097 /**
1098  *      get_highmem_buffer - if there are some highmem pages in the suspend
1099  *      image, we may need the buffer to copy them and/or load their data.
1100  */
1101
1102 static inline int get_highmem_buffer(int safe_needed)
1103 {
1104         buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1105         return buffer ? 0 : -ENOMEM;
1106 }
1107
1108 /**
1109  *      alloc_highmem_image_pages - allocate some highmem pages for the image.
1110  *      Try to allocate as many pages as needed, but if the number of free
1111  *      highmem pages is lesser than that, allocate them all.
1112  */
1113
1114 static inline unsigned int
1115 alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1116 {
1117         unsigned int to_alloc = count_free_highmem_pages();
1118
1119         if (to_alloc > nr_highmem)
1120                 to_alloc = nr_highmem;
1121
1122         nr_highmem -= to_alloc;
1123         while (to_alloc-- > 0) {
1124                 struct page *page;
1125
1126                 page = alloc_image_page(__GFP_HIGHMEM);
1127                 memory_bm_set_bit(bm, page_to_pfn(page));
1128         }
1129         return nr_highmem;
1130 }
1131 #else
1132 static inline int get_highmem_buffer(int safe_needed) { return 0; }
1133
1134 static inline unsigned int
1135 alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1136 #endif /* CONFIG_HIGHMEM */
1137
1138 /**
1139  *      swsusp_alloc - allocate memory for the suspend image
1140  *
1141  *      We first try to allocate as many highmem pages as there are
1142  *      saveable highmem pages in the system.  If that fails, we allocate
1143  *      non-highmem pages for the copies of the remaining highmem ones.
1144  *
1145  *      In this approach it is likely that the copies of highmem pages will
1146  *      also be located in the high memory, because of the way in which
1147  *      copy_data_pages() works.
1148  */
1149
1150 static int
1151 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1152                 unsigned int nr_pages, unsigned int nr_highmem)
1153 {
1154         int error;
1155
1156         error = memory_bm_create(orig_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
1157         if (error)
1158                 goto Free;
1159
1160         error = memory_bm_create(copy_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
1161         if (error)
1162                 goto Free;
1163
1164         if (nr_highmem > 0) {
1165                 error = get_highmem_buffer(PG_ANY);
1166                 if (error)
1167                         goto Free;
1168
1169                 nr_pages += alloc_highmem_image_pages(copy_bm, nr_highmem);
1170         }
1171         while (nr_pages-- > 0) {
1172                 struct page *page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1173
1174                 if (!page)
1175                         goto Free;
1176
1177                 memory_bm_set_bit(copy_bm, page_to_pfn(page));
1178         }
1179         return 0;
1180
1181  Free:
1182         swsusp_free();
1183         return -ENOMEM;
1184 }
1185
1186 /* Memory bitmap used for marking saveable pages (during suspend) or the
1187  * suspend image pages (during resume)
1188  */
1189 static struct memory_bitmap orig_bm;
1190 /* Memory bitmap used on suspend for marking allocated pages that will contain
1191  * the copies of saveable pages.  During resume it is initially used for
1192  * marking the suspend image pages, but then its set bits are duplicated in
1193  * @orig_bm and it is released.  Next, on systems with high memory, it may be
1194  * used for marking "safe" highmem pages, but it has to be reinitialized for
1195  * this purpose.
1196  */
1197 static struct memory_bitmap copy_bm;
1198
1199 asmlinkage int swsusp_save(void)
1200 {
1201         unsigned int nr_pages, nr_highmem;
1202
1203         printk("swsusp: critical section: \n");
1204
1205         drain_local_pages();
1206         nr_pages = count_data_pages();
1207         nr_highmem = count_highmem_pages();
1208         printk("swsusp: Need to copy %u pages\n", nr_pages + nr_highmem);
1209
1210         if (!enough_free_mem(nr_pages, nr_highmem)) {
1211                 printk(KERN_ERR "swsusp: Not enough free memory\n");
1212                 return -ENOMEM;
1213         }
1214
1215         if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1216                 printk(KERN_ERR "swsusp: Memory allocation failed\n");
1217                 return -ENOMEM;
1218         }
1219
1220         /* During allocating of suspend pagedir, new cold pages may appear.
1221          * Kill them.
1222          */
1223         drain_local_pages();
1224         copy_data_pages(&copy_bm, &orig_bm);
1225
1226         /*
1227          * End of critical section. From now on, we can write to memory,
1228          * but we should not touch disk. This specially means we must _not_
1229          * touch swap space! Except we must write out our image of course.
1230          */
1231
1232         nr_pages += nr_highmem;
1233         nr_copy_pages = nr_pages;
1234         nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1235
1236         printk("swsusp: critical section: done (%d pages copied)\n", nr_pages);
1237
1238         return 0;
1239 }
1240
1241 static void init_header(struct swsusp_info *info)
1242 {
1243         memset(info, 0, sizeof(struct swsusp_info));
1244         info->version_code = LINUX_VERSION_CODE;
1245         info->num_physpages = num_physpages;
1246         memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1247         info->cpus = num_online_cpus();
1248         info->image_pages = nr_copy_pages;
1249         info->pages = nr_copy_pages + nr_meta_pages + 1;
1250         info->size = info->pages;
1251         info->size <<= PAGE_SHIFT;
1252 }
1253
1254 /**
1255  *      pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1256  *      are stored in the array @buf[] (1 page at a time)
1257  */
1258
1259 static inline void
1260 pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1261 {
1262         int j;
1263
1264         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1265                 buf[j] = memory_bm_next_pfn(bm);
1266                 if (unlikely(buf[j] == BM_END_OF_MAP))
1267                         break;
1268         }
1269 }
1270
1271 /**
1272  *      snapshot_read_next - used for reading the system memory snapshot.
1273  *
1274  *      On the first call to it @handle should point to a zeroed
1275  *      snapshot_handle structure.  The structure gets updated and a pointer
1276  *      to it should be passed to this function every next time.
1277  *
1278  *      The @count parameter should contain the number of bytes the caller
1279  *      wants to read from the snapshot.  It must not be zero.
1280  *
1281  *      On success the function returns a positive number.  Then, the caller
1282  *      is allowed to read up to the returned number of bytes from the memory
1283  *      location computed by the data_of() macro.  The number returned
1284  *      may be smaller than @count, but this only happens if the read would
1285  *      cross a page boundary otherwise.
1286  *
1287  *      The function returns 0 to indicate the end of data stream condition,
1288  *      and a negative number is returned on error.  In such cases the
1289  *      structure pointed to by @handle is not updated and should not be used
1290  *      any more.
1291  */
1292
1293 int snapshot_read_next(struct snapshot_handle *handle, size_t count)
1294 {
1295         if (handle->cur > nr_meta_pages + nr_copy_pages)
1296                 return 0;
1297
1298         if (!buffer) {
1299                 /* This makes the buffer be freed by swsusp_free() */
1300                 buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1301                 if (!buffer)
1302                         return -ENOMEM;
1303         }
1304         if (!handle->offset) {
1305                 init_header((struct swsusp_info *)buffer);
1306                 handle->buffer = buffer;
1307                 memory_bm_position_reset(&orig_bm);
1308                 memory_bm_position_reset(&copy_bm);
1309         }
1310         if (handle->prev < handle->cur) {
1311                 if (handle->cur <= nr_meta_pages) {
1312                         memset(buffer, 0, PAGE_SIZE);
1313                         pack_pfns(buffer, &orig_bm);
1314                 } else {
1315                         struct page *page;
1316
1317                         page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1318                         if (PageHighMem(page)) {
1319                                 /* Highmem pages are copied to the buffer,
1320                                  * because we can't return with a kmapped
1321                                  * highmem page (we may not be called again).
1322                                  */
1323                                 void *kaddr;
1324
1325                                 kaddr = kmap_atomic(page, KM_USER0);
1326                                 memcpy(buffer, kaddr, PAGE_SIZE);
1327                                 kunmap_atomic(kaddr, KM_USER0);
1328                                 handle->buffer = buffer;
1329                         } else {
1330                                 handle->buffer = page_address(page);
1331                         }
1332                 }
1333                 handle->prev = handle->cur;
1334         }
1335         handle->buf_offset = handle->cur_offset;
1336         if (handle->cur_offset + count >= PAGE_SIZE) {
1337                 count = PAGE_SIZE - handle->cur_offset;
1338                 handle->cur_offset = 0;
1339                 handle->cur++;
1340         } else {
1341                 handle->cur_offset += count;
1342         }
1343         handle->offset += count;
1344         return count;
1345 }
1346
1347 /**
1348  *      mark_unsafe_pages - mark the pages that cannot be used for storing
1349  *      the image during resume, because they conflict with the pages that
1350  *      had been used before suspend
1351  */
1352
1353 static int mark_unsafe_pages(struct memory_bitmap *bm)
1354 {
1355         struct zone *zone;
1356         unsigned long pfn, max_zone_pfn;
1357
1358         /* Clear page flags */
1359         for_each_zone(zone) {
1360                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1361                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1362                         if (pfn_valid(pfn))
1363                                 swsusp_unset_page_free(pfn_to_page(pfn));
1364         }
1365
1366         /* Mark pages that correspond to the "original" pfns as "unsafe" */
1367         memory_bm_position_reset(bm);
1368         do {
1369                 pfn = memory_bm_next_pfn(bm);
1370                 if (likely(pfn != BM_END_OF_MAP)) {
1371                         if (likely(pfn_valid(pfn)))
1372                                 swsusp_set_page_free(pfn_to_page(pfn));
1373                         else
1374                                 return -EFAULT;
1375                 }
1376         } while (pfn != BM_END_OF_MAP);
1377
1378         allocated_unsafe_pages = 0;
1379
1380         return 0;
1381 }
1382
1383 static void
1384 duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1385 {
1386         unsigned long pfn;
1387
1388         memory_bm_position_reset(src);
1389         pfn = memory_bm_next_pfn(src);
1390         while (pfn != BM_END_OF_MAP) {
1391                 memory_bm_set_bit(dst, pfn);
1392                 pfn = memory_bm_next_pfn(src);
1393         }
1394 }
1395
1396 static inline int check_header(struct swsusp_info *info)
1397 {
1398         char *reason = NULL;
1399
1400         if (info->version_code != LINUX_VERSION_CODE)
1401                 reason = "kernel version";
1402         if (info->num_physpages != num_physpages)
1403                 reason = "memory size";
1404         if (strcmp(info->uts.sysname,init_utsname()->sysname))
1405                 reason = "system type";
1406         if (strcmp(info->uts.release,init_utsname()->release))
1407                 reason = "kernel release";
1408         if (strcmp(info->uts.version,init_utsname()->version))
1409                 reason = "version";
1410         if (strcmp(info->uts.machine,init_utsname()->machine))
1411                 reason = "machine";
1412         if (reason) {
1413                 printk(KERN_ERR "swsusp: Resume mismatch: %s\n", reason);
1414                 return -EPERM;
1415         }
1416         return 0;
1417 }
1418
1419 /**
1420  *      load header - check the image header and copy data from it
1421  */
1422
1423 static int
1424 load_header(struct swsusp_info *info)
1425 {
1426         int error;
1427
1428         restore_pblist = NULL;
1429         error = check_header(info);
1430         if (!error) {
1431                 nr_copy_pages = info->image_pages;
1432                 nr_meta_pages = info->pages - info->image_pages - 1;
1433         }
1434         return error;
1435 }
1436
1437 /**
1438  *      unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1439  *      the corresponding bit in the memory bitmap @bm
1440  */
1441
1442 static inline void
1443 unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1444 {
1445         int j;
1446
1447         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1448                 if (unlikely(buf[j] == BM_END_OF_MAP))
1449                         break;
1450
1451                 memory_bm_set_bit(bm, buf[j]);
1452         }
1453 }
1454
1455 /* List of "safe" pages that may be used to store data loaded from the suspend
1456  * image
1457  */
1458 static struct linked_page *safe_pages_list;
1459
1460 #ifdef CONFIG_HIGHMEM
1461 /* struct highmem_pbe is used for creating the list of highmem pages that
1462  * should be restored atomically during the resume from disk, because the page
1463  * frames they have occupied before the suspend are in use.
1464  */
1465 struct highmem_pbe {
1466         struct page *copy_page; /* data is here now */
1467         struct page *orig_page; /* data was here before the suspend */
1468         struct highmem_pbe *next;
1469 };
1470
1471 /* List of highmem PBEs needed for restoring the highmem pages that were
1472  * allocated before the suspend and included in the suspend image, but have
1473  * also been allocated by the "resume" kernel, so their contents cannot be
1474  * written directly to their "original" page frames.
1475  */
1476 static struct highmem_pbe *highmem_pblist;
1477
1478 /**
1479  *      count_highmem_image_pages - compute the number of highmem pages in the
1480  *      suspend image.  The bits in the memory bitmap @bm that correspond to the
1481  *      image pages are assumed to be set.
1482  */
1483
1484 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1485 {
1486         unsigned long pfn;
1487         unsigned int cnt = 0;
1488
1489         memory_bm_position_reset(bm);
1490         pfn = memory_bm_next_pfn(bm);
1491         while (pfn != BM_END_OF_MAP) {
1492                 if (PageHighMem(pfn_to_page(pfn)))
1493                         cnt++;
1494
1495                 pfn = memory_bm_next_pfn(bm);
1496         }
1497         return cnt;
1498 }
1499
1500 /**
1501  *      prepare_highmem_image - try to allocate as many highmem pages as
1502  *      there are highmem image pages (@nr_highmem_p points to the variable
1503  *      containing the number of highmem image pages).  The pages that are
1504  *      "safe" (ie. will not be overwritten when the suspend image is
1505  *      restored) have the corresponding bits set in @bm (it must be
1506  *      unitialized).
1507  *
1508  *      NOTE: This function should not be called if there are no highmem
1509  *      image pages.
1510  */
1511
1512 static unsigned int safe_highmem_pages;
1513
1514 static struct memory_bitmap *safe_highmem_bm;
1515
1516 static int
1517 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1518 {
1519         unsigned int to_alloc;
1520
1521         if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1522                 return -ENOMEM;
1523
1524         if (get_highmem_buffer(PG_SAFE))
1525                 return -ENOMEM;
1526
1527         to_alloc = count_free_highmem_pages();
1528         if (to_alloc > *nr_highmem_p)
1529                 to_alloc = *nr_highmem_p;
1530         else
1531                 *nr_highmem_p = to_alloc;
1532
1533         safe_highmem_pages = 0;
1534         while (to_alloc-- > 0) {
1535                 struct page *page;
1536
1537                 page = alloc_page(__GFP_HIGHMEM);
1538                 if (!swsusp_page_is_free(page)) {
1539                         /* The page is "safe", set its bit the bitmap */
1540                         memory_bm_set_bit(bm, page_to_pfn(page));
1541                         safe_highmem_pages++;
1542                 }
1543                 /* Mark the page as allocated */
1544                 swsusp_set_page_forbidden(page);
1545                 swsusp_set_page_free(page);
1546         }
1547         memory_bm_position_reset(bm);
1548         safe_highmem_bm = bm;
1549         return 0;
1550 }
1551
1552 /**
1553  *      get_highmem_page_buffer - for given highmem image page find the buffer
1554  *      that suspend_write_next() should set for its caller to write to.
1555  *
1556  *      If the page is to be saved to its "original" page frame or a copy of
1557  *      the page is to be made in the highmem, @buffer is returned.  Otherwise,
1558  *      the copy of the page is to be made in normal memory, so the address of
1559  *      the copy is returned.
1560  *
1561  *      If @buffer is returned, the caller of suspend_write_next() will write
1562  *      the page's contents to @buffer, so they will have to be copied to the
1563  *      right location on the next call to suspend_write_next() and it is done
1564  *      with the help of copy_last_highmem_page().  For this purpose, if
1565  *      @buffer is returned, @last_highmem page is set to the page to which
1566  *      the data will have to be copied from @buffer.
1567  */
1568
1569 static struct page *last_highmem_page;
1570
1571 static void *
1572 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1573 {
1574         struct highmem_pbe *pbe;
1575         void *kaddr;
1576
1577         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1578                 /* We have allocated the "original" page frame and we can
1579                  * use it directly to store the loaded page.
1580                  */
1581                 last_highmem_page = page;
1582                 return buffer;
1583         }
1584         /* The "original" page frame has not been allocated and we have to
1585          * use a "safe" page frame to store the loaded page.
1586          */
1587         pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1588         if (!pbe) {
1589                 swsusp_free();
1590                 return NULL;
1591         }
1592         pbe->orig_page = page;
1593         if (safe_highmem_pages > 0) {
1594                 struct page *tmp;
1595
1596                 /* Copy of the page will be stored in high memory */
1597                 kaddr = buffer;
1598                 tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1599                 safe_highmem_pages--;
1600                 last_highmem_page = tmp;
1601                 pbe->copy_page = tmp;
1602         } else {
1603                 /* Copy of the page will be stored in normal memory */
1604                 kaddr = safe_pages_list;
1605                 safe_pages_list = safe_pages_list->next;
1606                 pbe->copy_page = virt_to_page(kaddr);
1607         }
1608         pbe->next = highmem_pblist;
1609         highmem_pblist = pbe;
1610         return kaddr;
1611 }
1612
1613 /**
1614  *      copy_last_highmem_page - copy the contents of a highmem image from
1615  *      @buffer, where the caller of snapshot_write_next() has place them,
1616  *      to the right location represented by @last_highmem_page .
1617  */
1618
1619 static void copy_last_highmem_page(void)
1620 {
1621         if (last_highmem_page) {
1622                 void *dst;
1623
1624                 dst = kmap_atomic(last_highmem_page, KM_USER0);
1625                 memcpy(dst, buffer, PAGE_SIZE);
1626                 kunmap_atomic(dst, KM_USER0);
1627                 last_highmem_page = NULL;
1628         }
1629 }
1630
1631 static inline int last_highmem_page_copied(void)
1632 {
1633         return !last_highmem_page;
1634 }
1635
1636 static inline void free_highmem_data(void)
1637 {
1638         if (safe_highmem_bm)
1639                 memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
1640
1641         if (buffer)
1642                 free_image_page(buffer, PG_UNSAFE_CLEAR);
1643 }
1644 #else
1645 static inline int get_safe_write_buffer(void) { return 0; }
1646
1647 static unsigned int
1648 count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
1649
1650 static inline int
1651 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1652 {
1653         return 0;
1654 }
1655
1656 static inline void *
1657 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1658 {
1659         return NULL;
1660 }
1661
1662 static inline void copy_last_highmem_page(void) {}
1663 static inline int last_highmem_page_copied(void) { return 1; }
1664 static inline void free_highmem_data(void) {}
1665 #endif /* CONFIG_HIGHMEM */
1666
1667 /**
1668  *      prepare_image - use the memory bitmap @bm to mark the pages that will
1669  *      be overwritten in the process of restoring the system memory state
1670  *      from the suspend image ("unsafe" pages) and allocate memory for the
1671  *      image.
1672  *
1673  *      The idea is to allocate a new memory bitmap first and then allocate
1674  *      as many pages as needed for the image data, but not to assign these
1675  *      pages to specific tasks initially.  Instead, we just mark them as
1676  *      allocated and create a lists of "safe" pages that will be used
1677  *      later.  On systems with high memory a list of "safe" highmem pages is
1678  *      also created.
1679  */
1680
1681 #define PBES_PER_LINKED_PAGE    (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
1682
1683 static int
1684 prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
1685 {
1686         unsigned int nr_pages, nr_highmem;
1687         struct linked_page *sp_list, *lp;
1688         int error;
1689
1690         /* If there is no highmem, the buffer will not be necessary */
1691         free_image_page(buffer, PG_UNSAFE_CLEAR);
1692         buffer = NULL;
1693
1694         nr_highmem = count_highmem_image_pages(bm);
1695         error = mark_unsafe_pages(bm);
1696         if (error)
1697                 goto Free;
1698
1699         error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
1700         if (error)
1701                 goto Free;
1702
1703         duplicate_memory_bitmap(new_bm, bm);
1704         memory_bm_free(bm, PG_UNSAFE_KEEP);
1705         if (nr_highmem > 0) {
1706                 error = prepare_highmem_image(bm, &nr_highmem);
1707                 if (error)
1708                         goto Free;
1709         }
1710         /* Reserve some safe pages for potential later use.
1711          *
1712          * NOTE: This way we make sure there will be enough safe pages for the
1713          * chain_alloc() in get_buffer().  It is a bit wasteful, but
1714          * nr_copy_pages cannot be greater than 50% of the memory anyway.
1715          */
1716         sp_list = NULL;
1717         /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
1718         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
1719         nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
1720         while (nr_pages > 0) {
1721                 lp = get_image_page(GFP_ATOMIC, PG_SAFE);
1722                 if (!lp) {
1723                         error = -ENOMEM;
1724                         goto Free;
1725                 }
1726                 lp->next = sp_list;
1727                 sp_list = lp;
1728                 nr_pages--;
1729         }
1730         /* Preallocate memory for the image */
1731         safe_pages_list = NULL;
1732         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
1733         while (nr_pages > 0) {
1734                 lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
1735                 if (!lp) {
1736                         error = -ENOMEM;
1737                         goto Free;
1738                 }
1739                 if (!swsusp_page_is_free(virt_to_page(lp))) {
1740                         /* The page is "safe", add it to the list */
1741                         lp->next = safe_pages_list;
1742                         safe_pages_list = lp;
1743                 }
1744                 /* Mark the page as allocated */
1745                 swsusp_set_page_forbidden(virt_to_page(lp));
1746                 swsusp_set_page_free(virt_to_page(lp));
1747                 nr_pages--;
1748         }
1749         /* Free the reserved safe pages so that chain_alloc() can use them */
1750         while (sp_list) {
1751                 lp = sp_list->next;
1752                 free_image_page(sp_list, PG_UNSAFE_CLEAR);
1753                 sp_list = lp;
1754         }
1755         return 0;
1756
1757  Free:
1758         swsusp_free();
1759         return error;
1760 }
1761
1762 /**
1763  *      get_buffer - compute the address that snapshot_write_next() should
1764  *      set for its caller to write to.
1765  */
1766
1767 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
1768 {
1769         struct pbe *pbe;
1770         struct page *page = pfn_to_page(memory_bm_next_pfn(bm));
1771
1772         if (PageHighMem(page))
1773                 return get_highmem_page_buffer(page, ca);
1774
1775         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
1776                 /* We have allocated the "original" page frame and we can
1777                  * use it directly to store the loaded page.
1778                  */
1779                 return page_address(page);
1780
1781         /* The "original" page frame has not been allocated and we have to
1782          * use a "safe" page frame to store the loaded page.
1783          */
1784         pbe = chain_alloc(ca, sizeof(struct pbe));
1785         if (!pbe) {
1786                 swsusp_free();
1787                 return NULL;
1788         }
1789         pbe->orig_address = page_address(page);
1790         pbe->address = safe_pages_list;
1791         safe_pages_list = safe_pages_list->next;
1792         pbe->next = restore_pblist;
1793         restore_pblist = pbe;
1794         return pbe->address;
1795 }
1796
1797 /**
1798  *      snapshot_write_next - used for writing the system memory snapshot.
1799  *
1800  *      On the first call to it @handle should point to a zeroed
1801  *      snapshot_handle structure.  The structure gets updated and a pointer
1802  *      to it should be passed to this function every next time.
1803  *
1804  *      The @count parameter should contain the number of bytes the caller
1805  *      wants to write to the image.  It must not be zero.
1806  *
1807  *      On success the function returns a positive number.  Then, the caller
1808  *      is allowed to write up to the returned number of bytes to the memory
1809  *      location computed by the data_of() macro.  The number returned
1810  *      may be smaller than @count, but this only happens if the write would
1811  *      cross a page boundary otherwise.
1812  *
1813  *      The function returns 0 to indicate the "end of file" condition,
1814  *      and a negative number is returned on error.  In such cases the
1815  *      structure pointed to by @handle is not updated and should not be used
1816  *      any more.
1817  */
1818
1819 int snapshot_write_next(struct snapshot_handle *handle, size_t count)
1820 {
1821         static struct chain_allocator ca;
1822         int error = 0;
1823
1824         /* Check if we have already loaded the entire image */
1825         if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages)
1826                 return 0;
1827
1828         if (handle->offset == 0) {
1829                 if (!buffer)
1830                         /* This makes the buffer be freed by swsusp_free() */
1831                         buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1832
1833                 if (!buffer)
1834                         return -ENOMEM;
1835
1836                 handle->buffer = buffer;
1837         }
1838         handle->sync_read = 1;
1839         if (handle->prev < handle->cur) {
1840                 if (handle->prev == 0) {
1841                         error = load_header(buffer);
1842                         if (error)
1843                                 return error;
1844
1845                         error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
1846                         if (error)
1847                                 return error;
1848
1849                 } else if (handle->prev <= nr_meta_pages) {
1850                         unpack_orig_pfns(buffer, &copy_bm);
1851                         if (handle->prev == nr_meta_pages) {
1852                                 error = prepare_image(&orig_bm, &copy_bm);
1853                                 if (error)
1854                                         return error;
1855
1856                                 chain_init(&ca, GFP_ATOMIC, PG_SAFE);
1857                                 memory_bm_position_reset(&orig_bm);
1858                                 restore_pblist = NULL;
1859                                 handle->buffer = get_buffer(&orig_bm, &ca);
1860                                 handle->sync_read = 0;
1861                                 if (!handle->buffer)
1862                                         return -ENOMEM;
1863                         }
1864                 } else {
1865                         copy_last_highmem_page();
1866                         handle->buffer = get_buffer(&orig_bm, &ca);
1867                         if (handle->buffer != buffer)
1868                                 handle->sync_read = 0;
1869                 }
1870                 handle->prev = handle->cur;
1871         }
1872         handle->buf_offset = handle->cur_offset;
1873         if (handle->cur_offset + count >= PAGE_SIZE) {
1874                 count = PAGE_SIZE - handle->cur_offset;
1875                 handle->cur_offset = 0;
1876                 handle->cur++;
1877         } else {
1878                 handle->cur_offset += count;
1879         }
1880         handle->offset += count;
1881         return count;
1882 }
1883
1884 /**
1885  *      snapshot_write_finalize - must be called after the last call to
1886  *      snapshot_write_next() in case the last page in the image happens
1887  *      to be a highmem page and its contents should be stored in the
1888  *      highmem.  Additionally, it releases the memory that will not be
1889  *      used any more.
1890  */
1891
1892 void snapshot_write_finalize(struct snapshot_handle *handle)
1893 {
1894         copy_last_highmem_page();
1895         /* Free only if we have loaded the image entirely */
1896         if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages) {
1897                 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
1898                 free_highmem_data();
1899         }
1900 }
1901
1902 int snapshot_image_loaded(struct snapshot_handle *handle)
1903 {
1904         return !(!nr_copy_pages || !last_highmem_page_copied() ||
1905                         handle->cur <= nr_meta_pages + nr_copy_pages);
1906 }
1907
1908 #ifdef CONFIG_HIGHMEM
1909 /* Assumes that @buf is ready and points to a "safe" page */
1910 static inline void
1911 swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
1912 {
1913         void *kaddr1, *kaddr2;
1914
1915         kaddr1 = kmap_atomic(p1, KM_USER0);
1916         kaddr2 = kmap_atomic(p2, KM_USER1);
1917         memcpy(buf, kaddr1, PAGE_SIZE);
1918         memcpy(kaddr1, kaddr2, PAGE_SIZE);
1919         memcpy(kaddr2, buf, PAGE_SIZE);
1920         kunmap_atomic(kaddr1, KM_USER0);
1921         kunmap_atomic(kaddr2, KM_USER1);
1922 }
1923
1924 /**
1925  *      restore_highmem - for each highmem page that was allocated before
1926  *      the suspend and included in the suspend image, and also has been
1927  *      allocated by the "resume" kernel swap its current (ie. "before
1928  *      resume") contents with the previous (ie. "before suspend") one.
1929  *
1930  *      If the resume eventually fails, we can call this function once
1931  *      again and restore the "before resume" highmem state.
1932  */
1933
1934 int restore_highmem(void)
1935 {
1936         struct highmem_pbe *pbe = highmem_pblist;
1937         void *buf;
1938
1939         if (!pbe)
1940                 return 0;
1941
1942         buf = get_image_page(GFP_ATOMIC, PG_SAFE);
1943         if (!buf)
1944                 return -ENOMEM;
1945
1946         while (pbe) {
1947                 swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
1948                 pbe = pbe->next;
1949         }
1950         free_image_page(buf, PG_UNSAFE_CLEAR);
1951         return 0;
1952 }
1953 #endif /* CONFIG_HIGHMEM */