]> err.no Git - linux-2.6/blob - kernel/power/swsusp.c
[PATCH] swap: update swsusp use of swap_info
[linux-2.6] / kernel / power / swsusp.c
1 /*
2  * linux/kernel/power/swsusp.c
3  *
4  * This file is to realize architecture-independent
5  * machine suspend feature using pretty near only high-level routines
6  *
7  * Copyright (C) 1998-2001 Gabor Kuti <seasons@fornax.hu>
8  * Copyright (C) 1998,2001-2004 Pavel Machek <pavel@suse.cz>
9  *
10  * This file is released under the GPLv2.
11  *
12  * I'd like to thank the following people for their work:
13  *
14  * Pavel Machek <pavel@ucw.cz>:
15  * Modifications, defectiveness pointing, being with me at the very beginning,
16  * suspend to swap space, stop all tasks. Port to 2.4.18-ac and 2.5.17.
17  *
18  * Steve Doddi <dirk@loth.demon.co.uk>:
19  * Support the possibility of hardware state restoring.
20  *
21  * Raph <grey.havens@earthling.net>:
22  * Support for preserving states of network devices and virtual console
23  * (including X and svgatextmode)
24  *
25  * Kurt Garloff <garloff@suse.de>:
26  * Straightened the critical function in order to prevent compilers from
27  * playing tricks with local variables.
28  *
29  * Andreas Mohr <a.mohr@mailto.de>
30  *
31  * Alex Badea <vampire@go.ro>:
32  * Fixed runaway init
33  *
34  * More state savers are welcome. Especially for the scsi layer...
35  *
36  * For TODOs,FIXMEs also look in Documentation/power/swsusp.txt
37  */
38
39 #include <linux/module.h>
40 #include <linux/mm.h>
41 #include <linux/suspend.h>
42 #include <linux/smp_lock.h>
43 #include <linux/file.h>
44 #include <linux/utsname.h>
45 #include <linux/version.h>
46 #include <linux/delay.h>
47 #include <linux/reboot.h>
48 #include <linux/bitops.h>
49 #include <linux/vt_kern.h>
50 #include <linux/kbd_kern.h>
51 #include <linux/keyboard.h>
52 #include <linux/spinlock.h>
53 #include <linux/genhd.h>
54 #include <linux/kernel.h>
55 #include <linux/major.h>
56 #include <linux/swap.h>
57 #include <linux/pm.h>
58 #include <linux/device.h>
59 #include <linux/buffer_head.h>
60 #include <linux/swapops.h>
61 #include <linux/bootmem.h>
62 #include <linux/syscalls.h>
63 #include <linux/console.h>
64 #include <linux/highmem.h>
65 #include <linux/bio.h>
66 #include <linux/mount.h>
67
68 #include <asm/uaccess.h>
69 #include <asm/mmu_context.h>
70 #include <asm/pgtable.h>
71 #include <asm/tlbflush.h>
72 #include <asm/io.h>
73
74 #include "power.h"
75
76 /* References to section boundaries */
77 extern const void __nosave_begin, __nosave_end;
78
79 /* Variables to be preserved over suspend */
80 static int nr_copy_pages_check;
81
82 extern char resume_file[];
83
84 /* Local variables that should not be affected by save */
85 static unsigned int nr_copy_pages __nosavedata = 0;
86
87 /* Suspend pagedir is allocated before final copy, therefore it
88    must be freed after resume
89
90    Warning: this is evil. There are actually two pagedirs at time of
91    resume. One is "pagedir_save", which is empty frame allocated at
92    time of suspend, that must be freed. Second is "pagedir_nosave",
93    allocated at time of resume, that travels through memory not to
94    collide with anything.
95
96    Warning: this is even more evil than it seems. Pagedirs this file
97    talks about are completely different from page directories used by
98    MMU hardware.
99  */
100 suspend_pagedir_t *pagedir_nosave __nosavedata = NULL;
101 static suspend_pagedir_t *pagedir_save;
102
103 #define SWSUSP_SIG      "S1SUSPEND"
104
105 static struct swsusp_header {
106         char reserved[PAGE_SIZE - 20 - sizeof(swp_entry_t)];
107         swp_entry_t swsusp_info;
108         char    orig_sig[10];
109         char    sig[10];
110 } __attribute__((packed, aligned(PAGE_SIZE))) swsusp_header;
111
112 static struct swsusp_info swsusp_info;
113
114 /*
115  * XXX: We try to keep some more pages free so that I/O operations succeed
116  * without paging. Might this be more?
117  */
118 #define PAGES_FOR_IO    512
119
120 /*
121  * Saving part...
122  */
123
124 /* We memorize in swapfile_used what swap devices are used for suspension */
125 #define SWAPFILE_UNUSED    0
126 #define SWAPFILE_SUSPEND   1    /* This is the suspending device */
127 #define SWAPFILE_IGNORED   2    /* Those are other swap devices ignored for suspension */
128
129 static unsigned short swapfile_used[MAX_SWAPFILES];
130 static unsigned short root_swap;
131
132 static int mark_swapfiles(swp_entry_t prev)
133 {
134         int error;
135
136         rw_swap_page_sync(READ,
137                           swp_entry(root_swap, 0),
138                           virt_to_page((unsigned long)&swsusp_header));
139         if (!memcmp("SWAP-SPACE",swsusp_header.sig, 10) ||
140             !memcmp("SWAPSPACE2",swsusp_header.sig, 10)) {
141                 memcpy(swsusp_header.orig_sig,swsusp_header.sig, 10);
142                 memcpy(swsusp_header.sig,SWSUSP_SIG, 10);
143                 swsusp_header.swsusp_info = prev;
144                 error = rw_swap_page_sync(WRITE,
145                                           swp_entry(root_swap, 0),
146                                           virt_to_page((unsigned long)
147                                                        &swsusp_header));
148         } else {
149                 pr_debug("swsusp: Partition is not swap space.\n");
150                 error = -ENODEV;
151         }
152         return error;
153 }
154
155 /*
156  * Check whether the swap device is the specified resume
157  * device, irrespective of whether they are specified by
158  * identical names.
159  *
160  * (Thus, device inode aliasing is allowed.  You can say /dev/hda4
161  * instead of /dev/ide/host0/bus0/target0/lun0/part4 [if using devfs]
162  * and they'll be considered the same device.  This is *necessary* for
163  * devfs, since the resume code can only recognize the form /dev/hda4,
164  * but the suspend code would see the long name.)
165  */
166 static int is_resume_device(const struct swap_info_struct *swap_info)
167 {
168         struct file *file = swap_info->swap_file;
169         struct inode *inode = file->f_dentry->d_inode;
170
171         return S_ISBLK(inode->i_mode) &&
172                 swsusp_resume_device == MKDEV(imajor(inode), iminor(inode));
173 }
174
175 static int swsusp_swap_check(void) /* This is called before saving image */
176 {
177         int i, len;
178
179         len=strlen(resume_file);
180         root_swap = 0xFFFF;
181
182         spin_lock(&swap_lock);
183         for (i=0; i<MAX_SWAPFILES; i++) {
184                 if (!(swap_info[i].flags & SWP_WRITEOK)) {
185                         swapfile_used[i]=SWAPFILE_UNUSED;
186                 } else {
187                         if (!len) {
188                                 printk(KERN_WARNING "resume= option should be used to set suspend device" );
189                                 if (root_swap == 0xFFFF) {
190                                         swapfile_used[i] = SWAPFILE_SUSPEND;
191                                         root_swap = i;
192                                 } else
193                                         swapfile_used[i] = SWAPFILE_IGNORED;
194                         } else {
195                                 /* we ignore all swap devices that are not the resume_file */
196                                 if (is_resume_device(&swap_info[i])) {
197                                         swapfile_used[i] = SWAPFILE_SUSPEND;
198                                         root_swap = i;
199                                 } else {
200                                         swapfile_used[i] = SWAPFILE_IGNORED;
201                                 }
202                         }
203                 }
204         }
205         spin_unlock(&swap_lock);
206         return (root_swap != 0xffff) ? 0 : -ENODEV;
207 }
208
209 /**
210  * This is called after saving image so modification
211  * will be lost after resume... and that's what we want.
212  * we make the device unusable. A new call to
213  * lock_swapdevices can unlock the devices.
214  */
215 static void lock_swapdevices(void)
216 {
217         int i;
218
219         spin_lock(&swap_lock);
220         for (i = 0; i< MAX_SWAPFILES; i++)
221                 if (swapfile_used[i] == SWAPFILE_IGNORED) {
222                         swap_info[i].flags ^= SWP_WRITEOK;
223                 }
224         spin_unlock(&swap_lock);
225 }
226
227 /**
228  *      write_swap_page - Write one page to a fresh swap location.
229  *      @addr:  Address we're writing.
230  *      @loc:   Place to store the entry we used.
231  *
232  *      Allocate a new swap entry and 'sync' it. Note we discard -EIO
233  *      errors. That is an artifact left over from swsusp. It did not
234  *      check the return of rw_swap_page_sync() at all, since most pages
235  *      written back to swap would return -EIO.
236  *      This is a partial improvement, since we will at least return other
237  *      errors, though we need to eventually fix the damn code.
238  */
239 static int write_page(unsigned long addr, swp_entry_t * loc)
240 {
241         swp_entry_t entry;
242         int error = 0;
243
244         entry = get_swap_page();
245         if (swp_offset(entry) &&
246             swapfile_used[swp_type(entry)] == SWAPFILE_SUSPEND) {
247                 error = rw_swap_page_sync(WRITE, entry,
248                                           virt_to_page(addr));
249                 if (error == -EIO)
250                         error = 0;
251                 if (!error)
252                         *loc = entry;
253         } else
254                 error = -ENOSPC;
255         return error;
256 }
257
258 /**
259  *      data_free - Free the swap entries used by the saved image.
260  *
261  *      Walk the list of used swap entries and free each one.
262  *      This is only used for cleanup when suspend fails.
263  */
264 static void data_free(void)
265 {
266         swp_entry_t entry;
267         int i;
268
269         for (i = 0; i < nr_copy_pages; i++) {
270                 entry = (pagedir_nosave + i)->swap_address;
271                 if (entry.val)
272                         swap_free(entry);
273                 else
274                         break;
275                 (pagedir_nosave + i)->swap_address = (swp_entry_t){0};
276         }
277 }
278
279 /**
280  *      data_write - Write saved image to swap.
281  *
282  *      Walk the list of pages in the image and sync each one to swap.
283  */
284 static int data_write(void)
285 {
286         int error = 0, i = 0;
287         unsigned int mod = nr_copy_pages / 100;
288         struct pbe *p;
289
290         if (!mod)
291                 mod = 1;
292
293         printk( "Writing data to swap (%d pages)...     ", nr_copy_pages );
294         for_each_pbe (p, pagedir_nosave) {
295                 if (!(i%mod))
296                         printk( "\b\b\b\b%3d%%", i / mod );
297                 if ((error = write_page(p->address, &(p->swap_address))))
298                         return error;
299                 i++;
300         }
301         printk("\b\b\b\bdone\n");
302         return error;
303 }
304
305 static void dump_info(void)
306 {
307         pr_debug(" swsusp: Version: %u\n",swsusp_info.version_code);
308         pr_debug(" swsusp: Num Pages: %ld\n",swsusp_info.num_physpages);
309         pr_debug(" swsusp: UTS Sys: %s\n",swsusp_info.uts.sysname);
310         pr_debug(" swsusp: UTS Node: %s\n",swsusp_info.uts.nodename);
311         pr_debug(" swsusp: UTS Release: %s\n",swsusp_info.uts.release);
312         pr_debug(" swsusp: UTS Version: %s\n",swsusp_info.uts.version);
313         pr_debug(" swsusp: UTS Machine: %s\n",swsusp_info.uts.machine);
314         pr_debug(" swsusp: UTS Domain: %s\n",swsusp_info.uts.domainname);
315         pr_debug(" swsusp: CPUs: %d\n",swsusp_info.cpus);
316         pr_debug(" swsusp: Image: %ld Pages\n",swsusp_info.image_pages);
317         pr_debug(" swsusp: Pagedir: %ld Pages\n",swsusp_info.pagedir_pages);
318 }
319
320 static void init_header(void)
321 {
322         memset(&swsusp_info, 0, sizeof(swsusp_info));
323         swsusp_info.version_code = LINUX_VERSION_CODE;
324         swsusp_info.num_physpages = num_physpages;
325         memcpy(&swsusp_info.uts, &system_utsname, sizeof(system_utsname));
326
327         swsusp_info.suspend_pagedir = pagedir_nosave;
328         swsusp_info.cpus = num_online_cpus();
329         swsusp_info.image_pages = nr_copy_pages;
330 }
331
332 static int close_swap(void)
333 {
334         swp_entry_t entry;
335         int error;
336
337         dump_info();
338         error = write_page((unsigned long)&swsusp_info, &entry);
339         if (!error) {
340                 printk( "S" );
341                 error = mark_swapfiles(entry);
342                 printk( "|\n" );
343         }
344         return error;
345 }
346
347 /**
348  *      free_pagedir_entries - Free pages used by the page directory.
349  *
350  *      This is used during suspend for error recovery.
351  */
352
353 static void free_pagedir_entries(void)
354 {
355         int i;
356
357         for (i = 0; i < swsusp_info.pagedir_pages; i++)
358                 swap_free(swsusp_info.pagedir[i]);
359 }
360
361
362 /**
363  *      write_pagedir - Write the array of pages holding the page directory.
364  *      @last:  Last swap entry we write (needed for header).
365  */
366
367 static int write_pagedir(void)
368 {
369         int error = 0;
370         unsigned n = 0;
371         struct pbe * pbe;
372
373         printk( "Writing pagedir...");
374         for_each_pb_page (pbe, pagedir_nosave) {
375                 if ((error = write_page((unsigned long)pbe, &swsusp_info.pagedir[n++])))
376                         return error;
377         }
378
379         swsusp_info.pagedir_pages = n;
380         printk("done (%u pages)\n", n);
381         return error;
382 }
383
384 /**
385  *      write_suspend_image - Write entire image and metadata.
386  *
387  */
388
389 static int write_suspend_image(void)
390 {
391         int error;
392
393         init_header();
394         if ((error = data_write()))
395                 goto FreeData;
396
397         if ((error = write_pagedir()))
398                 goto FreePagedir;
399
400         if ((error = close_swap()))
401                 goto FreePagedir;
402  Done:
403         return error;
404  FreePagedir:
405         free_pagedir_entries();
406  FreeData:
407         data_free();
408         goto Done;
409 }
410
411
412 #ifdef CONFIG_HIGHMEM
413 struct highmem_page {
414         char *data;
415         struct page *page;
416         struct highmem_page *next;
417 };
418
419 static struct highmem_page *highmem_copy;
420
421 static int save_highmem_zone(struct zone *zone)
422 {
423         unsigned long zone_pfn;
424         mark_free_pages(zone);
425         for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
426                 struct page *page;
427                 struct highmem_page *save;
428                 void *kaddr;
429                 unsigned long pfn = zone_pfn + zone->zone_start_pfn;
430
431                 if (!(pfn%1000))
432                         printk(".");
433                 if (!pfn_valid(pfn))
434                         continue;
435                 page = pfn_to_page(pfn);
436                 /*
437                  * This condition results from rvmalloc() sans vmalloc_32()
438                  * and architectural memory reservations. This should be
439                  * corrected eventually when the cases giving rise to this
440                  * are better understood.
441                  */
442                 if (PageReserved(page)) {
443                         printk("highmem reserved page?!\n");
444                         continue;
445                 }
446                 BUG_ON(PageNosave(page));
447                 if (PageNosaveFree(page))
448                         continue;
449                 save = kmalloc(sizeof(struct highmem_page), GFP_ATOMIC);
450                 if (!save)
451                         return -ENOMEM;
452                 save->next = highmem_copy;
453                 save->page = page;
454                 save->data = (void *) get_zeroed_page(GFP_ATOMIC);
455                 if (!save->data) {
456                         kfree(save);
457                         return -ENOMEM;
458                 }
459                 kaddr = kmap_atomic(page, KM_USER0);
460                 memcpy(save->data, kaddr, PAGE_SIZE);
461                 kunmap_atomic(kaddr, KM_USER0);
462                 highmem_copy = save;
463         }
464         return 0;
465 }
466 #endif /* CONFIG_HIGHMEM */
467
468
469 static int save_highmem(void)
470 {
471 #ifdef CONFIG_HIGHMEM
472         struct zone *zone;
473         int res = 0;
474
475         pr_debug("swsusp: Saving Highmem\n");
476         for_each_zone (zone) {
477                 if (is_highmem(zone))
478                         res = save_highmem_zone(zone);
479                 if (res)
480                         return res;
481         }
482 #endif
483         return 0;
484 }
485
486 static int restore_highmem(void)
487 {
488 #ifdef CONFIG_HIGHMEM
489         printk("swsusp: Restoring Highmem\n");
490         while (highmem_copy) {
491                 struct highmem_page *save = highmem_copy;
492                 void *kaddr;
493                 highmem_copy = save->next;
494
495                 kaddr = kmap_atomic(save->page, KM_USER0);
496                 memcpy(kaddr, save->data, PAGE_SIZE);
497                 kunmap_atomic(kaddr, KM_USER0);
498                 free_page((long) save->data);
499                 kfree(save);
500         }
501 #endif
502         return 0;
503 }
504
505
506 static int pfn_is_nosave(unsigned long pfn)
507 {
508         unsigned long nosave_begin_pfn = __pa(&__nosave_begin) >> PAGE_SHIFT;
509         unsigned long nosave_end_pfn = PAGE_ALIGN(__pa(&__nosave_end)) >> PAGE_SHIFT;
510         return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
511 }
512
513 /**
514  *      saveable - Determine whether a page should be cloned or not.
515  *      @pfn:   The page
516  *
517  *      We save a page if it's Reserved, and not in the range of pages
518  *      statically defined as 'unsaveable', or if it isn't reserved, and
519  *      isn't part of a free chunk of pages.
520  */
521
522 static int saveable(struct zone * zone, unsigned long * zone_pfn)
523 {
524         unsigned long pfn = *zone_pfn + zone->zone_start_pfn;
525         struct page * page;
526
527         if (!pfn_valid(pfn))
528                 return 0;
529
530         page = pfn_to_page(pfn);
531         BUG_ON(PageReserved(page) && PageNosave(page));
532         if (PageNosave(page))
533                 return 0;
534         if (PageReserved(page) && pfn_is_nosave(pfn)) {
535                 pr_debug("[nosave pfn 0x%lx]", pfn);
536                 return 0;
537         }
538         if (PageNosaveFree(page))
539                 return 0;
540
541         return 1;
542 }
543
544 static void count_data_pages(void)
545 {
546         struct zone *zone;
547         unsigned long zone_pfn;
548
549         nr_copy_pages = 0;
550
551         for_each_zone (zone) {
552                 if (is_highmem(zone))
553                         continue;
554                 mark_free_pages(zone);
555                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
556                         nr_copy_pages += saveable(zone, &zone_pfn);
557         }
558 }
559
560
561 static void copy_data_pages(void)
562 {
563         struct zone *zone;
564         unsigned long zone_pfn;
565         struct pbe * pbe = pagedir_nosave;
566
567         pr_debug("copy_data_pages(): pages to copy: %d\n", nr_copy_pages);
568         for_each_zone (zone) {
569                 if (is_highmem(zone))
570                         continue;
571                 mark_free_pages(zone);
572                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
573                         if (saveable(zone, &zone_pfn)) {
574                                 struct page * page;
575                                 page = pfn_to_page(zone_pfn + zone->zone_start_pfn);
576                                 BUG_ON(!pbe);
577                                 pbe->orig_address = (long) page_address(page);
578                                 /* copy_page is not usable for copying task structs. */
579                                 memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE);
580                                 pbe = pbe->next;
581                         }
582                 }
583         }
584         BUG_ON(pbe);
585 }
586
587
588 /**
589  *      calc_nr - Determine the number of pages needed for a pbe list.
590  */
591
592 static int calc_nr(int nr_copy)
593 {
594         int extra = 0;
595         int mod = !!(nr_copy % PBES_PER_PAGE);
596         int diff = (nr_copy / PBES_PER_PAGE) + mod;
597
598         do {
599                 extra += diff;
600                 nr_copy += diff;
601                 mod = !!(nr_copy % PBES_PER_PAGE);
602                 diff = (nr_copy / PBES_PER_PAGE) + mod - extra;
603         } while (diff > 0);
604
605         return nr_copy;
606 }
607
608 /**
609  *      free_pagedir - free pages allocated with alloc_pagedir()
610  */
611
612 static inline void free_pagedir(struct pbe *pblist)
613 {
614         struct pbe *pbe;
615
616         while (pblist) {
617                 pbe = (pblist + PB_PAGE_SKIP)->next;
618                 free_page((unsigned long)pblist);
619                 pblist = pbe;
620         }
621 }
622
623 /**
624  *      fill_pb_page - Create a list of PBEs on a given memory page
625  */
626
627 static inline void fill_pb_page(struct pbe *pbpage)
628 {
629         struct pbe *p;
630
631         p = pbpage;
632         pbpage += PB_PAGE_SKIP;
633         do
634                 p->next = p + 1;
635         while (++p < pbpage);
636 }
637
638 /**
639  *      create_pbe_list - Create a list of PBEs on top of a given chain
640  *      of memory pages allocated with alloc_pagedir()
641  */
642
643 static void create_pbe_list(struct pbe *pblist, unsigned nr_pages)
644 {
645         struct pbe *pbpage, *p;
646         unsigned num = PBES_PER_PAGE;
647
648         for_each_pb_page (pbpage, pblist) {
649                 if (num >= nr_pages)
650                         break;
651
652                 fill_pb_page(pbpage);
653                 num += PBES_PER_PAGE;
654         }
655         if (pbpage) {
656                 for (num -= PBES_PER_PAGE - 1, p = pbpage; num < nr_pages; p++, num++)
657                         p->next = p + 1;
658                 p->next = NULL;
659         }
660         pr_debug("create_pbe_list(): initialized %d PBEs\n", num);
661 }
662
663 /**
664  *      alloc_pagedir - Allocate the page directory.
665  *
666  *      First, determine exactly how many pages we need and
667  *      allocate them.
668  *
669  *      We arrange the pages in a chain: each page is an array of PBES_PER_PAGE
670  *      struct pbe elements (pbes) and the last element in the page points
671  *      to the next page.
672  *
673  *      On each page we set up a list of struct_pbe elements.
674  */
675
676 static struct pbe * alloc_pagedir(unsigned nr_pages)
677 {
678         unsigned num;
679         struct pbe *pblist, *pbe;
680
681         if (!nr_pages)
682                 return NULL;
683
684         pr_debug("alloc_pagedir(): nr_pages = %d\n", nr_pages);
685         pblist = (struct pbe *)get_zeroed_page(GFP_ATOMIC | __GFP_COLD);
686         for (pbe = pblist, num = PBES_PER_PAGE; pbe && num < nr_pages;
687                         pbe = pbe->next, num += PBES_PER_PAGE) {
688                 pbe += PB_PAGE_SKIP;
689                 pbe->next = (struct pbe *)get_zeroed_page(GFP_ATOMIC | __GFP_COLD);
690         }
691         if (!pbe) { /* get_zeroed_page() failed */
692                 free_pagedir(pblist);
693                 pblist = NULL;
694         }
695         return pblist;
696 }
697
698 /**
699  *      free_image_pages - Free pages allocated for snapshot
700  */
701
702 static void free_image_pages(void)
703 {
704         struct pbe * p;
705
706         for_each_pbe (p, pagedir_save) {
707                 if (p->address) {
708                         ClearPageNosave(virt_to_page(p->address));
709                         free_page(p->address);
710                         p->address = 0;
711                 }
712         }
713 }
714
715 /**
716  *      alloc_image_pages - Allocate pages for the snapshot.
717  */
718
719 static int alloc_image_pages(void)
720 {
721         struct pbe * p;
722
723         for_each_pbe (p, pagedir_save) {
724                 p->address = get_zeroed_page(GFP_ATOMIC | __GFP_COLD);
725                 if (!p->address)
726                         return -ENOMEM;
727                 SetPageNosave(virt_to_page(p->address));
728         }
729         return 0;
730 }
731
732 void swsusp_free(void)
733 {
734         BUG_ON(PageNosave(virt_to_page(pagedir_save)));
735         BUG_ON(PageNosaveFree(virt_to_page(pagedir_save)));
736         free_image_pages();
737         free_pagedir(pagedir_save);
738 }
739
740
741 /**
742  *      enough_free_mem - Make sure we enough free memory to snapshot.
743  *
744  *      Returns TRUE or FALSE after checking the number of available
745  *      free pages.
746  */
747
748 static int enough_free_mem(void)
749 {
750         if (nr_free_pages() < (nr_copy_pages + PAGES_FOR_IO)) {
751                 pr_debug("swsusp: Not enough free pages: Have %d\n",
752                          nr_free_pages());
753                 return 0;
754         }
755         return 1;
756 }
757
758
759 /**
760  *      enough_swap - Make sure we have enough swap to save the image.
761  *
762  *      Returns TRUE or FALSE after checking the total amount of swap
763  *      space avaiable.
764  *
765  *      FIXME: si_swapinfo(&i) returns all swap devices information.
766  *      We should only consider resume_device.
767  */
768
769 static int enough_swap(void)
770 {
771         struct sysinfo i;
772
773         si_swapinfo(&i);
774         if (i.freeswap < (nr_copy_pages + PAGES_FOR_IO))  {
775                 pr_debug("swsusp: Not enough swap. Need %ld\n",i.freeswap);
776                 return 0;
777         }
778         return 1;
779 }
780
781 static int swsusp_alloc(void)
782 {
783         int error;
784
785         pagedir_nosave = NULL;
786         nr_copy_pages = calc_nr(nr_copy_pages);
787
788         pr_debug("suspend: (pages needed: %d + %d free: %d)\n",
789                  nr_copy_pages, PAGES_FOR_IO, nr_free_pages());
790
791         if (!enough_free_mem())
792                 return -ENOMEM;
793
794         if (!enough_swap())
795                 return -ENOSPC;
796
797         if (!(pagedir_save = alloc_pagedir(nr_copy_pages))) {
798                 printk(KERN_ERR "suspend: Allocating pagedir failed.\n");
799                 return -ENOMEM;
800         }
801         create_pbe_list(pagedir_save, nr_copy_pages);
802         pagedir_nosave = pagedir_save;
803         if ((error = alloc_image_pages())) {
804                 printk(KERN_ERR "suspend: Allocating image pages failed.\n");
805                 swsusp_free();
806                 return error;
807         }
808
809         nr_copy_pages_check = nr_copy_pages;
810         return 0;
811 }
812
813 static int suspend_prepare_image(void)
814 {
815         int error;
816
817         pr_debug("swsusp: critical section: \n");
818         if (save_highmem()) {
819                 printk(KERN_CRIT "Suspend machine: Not enough free pages for highmem\n");
820                 restore_highmem();
821                 return -ENOMEM;
822         }
823
824         drain_local_pages();
825         count_data_pages();
826         printk("swsusp: Need to copy %u pages\n", nr_copy_pages);
827
828         error = swsusp_alloc();
829         if (error)
830                 return error;
831
832         /* During allocating of suspend pagedir, new cold pages may appear.
833          * Kill them.
834          */
835         drain_local_pages();
836         copy_data_pages();
837
838         /*
839          * End of critical section. From now on, we can write to memory,
840          * but we should not touch disk. This specially means we must _not_
841          * touch swap space! Except we must write out our image of course.
842          */
843
844         printk("swsusp: critical section/: done (%d pages copied)\n", nr_copy_pages );
845         return 0;
846 }
847
848
849 /* It is important _NOT_ to umount filesystems at this point. We want
850  * them synced (in case something goes wrong) but we DO not want to mark
851  * filesystem clean: it is not. (And it does not matter, if we resume
852  * correctly, we'll mark system clean, anyway.)
853  */
854 int swsusp_write(void)
855 {
856         int error;
857         device_resume();
858         lock_swapdevices();
859         error = write_suspend_image();
860         /* This will unlock ignored swap devices since writing is finished */
861         lock_swapdevices();
862         return error;
863
864 }
865
866
867 extern asmlinkage int swsusp_arch_suspend(void);
868 extern asmlinkage int swsusp_arch_resume(void);
869
870
871 asmlinkage int swsusp_save(void)
872 {
873         return suspend_prepare_image();
874 }
875
876 int swsusp_suspend(void)
877 {
878         int error;
879         if ((error = arch_prepare_suspend()))
880                 return error;
881         local_irq_disable();
882         /* At this point, device_suspend() has been called, but *not*
883          * device_power_down(). We *must* device_power_down() now.
884          * Otherwise, drivers for some devices (e.g. interrupt controllers)
885          * become desynchronized with the actual state of the hardware
886          * at resume time, and evil weirdness ensues.
887          */
888         if ((error = device_power_down(PMSG_FREEZE))) {
889                 local_irq_enable();
890                 return error;
891         }
892
893         if ((error = swsusp_swap_check())) {
894                 printk(KERN_ERR "swsusp: FATAL: cannot find swap device, try "
895                                 "swapon -a!\n");
896                 local_irq_enable();
897                 return error;
898         }
899
900         save_processor_state();
901         if ((error = swsusp_arch_suspend()))
902                 printk("Error %d suspending\n", error);
903         /* Restore control flow magically appears here */
904         restore_processor_state();
905         BUG_ON (nr_copy_pages_check != nr_copy_pages);
906         restore_highmem();
907         device_power_up();
908         local_irq_enable();
909         return error;
910 }
911
912 int swsusp_resume(void)
913 {
914         int error;
915         local_irq_disable();
916         if (device_power_down(PMSG_FREEZE))
917                 printk(KERN_ERR "Some devices failed to power down, very bad\n");
918         /* We'll ignore saved state, but this gets preempt count (etc) right */
919         save_processor_state();
920         error = swsusp_arch_resume();
921         /* Code below is only ever reached in case of failure. Otherwise
922          * execution continues at place where swsusp_arch_suspend was called
923          */
924         BUG_ON(!error);
925         restore_processor_state();
926         restore_highmem();
927         device_power_up();
928         local_irq_enable();
929         return error;
930 }
931
932 /**
933  *      On resume, for storing the PBE list and the image,
934  *      we can only use memory pages that do not conflict with the pages
935  *      which had been used before suspend.
936  *
937  *      We don't know which pages are usable until we allocate them.
938  *
939  *      Allocated but unusable (ie eaten) memory pages are linked together
940  *      to create a list, so that we can free them easily
941  *
942  *      We could have used a type other than (void *)
943  *      for this purpose, but ...
944  */
945 static void **eaten_memory = NULL;
946
947 static inline void eat_page(void *page)
948 {
949         void **c;
950
951         c = eaten_memory;
952         eaten_memory = page;
953         *eaten_memory = c;
954 }
955
956 static unsigned long get_usable_page(unsigned gfp_mask)
957 {
958         unsigned long m;
959
960         m = get_zeroed_page(gfp_mask);
961         while (!PageNosaveFree(virt_to_page(m))) {
962                 eat_page((void *)m);
963                 m = get_zeroed_page(gfp_mask);
964                 if (!m)
965                         break;
966         }
967         return m;
968 }
969
970 static void free_eaten_memory(void)
971 {
972         unsigned long m;
973         void **c;
974         int i = 0;
975
976         c = eaten_memory;
977         while (c) {
978                 m = (unsigned long)c;
979                 c = *c;
980                 free_page(m);
981                 i++;
982         }
983         eaten_memory = NULL;
984         pr_debug("swsusp: %d unused pages freed\n", i);
985 }
986
987 /**
988  *      check_pagedir - We ensure here that pages that the PBEs point to
989  *      won't collide with pages where we're going to restore from the loaded
990  *      pages later
991  */
992
993 static int check_pagedir(struct pbe *pblist)
994 {
995         struct pbe *p;
996
997         /* This is necessary, so that we can free allocated pages
998          * in case of failure
999          */
1000         for_each_pbe (p, pblist)
1001                 p->address = 0UL;
1002
1003         for_each_pbe (p, pblist) {
1004                 p->address = get_usable_page(GFP_ATOMIC);
1005                 if (!p->address)
1006                         return -ENOMEM;
1007         }
1008         return 0;
1009 }
1010
1011 /**
1012  *      swsusp_pagedir_relocate - It is possible, that some memory pages
1013  *      occupied by the list of PBEs collide with pages where we're going to
1014  *      restore from the loaded pages later.  We relocate them here.
1015  */
1016
1017 static struct pbe * swsusp_pagedir_relocate(struct pbe *pblist)
1018 {
1019         struct zone *zone;
1020         unsigned long zone_pfn;
1021         struct pbe *pbpage, *tail, *p;
1022         void *m;
1023         int rel = 0, error = 0;
1024
1025         if (!pblist) /* a sanity check */
1026                 return NULL;
1027
1028         pr_debug("swsusp: Relocating pagedir (%lu pages to check)\n",
1029                         swsusp_info.pagedir_pages);
1030
1031         /* Set page flags */
1032
1033         for_each_zone (zone) {
1034                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
1035                         SetPageNosaveFree(pfn_to_page(zone_pfn +
1036                                         zone->zone_start_pfn));
1037         }
1038
1039         /* Clear orig addresses */
1040
1041         for_each_pbe (p, pblist)
1042                 ClearPageNosaveFree(virt_to_page(p->orig_address));
1043
1044         tail = pblist + PB_PAGE_SKIP;
1045
1046         /* Relocate colliding pages */
1047
1048         for_each_pb_page (pbpage, pblist) {
1049                 if (!PageNosaveFree(virt_to_page((unsigned long)pbpage))) {
1050                         m = (void *)get_usable_page(GFP_ATOMIC | __GFP_COLD);
1051                         if (!m) {
1052                                 error = -ENOMEM;
1053                                 break;
1054                         }
1055                         memcpy(m, (void *)pbpage, PAGE_SIZE);
1056                         if (pbpage == pblist)
1057                                 pblist = (struct pbe *)m;
1058                         else
1059                                 tail->next = (struct pbe *)m;
1060
1061                         eat_page((void *)pbpage);
1062                         pbpage = (struct pbe *)m;
1063
1064                         /* We have to link the PBEs again */
1065
1066                         for (p = pbpage; p < pbpage + PB_PAGE_SKIP; p++)
1067                                 if (p->next) /* needed to save the end */
1068                                         p->next = p + 1;
1069
1070                         rel++;
1071                 }
1072                 tail = pbpage + PB_PAGE_SKIP;
1073         }
1074
1075         if (error) {
1076                 printk("\nswsusp: Out of memory\n\n");
1077                 free_pagedir(pblist);
1078                 free_eaten_memory();
1079                 pblist = NULL;
1080         }
1081         else
1082                 printk("swsusp: Relocated %d pages\n", rel);
1083
1084         return pblist;
1085 }
1086
1087 /*
1088  *      Using bio to read from swap.
1089  *      This code requires a bit more work than just using buffer heads
1090  *      but, it is the recommended way for 2.5/2.6.
1091  *      The following are to signal the beginning and end of I/O. Bios
1092  *      finish asynchronously, while we want them to happen synchronously.
1093  *      A simple atomic_t, and a wait loop take care of this problem.
1094  */
1095
1096 static atomic_t io_done = ATOMIC_INIT(0);
1097
1098 static int end_io(struct bio * bio, unsigned int num, int err)
1099 {
1100         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1101                 panic("I/O error reading memory image");
1102         atomic_set(&io_done, 0);
1103         return 0;
1104 }
1105
1106 static struct block_device * resume_bdev;
1107
1108 /**
1109  *      submit - submit BIO request.
1110  *      @rw:    READ or WRITE.
1111  *      @off    physical offset of page.
1112  *      @page:  page we're reading or writing.
1113  *
1114  *      Straight from the textbook - allocate and initialize the bio.
1115  *      If we're writing, make sure the page is marked as dirty.
1116  *      Then submit it and wait.
1117  */
1118
1119 static int submit(int rw, pgoff_t page_off, void * page)
1120 {
1121         int error = 0;
1122         struct bio * bio;
1123
1124         bio = bio_alloc(GFP_ATOMIC, 1);
1125         if (!bio)
1126                 return -ENOMEM;
1127         bio->bi_sector = page_off * (PAGE_SIZE >> 9);
1128         bio_get(bio);
1129         bio->bi_bdev = resume_bdev;
1130         bio->bi_end_io = end_io;
1131
1132         if (bio_add_page(bio, virt_to_page(page), PAGE_SIZE, 0) < PAGE_SIZE) {
1133                 printk("swsusp: ERROR: adding page to bio at %ld\n",page_off);
1134                 error = -EFAULT;
1135                 goto Done;
1136         }
1137
1138         if (rw == WRITE)
1139                 bio_set_pages_dirty(bio);
1140
1141         atomic_set(&io_done, 1);
1142         submit_bio(rw | (1 << BIO_RW_SYNC), bio);
1143         while (atomic_read(&io_done))
1144                 yield();
1145
1146  Done:
1147         bio_put(bio);
1148         return error;
1149 }
1150
1151 static int bio_read_page(pgoff_t page_off, void * page)
1152 {
1153         return submit(READ, page_off, page);
1154 }
1155
1156 static int bio_write_page(pgoff_t page_off, void * page)
1157 {
1158         return submit(WRITE, page_off, page);
1159 }
1160
1161 /*
1162  * Sanity check if this image makes sense with this kernel/swap context
1163  * I really don't think that it's foolproof but more than nothing..
1164  */
1165
1166 static const char * sanity_check(void)
1167 {
1168         dump_info();
1169         if (swsusp_info.version_code != LINUX_VERSION_CODE)
1170                 return "kernel version";
1171         if (swsusp_info.num_physpages != num_physpages)
1172                 return "memory size";
1173         if (strcmp(swsusp_info.uts.sysname,system_utsname.sysname))
1174                 return "system type";
1175         if (strcmp(swsusp_info.uts.release,system_utsname.release))
1176                 return "kernel release";
1177         if (strcmp(swsusp_info.uts.version,system_utsname.version))
1178                 return "version";
1179         if (strcmp(swsusp_info.uts.machine,system_utsname.machine))
1180                 return "machine";
1181 #if 0
1182         if(swsusp_info.cpus != num_online_cpus())
1183                 return "number of cpus";
1184 #endif
1185         return NULL;
1186 }
1187
1188
1189 static int check_header(void)
1190 {
1191         const char * reason = NULL;
1192         int error;
1193
1194         if ((error = bio_read_page(swp_offset(swsusp_header.swsusp_info), &swsusp_info)))
1195                 return error;
1196
1197         /* Is this same machine? */
1198         if ((reason = sanity_check())) {
1199                 printk(KERN_ERR "swsusp: Resume mismatch: %s\n",reason);
1200                 return -EPERM;
1201         }
1202         nr_copy_pages = swsusp_info.image_pages;
1203         return error;
1204 }
1205
1206 static int check_sig(void)
1207 {
1208         int error;
1209
1210         memset(&swsusp_header, 0, sizeof(swsusp_header));
1211         if ((error = bio_read_page(0, &swsusp_header)))
1212                 return error;
1213         if (!memcmp(SWSUSP_SIG, swsusp_header.sig, 10)) {
1214                 memcpy(swsusp_header.sig, swsusp_header.orig_sig, 10);
1215
1216                 /*
1217                  * Reset swap signature now.
1218                  */
1219                 error = bio_write_page(0, &swsusp_header);
1220         } else { 
1221                 printk(KERN_ERR "swsusp: Suspend partition has wrong signature?\n");
1222                 return -EINVAL;
1223         }
1224         if (!error)
1225                 pr_debug("swsusp: Signature found, resuming\n");
1226         return error;
1227 }
1228
1229 /**
1230  *      data_read - Read image pages from swap.
1231  *
1232  *      You do not need to check for overlaps, check_pagedir()
1233  *      already did that.
1234  */
1235
1236 static int data_read(struct pbe *pblist)
1237 {
1238         struct pbe * p;
1239         int error = 0;
1240         int i = 0;
1241         int mod = swsusp_info.image_pages / 100;
1242
1243         if (!mod)
1244                 mod = 1;
1245
1246         printk("swsusp: Reading image data (%lu pages):     ",
1247                         swsusp_info.image_pages);
1248
1249         for_each_pbe (p, pblist) {
1250                 if (!(i % mod))
1251                         printk("\b\b\b\b%3d%%", i / mod);
1252
1253                 error = bio_read_page(swp_offset(p->swap_address),
1254                                   (void *)p->address);
1255                 if (error)
1256                         return error;
1257
1258                 i++;
1259         }
1260         printk("\b\b\b\bdone\n");
1261         return error;
1262 }
1263
1264 /**
1265  *      read_pagedir - Read page backup list pages from swap
1266  */
1267
1268 static int read_pagedir(struct pbe *pblist)
1269 {
1270         struct pbe *pbpage, *p;
1271         unsigned i = 0;
1272         int error;
1273
1274         if (!pblist)
1275                 return -EFAULT;
1276
1277         printk("swsusp: Reading pagedir (%lu pages)\n",
1278                         swsusp_info.pagedir_pages);
1279
1280         for_each_pb_page (pbpage, pblist) {
1281                 unsigned long offset = swp_offset(swsusp_info.pagedir[i++]);
1282
1283                 error = -EFAULT;
1284                 if (offset) {
1285                         p = (pbpage + PB_PAGE_SKIP)->next;
1286                         error = bio_read_page(offset, (void *)pbpage);
1287                         (pbpage + PB_PAGE_SKIP)->next = p;
1288                 }
1289                 if (error)
1290                         break;
1291         }
1292
1293         if (error)
1294                 free_page((unsigned long)pblist);
1295
1296         BUG_ON(i != swsusp_info.pagedir_pages);
1297
1298         return error;
1299 }
1300
1301
1302 static int check_suspend_image(void)
1303 {
1304         int error = 0;
1305
1306         if ((error = check_sig()))
1307                 return error;
1308
1309         if ((error = check_header()))
1310                 return error;
1311
1312         return 0;
1313 }
1314
1315 static int read_suspend_image(void)
1316 {
1317         int error = 0;
1318         struct pbe *p;
1319
1320         if (!(p = alloc_pagedir(nr_copy_pages)))
1321                 return -ENOMEM;
1322
1323         if ((error = read_pagedir(p)))
1324                 return error;
1325
1326         create_pbe_list(p, nr_copy_pages);
1327
1328         if (!(pagedir_nosave = swsusp_pagedir_relocate(p)))
1329                 return -ENOMEM;
1330
1331         /* Allocate memory for the image and read the data from swap */
1332
1333         error = check_pagedir(pagedir_nosave);
1334         free_eaten_memory();
1335         if (!error)
1336                 error = data_read(pagedir_nosave);
1337
1338         if (error) { /* We fail cleanly */
1339                 for_each_pbe (p, pagedir_nosave)
1340                         if (p->address) {
1341                                 free_page(p->address);
1342                                 p->address = 0UL;
1343                         }
1344                 free_pagedir(pagedir_nosave);
1345         }
1346         return error;
1347 }
1348
1349 /**
1350  *      swsusp_check - Check for saved image in swap
1351  */
1352
1353 int swsusp_check(void)
1354 {
1355         int error;
1356
1357         resume_bdev = open_by_devnum(swsusp_resume_device, FMODE_READ);
1358         if (!IS_ERR(resume_bdev)) {
1359                 set_blocksize(resume_bdev, PAGE_SIZE);
1360                 error = check_suspend_image();
1361                 if (error)
1362                     blkdev_put(resume_bdev);
1363         } else
1364                 error = PTR_ERR(resume_bdev);
1365
1366         if (!error)
1367                 pr_debug("swsusp: resume file found\n");
1368         else
1369                 pr_debug("swsusp: Error %d check for resume file\n", error);
1370         return error;
1371 }
1372
1373 /**
1374  *      swsusp_read - Read saved image from swap.
1375  */
1376
1377 int swsusp_read(void)
1378 {
1379         int error;
1380
1381         if (IS_ERR(resume_bdev)) {
1382                 pr_debug("swsusp: block device not initialised\n");
1383                 return PTR_ERR(resume_bdev);
1384         }
1385
1386         error = read_suspend_image();
1387         blkdev_put(resume_bdev);
1388
1389         if (!error)
1390                 pr_debug("swsusp: Reading resume file was successful\n");
1391         else
1392                 pr_debug("swsusp: Error %d resuming\n", error);
1393         return error;
1394 }
1395
1396 /**
1397  *      swsusp_close - close swap device.
1398  */
1399
1400 void swsusp_close(void)
1401 {
1402         if (IS_ERR(resume_bdev)) {
1403                 pr_debug("swsusp: block device not initialised\n");
1404                 return;
1405         }
1406
1407         blkdev_put(resume_bdev);
1408 }