[PATCH] swsusp: read speedup

Implement async reads for swsusp resuming.

Crufty old PIII testbox:
	15.7 MB/s -> 20.3 MB/s

Sony Vaio:
	14.6 MB/s -> 33.3 MB/s

I didn't implement the post-resume bio_set_pages_dirty().  I don't really
understand why resume needs to run set_page_dirty() against these pages.

It might be a worry that this code modifies PG_Uptodate, PG_Error and
PG_Locked against the image pages.  Can this possibly affect the resumed-into
kernel?  Hopefully not, if we're atomically restoring its mem_map?

Cc: Pavel Machek <pavel@ucw.cz>
Cc: "Rafael J. Wysocki" <rjw@sisk.pl>
Cc: Jens Axboe <axboe@suse.de>
Cc: Laurent Riffard <laurent.riffard@free.fr>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/linux/swap.h b/include/linux/swap.h
index 3d434cbffe26550a11bf6295885ed8141dbafe82..e7c36ba2a2dbe2466b5aa8ea4e4b6e406f4ed4e4 100644 (file)
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -220,6 +220,7 @@ extern int swap_readpage(struct file *, struct page *);
 extern int swap_writepage(struct page *page, struct writeback_control *wbc);
 extern int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
                                struct bio **bio_chain);
+extern int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err);
 
 /* linux/mm/swap_state.c */
 extern struct address_space swapper_space;

diff --git a/kernel/power/power.h b/kernel/power/power.h
index 57a792982fb921f329d2ac4edc0ee2462dd05497..59ce712f082d135eeab2625e4891546c87533f9a 100644 (file)
--- a/kernel/power/power.h
+++ b/kernel/power/power.h
@@ -58,6 +58,7 @@ struct snapshot_handle {
        struct pbe      *pbe, *last_pbe;
        void            *buffer;
        unsigned int    buf_offset;
+       int             sync_read;
 };
 
 #define data_of(handle)        ((handle).buffer + (handle).buf_offset)

diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 75d4886e648e7a6ae65258bc43a80fe6ec238ed7..591301ae8b7d9177873d10ff2630596898ea5177 100644 (file)
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -314,7 +314,7 @@ static unsigned int unsafe_pages;
  *     and we count them using unsafe_pages
  */
 
-static inline void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
+static void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
 {
        void *res;
 
@@ -828,13 +828,16 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count)
        }
        if (!handle->offset)
                handle->buffer = buffer;
+       handle->sync_read = 1;
        if (handle->prev < handle->page) {
                if (!handle->prev) {
-                       error = load_header(handle, (struct swsusp_info *)buffer);
+                       error = load_header(handle,
+                                       (struct swsusp_info *)buffer);
                        if (error)
                                return error;
                } else if (handle->prev <= nr_meta_pages) {
-                       handle->pbe = unpack_orig_addresses(buffer, handle->pbe);
+                       handle->pbe = unpack_orig_addresses(buffer,
+                                                       handle->pbe);
                        if (!handle->pbe) {
                                error = prepare_image(handle);
                                if (error)
@@ -842,10 +845,12 @@ int snapshot_write_next(struct snapshot_handle *handle, size_t count)
                                handle->pbe = pagedir_nosave;
                                handle->last_pbe = NULL;
                                handle->buffer = get_buffer(handle);
+                               handle->sync_read = 0;
                        }
                } else {
                        handle->pbe = handle->pbe->next;
                        handle->buffer = get_buffer(handle);
+                       handle->sync_read = 0;
                }
                handle->prev = handle->page;
        }

diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index 9ab9895721647382b6b6c492a0debf9967e3ea73..8309d20b2563d2783395b91f45cad020f97329f0 100644 (file)
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -22,6 +22,7 @@
 #include <linux/device.h>
 #include <linux/buffer_head.h>
 #include <linux/bio.h>
+#include <linux/blkdev.h>
 #include <linux/swap.h>
 #include <linux/swapops.h>
 #include <linux/pm.h>
@@ -214,6 +215,8 @@ static int wait_on_bio_chain(struct bio **bio_chain)
                return 0;
 
        bio = *bio_chain;
+       if (bio == NULL)
+               return 0;
        while (bio) {
                struct page *page;
 
@@ -349,7 +352,8 @@ int swsusp_write(void)
        int error;
 
        if ((error = swsusp_swap_check())) {
-               printk(KERN_ERR "swsusp: Cannot find swap device, try swapon -a.\n");
+               printk(KERN_ERR "swsusp: Cannot find swap device, try "
+                               "swapon -a.\n");
                return error;
        }
        memset(&snapshot, 0, sizeof(struct snapshot_handle));
@@ -381,27 +385,6 @@ int swsusp_write(void)
        return error;
 }
 
-/*
- *     Using bio to read from swap.
- *     This code requires a bit more work than just using buffer heads
- *     but, it is the recommended way for 2.5/2.6.
- *     The following are to signal the beginning and end of I/O. Bios
- *     finish asynchronously, while we want them to happen synchronously.
- *     A simple atomic_t, and a wait loop take care of this problem.
- */
-
-static atomic_t io_done = ATOMIC_INIT(0);
-
-static int end_io(struct bio *bio, unsigned int num, int err)
-{
-       if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
-               printk(KERN_ERR "I/O error reading swsusp image.\n");
-               return -EIO;
-       }
-       atomic_set(&io_done, 0);
-       return 0;
-}
-
 static struct block_device *resume_bdev;
 
 /**
@@ -409,15 +392,15 @@ static struct block_device *resume_bdev;
  *     @rw:    READ or WRITE.
  *     @off    physical offset of page.
  *     @page:  page we're reading or writing.
+ *     @bio_chain: list of pending biod (for async reading)
  *
  *     Straight from the textbook - allocate and initialize the bio.
- *     If we're writing, make sure the page is marked as dirty.
- *     Then submit it and wait.
+ *     If we're reading, make sure the page is marked as dirty.
+ *     Then submit it and, if @bio_chain == NULL, wait.
  */
-
-static int submit(int rw, pgoff_t page_off, void *page)
+static int submit(int rw, pgoff_t page_off, struct page *page,
+                       struct bio **bio_chain)
 {
-       int error = 0;
        struct bio *bio;
 
        bio = bio_alloc(GFP_ATOMIC, 1);
@@ -425,33 +408,40 @@ static int submit(int rw, pgoff_t page_off, void *page)
                return -ENOMEM;
        bio->bi_sector = page_off * (PAGE_SIZE >> 9);
        bio->bi_bdev = resume_bdev;
-       bio->bi_end_io = end_io;
+       bio->bi_end_io = end_swap_bio_read;
 
-       if (bio_add_page(bio, virt_to_page(page), PAGE_SIZE, 0) < PAGE_SIZE) {
-               printk("swsusp: ERROR: adding page to bio at %ld\n",page_off);
-               error = -EFAULT;
-               goto Done;
+       if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
+               printk("swsusp: ERROR: adding page to bio at %ld\n", page_off);
+               bio_put(bio);
+               return -EFAULT;
        }
 
-       atomic_set(&io_done, 1);
-       submit_bio(rw | (1 << BIO_RW_SYNC), bio);
-       while (atomic_read(&io_done))
-               yield();
-       if (rw == READ)
-               bio_set_pages_dirty(bio);
- Done:
-       bio_put(bio);
-       return error;
+       lock_page(page);
+       bio_get(bio);
+
+       if (bio_chain == NULL) {
+               submit_bio(rw | (1 << BIO_RW_SYNC), bio);
+               wait_on_page_locked(page);
+               if (rw == READ)
+                       bio_set_pages_dirty(bio);
+               bio_put(bio);
+       } else {
+               get_page(page);
+               bio->bi_private = *bio_chain;
+               *bio_chain = bio;
+               submit_bio(rw | (1 << BIO_RW_SYNC), bio);
+       }
+       return 0;
 }
 
-static int bio_read_page(pgoff_t page_off, void *page)
+static int bio_read_page(pgoff_t page_off, void *addr, struct bio **bio_chain)
 {
-       return submit(READ, page_off, page);
+       return submit(READ, page_off, virt_to_page(addr), bio_chain);
 }
 
-static int bio_write_page(pgoff_t page_off, void *page)
+static int bio_write_page(pgoff_t page_off, void *addr)
 {
-       return submit(WRITE, page_off, page);
+       return submit(WRITE, page_off, virt_to_page(addr), NULL);
 }
 
 /**
@@ -476,7 +466,7 @@ static int get_swap_reader(struct swap_map_handle *handle,
        handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_ATOMIC);
        if (!handle->cur)
                return -ENOMEM;
-       error = bio_read_page(swp_offset(start), handle->cur);
+       error = bio_read_page(swp_offset(start), handle->cur, NULL);
        if (error) {
                release_swap_reader(handle);
                return error;
@@ -485,7 +475,8 @@ static int get_swap_reader(struct swap_map_handle *handle,
        return 0;
 }
 
-static int swap_read_page(struct swap_map_handle *handle, void *buf)
+static int swap_read_page(struct swap_map_handle *handle, void *buf,
+                               struct bio **bio_chain)
 {
        unsigned long offset;
        int error;
@@ -495,16 +486,17 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf)
        offset = handle->cur->entries[handle->k];
        if (!offset)
                return -EFAULT;
-       error = bio_read_page(offset, buf);
+       error = bio_read_page(offset, buf, bio_chain);
        if (error)
                return error;
        if (++handle->k >= MAP_PAGE_ENTRIES) {
+               error = wait_on_bio_chain(bio_chain);
                handle->k = 0;
                offset = handle->cur->next_swap;
                if (!offset)
                        release_swap_reader(handle);
-               else
-                       error = bio_read_page(offset, handle->cur);
+               else if (!error)
+                       error = bio_read_page(offset, handle->cur, NULL);
        }
        return error;
 }
@@ -517,38 +509,48 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf)
 
 static int load_image(struct swap_map_handle *handle,
                       struct snapshot_handle *snapshot,
-                      unsigned int nr_pages)
+                      unsigned int nr_to_read)
 {
        unsigned int m;
-       int ret;
        int error = 0;
        struct timeval start;
        struct timeval stop;
+       struct bio *bio;
+       int err2;
+       unsigned nr_pages;
 
-       printk("Loading image data pages (%u pages) ...     ", nr_pages);
-       m = nr_pages / 100;
+       printk("Loading image data pages (%u pages) ...     ", nr_to_read);
+       m = nr_to_read / 100;
        if (!m)
                m = 1;
        nr_pages = 0;
+       bio = NULL;
        do_gettimeofday(&start);
-       do {
-               ret = snapshot_write_next(snapshot, PAGE_SIZE);
-               if (ret > 0) {
-                       error = swap_read_page(handle, data_of(*snapshot));
-                       if (error)
-                               break;
-                       if (!(nr_pages % m))
-                               printk("\b\b\b\b%3d%%", nr_pages / m);
-                       nr_pages++;
-               }
-       } while (ret > 0);
+       for ( ; ; ) {
+               error = snapshot_write_next(snapshot, PAGE_SIZE);
+               if (error <= 0)
+                       break;
+               error = swap_read_page(handle, data_of(*snapshot), &bio);
+               if (error)
+                       break;
+               if (snapshot->sync_read)
+                       error = wait_on_bio_chain(&bio);
+               if (error)
+                       break;
+               if (!(nr_pages % m))
+                       printk("\b\b\b\b%3d%%", nr_pages / m);
+               nr_pages++;
+       }
+       err2 = wait_on_bio_chain(&bio);
        do_gettimeofday(&stop);
+       if (!error)
+               error = err2;
        if (!error) {
                printk("\b\b\b\bdone\n");
                if (!snapshot_image_loaded(snapshot))
                        error = -ENODATA;
        }
-       show_speed(&start, &stop, nr_pages, "Read");
+       show_speed(&start, &stop, nr_to_read, "Read");
        return error;
 }
 
@@ -571,7 +573,7 @@ int swsusp_read(void)
        header = (struct swsusp_info *)data_of(snapshot);
        error = get_swap_reader(&handle, swsusp_header.image);
        if (!error)
-               error = swap_read_page(&handle, header);
+               error = swap_read_page(&handle, header, NULL);
        if (!error)
                error = load_image(&handle, &snapshot, header->pages - 1);
        release_swap_reader(&handle);
@@ -597,7 +599,7 @@ int swsusp_check(void)
        if (!IS_ERR(resume_bdev)) {
                set_blocksize(resume_bdev, PAGE_SIZE);
                memset(&swsusp_header, 0, sizeof(swsusp_header));
-               if ((error = bio_read_page(0, &swsusp_header)))
+               if ((error = bio_read_page(0, &swsusp_header, NULL)))
                        return error;
                if (!memcmp(SWSUSP_SIG, swsusp_header.sig, 10)) {
                        memcpy(swsusp_header.sig, swsusp_header.orig_sig, 10);

diff --git a/mm/page_io.c b/mm/page_io.c
index f46a9862b7efa5cbc4c812056241ec51efeea670..d4840ecbf8f9358e1dfc5918802f3604e1f06b93 100644 (file)
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -74,7 +74,7 @@ static int end_swap_bio_write(struct bio *bio, unsigned int bytes_done, int err)
        return 0;
 }
 
-static int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err)
+int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err)
 {
        const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
        struct page *page = bio->bi_io_vec[0].bv_page;