X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=mm%2Fswapfile.c;h=fdee145afc6ff187f407f8ada1dd05e0a40c5cc4;hb=52b7efdbe5f5696fc80338560a3fc51e0b0a993c;hp=da48405cd9a3199d9088d1221864fa77a76a108d;hpb=0c2fc10901e1b516526c92849ccd03327f119dbe;p=linux-2.6 diff --git a/mm/swapfile.c b/mm/swapfile.c index da48405cd9..fdee145afc 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -84,116 +84,125 @@ void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page) up_read(&swap_unplug_sem); } -static inline int scan_swap_map(struct swap_info_struct *si) +static inline unsigned long scan_swap_map(struct swap_info_struct *si) { - unsigned long offset; + unsigned long offset, last_in_cluster; + /* - * We try to cluster swap pages by allocating them - * sequentially in swap. Once we've allocated - * SWAPFILE_CLUSTER pages this way, however, we resort to - * first-free allocation, starting a new cluster. This - * prevents us from scattering swap pages all over the entire - * swap partition, so that we reduce overall disk seek times - * between swap pages. -- sct */ - if (si->cluster_nr) { - while (si->cluster_next <= si->highest_bit) { - offset = si->cluster_next++; + * We try to cluster swap pages by allocating them sequentially + * in swap. Once we've allocated SWAPFILE_CLUSTER pages this + * way, however, we resort to first-free allocation, starting + * a new cluster. This prevents us from scattering swap pages + * all over the entire swap partition, so that we reduce + * overall disk seek times between swap pages. -- sct + * But we do now try to find an empty cluster. -Andrea + */ + + si->flags += SWP_SCANNING; + if (unlikely(!si->cluster_nr)) { + si->cluster_nr = SWAPFILE_CLUSTER - 1; + if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) + goto lowest; + swap_device_unlock(si); + + offset = si->lowest_bit; + last_in_cluster = offset + SWAPFILE_CLUSTER - 1; + + /* Locate the first empty (unaligned) cluster */ + for (; last_in_cluster <= si->highest_bit; offset++) { if (si->swap_map[offset]) - continue; - si->cluster_nr--; - goto got_page; - } - } - si->cluster_nr = SWAPFILE_CLUSTER; - - /* try to find an empty (even not aligned) cluster. */ - offset = si->lowest_bit; - check_next_cluster: - if (offset+SWAPFILE_CLUSTER-1 <= si->highest_bit) - { - unsigned long nr; - for (nr = offset; nr < offset+SWAPFILE_CLUSTER; nr++) - if (si->swap_map[nr]) - { - offset = nr+1; - goto check_next_cluster; + last_in_cluster = offset + SWAPFILE_CLUSTER; + else if (offset == last_in_cluster) { + swap_device_lock(si); + si->cluster_next = offset-SWAPFILE_CLUSTER-1; + goto cluster; } - /* We found a completly empty cluster, so start - * using it. - */ - goto got_page; + } + swap_device_lock(si); + goto lowest; } - /* No luck, so now go finegrined as usual. -Andrea */ - for (offset = si->lowest_bit; offset <= si->highest_bit ; offset++) { - if (si->swap_map[offset]) - continue; - si->lowest_bit = offset+1; - got_page: + + si->cluster_nr--; +cluster: + offset = si->cluster_next; + if (offset > si->highest_bit) +lowest: offset = si->lowest_bit; +checks: if (!(si->flags & SWP_WRITEOK)) + goto no_page; + if (!si->highest_bit) + goto no_page; + if (!si->swap_map[offset]) { if (offset == si->lowest_bit) si->lowest_bit++; if (offset == si->highest_bit) si->highest_bit--; - if (si->lowest_bit > si->highest_bit) { + si->inuse_pages++; + if (si->inuse_pages == si->pages) { si->lowest_bit = si->max; si->highest_bit = 0; } si->swap_map[offset] = 1; - si->inuse_pages++; - nr_swap_pages--; - si->cluster_next = offset+1; + si->cluster_next = offset + 1; + si->flags -= SWP_SCANNING; return offset; } - si->lowest_bit = si->max; - si->highest_bit = 0; + + swap_device_unlock(si); + while (++offset <= si->highest_bit) { + if (!si->swap_map[offset]) { + swap_device_lock(si); + goto checks; + } + } + swap_device_lock(si); + goto lowest; + +no_page: + si->flags -= SWP_SCANNING; return 0; } swp_entry_t get_swap_page(void) { - struct swap_info_struct * p; - unsigned long offset; - swp_entry_t entry; - int type, wrapped = 0; + struct swap_info_struct *si; + pgoff_t offset; + int type, next; + int wrapped = 0; - entry.val = 0; /* Out of memory */ swap_list_lock(); - type = swap_list.next; - if (type < 0) - goto out; if (nr_swap_pages <= 0) - goto out; - - while (1) { - p = &swap_info[type]; - if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) { - swap_device_lock(p); - offset = scan_swap_map(p); - swap_device_unlock(p); - if (offset) { - entry = swp_entry(type,offset); - type = swap_info[type].next; - if (type < 0 || - p->prio != swap_info[type].prio) { - swap_list.next = swap_list.head; - } else { - swap_list.next = type; - } - goto out; - } + goto noswap; + nr_swap_pages--; + + for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) { + si = swap_info + type; + next = si->next; + if (next < 0 || + (!wrapped && si->prio != swap_info[next].prio)) { + next = swap_list.head; + wrapped++; } - type = p->next; - if (!wrapped) { - if (type < 0 || p->prio != swap_info[type].prio) { - type = swap_list.head; - wrapped = 1; - } - } else - if (type < 0) - goto out; /* out of swap space */ + + if (!si->highest_bit) + continue; + if (!(si->flags & SWP_WRITEOK)) + continue; + + swap_list.next = next; + swap_device_lock(si); + swap_list_unlock(); + offset = scan_swap_map(si); + swap_device_unlock(si); + if (offset) + return swp_entry(type, offset); + swap_list_lock(); + next = swap_list.next; } -out: + + nr_swap_pages++; +noswap: swap_list_unlock(); - return entry; + return (swp_entry_t) {0}; } static struct swap_info_struct * swap_info_get(swp_entry_t entry) @@ -215,8 +224,6 @@ static struct swap_info_struct * swap_info_get(swp_entry_t entry) if (!p->swap_map[offset]) goto bad_free; swap_list_lock(); - if (p->prio > swap_info[swap_list.next].prio) - swap_list.next = type; swap_device_lock(p); return p; @@ -253,6 +260,8 @@ static int swap_entry_free(struct swap_info_struct *p, unsigned long offset) p->lowest_bit = offset; if (offset > p->highest_bit) p->highest_bit = offset; + if (p->prio > swap_info[swap_list.next].prio) + swap_list.next = p - swap_info; nr_swap_pages++; p->inuse_pages--; } @@ -276,61 +285,37 @@ void swap_free(swp_entry_t entry) } /* - * Check if we're the only user of a swap page, - * when the page is locked. + * How many references to page are currently swapped out? */ -static int exclusive_swap_page(struct page *page) +static inline int page_swapcount(struct page *page) { - int retval = 0; - struct swap_info_struct * p; + int count = 0; + struct swap_info_struct *p; swp_entry_t entry; entry.val = page->private; p = swap_info_get(entry); if (p) { - /* Is the only swap cache user the cache itself? */ - if (p->swap_map[swp_offset(entry)] == 1) { - /* Recheck the page count with the swapcache lock held.. */ - write_lock_irq(&swapper_space.tree_lock); - if (page_count(page) == 2) - retval = 1; - write_unlock_irq(&swapper_space.tree_lock); - } + /* Subtract the 1 for the swap cache itself */ + count = p->swap_map[swp_offset(entry)] - 1; swap_info_put(p); } - return retval; + return count; } /* * We can use this swap cache entry directly * if there are no other references to it. - * - * Here "exclusive_swap_page()" does the real - * work, but we opportunistically check whether - * we need to get all the locks first.. */ int can_share_swap_page(struct page *page) { - int retval = 0; + int count; - if (!PageLocked(page)) - BUG(); - switch (page_count(page)) { - case 3: - if (!PagePrivate(page)) - break; - /* Fallthrough */ - case 2: - if (!PageSwapCache(page)) - break; - retval = exclusive_swap_page(page); - break; - case 1: - if (PageReserved(page)) - break; - retval = 1; - } - return retval; + BUG_ON(!PageLocked(page)); + count = page_mapcount(page); + if (count <= 1 && PageSwapCache(page)) + count += page_swapcount(page); + return count == 1; } /* @@ -529,9 +514,10 @@ static int unuse_mm(struct mm_struct *mm, if (!down_read_trylock(&mm->mmap_sem)) { /* - * Our reference to the page stops try_to_unmap_one from - * unmapping its ptes, so swapoff can make progress. + * Activate page so shrink_cache is unlikely to unmap its + * ptes while lock is dropped, so swapoff can make progress. */ + activate_page(page); unlock_page(page); down_read(&mm->mmap_sem); lock_page(page); @@ -554,10 +540,11 @@ static int unuse_mm(struct mm_struct *mm, * Scan swap_map from current position to next entry still in use. * Recycle to start on reaching the end, returning 0 when empty. */ -static int find_next_to_unuse(struct swap_info_struct *si, int prev) +static unsigned int find_next_to_unuse(struct swap_info_struct *si, + unsigned int prev) { - int max = si->max; - int i = prev; + unsigned int max = si->max; + unsigned int i = prev; int count; /* @@ -600,7 +587,7 @@ static int try_to_unuse(unsigned int type) unsigned short swcount; struct page *page; swp_entry_t entry; - int i = 0; + unsigned int i = 0; int retval = 0; int reset_overflow = 0; int shmem; @@ -855,9 +842,9 @@ sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset) offset < (se->start_page + se->nr_pages)) { return se->start_block + (offset - se->start_page); } - lh = se->list.prev; + lh = se->list.next; if (lh == &sis->extent_list) - lh = lh->prev; + lh = lh->next; se = list_entry(lh, struct swap_extent, list); sis->curr_swap_extent = se; BUG_ON(se == start_se); /* It *must* be present */ @@ -877,15 +864,13 @@ static void destroy_swap_extents(struct swap_info_struct *sis) list_del(&se->list); kfree(se); } - sis->nr_extents = 0; } /* * Add a block range (and the corresponding page range) into this swapdev's - * extent list. The extent list is kept sorted in block order. + * extent list. The extent list is kept sorted in page order. * - * This function rather assumes that it is called in ascending sector_t order. - * It doesn't look for extent coalescing opportunities. + * This function rather assumes that it is called in ascending page order. */ static int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, @@ -895,16 +880,15 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, struct swap_extent *new_se; struct list_head *lh; - lh = sis->extent_list.next; /* The highest-addressed block */ - while (lh != &sis->extent_list) { + lh = sis->extent_list.prev; /* The highest page extent */ + if (lh != &sis->extent_list) { se = list_entry(lh, struct swap_extent, list); - if (se->start_block + se->nr_pages == start_block && - se->start_page + se->nr_pages == start_page) { + BUG_ON(se->start_page + se->nr_pages != start_page); + if (se->start_block + se->nr_pages == start_block) { /* Merge it */ se->nr_pages += nr_pages; return 0; } - lh = lh->next; } /* @@ -917,16 +901,8 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, new_se->nr_pages = nr_pages; new_se->start_block = start_block; - lh = sis->extent_list.prev; /* The lowest block */ - while (lh != &sis->extent_list) { - se = list_entry(lh, struct swap_extent, list); - if (se->start_block > start_block) - break; - lh = lh->prev; - } - list_add_tail(&new_se->list, lh); - sis->nr_extents++; - return 0; + list_add_tail(&new_se->list, &sis->extent_list); + return 1; } /* @@ -949,7 +925,7 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, * requirements, they are simply tossed out - we will never use those blocks * for swapping. * - * For S_ISREG swapfiles we hold i_sem across the life of the swapon. This + * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This * prevents root from shooting her foot off by ftruncating an in-use swapfile, * which will scribble on the fs. * @@ -960,7 +936,7 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, * This is extremely effective. The average number of iterations in * map_swap_page() has been measured at about 0.3 per page. - akpm. */ -static int setup_swap_extents(struct swap_info_struct *sis) +static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) { struct inode *inode; unsigned blocks_per_page; @@ -968,11 +944,15 @@ static int setup_swap_extents(struct swap_info_struct *sis) unsigned blkbits; sector_t probe_block; sector_t last_block; + sector_t lowest_block = -1; + sector_t highest_block = 0; + int nr_extents = 0; int ret; inode = sis->swap_file->f_mapping->host; if (S_ISBLK(inode->i_mode)) { ret = add_swap_extent(sis, 0, sis->max, 0); + *span = sis->pages; goto done; } @@ -1017,22 +997,32 @@ static int setup_swap_extents(struct swap_info_struct *sis) } } + first_block >>= (PAGE_SHIFT - blkbits); + if (page_no) { /* exclude the header page */ + if (first_block < lowest_block) + lowest_block = first_block; + if (first_block > highest_block) + highest_block = first_block; + } + /* * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks */ - ret = add_swap_extent(sis, page_no, 1, - first_block >> (PAGE_SHIFT - blkbits)); - if (ret) + ret = add_swap_extent(sis, page_no, 1, first_block); + if (ret < 0) goto out; + nr_extents += ret; page_no++; probe_block += blocks_per_page; reprobe: continue; } - ret = 0; + ret = nr_extents; + *span = 1 + highest_block - lowest_block; if (page_no == 0) - ret = -EINVAL; + page_no = 1; /* force Empty message */ sis->max = page_no; + sis->pages = page_no - 1; sis->highest_bit = page_no - 1; done: sis->curr_swap_extent = list_entry(sis->extent_list.prev, @@ -1124,16 +1114,15 @@ asmlinkage long sys_swapoff(const char __user * specialfile) } nr_swap_pages -= p->pages; total_swap_pages -= p->pages; + swap_device_lock(p); p->flags &= ~SWP_WRITEOK; + swap_device_unlock(p); swap_list_unlock(); + current->flags |= PF_SWAPOFF; err = try_to_unuse(type); current->flags &= ~PF_SWAPOFF; - /* wait for any unplug function to finish */ - down_write(&swap_unplug_sem); - up_write(&swap_unplug_sem); - if (err) { /* re-insert swap space back into swap_list */ swap_list_lock(); @@ -1147,10 +1136,29 @@ asmlinkage long sys_swapoff(const char __user * specialfile) swap_info[prev].next = p - swap_info; nr_swap_pages += p->pages; total_swap_pages += p->pages; + swap_device_lock(p); p->flags |= SWP_WRITEOK; + swap_device_unlock(p); swap_list_unlock(); goto out_dput; } + + /* wait for any unplug function to finish */ + down_write(&swap_unplug_sem); + up_write(&swap_unplug_sem); + + /* wait for anyone still in scan_swap_map */ + swap_device_lock(p); + p->highest_bit = 0; /* cuts scans short */ + while (p->flags >= SWP_SCANNING) { + swap_device_unlock(p); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(1); + swap_device_lock(p); + } + swap_device_unlock(p); + + destroy_swap_extents(p); down(&swapon_sem); swap_list_lock(); drain_mmlist(); @@ -1161,7 +1169,6 @@ asmlinkage long sys_swapoff(const char __user * specialfile) swap_map = p->swap_map; p->swap_map = NULL; p->flags = 0; - destroy_swap_extents(p); swap_device_unlock(p); swap_list_unlock(); up(&swapon_sem); @@ -1236,7 +1243,7 @@ static int swap_show(struct seq_file *swap, void *v) file = ptr->swap_file; len = seq_path(swap, file->f_vfsmnt, file->f_dentry, " \t\n\\"); - seq_printf(swap, "%*s%s\t%d\t%ld\t%d\n", + seq_printf(swap, "%*s%s\t%u\t%u\t%d\n", len < 40 ? 40 - len : 1, " ", S_ISBLK(file->f_dentry->d_inode->i_mode) ? "partition" : "file\t", @@ -1295,7 +1302,9 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) static int least_priority; union swap_header *swap_header = NULL; int swap_header_version; - int nr_good_pages = 0; + unsigned int nr_good_pages = 0; + int nr_extents = 0; + sector_t span; unsigned long maxpages = 1; int swapfilesize; unsigned short *swap_map; @@ -1331,7 +1340,6 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) nr_swapfiles = type+1; INIT_LIST_HEAD(&p->extent_list); p->flags = SWP_USED; - p->nr_extents = 0; p->swap_file = NULL; p->old_block_size = 0; p->swap_map = NULL; @@ -1449,6 +1457,8 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) } p->lowest_bit = 1; + p->cluster_next = 1; + /* * Find out how many pages are allowed for a single swap * device. There are two limiting factors: 1) the number of @@ -1469,6 +1479,10 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) p->highest_bit = maxpages - 1; error = -EINVAL; + if (!maxpages) + goto bad_swap; + if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) + goto bad_swap; if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) goto bad_swap; @@ -1493,25 +1507,29 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) if (error) goto bad_swap; } - + if (swapfilesize && maxpages > swapfilesize) { printk(KERN_WARNING "Swap area shorter than signature indicates\n"); error = -EINVAL; goto bad_swap; } + if (nr_good_pages) { + p->swap_map[0] = SWAP_MAP_BAD; + p->max = maxpages; + p->pages = nr_good_pages; + nr_extents = setup_swap_extents(p, &span); + if (nr_extents < 0) { + error = nr_extents; + goto bad_swap; + } + nr_good_pages = p->pages; + } if (!nr_good_pages) { printk(KERN_WARNING "Empty swap-file\n"); error = -EINVAL; goto bad_swap; } - p->swap_map[0] = SWAP_MAP_BAD; - p->max = maxpages; - p->pages = nr_good_pages; - - error = setup_swap_extents(p); - if (error) - goto bad_swap; down(&swapon_sem); swap_list_lock(); @@ -1519,9 +1537,11 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) p->flags = SWP_ACTIVE; nr_swap_pages += nr_good_pages; total_swap_pages += nr_good_pages; - printk(KERN_INFO "Adding %dk swap on %s. Priority:%d extents:%d\n", - nr_good_pages<<(PAGE_SHIFT-10), name, - p->prio, p->nr_extents); + + printk(KERN_INFO "Adding %uk swap on %s. " + "Priority:%d extents:%d across:%lluk\n", + nr_good_pages<<(PAGE_SHIFT-10), name, p->prio, + nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10)); /* insert swap space into swap_list: */ prev = -1; @@ -1547,6 +1567,7 @@ bad_swap: set_blocksize(bdev, p->old_block_size); bd_release(bdev); } + destroy_swap_extents(p); bad_swap_2: swap_list_lock(); swap_map = p->swap_map; @@ -1556,7 +1577,6 @@ bad_swap_2: if (!(swap_flags & SWAP_FLAG_PREFER)) ++least_priority; swap_list_unlock(); - destroy_swap_extents(p); vfree(swap_map); if (swap_file) filp_close(swap_file, NULL);