#include <linux/module.h>
#include <linux/syscalls.h>
#include <linux/uio.h>
+#include <linux/security.h>
/*
* Attempt to steal a page from a pipe buffer. This should perhaps go into
* @spd: data to fill
*
* Description:
- * @spd contains a map of pages and len/offset tupples, a long with
+ * @spd contains a map of pages and len/offset tuples, along with
* the struct pipe_buf_operations associated with these pages. This
* function will link that data to the pipe.
*
unsigned int flags)
{
struct address_space *mapping = in->f_mapping;
- unsigned int loff, nr_pages;
+ unsigned int loff, nr_pages, req_pages;
struct page *pages[PIPE_BUFFERS];
struct partial_page partial[PIPE_BUFFERS];
struct page *page;
index = *ppos >> PAGE_CACHE_SHIFT;
loff = *ppos & ~PAGE_CACHE_MASK;
- nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
-
- if (nr_pages > PIPE_BUFFERS)
- nr_pages = PIPE_BUFFERS;
-
- /*
- * Don't try to 2nd guess the read-ahead logic, call into
- * page_cache_readahead() like the page cache reads would do.
- */
- page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages);
+ req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+ nr_pages = min(req_pages, (unsigned)PIPE_BUFFERS);
/*
* Lookup the (hopefully) full range of pages we need.
*/
spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
+ index += spd.nr_pages;
/*
* If find_get_pages_contig() returned fewer pages than we needed,
- * allocate the rest and fill in the holes.
+ * readahead/allocate the rest and fill in the holes.
*/
+ if (spd.nr_pages < nr_pages)
+ page_cache_sync_readahead(mapping, &in->f_ra, in,
+ index, req_pages - spd.nr_pages);
+
error = 0;
- index += spd.nr_pages;
while (spd.nr_pages < nr_pages) {
/*
* Page could be there, find_get_pages_contig() breaks on
*/
page = find_get_page(mapping, index);
if (!page) {
- /*
- * Make sure the read-ahead engine is notified
- * about this failure.
- */
- handle_ra_miss(mapping, &in->f_ra, index);
-
/*
* page didn't exist, allocate one.
*/
this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
page = pages[page_nr];
+ if (PageReadahead(page))
+ page_cache_async_readahead(mapping, &in->f_ra, in,
+ page, index, req_pages - page_nr);
+
/*
* If the page isn't uptodate, we may need to start io on it
*/
*/
while (page_nr < nr_pages)
page_cache_release(pages[page_nr++]);
+ in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT;
if (spd.nr_pages)
return splice_to_pipe(pipe, &spd);
ret = 0;
spliced = 0;
- while (len) {
+ while (len && !spliced) {
ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
if (ret < 0)
struct address_space *mapping = file->f_mapping;
unsigned int offset, this_len;
struct page *page;
- pgoff_t index;
+ void *fsdata;
int ret;
/*
if (unlikely(ret))
return ret;
- index = sd->pos >> PAGE_CACHE_SHIFT;
offset = sd->pos & ~PAGE_CACHE_MASK;
this_len = sd->len;
if (this_len + offset > PAGE_CACHE_SIZE)
this_len = PAGE_CACHE_SIZE - offset;
-find_page:
- page = find_lock_page(mapping, index);
- if (!page) {
- ret = -ENOMEM;
- page = page_cache_alloc_cold(mapping);
- if (unlikely(!page))
- goto out_ret;
-
- /*
- * This will also lock the page
- */
- ret = add_to_page_cache_lru(page, mapping, index,
- GFP_KERNEL);
- if (unlikely(ret))
- goto out;
- }
-
- ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
- if (unlikely(ret)) {
- loff_t isize = i_size_read(mapping->host);
-
- if (ret != AOP_TRUNCATED_PAGE)
- unlock_page(page);
- page_cache_release(page);
- if (ret == AOP_TRUNCATED_PAGE)
- goto find_page;
-
- /*
- * prepare_write() may have instantiated a few blocks
- * outside i_size. Trim these off again.
- */
- if (sd->pos + this_len > isize)
- vmtruncate(mapping->host, isize);
-
- goto out_ret;
- }
+ ret = pagecache_write_begin(file, mapping, sd->pos, this_len,
+ AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata);
+ if (unlikely(ret))
+ goto out;
if (buf->page != page) {
/*
kunmap_atomic(dst, KM_USER1);
buf->ops->unmap(pipe, buf, src);
}
-
- ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
- if (ret) {
- if (ret == AOP_TRUNCATED_PAGE) {
- page_cache_release(page);
- goto find_page;
- }
- if (ret < 0)
- goto out;
- /*
- * Partial write has happened, so 'ret' already initialized by
- * number of bytes written, Where is nothing we have to do here.
- */
- } else
- ret = this_len;
- /*
- * Return the number of bytes written and mark page as
- * accessed, we are now done!
- */
- mark_page_accessed(page);
+ ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len,
+ page, fsdata);
out:
- page_cache_release(page);
- unlock_page(page);
-out_ret:
return ret;
}
if (unlikely(ret < 0))
return ret;
+ ret = security_file_permission(out, MAY_WRITE);
+ if (unlikely(ret < 0))
+ return ret;
+
return out->f_op->splice_write(pipe, out, ppos, len, flags);
}
if (unlikely(ret < 0))
return ret;
+ ret = security_file_permission(in, MAY_READ);
+ if (unlikely(ret < 0))
+ return ret;
+
return in->f_op->splice_read(in, ppos, pipe, len, flags);
}
* Description:
* This is a special case helper to splice directly between two
* points, without requiring an explicit pipe. Internally an allocated
- * pipe is cached in the process, and reused during the life time of
+ * pipe is cached in the process, and reused during the lifetime of
* that process.
*
*/
sd->flags &= ~SPLICE_F_NONBLOCK;
while (len) {
- size_t read_len, max_read_len;
-
- /*
- * Do at most PIPE_BUFFERS pages worth of transfer:
- */
- max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
+ size_t read_len;
+ loff_t pos = sd->pos;
- ret = do_splice_to(in, &sd->pos, pipe, max_read_len, flags);
- if (unlikely(ret < 0))
+ ret = do_splice_to(in, &pos, pipe, len, flags);
+ if (unlikely(ret <= 0))
goto out_release;
read_len = ret;
* could get stuck data in the internal pipe:
*/
ret = actor(pipe, sd);
- if (unlikely(ret < 0))
+ if (unlikely(ret <= 0))
goto out_release;
bytes += ret;
len -= ret;
+ sd->pos = pos;
- /*
- * In nonblocking mode, if we got back a short read then
- * that was due to either an IO error or due to the
- * pagecache entry not being there. In the IO error case
- * the _next_ splice attempt will produce a clean IO error
- * return value (not a short read), so in both cases it's
- * correct to break out of the loop here:
- */
- if ((flags & SPLICE_F_NONBLOCK) && (read_len < max_read_len))
- break;
+ if (ret < read_len)
+ goto out_release;
}
pipe->nrbufs = pipe->curbuf = 0;
-
return bytes;
out_release:
.pos = *ppos,
.u.file = out,
};
- size_t ret;
+ long ret;
ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
- *ppos = sd.pos;
+ if (ret > 0)
+ *ppos += ret;
+
return ret;
}
return -EINVAL;
}
+/*
+ * Do a copy-from-user while holding the mmap_semaphore for reading, in a
+ * manner safe from deadlocking with simultaneous mmap() (grabbing mmap_sem
+ * for writing) and page faulting on the user memory pointed to by src.
+ * This assumes that we will very rarely hit the partial != 0 path, or this
+ * will not be a win.
+ */
+static int copy_from_user_mmap_sem(void *dst, const void __user *src, size_t n)
+{
+ int partial;
+
+ pagefault_disable();
+ partial = __copy_from_user_inatomic(dst, src, n);
+ pagefault_enable();
+
+ /*
+ * Didn't copy everything, drop the mmap_sem and do a faulting copy
+ */
+ if (unlikely(partial)) {
+ up_read(¤t->mm->mmap_sem);
+ partial = copy_from_user(dst, src, n);
+ down_read(¤t->mm->mmap_sem);
+ }
+
+ return partial;
+}
+
/*
* Map an iov into an array of pages and offset/length tupples. With the
* partial_page structure, we can map several non-contiguous ranges into
{
int buffers = 0, error = 0;
- /*
- * It's ok to take the mmap_sem for reading, even
- * across a "get_user()".
- */
down_read(¤t->mm->mmap_sem);
while (nr_vecs) {
unsigned long off, npages;
+ struct iovec entry;
void __user *base;
size_t len;
int i;
- /*
- * Get user address base and length for this iovec.
- */
- error = get_user(base, &iov->iov_base);
- if (unlikely(error))
- break;
- error = get_user(len, &iov->iov_len);
- if (unlikely(error))
+ error = -EFAULT;
+ if (copy_from_user_mmap_sem(&entry, iov, sizeof(entry)))
break;
+ base = entry.iov_base;
+ len = entry.iov_len;
+
/*
* Sanity check this iovec. 0 read succeeds.
*/
+ error = 0;
if (unlikely(!len))
break;
error = -EFAULT;