4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
14 #include <linux/mount.h>
15 #include <linux/pipe_fs_i.h>
16 #include <linux/uio.h>
17 #include <linux/highmem.h>
18 #include <linux/pagemap.h>
20 #include <asm/uaccess.h>
21 #include <asm/ioctls.h>
24 * We use a start+len construction, which provides full use of the
26 * -- Florian Coosmann (FGC)
28 * Reads with count = 0 should always return 0.
29 * -- Julian Bradfield 1999-06-07.
31 * FIFOs and Pipes now generate SIGIO for both readers and writers.
32 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
34 * pipe_read & write cleanup
35 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
38 /* Drop the inode semaphore and wait for a pipe event, atomically */
39 void pipe_wait(struct pipe_inode_info *pipe)
44 * Pipes are system-local resources, so sleeping on them
45 * is considered a noninteractive wait:
47 prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE|TASK_NONINTERACTIVE);
49 mutex_unlock(&pipe->inode->i_mutex);
51 finish_wait(&pipe->wait, &wait);
53 mutex_lock(&pipe->inode->i_mutex);
57 pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len)
64 copy = min_t(unsigned long, len, iov->iov_len);
66 if (copy_from_user(to, iov->iov_base, copy))
70 iov->iov_base += copy;
77 pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len)
84 copy = min_t(unsigned long, len, iov->iov_len);
86 if (copy_to_user(iov->iov_base, from, copy))
90 iov->iov_base += copy;
96 static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buffer *buf)
98 struct page *page = buf->page;
100 buf->flags &= ~PIPE_BUF_FLAG_STOLEN;
103 * If nobody else uses this page, and we don't already have a
104 * temporary page, let's keep track of it as a one-deep
107 if (page_count(page) == 1 && !info->tmp_page) {
108 info->tmp_page = page;
113 * Otherwise just release our reference to it
115 page_cache_release(page);
118 static void *anon_pipe_buf_map(struct file *file, struct pipe_inode_info *info, struct pipe_buffer *buf)
120 return kmap(buf->page);
123 static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer *buf)
128 static int anon_pipe_buf_steal(struct pipe_inode_info *info,
129 struct pipe_buffer *buf)
131 buf->flags |= PIPE_BUF_FLAG_STOLEN;
135 static struct pipe_buf_operations anon_pipe_buf_ops = {
137 .map = anon_pipe_buf_map,
138 .unmap = anon_pipe_buf_unmap,
139 .release = anon_pipe_buf_release,
140 .steal = anon_pipe_buf_steal,
144 pipe_readv(struct file *filp, const struct iovec *_iov,
145 unsigned long nr_segs, loff_t *ppos)
147 struct inode *inode = filp->f_dentry->d_inode;
148 struct pipe_inode_info *info;
151 struct iovec *iov = (struct iovec *)_iov;
154 total_len = iov_length(iov, nr_segs);
155 /* Null read succeeds. */
156 if (unlikely(total_len == 0))
161 mutex_lock(PIPE_MUTEX(*inode));
162 info = inode->i_pipe;
164 int bufs = info->nrbufs;
166 int curbuf = info->curbuf;
167 struct pipe_buffer *buf = info->bufs + curbuf;
168 struct pipe_buf_operations *ops = buf->ops;
170 size_t chars = buf->len;
173 if (chars > total_len)
176 addr = ops->map(filp, info, buf);
182 error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars);
183 ops->unmap(info, buf);
184 if (unlikely(error)) {
185 if (!ret) ret = -EFAULT;
189 buf->offset += chars;
193 ops->release(info, buf);
194 curbuf = (curbuf + 1) & (PIPE_BUFFERS-1);
195 info->curbuf = curbuf;
196 info->nrbufs = --bufs;
201 break; /* common path: read succeeded */
203 if (bufs) /* More to do? */
205 if (!PIPE_WRITERS(*inode))
207 if (!PIPE_WAITING_WRITERS(*inode)) {
208 /* syscall merging: Usually we must not sleep
209 * if O_NONBLOCK is set, or if we got some data.
210 * But if a writer sleeps in kernel space, then
211 * we can wait for that data without violating POSIX.
215 if (filp->f_flags & O_NONBLOCK) {
220 if (signal_pending(current)) {
221 if (!ret) ret = -ERESTARTSYS;
225 wake_up_interruptible_sync(PIPE_WAIT(*inode));
226 kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
228 pipe_wait(inode->i_pipe);
230 mutex_unlock(PIPE_MUTEX(*inode));
231 /* Signal writers asynchronously that there is more room. */
233 wake_up_interruptible(PIPE_WAIT(*inode));
234 kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
242 pipe_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
244 struct iovec iov = { .iov_base = buf, .iov_len = count };
245 return pipe_readv(filp, &iov, 1, ppos);
249 pipe_writev(struct file *filp, const struct iovec *_iov,
250 unsigned long nr_segs, loff_t *ppos)
252 struct inode *inode = filp->f_dentry->d_inode;
253 struct pipe_inode_info *info;
256 struct iovec *iov = (struct iovec *)_iov;
260 total_len = iov_length(iov, nr_segs);
261 /* Null write succeeds. */
262 if (unlikely(total_len == 0))
267 mutex_lock(PIPE_MUTEX(*inode));
268 info = inode->i_pipe;
270 if (!PIPE_READERS(*inode)) {
271 send_sig(SIGPIPE, current, 0);
276 /* We try to merge small writes */
277 chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
278 if (info->nrbufs && chars != 0) {
279 int lastbuf = (info->curbuf + info->nrbufs - 1) & (PIPE_BUFFERS-1);
280 struct pipe_buffer *buf = info->bufs + lastbuf;
281 struct pipe_buf_operations *ops = buf->ops;
282 int offset = buf->offset + buf->len;
283 if (ops->can_merge && offset + chars <= PAGE_SIZE) {
287 addr = ops->map(filp, info, buf);
289 error = PTR_ERR(addr);
292 error = pipe_iov_copy_from_user(offset + addr, iov,
294 ops->unmap(info, buf);
309 if (!PIPE_READERS(*inode)) {
310 send_sig(SIGPIPE, current, 0);
311 if (!ret) ret = -EPIPE;
315 if (bufs < PIPE_BUFFERS) {
316 int newbuf = (info->curbuf + bufs) & (PIPE_BUFFERS-1);
317 struct pipe_buffer *buf = info->bufs + newbuf;
318 struct page *page = info->tmp_page;
322 page = alloc_page(GFP_HIGHUSER);
323 if (unlikely(!page)) {
324 ret = ret ? : -ENOMEM;
327 info->tmp_page = page;
329 /* Always wakeup, even if the copy fails. Otherwise
330 * we lock up (O_NONBLOCK-)readers that sleep due to
332 * FIXME! Is this really true?
336 if (chars > total_len)
339 error = pipe_iov_copy_from_user(kmap(page), iov, chars);
341 if (unlikely(error)) {
342 if (!ret) ret = -EFAULT;
347 /* Insert it into the buffer array */
349 buf->ops = &anon_pipe_buf_ops;
352 info->nrbufs = ++bufs;
353 info->tmp_page = NULL;
359 if (bufs < PIPE_BUFFERS)
361 if (filp->f_flags & O_NONBLOCK) {
362 if (!ret) ret = -EAGAIN;
365 if (signal_pending(current)) {
366 if (!ret) ret = -ERESTARTSYS;
370 wake_up_interruptible_sync(PIPE_WAIT(*inode));
371 kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN);
374 PIPE_WAITING_WRITERS(*inode)++;
375 pipe_wait(inode->i_pipe);
376 PIPE_WAITING_WRITERS(*inode)--;
379 mutex_unlock(PIPE_MUTEX(*inode));
381 wake_up_interruptible(PIPE_WAIT(*inode));
382 kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN);
385 file_update_time(filp);
390 pipe_write(struct file *filp, const char __user *buf,
391 size_t count, loff_t *ppos)
393 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = count };
394 return pipe_writev(filp, &iov, 1, ppos);
398 bad_pipe_r(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
404 bad_pipe_w(struct file *filp, const char __user *buf, size_t count, loff_t *ppos)
410 pipe_ioctl(struct inode *pino, struct file *filp,
411 unsigned int cmd, unsigned long arg)
413 struct inode *inode = filp->f_dentry->d_inode;
414 struct pipe_inode_info *info;
415 int count, buf, nrbufs;
419 mutex_lock(PIPE_MUTEX(*inode));
420 info = inode->i_pipe;
423 nrbufs = info->nrbufs;
424 while (--nrbufs >= 0) {
425 count += info->bufs[buf].len;
426 buf = (buf+1) & (PIPE_BUFFERS-1);
428 mutex_unlock(PIPE_MUTEX(*inode));
429 return put_user(count, (int __user *)arg);
435 /* No kernel lock held - fine */
437 pipe_poll(struct file *filp, poll_table *wait)
440 struct inode *inode = filp->f_dentry->d_inode;
441 struct pipe_inode_info *info = inode->i_pipe;
444 poll_wait(filp, PIPE_WAIT(*inode), wait);
446 /* Reading only -- no need for acquiring the semaphore. */
447 nrbufs = info->nrbufs;
449 if (filp->f_mode & FMODE_READ) {
450 mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0;
451 if (!PIPE_WRITERS(*inode) && filp->f_version != PIPE_WCOUNTER(*inode))
455 if (filp->f_mode & FMODE_WRITE) {
456 mask |= (nrbufs < PIPE_BUFFERS) ? POLLOUT | POLLWRNORM : 0;
458 * Most Unices do not set POLLERR for FIFOs but on Linux they
459 * behave exactly like pipes for poll().
461 if (!PIPE_READERS(*inode))
469 pipe_release(struct inode *inode, int decr, int decw)
471 mutex_lock(PIPE_MUTEX(*inode));
472 PIPE_READERS(*inode) -= decr;
473 PIPE_WRITERS(*inode) -= decw;
474 if (!PIPE_READERS(*inode) && !PIPE_WRITERS(*inode)) {
475 free_pipe_info(inode);
477 wake_up_interruptible(PIPE_WAIT(*inode));
478 kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN);
479 kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
481 mutex_unlock(PIPE_MUTEX(*inode));
487 pipe_read_fasync(int fd, struct file *filp, int on)
489 struct inode *inode = filp->f_dentry->d_inode;
492 mutex_lock(PIPE_MUTEX(*inode));
493 retval = fasync_helper(fd, filp, on, PIPE_FASYNC_READERS(*inode));
494 mutex_unlock(PIPE_MUTEX(*inode));
504 pipe_write_fasync(int fd, struct file *filp, int on)
506 struct inode *inode = filp->f_dentry->d_inode;
509 mutex_lock(PIPE_MUTEX(*inode));
510 retval = fasync_helper(fd, filp, on, PIPE_FASYNC_WRITERS(*inode));
511 mutex_unlock(PIPE_MUTEX(*inode));
521 pipe_rdwr_fasync(int fd, struct file *filp, int on)
523 struct inode *inode = filp->f_dentry->d_inode;
526 mutex_lock(PIPE_MUTEX(*inode));
528 retval = fasync_helper(fd, filp, on, PIPE_FASYNC_READERS(*inode));
531 retval = fasync_helper(fd, filp, on, PIPE_FASYNC_WRITERS(*inode));
533 mutex_unlock(PIPE_MUTEX(*inode));
543 pipe_read_release(struct inode *inode, struct file *filp)
545 pipe_read_fasync(-1, filp, 0);
546 return pipe_release(inode, 1, 0);
550 pipe_write_release(struct inode *inode, struct file *filp)
552 pipe_write_fasync(-1, filp, 0);
553 return pipe_release(inode, 0, 1);
557 pipe_rdwr_release(struct inode *inode, struct file *filp)
561 pipe_rdwr_fasync(-1, filp, 0);
562 decr = (filp->f_mode & FMODE_READ) != 0;
563 decw = (filp->f_mode & FMODE_WRITE) != 0;
564 return pipe_release(inode, decr, decw);
568 pipe_read_open(struct inode *inode, struct file *filp)
570 /* We could have perhaps used atomic_t, but this and friends
571 below are the only places. So it doesn't seem worthwhile. */
572 mutex_lock(PIPE_MUTEX(*inode));
573 PIPE_READERS(*inode)++;
574 mutex_unlock(PIPE_MUTEX(*inode));
580 pipe_write_open(struct inode *inode, struct file *filp)
582 mutex_lock(PIPE_MUTEX(*inode));
583 PIPE_WRITERS(*inode)++;
584 mutex_unlock(PIPE_MUTEX(*inode));
590 pipe_rdwr_open(struct inode *inode, struct file *filp)
592 mutex_lock(PIPE_MUTEX(*inode));
593 if (filp->f_mode & FMODE_READ)
594 PIPE_READERS(*inode)++;
595 if (filp->f_mode & FMODE_WRITE)
596 PIPE_WRITERS(*inode)++;
597 mutex_unlock(PIPE_MUTEX(*inode));
603 * The file_operations structs are not static because they
604 * are also used in linux/fs/fifo.c to do operations on FIFOs.
606 const struct file_operations read_fifo_fops = {
613 .open = pipe_read_open,
614 .release = pipe_read_release,
615 .fasync = pipe_read_fasync,
618 const struct file_operations write_fifo_fops = {
622 .writev = pipe_writev,
625 .open = pipe_write_open,
626 .release = pipe_write_release,
627 .fasync = pipe_write_fasync,
630 const struct file_operations rdwr_fifo_fops = {
635 .writev = pipe_writev,
638 .open = pipe_rdwr_open,
639 .release = pipe_rdwr_release,
640 .fasync = pipe_rdwr_fasync,
643 static struct file_operations read_pipe_fops = {
650 .open = pipe_read_open,
651 .release = pipe_read_release,
652 .fasync = pipe_read_fasync,
655 static struct file_operations write_pipe_fops = {
659 .writev = pipe_writev,
662 .open = pipe_write_open,
663 .release = pipe_write_release,
664 .fasync = pipe_write_fasync,
667 static struct file_operations rdwr_pipe_fops = {
672 .writev = pipe_writev,
675 .open = pipe_rdwr_open,
676 .release = pipe_rdwr_release,
677 .fasync = pipe_rdwr_fasync,
680 struct pipe_inode_info * alloc_pipe_info(struct inode *inode)
682 struct pipe_inode_info *info;
684 info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
686 init_waitqueue_head(&info->wait);
687 info->r_counter = info->w_counter = 1;
694 void free_pipe_info(struct inode *inode)
697 struct pipe_inode_info *info = inode->i_pipe;
699 inode->i_pipe = NULL;
700 for (i = 0; i < PIPE_BUFFERS; i++) {
701 struct pipe_buffer *buf = info->bufs + i;
703 buf->ops->release(info, buf);
706 __free_page(info->tmp_page);
710 static struct vfsmount *pipe_mnt __read_mostly;
711 static int pipefs_delete_dentry(struct dentry *dentry)
715 static struct dentry_operations pipefs_dentry_operations = {
716 .d_delete = pipefs_delete_dentry,
719 static struct inode * get_pipe_inode(void)
721 struct inode *inode = new_inode(pipe_mnt->mnt_sb);
726 inode->i_pipe = alloc_pipe_info(inode);
730 PIPE_READERS(*inode) = PIPE_WRITERS(*inode) = 1;
731 inode->i_fop = &rdwr_pipe_fops;
734 * Mark the inode dirty from the very beginning,
735 * that way it will never be moved to the dirty
736 * list because "mark_inode_dirty()" will think
737 * that it already _is_ on the dirty list.
739 inode->i_state = I_DIRTY;
740 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
741 inode->i_uid = current->fsuid;
742 inode->i_gid = current->fsgid;
743 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
744 inode->i_blksize = PAGE_SIZE;
757 struct dentry *dentry;
758 struct inode * inode;
759 struct file *f1, *f2;
764 f1 = get_empty_filp();
768 f2 = get_empty_filp();
772 inode = get_pipe_inode();
776 error = get_unused_fd();
778 goto close_f12_inode;
781 error = get_unused_fd();
783 goto close_f12_inode_i;
787 sprintf(name, "[%lu]", inode->i_ino);
789 this.len = strlen(name);
790 this.hash = inode->i_ino; /* will go */
791 dentry = d_alloc(pipe_mnt->mnt_sb->s_root, &this);
793 goto close_f12_inode_i_j;
794 dentry->d_op = &pipefs_dentry_operations;
795 d_add(dentry, inode);
796 f1->f_vfsmnt = f2->f_vfsmnt = mntget(mntget(pipe_mnt));
797 f1->f_dentry = f2->f_dentry = dget(dentry);
798 f1->f_mapping = f2->f_mapping = inode->i_mapping;
801 f1->f_pos = f2->f_pos = 0;
802 f1->f_flags = O_RDONLY;
803 f1->f_op = &read_pipe_fops;
804 f1->f_mode = FMODE_READ;
808 f2->f_flags = O_WRONLY;
809 f2->f_op = &write_pipe_fops;
810 f2->f_mode = FMODE_WRITE;
824 free_pipe_info(inode);
835 * pipefs should _never_ be mounted by userland - too much of security hassle,
836 * no real gain from having the whole whorehouse mounted. So we don't need
837 * any operations on the root directory. However, we need a non-trivial
838 * d_name - pipe: will go nicely and kill the special-casing in procfs.
841 static struct super_block *pipefs_get_sb(struct file_system_type *fs_type,
842 int flags, const char *dev_name, void *data)
844 return get_sb_pseudo(fs_type, "pipe:", NULL, PIPEFS_MAGIC);
847 static struct file_system_type pipe_fs_type = {
849 .get_sb = pipefs_get_sb,
850 .kill_sb = kill_anon_super,
853 static int __init init_pipe_fs(void)
855 int err = register_filesystem(&pipe_fs_type);
857 pipe_mnt = kern_mount(&pipe_fs_type);
858 if (IS_ERR(pipe_mnt)) {
859 err = PTR_ERR(pipe_mnt);
860 unregister_filesystem(&pipe_fs_type);
866 static void __exit exit_pipe_fs(void)
868 unregister_filesystem(&pipe_fs_type);
872 fs_initcall(init_pipe_fs);
873 module_exit(exit_pipe_fs);