From 027445c37282bc1ed26add45e573ad2d3e4860a5 Mon Sep 17 00:00:00 2001 From: Badari Pulavarty Date: Sat, 30 Sep 2006 23:28:46 -0700 Subject: [PATCH] Vectorize aio_read/aio_write fileop methods This patch vectorizes aio_read() and aio_write() methods to prepare for collapsing all aio & vectored operations into one interface - which is aio_read()/aio_write(). Signed-off-by: Badari Pulavarty Signed-off-by: Christoph Hellwig Cc: Michael Holzheu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/read_write.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'fs/read_write.c') diff --git a/fs/read_write.c b/fs/read_write.c index d4cb3183c99..679dd535380 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -227,14 +227,20 @@ static void wait_on_retry_sync_kiocb(struct kiocb *iocb) ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos) { + struct iovec iov = { .iov_base = buf, .iov_len = len }; struct kiocb kiocb; ssize_t ret; init_sync_kiocb(&kiocb, filp); kiocb.ki_pos = *ppos; - while (-EIOCBRETRY == - (ret = filp->f_op->aio_read(&kiocb, buf, len, kiocb.ki_pos))) + kiocb.ki_left = len; + + for (;;) { + ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos); + if (ret != -EIOCBRETRY) + break; wait_on_retry_sync_kiocb(&kiocb); + } if (-EIOCBQUEUED == ret) ret = wait_on_sync_kiocb(&kiocb); @@ -279,14 +285,20 @@ EXPORT_SYMBOL(vfs_read); ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) { + struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len }; struct kiocb kiocb; ssize_t ret; init_sync_kiocb(&kiocb, filp); kiocb.ki_pos = *ppos; - while (-EIOCBRETRY == - (ret = filp->f_op->aio_write(&kiocb, buf, len, kiocb.ki_pos))) + kiocb.ki_left = len; + + for (;;) { + ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos); + if (ret != -EIOCBRETRY) + break; wait_on_retry_sync_kiocb(&kiocb); + } if (-EIOCBQUEUED == ret) ret = wait_on_sync_kiocb(&kiocb); -- cgit v1.2.3-70-g09d2 From ee0b3e671baff681d69fbf0db33b47603c0a8280 Mon Sep 17 00:00:00 2001 From: Badari Pulavarty Date: Sat, 30 Sep 2006 23:28:47 -0700 Subject: [PATCH] Remove readv/writev methods and use aio_read/aio_write instead This patch removes readv() and writev() methods and replaces them with aio_read()/aio_write() methods. Signed-off-by: Badari Pulavarty Signed-off-by: Christoph Hellwig Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/char/raw.c | 2 - drivers/net/tun.c | 37 +++++----------- fs/bad_inode.c | 2 - fs/block_dev.c | 2 - fs/cifs/cifsfs.c | 16 ------- fs/compat.c | 44 +++++-------------- fs/ext2/file.c | 2 - fs/ext3/file.c | 2 - fs/fat/file.c | 2 - fs/fuse/dev.c | 37 +++++----------- fs/hostfs/hostfs_kern.c | 2 - fs/jfs/file.c | 2 - fs/ntfs/file.c | 2 - fs/pipe.c | 59 ++++++++++---------------- fs/read_write.c | 101 +++++++++++++++++++++++++++++--------------- fs/read_write.h | 14 ++++++ fs/xfs/linux-2.6/xfs_file.c | 94 ----------------------------------------- include/linux/fs.h | 6 --- mm/filemap.c | 36 ---------------- net/socket.c | 40 ------------------ sound/core/pcm_native.c | 40 +++++++++--------- 21 files changed, 154 insertions(+), 388 deletions(-) create mode 100644 fs/read_write.h (limited to 'fs/read_write.c') diff --git a/drivers/char/raw.c b/drivers/char/raw.c index 173fb08555d..490db531e2d 100644 --- a/drivers/char/raw.c +++ b/drivers/char/raw.c @@ -257,8 +257,6 @@ static const struct file_operations raw_fops = { .open = raw_open, .release= raw_release, .ioctl = raw_ioctl, - .readv = generic_file_readv, - .writev = generic_file_writev, .owner = THIS_MODULE, }; diff --git a/drivers/net/tun.c b/drivers/net/tun.c index de8da6dee1b..bc0face01d2 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -288,11 +288,10 @@ static inline size_t iov_total(const struct iovec *iv, unsigned long count) return len; } -/* Writev */ -static ssize_t tun_chr_writev(struct file * file, const struct iovec *iv, - unsigned long count, loff_t *pos) +static ssize_t tun_chr_aio_write(struct kiocb *iocb, const struct iovec *iv, + unsigned long count, loff_t pos) { - struct tun_struct *tun = file->private_data; + struct tun_struct *tun = iocb->ki_filp->private_data; if (!tun) return -EBADFD; @@ -302,14 +301,6 @@ static ssize_t tun_chr_writev(struct file * file, const struct iovec *iv, return tun_get_user(tun, (struct iovec *) iv, iov_total(iv, count)); } -/* Write */ -static ssize_t tun_chr_write(struct file * file, const char __user * buf, - size_t count, loff_t *pos) -{ - struct iovec iv = { (void __user *) buf, count }; - return tun_chr_writev(file, &iv, 1, pos); -} - /* Put packet to the user space buffer */ static __inline__ ssize_t tun_put_user(struct tun_struct *tun, struct sk_buff *skb, @@ -343,10 +334,10 @@ static __inline__ ssize_t tun_put_user(struct tun_struct *tun, return total; } -/* Readv */ -static ssize_t tun_chr_readv(struct file *file, const struct iovec *iv, - unsigned long count, loff_t *pos) +static ssize_t tun_chr_aio_read(struct kiocb *iocb, const struct iovec *iv, + unsigned long count, loff_t pos) { + struct file *file = iocb->ki_filp; struct tun_struct *tun = file->private_data; DECLARE_WAITQUEUE(wait, current); struct sk_buff *skb; @@ -426,14 +417,6 @@ static ssize_t tun_chr_readv(struct file *file, const struct iovec *iv, return ret; } -/* Read */ -static ssize_t tun_chr_read(struct file * file, char __user * buf, - size_t count, loff_t *pos) -{ - struct iovec iv = { buf, count }; - return tun_chr_readv(file, &iv, 1, pos); -} - static void tun_setup(struct net_device *dev) { struct tun_struct *tun = netdev_priv(dev); @@ -764,10 +747,10 @@ static int tun_chr_close(struct inode *inode, struct file *file) static struct file_operations tun_fops = { .owner = THIS_MODULE, .llseek = no_llseek, - .read = tun_chr_read, - .readv = tun_chr_readv, - .write = tun_chr_write, - .writev = tun_chr_writev, + .read = do_sync_read, + .aio_read = tun_chr_aio_read, + .write = do_sync_write, + .aio_write = tun_chr_aio_write, .poll = tun_chr_poll, .ioctl = tun_chr_ioctl, .open = tun_chr_open, diff --git a/fs/bad_inode.c b/fs/bad_inode.c index 80599ae3396..34e6d7b220c 100644 --- a/fs/bad_inode.c +++ b/fs/bad_inode.c @@ -40,8 +40,6 @@ static const struct file_operations bad_file_ops = .aio_fsync = EIO_ERROR, .fasync = EIO_ERROR, .lock = EIO_ERROR, - .readv = EIO_ERROR, - .writev = EIO_ERROR, .sendfile = EIO_ERROR, .sendpage = EIO_ERROR, .get_unmapped_area = EIO_ERROR, diff --git a/fs/block_dev.c b/fs/block_dev.c index 8c819117310..0f143094ef1 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1191,8 +1191,6 @@ const struct file_operations def_blk_fops = { #ifdef CONFIG_COMPAT .compat_ioctl = compat_blkdev_ioctl, #endif - .readv = generic_file_readv, - .writev = generic_file_write_nolock, .sendfile = generic_file_sendfile, .splice_read = generic_file_splice_read, .splice_write = generic_file_splice_write, diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index 5abb42a7c53..c00c654f2e1 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -480,18 +480,6 @@ cifs_get_sb(struct file_system_type *fs_type, return simple_set_mnt(mnt, sb); } -static ssize_t cifs_file_writev(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) -{ - struct inode *inode = file->f_dentry->d_inode; - ssize_t written; - - written = generic_file_writev(file, iov, nr_segs, ppos); - if (!CIFS_I(inode)->clientCanCacheAll) - filemap_fdatawrite(inode->i_mapping); - return written; -} - static ssize_t cifs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { @@ -577,8 +565,6 @@ struct inode_operations cifs_symlink_inode_ops = { const struct file_operations cifs_file_ops = { .read = do_sync_read, .write = do_sync_write, - .readv = generic_file_readv, - .writev = cifs_file_writev, .aio_read = generic_file_aio_read, .aio_write = cifs_file_aio_write, .open = cifs_open, @@ -620,8 +606,6 @@ const struct file_operations cifs_file_direct_ops = { const struct file_operations cifs_file_nobrl_ops = { .read = do_sync_read, .write = do_sync_write, - .readv = generic_file_readv, - .writev = cifs_file_writev, .aio_read = generic_file_aio_read, .aio_write = cifs_file_aio_write, .open = cifs_open, diff --git a/fs/compat.c b/fs/compat.c index 122b4e3992b..6b90bf35f61 100644 --- a/fs/compat.c +++ b/fs/compat.c @@ -70,6 +70,8 @@ int compat_printk(const char *fmt, ...) return ret; } +#include "read_write.h" + /* * Not all architectures have sys_utime, so implement this in terms * of sys_utimes. @@ -1149,9 +1151,6 @@ static ssize_t compat_do_readv_writev(int type, struct file *file, const struct compat_iovec __user *uvector, unsigned long nr_segs, loff_t *pos) { - typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *); - typedef ssize_t (*iov_fn_t)(struct file *, const struct iovec *, unsigned long, loff_t *); - compat_ssize_t tot_len; struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov=iovstack, *vector; @@ -1234,39 +1233,18 @@ static ssize_t compat_do_readv_writev(int type, struct file *file, fnv = NULL; if (type == READ) { fn = file->f_op->read; - fnv = file->f_op->readv; + fnv = file->f_op->aio_read; } else { fn = (io_fn_t)file->f_op->write; - fnv = file->f_op->writev; - } - if (fnv) { - ret = fnv(file, iov, nr_segs, pos); - goto out; + fnv = file->f_op->aio_write; } - /* Do it by hand, with file-ops */ - ret = 0; - vector = iov; - while (nr_segs > 0) { - void __user * base; - size_t len; - ssize_t nr; - - base = vector->iov_base; - len = vector->iov_len; - vector++; - nr_segs--; - - nr = fn(file, base, len, pos); + if (fnv) + ret = do_sync_readv_writev(file, iov, nr_segs, tot_len, + pos, fnv); + else + ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn); - if (nr < 0) { - if (!ret) ret = nr; - break; - } - ret += nr; - if (nr != len) - break; - } out: if (iov != iovstack) kfree(iov); @@ -1294,7 +1272,7 @@ compat_sys_readv(unsigned long fd, const struct compat_iovec __user *vec, unsign goto out; ret = -EINVAL; - if (!file->f_op || (!file->f_op->readv && !file->f_op->read)) + if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read)) goto out; ret = compat_do_readv_writev(READ, file, vec, vlen, &file->f_pos); @@ -1317,7 +1295,7 @@ compat_sys_writev(unsigned long fd, const struct compat_iovec __user *vec, unsig goto out; ret = -EINVAL; - if (!file->f_op || (!file->f_op->writev && !file->f_op->write)) + if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write)) goto out; ret = compat_do_readv_writev(WRITE, file, vec, vlen, &file->f_pos); diff --git a/fs/ext2/file.c b/fs/ext2/file.c index e8bbed9dd26..e893e2be9ed 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -53,8 +53,6 @@ const struct file_operations ext2_file_operations = { .open = generic_file_open, .release = ext2_release_file, .fsync = ext2_sync_file, - .readv = generic_file_readv, - .writev = generic_file_writev, .sendfile = generic_file_sendfile, .splice_read = generic_file_splice_read, .splice_write = generic_file_splice_write, diff --git a/fs/ext3/file.c b/fs/ext3/file.c index 5c762457bc8..e96c388047e 100644 --- a/fs/ext3/file.c +++ b/fs/ext3/file.c @@ -112,8 +112,6 @@ const struct file_operations ext3_file_operations = { .write = do_sync_write, .aio_read = generic_file_aio_read, .aio_write = ext3_file_write, - .readv = generic_file_readv, - .writev = generic_file_writev, .ioctl = ext3_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = ext3_compat_ioctl, diff --git a/fs/fat/file.c b/fs/fat/file.c index d50fc47169c..f4b8f8b3fbd 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -127,8 +127,6 @@ const struct file_operations fat_file_operations = { .llseek = generic_file_llseek, .read = do_sync_read, .write = do_sync_write, - .readv = generic_file_readv, - .writev = generic_file_writev, .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 4fc557c40cc..66571eafbb1 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -680,14 +680,15 @@ static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_req *req, * request_end(). Otherwise add it to the processing list, and set * the 'sent' flag. */ -static ssize_t fuse_dev_readv(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *off) +static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) { int err; struct fuse_req *req; struct fuse_in *in; struct fuse_copy_state cs; unsigned reqsize; + struct file *file = iocb->ki_filp; struct fuse_conn *fc = fuse_get_conn(file); if (!fc) return -EPERM; @@ -761,15 +762,6 @@ static ssize_t fuse_dev_readv(struct file *file, const struct iovec *iov, return err; } -static ssize_t fuse_dev_read(struct file *file, char __user *buf, - size_t nbytes, loff_t *off) -{ - struct iovec iov; - iov.iov_len = nbytes; - iov.iov_base = buf; - return fuse_dev_readv(file, &iov, 1, off); -} - /* Look up request on processing list by unique ID */ static struct fuse_req *request_find(struct fuse_conn *fc, u64 unique) { @@ -814,15 +806,15 @@ static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out, * it from the list and copy the rest of the buffer to the request. * The request is finished by calling request_end() */ -static ssize_t fuse_dev_writev(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *off) +static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) { int err; unsigned nbytes = iov_length(iov, nr_segs); struct fuse_req *req; struct fuse_out_header oh; struct fuse_copy_state cs; - struct fuse_conn *fc = fuse_get_conn(file); + struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp); if (!fc) return -EPERM; @@ -898,15 +890,6 @@ static ssize_t fuse_dev_writev(struct file *file, const struct iovec *iov, return err; } -static ssize_t fuse_dev_write(struct file *file, const char __user *buf, - size_t nbytes, loff_t *off) -{ - struct iovec iov; - iov.iov_len = nbytes; - iov.iov_base = (char __user *) buf; - return fuse_dev_writev(file, &iov, 1, off); -} - static unsigned fuse_dev_poll(struct file *file, poll_table *wait) { unsigned mask = POLLOUT | POLLWRNORM; @@ -1041,10 +1024,10 @@ static int fuse_dev_fasync(int fd, struct file *file, int on) const struct file_operations fuse_dev_operations = { .owner = THIS_MODULE, .llseek = no_llseek, - .read = fuse_dev_read, - .readv = fuse_dev_readv, - .write = fuse_dev_write, - .writev = fuse_dev_writev, + .read = do_sync_read, + .aio_read = fuse_dev_read, + .write = do_sync_write, + .aio_write = fuse_dev_write, .poll = fuse_dev_poll, .release = fuse_dev_release, .fasync = fuse_dev_fasync, diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index 322e876c35e..4908c38a588 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -389,8 +389,6 @@ static const struct file_operations hostfs_file_fops = { .sendfile = generic_file_sendfile, .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, - .readv = generic_file_readv, - .writev = generic_file_writev, .write = generic_file_write, .mmap = generic_file_mmap, .open = hostfs_file_open, diff --git a/fs/jfs/file.c b/fs/jfs/file.c index 1c9745be5ad..f535f2911c1 100644 --- a/fs/jfs/file.c +++ b/fs/jfs/file.c @@ -108,8 +108,6 @@ const struct file_operations jfs_file_operations = { .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, - .readv = generic_file_readv, - .writev = generic_file_writev, .sendfile = generic_file_sendfile, .fsync = jfs_fsync, .release = jfs_release, diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index 0c46f5c86b7..2f9b5a0953f 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -2298,11 +2298,9 @@ const struct file_operations ntfs_file_ops = { .llseek = generic_file_llseek, /* Seek inside file. */ .read = generic_file_read, /* Read from file. */ .aio_read = generic_file_aio_read, /* Async read from file. */ - .readv = generic_file_readv, /* Read from file. */ #ifdef NTFS_RW .write = ntfs_file_write, /* Write to file. */ .aio_write = ntfs_file_aio_write, /* Async write to file. */ - .writev = ntfs_file_writev, /* Write to file. */ /*.release = ,*/ /* Last file is closed. See fs/ext2/file.c:: ext2_release_file() for diff --git a/fs/pipe.c b/fs/pipe.c index f3b6f71e9d0..2e60e1c8815 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -218,9 +218,10 @@ static struct pipe_buf_operations anon_pipe_buf_ops = { }; static ssize_t -pipe_readv(struct file *filp, const struct iovec *_iov, - unsigned long nr_segs, loff_t *ppos) +pipe_read(struct kiocb *iocb, const struct iovec *_iov, + unsigned long nr_segs, loff_t pos) { + struct file *filp = iocb->ki_filp; struct inode *inode = filp->f_dentry->d_inode; struct pipe_inode_info *pipe; int do_wakeup; @@ -330,17 +331,10 @@ redo: } static ssize_t -pipe_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos) -{ - struct iovec iov = { .iov_base = buf, .iov_len = count }; - - return pipe_readv(filp, &iov, 1, ppos); -} - -static ssize_t -pipe_writev(struct file *filp, const struct iovec *_iov, - unsigned long nr_segs, loff_t *ppos) +pipe_write(struct kiocb *iocb, const struct iovec *_iov, + unsigned long nr_segs, loff_t ppos) { + struct file *filp = iocb->ki_filp; struct inode *inode = filp->f_dentry->d_inode; struct pipe_inode_info *pipe; ssize_t ret; @@ -509,15 +503,6 @@ out: return ret; } -static ssize_t -pipe_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = count }; - - return pipe_writev(filp, &iov, 1, ppos); -} - static ssize_t bad_pipe_r(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { @@ -736,8 +721,8 @@ pipe_rdwr_open(struct inode *inode, struct file *filp) */ const struct file_operations read_fifo_fops = { .llseek = no_llseek, - .read = pipe_read, - .readv = pipe_readv, + .read = do_sync_read, + .aio_read = pipe_read, .write = bad_pipe_w, .poll = pipe_poll, .ioctl = pipe_ioctl, @@ -749,8 +734,8 @@ const struct file_operations read_fifo_fops = { const struct file_operations write_fifo_fops = { .llseek = no_llseek, .read = bad_pipe_r, - .write = pipe_write, - .writev = pipe_writev, + .write = do_sync_write, + .aio_write = pipe_write, .poll = pipe_poll, .ioctl = pipe_ioctl, .open = pipe_write_open, @@ -760,10 +745,10 @@ const struct file_operations write_fifo_fops = { const struct file_operations rdwr_fifo_fops = { .llseek = no_llseek, - .read = pipe_read, - .readv = pipe_readv, - .write = pipe_write, - .writev = pipe_writev, + .read = do_sync_read, + .aio_read = pipe_read, + .write = do_sync_write, + .aio_write = pipe_write, .poll = pipe_poll, .ioctl = pipe_ioctl, .open = pipe_rdwr_open, @@ -773,8 +758,8 @@ const struct file_operations rdwr_fifo_fops = { static struct file_operations read_pipe_fops = { .llseek = no_llseek, - .read = pipe_read, - .readv = pipe_readv, + .read = do_sync_read, + .aio_read = pipe_read, .write = bad_pipe_w, .poll = pipe_poll, .ioctl = pipe_ioctl, @@ -786,8 +771,8 @@ static struct file_operations read_pipe_fops = { static struct file_operations write_pipe_fops = { .llseek = no_llseek, .read = bad_pipe_r, - .write = pipe_write, - .writev = pipe_writev, + .write = do_sync_write, + .aio_write = pipe_write, .poll = pipe_poll, .ioctl = pipe_ioctl, .open = pipe_write_open, @@ -797,10 +782,10 @@ static struct file_operations write_pipe_fops = { static struct file_operations rdwr_pipe_fops = { .llseek = no_llseek, - .read = pipe_read, - .readv = pipe_readv, - .write = pipe_write, - .writev = pipe_writev, + .read = do_sync_read, + .aio_read = pipe_read, + .write = do_sync_write, + .aio_write = pipe_write, .poll = pipe_poll, .ioctl = pipe_ioctl, .open = pipe_rdwr_open, diff --git a/fs/read_write.c b/fs/read_write.c index 679dd535380..32d54cca9bd 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -15,6 +15,7 @@ #include #include #include +#include "read_write.h" #include #include @@ -450,6 +451,62 @@ unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to) EXPORT_UNUSED_SYMBOL(iov_shorten); /* June 2006 */ +ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov, + unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn) +{ + struct kiocb kiocb; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + kiocb.ki_pos = *ppos; + kiocb.ki_left = len; + kiocb.ki_nbytes = len; + + for (;;) { + ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos); + if (ret != -EIOCBRETRY) + break; + wait_on_retry_sync_kiocb(&kiocb); + } + + if (ret == -EIOCBQUEUED) + ret = wait_on_sync_kiocb(&kiocb); + *ppos = kiocb.ki_pos; + return ret; +} + +/* Do it by hand, with file-ops */ +ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov, + unsigned long nr_segs, loff_t *ppos, io_fn_t fn) +{ + struct iovec *vector = iov; + ssize_t ret = 0; + + while (nr_segs > 0) { + void __user *base; + size_t len; + ssize_t nr; + + base = vector->iov_base; + len = vector->iov_len; + vector++; + nr_segs--; + + nr = fn(filp, base, len, ppos); + + if (nr < 0) { + if (!ret) + ret = nr; + break; + } + ret += nr; + if (nr != len) + break; + } + + return ret; +} + /* A write operation does a read from user space and vice versa */ #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ) @@ -457,12 +514,9 @@ static ssize_t do_readv_writev(int type, struct file *file, const struct iovec __user * uvector, unsigned long nr_segs, loff_t *pos) { - typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *); - typedef ssize_t (*iov_fn_t)(struct file *, const struct iovec *, unsigned long, loff_t *); - size_t tot_len; struct iovec iovstack[UIO_FASTIOV]; - struct iovec *iov=iovstack, *vector; + struct iovec *iov = iovstack; ssize_t ret; int seg; io_fn_t fn; @@ -532,39 +586,18 @@ static ssize_t do_readv_writev(int type, struct file *file, fnv = NULL; if (type == READ) { fn = file->f_op->read; - fnv = file->f_op->readv; + fnv = file->f_op->aio_read; } else { fn = (io_fn_t)file->f_op->write; - fnv = file->f_op->writev; - } - if (fnv) { - ret = fnv(file, iov, nr_segs, pos); - goto out; + fnv = file->f_op->aio_write; } - /* Do it by hand, with file-ops */ - ret = 0; - vector = iov; - while (nr_segs > 0) { - void __user * base; - size_t len; - ssize_t nr; - - base = vector->iov_base; - len = vector->iov_len; - vector++; - nr_segs--; - - nr = fn(file, base, len, pos); + if (fnv) + ret = do_sync_readv_writev(file, iov, nr_segs, tot_len, + pos, fnv); + else + ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn); - if (nr < 0) { - if (!ret) ret = nr; - break; - } - ret += nr; - if (nr != len) - break; - } out: if (iov != iovstack) kfree(iov); @@ -585,7 +618,7 @@ ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, { if (!(file->f_mode & FMODE_READ)) return -EBADF; - if (!file->f_op || (!file->f_op->readv && !file->f_op->read)) + if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read)) return -EINVAL; return do_readv_writev(READ, file, vec, vlen, pos); @@ -598,7 +631,7 @@ ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, { if (!(file->f_mode & FMODE_WRITE)) return -EBADF; - if (!file->f_op || (!file->f_op->writev && !file->f_op->write)) + if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write)) return -EINVAL; return do_readv_writev(WRITE, file, vec, vlen, pos); diff --git a/fs/read_write.h b/fs/read_write.h new file mode 100644 index 00000000000..d07b954c6e0 --- /dev/null +++ b/fs/read_write.h @@ -0,0 +1,14 @@ +/* + * This file is only for sharing some helpers from read_write.c with compat.c. + * Don't use anywhere else. + */ + + +typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *); +typedef ssize_t (*iov_fn_t)(struct kiocb *, const struct iovec *, + unsigned long, loff_t); + +ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov, + unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn); +ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov, + unsigned long nr_segs, loff_t *ppos, io_fn_t fn); diff --git a/fs/xfs/linux-2.6/xfs_file.c b/fs/xfs/linux-2.6/xfs_file.c index 4737971c6a3..d93d8dd1958 100644 --- a/fs/xfs/linux-2.6/xfs_file.c +++ b/fs/xfs/linux-2.6/xfs_file.c @@ -123,96 +123,6 @@ xfs_file_aio_write_invis( return __xfs_file_write(iocb, iov, nr_segs, IO_ISAIO|IO_INVIS, pos); } -STATIC inline ssize_t -__xfs_file_readv( - struct file *file, - const struct iovec *iov, - int ioflags, - unsigned long nr_segs, - loff_t *ppos) -{ - struct inode *inode = file->f_mapping->host; - bhv_vnode_t *vp = vn_from_inode(inode); - struct kiocb kiocb; - ssize_t rval; - - init_sync_kiocb(&kiocb, file); - kiocb.ki_pos = *ppos; - - if (unlikely(file->f_flags & O_DIRECT)) - ioflags |= IO_ISDIRECT; - rval = bhv_vop_read(vp, &kiocb, iov, nr_segs, - &kiocb.ki_pos, ioflags, NULL); - - *ppos = kiocb.ki_pos; - return rval; -} - -STATIC ssize_t -xfs_file_readv( - struct file *file, - const struct iovec *iov, - unsigned long nr_segs, - loff_t *ppos) -{ - return __xfs_file_readv(file, iov, 0, nr_segs, ppos); -} - -STATIC ssize_t -xfs_file_readv_invis( - struct file *file, - const struct iovec *iov, - unsigned long nr_segs, - loff_t *ppos) -{ - return __xfs_file_readv(file, iov, IO_INVIS, nr_segs, ppos); -} - -STATIC inline ssize_t -__xfs_file_writev( - struct file *file, - const struct iovec *iov, - int ioflags, - unsigned long nr_segs, - loff_t *ppos) -{ - struct inode *inode = file->f_mapping->host; - bhv_vnode_t *vp = vn_from_inode(inode); - struct kiocb kiocb; - ssize_t rval; - - init_sync_kiocb(&kiocb, file); - kiocb.ki_pos = *ppos; - if (unlikely(file->f_flags & O_DIRECT)) - ioflags |= IO_ISDIRECT; - - rval = bhv_vop_write(vp, &kiocb, iov, nr_segs, - &kiocb.ki_pos, ioflags, NULL); - - *ppos = kiocb.ki_pos; - return rval; -} - -STATIC ssize_t -xfs_file_writev( - struct file *file, - const struct iovec *iov, - unsigned long nr_segs, - loff_t *ppos) -{ - return __xfs_file_writev(file, iov, 0, nr_segs, ppos); -} - -STATIC ssize_t -xfs_file_writev_invis( - struct file *file, - const struct iovec *iov, - unsigned long nr_segs, - loff_t *ppos) -{ - return __xfs_file_writev(file, iov, IO_INVIS, nr_segs, ppos); -} - STATIC ssize_t xfs_file_sendfile( struct file *filp, @@ -540,8 +450,6 @@ const struct file_operations xfs_file_operations = { .llseek = generic_file_llseek, .read = do_sync_read, .write = do_sync_write, - .readv = xfs_file_readv, - .writev = xfs_file_writev, .aio_read = xfs_file_aio_read, .aio_write = xfs_file_aio_write, .sendfile = xfs_file_sendfile, @@ -565,8 +473,6 @@ const struct file_operations xfs_invis_file_operations = { .llseek = generic_file_llseek, .read = do_sync_read, .write = do_sync_write, - .readv = xfs_file_readv_invis, - .writev = xfs_file_writev_invis, .aio_read = xfs_file_aio_read_invis, .aio_write = xfs_file_aio_write_invis, .sendfile = xfs_file_sendfile_invis, diff --git a/include/linux/fs.h b/include/linux/fs.h index 257bae16f54..afb6e6f89a0 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1113,8 +1113,6 @@ struct file_operations { int (*aio_fsync) (struct kiocb *, int datasync); int (*fasync) (int, struct file *, int); int (*lock) (struct file *, int, struct file_lock *); - ssize_t (*readv) (struct file *, const struct iovec *, unsigned long, loff_t *); - ssize_t (*writev) (struct file *, const struct iovec *, unsigned long, loff_t *); ssize_t (*sendfile) (struct file *, loff_t *, size_t, read_actor_t, void *); ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); @@ -1734,10 +1732,6 @@ extern long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, extern void file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping); -extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos); -ssize_t generic_file_writev(struct file *filp, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos); extern loff_t no_llseek(struct file *file, loff_t offset, int origin); extern loff_t generic_file_llseek(struct file *file, loff_t offset, int origin); extern loff_t remote_llseek(struct file *file, loff_t offset, int origin); diff --git a/mm/filemap.c b/mm/filemap.c index f6c1d22b504..48497094ae8 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2421,42 +2421,6 @@ ssize_t generic_file_write(struct file *file, const char __user *buf, } EXPORT_SYMBOL(generic_file_write); -ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) -{ - struct kiocb kiocb; - ssize_t ret; - - init_sync_kiocb(&kiocb, filp); - ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos); - if (-EIOCBQUEUED == ret) - ret = wait_on_sync_kiocb(&kiocb); - return ret; -} -EXPORT_SYMBOL(generic_file_readv); - -ssize_t generic_file_writev(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) -{ - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - ssize_t ret; - - mutex_lock(&inode->i_mutex); - ret = __generic_file_write_nolock(file, iov, nr_segs, ppos); - mutex_unlock(&inode->i_mutex); - - if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { - int err; - - err = sync_page_range(inode, mapping, *ppos - ret, ret); - if (err < 0) - ret = err; - } - return ret; -} -EXPORT_SYMBOL(generic_file_writev); - /* * Called under i_mutex for writes to S_ISREG files. Returns -EIO if something * went wrong during pagecache shootdown. diff --git a/net/socket.c b/net/socket.c index df92e425274..01918f7a301 100644 --- a/net/socket.c +++ b/net/socket.c @@ -110,10 +110,6 @@ static long compat_sock_ioctl(struct file *file, unsigned int cmd, unsigned long arg); #endif static int sock_fasync(int fd, struct file *filp, int on); -static ssize_t sock_readv(struct file *file, const struct iovec *vector, - unsigned long count, loff_t *ppos); -static ssize_t sock_writev(struct file *file, const struct iovec *vector, - unsigned long count, loff_t *ppos); static ssize_t sock_sendpage(struct file *file, struct page *page, int offset, size_t size, loff_t *ppos, int more); @@ -136,8 +132,6 @@ static struct file_operations socket_file_ops = { .open = sock_no_open, /* special open code to disallow open via /proc */ .release = sock_close, .fasync = sock_fasync, - .readv = sock_readv, - .writev = sock_writev, .sendpage = sock_sendpage, .splice_write = generic_splice_sendpage, }; @@ -700,23 +694,6 @@ static ssize_t do_sock_read(struct msghdr *msg, struct kiocb *iocb, return __sock_recvmsg(iocb, sock, msg, size, msg->msg_flags); } -static ssize_t sock_readv(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) -{ - struct kiocb iocb; - struct sock_iocb siocb; - struct msghdr msg; - int ret; - - init_sync_kiocb(&iocb, NULL); - iocb.private = &siocb; - - ret = do_sock_read(&msg, &iocb, file, iov, nr_segs); - if (-EIOCBQUEUED == ret) - ret = wait_on_sync_kiocb(&iocb); - return ret; -} - static ssize_t sock_aio_read(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { @@ -759,23 +736,6 @@ static ssize_t do_sock_write(struct msghdr *msg, struct kiocb *iocb, return __sock_sendmsg(iocb, sock, msg, size); } -static ssize_t sock_writev(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) -{ - struct msghdr msg; - struct kiocb iocb; - struct sock_iocb siocb; - int ret; - - init_sync_kiocb(&iocb, NULL); - iocb.private = &siocb; - - ret = do_sock_write(&msg, &iocb, file, iov, nr_segs); - if (-EIOCBQUEUED == ret) - ret = wait_on_sync_kiocb(&iocb); - return ret; -} - static ssize_t sock_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 891d7140553..37b4b10850a 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -2852,8 +2852,8 @@ static ssize_t snd_pcm_write(struct file *file, const char __user *buf, return result; } -static ssize_t snd_pcm_readv(struct file *file, const struct iovec *_vector, - unsigned long count, loff_t * offset) +static ssize_t snd_pcm_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) { struct snd_pcm_file *pcm_file; @@ -2864,22 +2864,22 @@ static ssize_t snd_pcm_readv(struct file *file, const struct iovec *_vector, void __user **bufs; snd_pcm_uframes_t frames; - pcm_file = file->private_data; + pcm_file = iocb->ki_filp->private_data; substream = pcm_file->substream; snd_assert(substream != NULL, return -ENXIO); runtime = substream->runtime; if (runtime->status->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; - if (count > 1024 || count != runtime->channels) + if (nr_segs > 1024 || nr_segs != runtime->channels) return -EINVAL; - if (!frame_aligned(runtime, _vector->iov_len)) + if (!frame_aligned(runtime, iov->iov_len)) return -EINVAL; - frames = bytes_to_samples(runtime, _vector->iov_len); - bufs = kmalloc(sizeof(void *) * count, GFP_KERNEL); + frames = bytes_to_samples(runtime, iov->iov_len); + bufs = kmalloc(sizeof(void *) * nr_segs, GFP_KERNEL); if (bufs == NULL) return -ENOMEM; - for (i = 0; i < count; ++i) - bufs[i] = _vector[i].iov_base; + for (i = 0; i < nr_segs; ++i) + bufs[i] = iov[i].iov_base; result = snd_pcm_lib_readv(substream, bufs, frames); if (result > 0) result = frames_to_bytes(runtime, result); @@ -2887,8 +2887,8 @@ static ssize_t snd_pcm_readv(struct file *file, const struct iovec *_vector, return result; } -static ssize_t snd_pcm_writev(struct file *file, const struct iovec *_vector, - unsigned long count, loff_t * offset) +static ssize_t snd_pcm_aio_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) { struct snd_pcm_file *pcm_file; struct snd_pcm_substream *substream; @@ -2898,7 +2898,7 @@ static ssize_t snd_pcm_writev(struct file *file, const struct iovec *_vector, void __user **bufs; snd_pcm_uframes_t frames; - pcm_file = file->private_data; + pcm_file = iocb->ki_filp->private_data; substream = pcm_file->substream; snd_assert(substream != NULL, result = -ENXIO; goto end); runtime = substream->runtime; @@ -2906,17 +2906,17 @@ static ssize_t snd_pcm_writev(struct file *file, const struct iovec *_vector, result = -EBADFD; goto end; } - if (count > 128 || count != runtime->channels || - !frame_aligned(runtime, _vector->iov_len)) { + if (nr_segs > 128 || nr_segs != runtime->channels || + !frame_aligned(runtime, iov->iov_len)) { result = -EINVAL; goto end; } - frames = bytes_to_samples(runtime, _vector->iov_len); - bufs = kmalloc(sizeof(void *) * count, GFP_KERNEL); + frames = bytes_to_samples(runtime, iov->iov_len); + bufs = kmalloc(sizeof(void *) * nr_segs, GFP_KERNEL); if (bufs == NULL) return -ENOMEM; - for (i = 0; i < count; ++i) - bufs[i] = _vector[i].iov_base; + for (i = 0; i < nr_segs; ++i) + bufs[i] = iov[i].iov_base; result = snd_pcm_lib_writev(substream, bufs, frames); if (result > 0) result = frames_to_bytes(runtime, result); @@ -3426,7 +3426,7 @@ struct file_operations snd_pcm_f_ops[2] = { { .owner = THIS_MODULE, .write = snd_pcm_write, - .writev = snd_pcm_writev, + .aio_write = snd_pcm_aio_write, .open = snd_pcm_playback_open, .release = snd_pcm_release, .poll = snd_pcm_playback_poll, @@ -3438,7 +3438,7 @@ struct file_operations snd_pcm_f_ops[2] = { { .owner = THIS_MODULE, .read = snd_pcm_read, - .readv = snd_pcm_readv, + .aio_read = snd_pcm_aio_read, .open = snd_pcm_capture_open, .release = snd_pcm_release, .poll = snd_pcm_capture_poll, -- cgit v1.2.3-70-g09d2 From 543ade1fc901db4c3dbe9fb27241fb977f1f3eea Mon Sep 17 00:00:00 2001 From: Badari Pulavarty Date: Sat, 30 Sep 2006 23:28:48 -0700 Subject: [PATCH] Streamline generic_file_* interfaces and filemap cleanups This patch cleans up generic_file_*_read/write() interfaces. Christoph Hellwig gave me the idea for this clean ups. In a nutshell, all filesystems should set .aio_read/.aio_write methods and use do_sync_read/ do_sync_write() as their .read/.write methods. This allows us to cleanup all variants of generic_file_* routines. Final available interfaces: generic_file_aio_read() - read handler generic_file_aio_write() - write handler generic_file_aio_write_nolock() - no lock write handler __generic_file_aio_write_nolock() - internal worker routine Signed-off-by: Badari Pulavarty Signed-off-by: Christoph Hellwig Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/char/raw.c | 15 ++------ fs/adfs/file.c | 6 ++-- fs/affs/file.c | 6 ++-- fs/bfs/file.c | 6 ++-- fs/block_dev.c | 12 ++----- fs/ext2/file.c | 4 +-- fs/fuse/file.c | 6 ++-- fs/hfs/inode.c | 6 ++-- fs/hfsplus/inode.c | 6 ++-- fs/hostfs/hostfs_kern.c | 4 +-- fs/hpfs/file.c | 6 ++-- fs/jffs/inode-v23.c | 6 ++-- fs/jffs2/file.c | 6 ++-- fs/jfs/file.c | 4 +-- fs/minix/file.c | 6 ++-- fs/ntfs/file.c | 2 +- fs/qnx4/file.c | 6 ++-- fs/ramfs/file-mmu.c | 6 ++-- fs/ramfs/file-nommu.c | 6 ++-- fs/read_write.c | 3 +- fs/smbfs/file.c | 24 ++++++++----- fs/sysv/file.c | 6 ++-- fs/udf/file.c | 16 +++++---- fs/ufs/file.c | 6 ++-- fs/xfs/linux-2.6/xfs_lrw.c | 4 ++- include/linux/fs.h | 5 --- mm/filemap.c | 87 +++------------------------------------------- 27 files changed, 105 insertions(+), 165 deletions(-) (limited to 'fs/read_write.c') diff --git a/drivers/char/raw.c b/drivers/char/raw.c index 490db531e2d..89b718e326e 100644 --- a/drivers/char/raw.c +++ b/drivers/char/raw.c @@ -238,21 +238,10 @@ out: return err; } -static ssize_t raw_file_write(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct iovec local_iov = { - .iov_base = (char __user *)buf, - .iov_len = count - }; - - return generic_file_write_nolock(file, &local_iov, 1, ppos); -} - static const struct file_operations raw_fops = { - .read = generic_file_read, + .read = do_sync_read, .aio_read = generic_file_aio_read, - .write = raw_file_write, + .write = do_sync_write, .aio_write = generic_file_aio_write_nolock, .open = raw_open, .release= raw_release, diff --git a/fs/adfs/file.c b/fs/adfs/file.c index 1014b9f2117..6101ea679cb 100644 --- a/fs/adfs/file.c +++ b/fs/adfs/file.c @@ -27,10 +27,12 @@ const struct file_operations adfs_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, + .read = do_sync_read, + .aio_read = generic_file_aio_read, .mmap = generic_file_mmap, .fsync = file_fsync, - .write = generic_file_write, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .sendfile = generic_file_sendfile, }; diff --git a/fs/affs/file.c b/fs/affs/file.c index 3de8590e4f6..05b5e22de75 100644 --- a/fs/affs/file.c +++ b/fs/affs/file.c @@ -27,8 +27,10 @@ static int affs_file_release(struct inode *inode, struct file *filp); const struct file_operations affs_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .open = affs_file_open, .release = affs_file_release, diff --git a/fs/bfs/file.c b/fs/bfs/file.c index 3d5aca28a0a..a9164a87f8d 100644 --- a/fs/bfs/file.c +++ b/fs/bfs/file.c @@ -19,8 +19,10 @@ const struct file_operations bfs_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .sendfile = generic_file_sendfile, }; diff --git a/fs/block_dev.c b/fs/block_dev.c index 0f143094ef1..bc8f27cc448 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1154,14 +1154,6 @@ static int blkdev_close(struct inode * inode, struct file * filp) return blkdev_put(bdev); } -static ssize_t blkdev_file_write(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct iovec local_iov = { .iov_base = (void __user *)buf, .iov_len = count }; - - return generic_file_write_nolock(file, &local_iov, 1, ppos); -} - static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg) { return blkdev_ioctl(file->f_mapping->host, file, cmd, arg); @@ -1181,8 +1173,8 @@ const struct file_operations def_blk_fops = { .open = blkdev_open, .release = blkdev_close, .llseek = block_llseek, - .read = generic_file_read, - .write = blkdev_file_write, + .read = do_sync_read, + .write = do_sync_write, .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write_nolock, .mmap = generic_file_mmap, diff --git a/fs/ext2/file.c b/fs/ext2/file.c index e893e2be9ed..2dba473c524 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -41,8 +41,8 @@ static int ext2_release_file (struct inode * inode, struct file * filp) */ const struct file_operations ext2_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .write = do_sync_write, .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, .ioctl = ext2_ioctl, diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5c4fcd1dbf5..183626868ee 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -753,8 +753,10 @@ static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl) static const struct file_operations fuse_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = fuse_file_mmap, .open = fuse_open, .flush = fuse_flush, diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index d05641c35fc..02f5573e034 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -601,8 +601,10 @@ int hfs_inode_setattr(struct dentry *dentry, struct iattr * attr) static const struct file_operations hfs_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .sendfile = generic_file_sendfile, .fsync = file_fsync, diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 0eb1a609266..9e367524963 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -282,8 +282,10 @@ static struct inode_operations hfsplus_file_inode_operations = { static const struct file_operations hfsplus_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .sendfile = generic_file_sendfile, .fsync = file_fsync, diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index 4908c38a588..b6bd33ca373 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -385,11 +385,11 @@ int hostfs_fsync(struct file *file, struct dentry *dentry, int datasync) static const struct file_operations hostfs_file_fops = { .llseek = generic_file_llseek, - .read = generic_file_read, + .read = do_sync_read, .sendfile = generic_file_sendfile, .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, - .write = generic_file_write, + .write = do_sync_write, .mmap = generic_file_mmap, .open = hostfs_file_open, .release = NULL, diff --git a/fs/hpfs/file.c b/fs/hpfs/file.c index d9eb19b7b8a..8b94d24855f 100644 --- a/fs/hpfs/file.c +++ b/fs/hpfs/file.c @@ -113,7 +113,7 @@ static ssize_t hpfs_file_write(struct file *file, const char __user *buf, { ssize_t retval; - retval = generic_file_write(file, buf, count, ppos); + retval = do_sync_write(file, buf, count, ppos); if (retval > 0) hpfs_i(file->f_dentry->d_inode)->i_dirty = 1; return retval; @@ -122,8 +122,10 @@ static ssize_t hpfs_file_write(struct file *file, const char __user *buf, const struct file_operations hpfs_file_ops = { .llseek = generic_file_llseek, - .read = generic_file_read, + .read = do_sync_read, + .aio_read = generic_file_aio_read, .write = hpfs_file_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .release = hpfs_file_release, .fsync = hpfs_file_fsync, diff --git a/fs/jffs/inode-v23.c b/fs/jffs/inode-v23.c index f5cf9c93e24..068ef0de8de 100644 --- a/fs/jffs/inode-v23.c +++ b/fs/jffs/inode-v23.c @@ -1632,8 +1632,10 @@ static const struct file_operations jffs_file_operations = { .open = generic_file_open, .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .ioctl = jffs_ioctl, .mmap = generic_file_readonly_mmap, .fsync = jffs_fsync, diff --git a/fs/jffs2/file.c b/fs/jffs2/file.c index 3ed6e3e120b..242875f77cb 100644 --- a/fs/jffs2/file.c +++ b/fs/jffs2/file.c @@ -42,8 +42,10 @@ const struct file_operations jffs2_file_operations = { .llseek = generic_file_llseek, .open = generic_file_open, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .ioctl = jffs2_ioctl, .mmap = generic_file_readonly_mmap, .fsync = jffs2_fsync, diff --git a/fs/jfs/file.c b/fs/jfs/file.c index f535f2911c1..976e90dc2d1 100644 --- a/fs/jfs/file.c +++ b/fs/jfs/file.c @@ -103,8 +103,8 @@ struct inode_operations jfs_file_inode_operations = { const struct file_operations jfs_file_operations = { .open = jfs_open, .llseek = generic_file_llseek, - .write = generic_file_write, - .read = generic_file_read, + .write = do_sync_write, + .read = do_sync_read, .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, diff --git a/fs/minix/file.c b/fs/minix/file.c index 420b32882a1..40eac2e60d2 100644 --- a/fs/minix/file.c +++ b/fs/minix/file.c @@ -17,8 +17,10 @@ int minix_sync_file(struct file *, struct dentry *, int); const struct file_operations minix_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .fsync = minix_sync_file, .sendfile = generic_file_sendfile, diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index 2f9b5a0953f..ae2fe0016d2 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -2296,7 +2296,7 @@ static int ntfs_file_fsync(struct file *filp, struct dentry *dentry, const struct file_operations ntfs_file_ops = { .llseek = generic_file_llseek, /* Seek inside file. */ - .read = generic_file_read, /* Read from file. */ + .read = do_sync_read, /* Read from file. */ .aio_read = generic_file_aio_read, /* Async read from file. */ #ifdef NTFS_RW .write = ntfs_file_write, /* Write to file. */ diff --git a/fs/qnx4/file.c b/fs/qnx4/file.c index 62af4b1348b..467e5ac7280 100644 --- a/fs/qnx4/file.c +++ b/fs/qnx4/file.c @@ -22,11 +22,13 @@ const struct file_operations qnx4_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, + .read = do_sync_read, + .aio_read = generic_file_aio_read, .mmap = generic_file_mmap, .sendfile = generic_file_sendfile, #ifdef CONFIG_QNX4FS_RW - .write = generic_file_write, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .fsync = qnx4_sync_file, #endif }; diff --git a/fs/ramfs/file-mmu.c b/fs/ramfs/file-mmu.c index 86f14cacf64..0947fb57dcf 100644 --- a/fs/ramfs/file-mmu.c +++ b/fs/ramfs/file-mmu.c @@ -33,8 +33,10 @@ const struct address_space_operations ramfs_aops = { }; const struct file_operations ramfs_file_operations = { - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .fsync = simple_sync_file, .sendfile = generic_file_sendfile, diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c index 677139b48e0..bfe5dbf1002 100644 --- a/fs/ramfs/file-nommu.c +++ b/fs/ramfs/file-nommu.c @@ -36,8 +36,10 @@ const struct address_space_operations ramfs_aops = { const struct file_operations ramfs_file_operations = { .mmap = ramfs_nommu_mmap, .get_unmapped_area = ramfs_nommu_get_unmapped_area, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .fsync = simple_sync_file, .sendfile = generic_file_sendfile, .llseek = generic_file_llseek, diff --git a/fs/read_write.c b/fs/read_write.c index 32d54cca9bd..4ed839bcb91 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -22,7 +22,8 @@ const struct file_operations generic_ro_fops = { .llseek = generic_file_llseek, - .read = generic_file_read, + .read = do_sync_read, + .aio_read = generic_file_aio_read, .mmap = generic_file_readonly_mmap, .sendfile = generic_file_sendfile, }; diff --git a/fs/smbfs/file.c b/fs/smbfs/file.c index dae67048bab..50784d13c87 100644 --- a/fs/smbfs/file.c +++ b/fs/smbfs/file.c @@ -214,13 +214,15 @@ smb_updatepage(struct file *file, struct page *page, unsigned long offset, } static ssize_t -smb_file_read(struct file * file, char __user * buf, size_t count, loff_t *ppos) +smb_file_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) { + struct file * file = iocb->ki_filp; struct dentry * dentry = file->f_dentry; ssize_t status; VERBOSE("file %s/%s, count=%lu@%lu\n", DENTRY_PATH(dentry), - (unsigned long) count, (unsigned long) *ppos); + (unsigned long) iocb->ki_left, (unsigned long) pos); status = smb_revalidate_inode(dentry); if (status) { @@ -233,7 +235,7 @@ smb_file_read(struct file * file, char __user * buf, size_t count, loff_t *ppos) (long)dentry->d_inode->i_size, dentry->d_inode->i_flags, dentry->d_inode->i_atime); - status = generic_file_read(file, buf, count, ppos); + status = generic_file_aio_read(iocb, iov, nr_segs, pos); out: return status; } @@ -317,14 +319,16 @@ const struct address_space_operations smb_file_aops = { * Write to a file (through the page cache). */ static ssize_t -smb_file_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) +smb_file_aio_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) { + struct file * file = iocb->ki_filp; struct dentry * dentry = file->f_dentry; ssize_t result; VERBOSE("file %s/%s, count=%lu@%lu\n", DENTRY_PATH(dentry), - (unsigned long) count, (unsigned long) *ppos); + (unsigned long) iocb->ki_left, (unsigned long) pos); result = smb_revalidate_inode(dentry); if (result) { @@ -337,8 +341,8 @@ smb_file_write(struct file *file, const char __user *buf, size_t count, loff_t * if (result) goto out; - if (count > 0) { - result = generic_file_write(file, buf, count, ppos); + if (iocb->ki_left > 0) { + result = generic_file_aio_write(iocb, iov, nr_segs, pos); VERBOSE("pos=%ld, size=%ld, mtime=%ld, atime=%ld\n", (long) file->f_pos, (long) dentry->d_inode->i_size, dentry->d_inode->i_mtime, dentry->d_inode->i_atime); @@ -402,8 +406,10 @@ smb_file_permission(struct inode *inode, int mask, struct nameidata *nd) const struct file_operations smb_file_operations = { .llseek = remote_llseek, - .read = smb_file_read, - .write = smb_file_write, + .read = do_sync_read, + .aio_read = smb_file_aio_read, + .write = do_sync_write, + .aio_write = smb_file_aio_write, .ioctl = smb_ioctl, .mmap = smb_file_mmap, .open = smb_file_open, diff --git a/fs/sysv/file.c b/fs/sysv/file.c index a59e303135f..47a4b728f15 100644 --- a/fs/sysv/file.c +++ b/fs/sysv/file.c @@ -21,8 +21,10 @@ */ const struct file_operations sysv_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .fsync = sysv_sync_file, .sendfile = generic_file_sendfile, diff --git a/fs/udf/file.c b/fs/udf/file.c index a59e5f33daf..7aedd552cba 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -103,19 +103,21 @@ const struct address_space_operations udf_adinicb_aops = { .commit_write = udf_adinicb_commit_write, }; -static ssize_t udf_file_write(struct file * file, const char __user * buf, - size_t count, loff_t *ppos) +static ssize_t udf_file_aio_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t ppos) { ssize_t retval; + struct file *file = iocb->ki_filp; struct inode *inode = file->f_dentry->d_inode; int err, pos; + size_t count = iocb->ki_left; if (UDF_I_ALLOCTYPE(inode) == ICBTAG_FLAG_AD_IN_ICB) { if (file->f_flags & O_APPEND) pos = inode->i_size; else - pos = *ppos; + pos = ppos; if (inode->i_sb->s_blocksize < (udf_file_entry_alloc_offset(inode) + pos + count)) @@ -136,7 +138,7 @@ static ssize_t udf_file_write(struct file * file, const char __user * buf, } } - retval = generic_file_write(file, buf, count, ppos); + retval = generic_file_aio_write(iocb, iov, nr_segs, ppos); if (retval > 0) mark_inode_dirty(inode); @@ -249,11 +251,13 @@ static int udf_release_file(struct inode * inode, struct file * filp) } const struct file_operations udf_file_operations = { - .read = generic_file_read, + .read = do_sync_read, + .aio_read = generic_file_aio_read, .ioctl = udf_ioctl, .open = generic_file_open, .mmap = generic_file_mmap, - .write = udf_file_write, + .write = do_sync_write, + .aio_write = udf_file_aio_write, .release = udf_release_file, .fsync = udf_fsync_file, .sendfile = generic_file_sendfile, diff --git a/fs/ufs/file.c b/fs/ufs/file.c index a9c6e5f04fa..1e096323bad 100644 --- a/fs/ufs/file.c +++ b/fs/ufs/file.c @@ -53,8 +53,10 @@ static int ufs_sync_file(struct file *file, struct dentry *dentry, int datasync) const struct file_operations ufs_file_operations = { .llseek = generic_file_llseek, - .read = generic_file_read, - .write = generic_file_write, + .read = do_sync_read, + .aio_read = generic_file_aio_read, + .write = do_sync_write, + .aio_write = generic_file_aio_write, .mmap = generic_file_mmap, .open = generic_file_open, .fsync = ufs_sync_file, diff --git a/fs/xfs/linux-2.6/xfs_lrw.c b/fs/xfs/linux-2.6/xfs_lrw.c index 55992b40353..fa842f1c9fa 100644 --- a/fs/xfs/linux-2.6/xfs_lrw.c +++ b/fs/xfs/linux-2.6/xfs_lrw.c @@ -279,7 +279,9 @@ xfs_read( xfs_rw_enter_trace(XFS_READ_ENTER, &ip->i_iocore, (void *)iovp, segs, *offset, ioflags); - ret = __generic_file_aio_read(iocb, iovp, segs, offset); + + iocb->ki_pos = *offset; + ret = generic_file_aio_read(iocb, iovp, segs, *offset); if (ret == -EIOCBQUEUED && !(ioflags & IO_ISAIO)) ret = wait_on_sync_kiocb(iocb); if (ret > 0) diff --git a/include/linux/fs.h b/include/linux/fs.h index afb6e6f89a0..011129f8803 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1699,11 +1699,8 @@ extern int generic_file_mmap(struct file *, struct vm_area_struct *); extern int generic_file_readonly_mmap(struct file *, struct vm_area_struct *); extern int file_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size); extern int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size); -extern ssize_t generic_file_read(struct file *, char __user *, size_t, loff_t *); int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk); -extern ssize_t generic_file_write(struct file *, const char __user *, size_t, loff_t *); extern ssize_t generic_file_aio_read(struct kiocb *, const struct iovec *, unsigned long, loff_t); -extern ssize_t __generic_file_aio_read(struct kiocb *, const struct iovec *, unsigned long, loff_t *); extern ssize_t generic_file_aio_write(struct kiocb *, const struct iovec *, unsigned long, loff_t); extern ssize_t generic_file_aio_write_nolock(struct kiocb *, const struct iovec *, unsigned long, loff_t); @@ -1713,8 +1710,6 @@ extern ssize_t generic_file_buffered_write(struct kiocb *, const struct iovec *, unsigned long, loff_t, loff_t *, size_t, ssize_t); extern ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos); extern ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos); -ssize_t generic_file_write_nolock(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos); extern ssize_t generic_file_sendfile(struct file *, loff_t *, size_t, read_actor_t, void *); extern void do_generic_mapping_read(struct address_space *mapping, struct file_ra_state *, struct file *, diff --git a/mm/filemap.c b/mm/filemap.c index 48497094ae8..ec469235985 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1149,13 +1149,14 @@ success: * that can use the page cache directly. */ ssize_t -__generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) +generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) { struct file *filp = iocb->ki_filp; ssize_t retval; unsigned long seg; size_t count; + loff_t *ppos = &iocb->ki_pos; count = 0; for (seg = 0; seg < nr_segs; seg++) { @@ -1179,7 +1180,7 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ if (filp->f_flags & O_DIRECT) { - loff_t pos = *ppos, size; + loff_t size; struct address_space *mapping; struct inode *inode; @@ -1223,32 +1224,8 @@ __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, out: return retval; } -EXPORT_SYMBOL(__generic_file_aio_read); - -ssize_t -generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, - unsigned long nr_segs, loff_t pos) -{ - BUG_ON(iocb->ki_pos != pos); - return __generic_file_aio_read(iocb, iov, nr_segs, &iocb->ki_pos); -} EXPORT_SYMBOL(generic_file_aio_read); -ssize_t -generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos) -{ - struct iovec local_iov = { .iov_base = buf, .iov_len = count }; - struct kiocb kiocb; - ssize_t ret; - - init_sync_kiocb(&kiocb, filp); - ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos); - if (-EIOCBQUEUED == ret) - ret = wait_on_sync_kiocb(&kiocb); - return ret; -} -EXPORT_SYMBOL(generic_file_read); - int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size) { ssize_t written; @@ -2339,38 +2316,6 @@ ssize_t generic_file_aio_write_nolock(struct kiocb *iocb, } EXPORT_SYMBOL(generic_file_aio_write_nolock); -static ssize_t -__generic_file_write_nolock(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) -{ - struct kiocb kiocb; - ssize_t ret; - - init_sync_kiocb(&kiocb, file); - kiocb.ki_pos = *ppos; - ret = __generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos); - if (-EIOCBQUEUED == ret) - ret = wait_on_sync_kiocb(&kiocb); - return ret; -} - -ssize_t -generic_file_write_nolock(struct file *file, const struct iovec *iov, - unsigned long nr_segs, loff_t *ppos) -{ - struct kiocb kiocb; - ssize_t ret; - - init_sync_kiocb(&kiocb, file); - kiocb.ki_pos = *ppos; - ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, *ppos); - if (-EIOCBQUEUED == ret) - ret = wait_on_sync_kiocb(&kiocb); - *ppos = kiocb.ki_pos; - return ret; -} -EXPORT_SYMBOL(generic_file_write_nolock); - ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { @@ -2397,30 +2342,6 @@ ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, } EXPORT_SYMBOL(generic_file_aio_write); -ssize_t generic_file_write(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - ssize_t ret; - struct iovec local_iov = { .iov_base = (void __user *)buf, - .iov_len = count }; - - mutex_lock(&inode->i_mutex); - ret = __generic_file_write_nolock(file, &local_iov, 1, ppos); - mutex_unlock(&inode->i_mutex); - - if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { - ssize_t err; - - err = sync_page_range(inode, mapping, *ppos - ret, ret); - if (err < 0) - ret = err; - } - return ret; -} -EXPORT_SYMBOL(generic_file_write); - /* * Called under i_mutex for writes to S_ISREG files. Returns -EIO if something * went wrong during pagecache shootdown. -- cgit v1.2.3-70-g09d2 From eed4e51fb60c3863c134a5e9f6006b29805ead97 Mon Sep 17 00:00:00 2001 From: Badari Pulavarty Date: Sat, 30 Sep 2006 23:28:49 -0700 Subject: [PATCH] Add vector AIO support This work is initially done by Zach Brown to add support for vectored aio. These are the core changes for AIO to support IOCB_CMD_PREADV/IOCB_CMD_PWRITEV. [akpm@osdl.org: huge build fix] Signed-off-by: Zach Brown Signed-off-by: Christoph Hellwig Signed-off-by: Badari Pulavarty Acked-by: Benjamin LaHaise Acked-by: James Morris Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/aio.c | 171 +++++++++++++++++++++++++++++++++--------------- fs/read_write.c | 129 +++++++++++++++++++++--------------- include/linux/aio.h | 4 ++ include/linux/aio_abi.h | 2 + include/linux/fs.h | 5 ++ 5 files changed, 203 insertions(+), 108 deletions(-) (limited to 'fs/read_write.c') diff --git a/fs/aio.c b/fs/aio.c index 27ff56540c7..2e0d1505ee3 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -415,6 +415,7 @@ static struct kiocb fastcall *__aio_get_req(struct kioctx *ctx) req->ki_retry = NULL; req->ki_dtor = NULL; req->private = NULL; + req->ki_iovec = NULL; INIT_LIST_HEAD(&req->ki_run_list); /* Check if the completion queue has enough free space to @@ -460,6 +461,8 @@ static inline void really_put_req(struct kioctx *ctx, struct kiocb *req) if (req->ki_dtor) req->ki_dtor(req); + if (req->ki_iovec != &req->ki_inline_vec) + kfree(req->ki_iovec); kmem_cache_free(kiocb_cachep, req); ctx->reqs_active--; @@ -1301,69 +1304,63 @@ asmlinkage long sys_io_destroy(aio_context_t ctx) return -EINVAL; } -/* - * aio_p{read,write} are the default ki_retry methods for - * IO_CMD_P{READ,WRITE}. They maintains kiocb retry state around potentially - * multiple calls to f_op->aio_read(). They loop around partial progress - * instead of returning -EIOCBRETRY because they don't have the means to call - * kick_iocb(). - */ -static ssize_t aio_pread(struct kiocb *iocb) +static void aio_advance_iovec(struct kiocb *iocb, ssize_t ret) { - struct file *file = iocb->ki_filp; - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - ssize_t ret = 0; - - do { - iocb->ki_inline_vec.iov_base = iocb->ki_buf; - iocb->ki_inline_vec.iov_len = iocb->ki_left; - - ret = file->f_op->aio_read(iocb, &iocb->ki_inline_vec, - 1, iocb->ki_pos); - /* - * Can't just depend on iocb->ki_left to determine - * whether we are done. This may have been a short read. - */ - if (ret > 0) { - iocb->ki_buf += ret; - iocb->ki_left -= ret; + struct iovec *iov = &iocb->ki_iovec[iocb->ki_cur_seg]; + + BUG_ON(ret <= 0); + + while (iocb->ki_cur_seg < iocb->ki_nr_segs && ret > 0) { + ssize_t this = min((ssize_t)iov->iov_len, ret); + iov->iov_base += this; + iov->iov_len -= this; + iocb->ki_left -= this; + ret -= this; + if (iov->iov_len == 0) { + iocb->ki_cur_seg++; + iov++; } + } - /* - * For pipes and sockets we return once we have some data; for - * regular files we retry till we complete the entire read or - * find that we can't read any more data (e.g short reads). - */ - } while (ret > 0 && iocb->ki_left > 0 && - !S_ISFIFO(inode->i_mode) && !S_ISSOCK(inode->i_mode)); - - /* This means we must have transferred all that we could */ - /* No need to retry anymore */ - if ((ret == 0) || (iocb->ki_left == 0)) - ret = iocb->ki_nbytes - iocb->ki_left; - - return ret; + /* the caller should not have done more io than what fit in + * the remaining iovecs */ + BUG_ON(ret > 0 && iocb->ki_left == 0); } -/* see aio_pread() */ -static ssize_t aio_pwrite(struct kiocb *iocb) +static ssize_t aio_rw_vect_retry(struct kiocb *iocb) { struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + ssize_t (*rw_op)(struct kiocb *, const struct iovec *, + unsigned long, loff_t); ssize_t ret = 0; + unsigned short opcode; + + if ((iocb->ki_opcode == IOCB_CMD_PREADV) || + (iocb->ki_opcode == IOCB_CMD_PREAD)) { + rw_op = file->f_op->aio_read; + opcode = IOCB_CMD_PREADV; + } else { + rw_op = file->f_op->aio_write; + opcode = IOCB_CMD_PWRITEV; + } do { - iocb->ki_inline_vec.iov_base = iocb->ki_buf; - iocb->ki_inline_vec.iov_len = iocb->ki_left; - - ret = file->f_op->aio_write(iocb, &iocb->ki_inline_vec, - 1, iocb->ki_pos); - if (ret > 0) { - iocb->ki_buf += ret; - iocb->ki_left -= ret; - } - } while (ret > 0 && iocb->ki_left > 0); + ret = rw_op(iocb, &iocb->ki_iovec[iocb->ki_cur_seg], + iocb->ki_nr_segs - iocb->ki_cur_seg, + iocb->ki_pos); + if (ret > 0) + aio_advance_iovec(iocb, ret); + + /* retry all partial writes. retry partial reads as long as its a + * regular file. */ + } while (ret > 0 && iocb->ki_left > 0 && + (opcode == IOCB_CMD_PWRITEV || + (!S_ISFIFO(inode->i_mode) && !S_ISSOCK(inode->i_mode)))); + /* This means we must have transferred all that we could */ + /* No need to retry anymore */ if ((ret == 0) || (iocb->ki_left == 0)) ret = iocb->ki_nbytes - iocb->ki_left; @@ -1390,6 +1387,38 @@ static ssize_t aio_fsync(struct kiocb *iocb) return ret; } +static ssize_t aio_setup_vectored_rw(int type, struct kiocb *kiocb) +{ + ssize_t ret; + + ret = rw_copy_check_uvector(type, (struct iovec __user *)kiocb->ki_buf, + kiocb->ki_nbytes, 1, + &kiocb->ki_inline_vec, &kiocb->ki_iovec); + if (ret < 0) + goto out; + + kiocb->ki_nr_segs = kiocb->ki_nbytes; + kiocb->ki_cur_seg = 0; + /* ki_nbytes/left now reflect bytes instead of segs */ + kiocb->ki_nbytes = ret; + kiocb->ki_left = ret; + + ret = 0; +out: + return ret; +} + +static ssize_t aio_setup_single_vector(struct kiocb *kiocb) +{ + kiocb->ki_iovec = &kiocb->ki_inline_vec; + kiocb->ki_iovec->iov_base = kiocb->ki_buf; + kiocb->ki_iovec->iov_len = kiocb->ki_left; + kiocb->ki_nr_segs = 1; + kiocb->ki_cur_seg = 0; + kiocb->ki_nbytes = kiocb->ki_left; + return 0; +} + /* * aio_setup_iocb: * Performs the initial checks and aio retry method @@ -1412,9 +1441,12 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb) ret = security_file_permission(file, MAY_READ); if (unlikely(ret)) break; + ret = aio_setup_single_vector(kiocb); + if (ret) + break; ret = -EINVAL; if (file->f_op->aio_read) - kiocb->ki_retry = aio_pread; + kiocb->ki_retry = aio_rw_vect_retry; break; case IOCB_CMD_PWRITE: ret = -EBADF; @@ -1427,9 +1459,40 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb) ret = security_file_permission(file, MAY_WRITE); if (unlikely(ret)) break; + ret = aio_setup_single_vector(kiocb); + if (ret) + break; + ret = -EINVAL; + if (file->f_op->aio_write) + kiocb->ki_retry = aio_rw_vect_retry; + break; + case IOCB_CMD_PREADV: + ret = -EBADF; + if (unlikely(!(file->f_mode & FMODE_READ))) + break; + ret = security_file_permission(file, MAY_READ); + if (unlikely(ret)) + break; + ret = aio_setup_vectored_rw(READ, kiocb); + if (ret) + break; + ret = -EINVAL; + if (file->f_op->aio_read) + kiocb->ki_retry = aio_rw_vect_retry; + break; + case IOCB_CMD_PWRITEV: + ret = -EBADF; + if (unlikely(!(file->f_mode & FMODE_WRITE))) + break; + ret = security_file_permission(file, MAY_WRITE); + if (unlikely(ret)) + break; + ret = aio_setup_vectored_rw(WRITE, kiocb); + if (ret) + break; ret = -EINVAL; if (file->f_op->aio_write) - kiocb->ki_retry = aio_pwrite; + kiocb->ki_retry = aio_rw_vect_retry; break; case IOCB_CMD_FDSYNC: ret = -EINVAL; diff --git a/fs/read_write.c b/fs/read_write.c index 4ed839bcb91..f792000a28e 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -511,6 +511,74 @@ ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov, /* A write operation does a read from user space and vice versa */ #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ) +ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, + unsigned long nr_segs, unsigned long fast_segs, + struct iovec *fast_pointer, + struct iovec **ret_pointer) + { + unsigned long seg; + ssize_t ret; + struct iovec *iov = fast_pointer; + + /* + * SuS says "The readv() function *may* fail if the iovcnt argument + * was less than or equal to 0, or greater than {IOV_MAX}. Linux has + * traditionally returned zero for zero segments, so... + */ + if (nr_segs == 0) { + ret = 0; + goto out; + } + + /* + * First get the "struct iovec" from user memory and + * verify all the pointers + */ + if (nr_segs > UIO_MAXIOV) { + ret = -EINVAL; + goto out; + } + if (nr_segs > fast_segs) { + iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL); + if (iov == NULL) { + ret = -ENOMEM; + goto out; + } + } + if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) { + ret = -EFAULT; + goto out; + } + + /* + * According to the Single Unix Specification we should return EINVAL + * if an element length is < 0 when cast to ssize_t or if the + * total length would overflow the ssize_t return value of the + * system call. + */ + ret = 0; + for (seg = 0; seg < nr_segs; seg++) { + void __user *buf = iov[seg].iov_base; + ssize_t len = (ssize_t)iov[seg].iov_len; + + /* see if we we're about to use an invalid len or if + * it's about to overflow ssize_t */ + if (len < 0 || (ret + len < ret)) { + ret = -EINVAL; + goto out; + } + if (unlikely(!access_ok(vrfy_dir(type), buf, len))) { + ret = -EFAULT; + goto out; + } + + ret += len; + } +out: + *ret_pointer = iov; + return ret; +} + static ssize_t do_readv_writev(int type, struct file *file, const struct iovec __user * uvector, unsigned long nr_segs, loff_t *pos) @@ -519,64 +587,20 @@ static ssize_t do_readv_writev(int type, struct file *file, struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov = iovstack; ssize_t ret; - int seg; io_fn_t fn; iov_fn_t fnv; - /* - * SuS says "The readv() function *may* fail if the iovcnt argument - * was less than or equal to 0, or greater than {IOV_MAX}. Linux has - * traditionally returned zero for zero segments, so... - */ - ret = 0; - if (nr_segs == 0) - goto out; - - /* - * First get the "struct iovec" from user memory and - * verify all the pointers - */ - ret = -EINVAL; - if (nr_segs > UIO_MAXIOV) - goto out; - if (!file->f_op) - goto out; - if (nr_segs > UIO_FASTIOV) { - ret = -ENOMEM; - iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL); - if (!iov) - goto out; - } - ret = -EFAULT; - if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) + if (!file->f_op) { + ret = -EINVAL; goto out; + } - /* - * Single unix specification: - * We should -EINVAL if an element length is not >= 0 and fitting an - * ssize_t. The total length is fitting an ssize_t - * - * Be careful here because iov_len is a size_t not an ssize_t - */ - tot_len = 0; - ret = -EINVAL; - for (seg = 0; seg < nr_segs; seg++) { - void __user *buf = iov[seg].iov_base; - ssize_t len = (ssize_t)iov[seg].iov_len; - - if (len < 0) /* size_t not fitting an ssize_t .. */ - goto out; - if (unlikely(!access_ok(vrfy_dir(type), buf, len))) - goto Efault; - tot_len += len; - if ((ssize_t)tot_len < 0) /* maths overflow on the ssize_t */ - goto out; - } - if (tot_len == 0) { - ret = 0; + ret = rw_copy_check_uvector(type, uvector, nr_segs, + ARRAY_SIZE(iovstack), iovstack, &iov); + if (ret <= 0) goto out; - } + tot_len = ret; ret = rw_verify_area(type, file, pos, tot_len); if (ret < 0) goto out; @@ -609,9 +633,6 @@ out: fsnotify_modify(file->f_dentry); } return ret; -Efault: - ret = -EFAULT; - goto out; } ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, diff --git a/include/linux/aio.h b/include/linux/aio.h index 58349e58b74..5722568fc71 100644 --- a/include/linux/aio.h +++ b/include/linux/aio.h @@ -7,6 +7,7 @@ #include #include +#include #define AIO_MAXSEGS 4 #define AIO_KIOGRP_NR_ATOMIC 8 @@ -114,6 +115,9 @@ struct kiocb { long ki_kicked; /* just for testing */ long ki_queued; /* just for testing */ struct iovec ki_inline_vec; /* inline vector */ + struct iovec *ki_iovec; + unsigned long ki_nr_segs; + unsigned long ki_cur_seg; struct list_head ki_list; /* the aio core uses this * for cancellation */ diff --git a/include/linux/aio_abi.h b/include/linux/aio_abi.h index 30fdcc89d14..3466b1d0ffd 100644 --- a/include/linux/aio_abi.h +++ b/include/linux/aio_abi.h @@ -41,6 +41,8 @@ enum { * IOCB_CMD_POLL = 5, */ IOCB_CMD_NOOP = 6, + IOCB_CMD_PREADV = 7, + IOCB_CMD_PWRITEV = 8, }; /* read() from /dev/aio returns these structures. */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 011129f8803..4bb70871873 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1150,6 +1150,11 @@ struct inode_operations { struct seq_file; +ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, + unsigned long nr_segs, unsigned long fast_segs, + struct iovec *fast_pointer, + struct iovec **ret_pointer); + extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *); extern ssize_t vfs_readv(struct file *, const struct iovec __user *, -- cgit v1.2.3-70-g09d2