#ifdef RPC_DEBUG
#define RPCDBG_FACILITY RPCDBG_SCHED
#define RPC_TASK_MAGIC_ID 0xf00baa
-static int rpc_task_id;
#endif
/*
static mempool_t *rpc_buffer_mempool __read_mostly;
static void __rpc_default_timer(struct rpc_task *task);
-static void rpciod_killall(void);
static void rpc_async_schedule(struct work_struct *);
static void rpc_release_task(struct rpc_task *task);
*/
static RPC_WAITQ(delay_queue, "delayq");
-/*
- * All RPC tasks are linked into this list
- */
-static LIST_HEAD(all_tasks);
-
/*
* rpciod-related stuff
*/
static DEFINE_MUTEX(rpciod_mutex);
-static unsigned int rpciod_users;
+static atomic_t rpciod_users = ATOMIC_INIT(0);
struct workqueue_struct *rpciod_workqueue;
-/*
- * Spinlock for other critical sections of code.
- */
-static DEFINE_SPINLOCK(rpc_sched_lock);
-
/*
* Disable the timer for a given RPC task. Should be called with
* queue->lock and bh_disabled in order to avoid races within
static inline void
__rpc_disable_timer(struct rpc_task *task)
{
- dprintk("RPC: %4d disabling timer\n", task->tk_pid);
+ dprintk("RPC: %5u disabling timer\n", task->tk_pid);
task->tk_timeout_fn = NULL;
task->tk_timeout = 0;
}
callback = task->tk_timeout_fn;
task->tk_timeout_fn = NULL;
if (callback && RPC_IS_QUEUED(task)) {
- dprintk("RPC: %4d running timer\n", task->tk_pid);
+ dprintk("RPC: %5u running timer\n", task->tk_pid);
callback(task);
}
smp_mb__before_clear_bit();
if (!task->tk_timeout)
return;
- dprintk("RPC: %4d setting alarm for %lu ms\n",
+ dprintk("RPC: %5u setting alarm for %lu ms\n",
task->tk_pid, task->tk_timeout * 1000 / HZ);
if (timer)
return;
if (test_and_clear_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate)) {
del_singleshot_timer_sync(&task->tk_timer);
- dprintk("RPC: %4d deleting timer\n", task->tk_pid);
+ dprintk("RPC: %5u deleting timer\n", task->tk_pid);
}
}
queue->qlen++;
rpc_set_queued(task);
- dprintk("RPC: %4d added to queue %p \"%s\"\n",
- task->tk_pid, queue, rpc_qname(queue));
+ dprintk("RPC: %5u added to queue %p \"%s\"\n",
+ task->tk_pid, queue, rpc_qname(queue));
}
/*
else
list_del(&task->u.tk_wait.list);
queue->qlen--;
- dprintk("RPC: %4d removed from queue %p \"%s\"\n",
- task->tk_pid, queue, rpc_qname(queue));
+ dprintk("RPC: %5u removed from queue %p \"%s\"\n",
+ task->tk_pid, queue, rpc_qname(queue));
}
static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
return 0;
}
+#ifdef RPC_DEBUG
+static void rpc_task_set_debuginfo(struct rpc_task *task)
+{
+ static atomic_t rpc_pid;
+
+ task->tk_magic = RPC_TASK_MAGIC_ID;
+ task->tk_pid = atomic_inc_return(&rpc_pid);
+}
+#else
+static inline void rpc_task_set_debuginfo(struct rpc_task *task)
+{
+}
+#endif
+
static void rpc_set_active(struct rpc_task *task)
{
+ struct rpc_clnt *clnt;
if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0)
return;
- spin_lock(&rpc_sched_lock);
-#ifdef RPC_DEBUG
- task->tk_magic = RPC_TASK_MAGIC_ID;
- task->tk_pid = rpc_task_id++;
-#endif
+ rpc_task_set_debuginfo(task);
/* Add to global list of all tasks */
- list_add_tail(&task->tk_task, &all_tasks);
- spin_unlock(&rpc_sched_lock);
+ clnt = task->tk_client;
+ if (clnt != NULL) {
+ spin_lock(&clnt->cl_lock);
+ list_add_tail(&task->tk_task, &clnt->cl_tasks);
+ spin_unlock(&clnt->cl_lock);
+ }
}
/*
static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
rpc_action action, rpc_action timer)
{
- dprintk("RPC: %4d sleep_on(queue \"%s\" time %ld)\n", task->tk_pid,
- rpc_qname(q), jiffies);
+ dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
+ task->tk_pid, rpc_qname(q), jiffies);
if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) {
printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n");
*/
static void __rpc_do_wake_up_task(struct rpc_task *task)
{
- dprintk("RPC: %4d __rpc_wake_up_task (now %ld)\n", task->tk_pid, jiffies);
+ dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
+ task->tk_pid, jiffies);
#ifdef RPC_DEBUG
BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
rpc_make_runnable(task);
- dprintk("RPC: __rpc_wake_up_task done\n");
+ dprintk("RPC: __rpc_wake_up_task done\n");
}
/*
static void
__rpc_default_timer(struct rpc_task *task)
{
- dprintk("RPC: %d timeout (default timer)\n", task->tk_pid);
+ dprintk("RPC: %5u timeout (default timer)\n", task->tk_pid);
task->tk_status = -ETIMEDOUT;
rpc_wake_up_task(task);
}
{
struct rpc_task *task = NULL;
- dprintk("RPC: wake_up_next(%p \"%s\")\n", queue, rpc_qname(queue));
+ dprintk("RPC: wake_up_next(%p \"%s\")\n",
+ queue, rpc_qname(queue));
rcu_read_lock_bh();
spin_lock(&queue->lock);
if (RPC_IS_PRIORITY(queue))
/*
* This is the RPC `scheduler' (or rather, the finite state machine).
*/
-static int __rpc_execute(struct rpc_task *task)
+static void __rpc_execute(struct rpc_task *task)
{
int status = 0;
- dprintk("RPC: %4d rpc_execute flgs %x\n",
- task->tk_pid, task->tk_flags);
+ dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
+ task->tk_pid, task->tk_flags);
BUG_ON(RPC_IS_QUEUED(task));
if (RPC_IS_ASYNC(task)) {
/* Careful! we may have raced... */
if (RPC_IS_QUEUED(task))
- return 0;
+ return;
if (rpc_test_and_set_running(task))
- return 0;
+ return;
continue;
}
/* sync task: sleep here */
- dprintk("RPC: %4d sync task going to sleep\n", task->tk_pid);
+ dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
/* Note: Caller should be using rpc_clnt_sigmask() */
status = out_of_line_wait_on_bit(&task->tk_runstate,
RPC_TASK_QUEUED, rpc_wait_bit_interruptible,
* clean up after sleeping on some queue, we don't
* break the loop here, but go around once more.
*/
- dprintk("RPC: %4d got signal\n", task->tk_pid);
+ dprintk("RPC: %5u got signal\n", task->tk_pid);
task->tk_flags |= RPC_TASK_KILLED;
rpc_exit(task, -ERESTARTSYS);
rpc_wake_up_task(task);
}
rpc_set_running(task);
- dprintk("RPC: %4d sync task resuming\n", task->tk_pid);
+ dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
}
- dprintk("RPC: %4d, return %d, status %d\n", task->tk_pid, status, task->tk_status);
+ dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
+ task->tk_status);
/* Release all resources associated with the task */
rpc_release_task(task);
- return status;
}
/*
* released. In particular note that tk_release() will have
* been called, so your task memory may have been freed.
*/
-int
-rpc_execute(struct rpc_task *task)
+void rpc_execute(struct rpc_task *task)
{
rpc_set_active(task);
rpc_set_running(task);
- return __rpc_execute(task);
+ __rpc_execute(task);
}
static void rpc_async_schedule(struct work_struct *work)
__rpc_execute(container_of(work, struct rpc_task, u.tk_work));
}
+struct rpc_buffer {
+ size_t len;
+ char data[];
+};
+
/**
* rpc_malloc - allocate an RPC buffer
* @task: RPC task that will use this buffer
* @size: requested byte size
*
- * We try to ensure that some NFS reads and writes can always proceed
- * by using a mempool when allocating 'small' buffers.
+ * To prevent rpciod from hanging, this allocator never sleeps,
+ * returning NULL if the request cannot be serviced immediately.
+ * The caller can arrange to sleep in a way that is safe for rpciod.
+ *
+ * Most requests are 'small' (under 2KiB) and can be serviced from a
+ * mempool, ensuring that NFS reads and writes can always proceed,
+ * and that there is good locality of reference for these buffers.
+ *
* In order to avoid memory starvation triggering more writebacks of
- * NFS requests, we use GFP_NOFS rather than GFP_KERNEL.
+ * NFS requests, we avoid using GFP_KERNEL.
*/
-void * rpc_malloc(struct rpc_task *task, size_t size)
+void *rpc_malloc(struct rpc_task *task, size_t size)
{
- struct rpc_rqst *req = task->tk_rqstp;
- gfp_t gfp;
+ struct rpc_buffer *buf;
+ gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
- if (task->tk_flags & RPC_TASK_SWAPPER)
- gfp = GFP_ATOMIC;
+ size += sizeof(struct rpc_buffer);
+ if (size <= RPC_BUFFER_MAXSIZE)
+ buf = mempool_alloc(rpc_buffer_mempool, gfp);
else
- gfp = GFP_NOFS;
-
- if (size > RPC_BUFFER_MAXSIZE) {
- req->rq_buffer = kmalloc(size, gfp);
- if (req->rq_buffer)
- req->rq_bufsize = size;
- } else {
- req->rq_buffer = mempool_alloc(rpc_buffer_mempool, gfp);
- if (req->rq_buffer)
- req->rq_bufsize = RPC_BUFFER_MAXSIZE;
- }
- return req->rq_buffer;
+ buf = kmalloc(size, gfp);
+
+ if (!buf)
+ return NULL;
+
+ buf->len = size;
+ dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
+ task->tk_pid, size, buf);
+ return &buf->data;
}
/**
* rpc_free - free buffer allocated via rpc_malloc
- * @task: RPC task with a buffer to be freed
+ * @buffer: buffer to free
*
*/
-void rpc_free(struct rpc_task *task)
+void rpc_free(void *buffer)
{
- struct rpc_rqst *req = task->tk_rqstp;
+ size_t size;
+ struct rpc_buffer *buf;
- if (req->rq_buffer) {
- if (req->rq_bufsize == RPC_BUFFER_MAXSIZE)
- mempool_free(req->rq_buffer, rpc_buffer_mempool);
- else
- kfree(req->rq_buffer);
- req->rq_buffer = NULL;
- req->rq_bufsize = 0;
- }
+ if (!buffer)
+ return;
+
+ buf = container_of(buffer, struct rpc_buffer, data);
+ size = buf->len;
+
+ dprintk("RPC: freeing buffer of size %zu at %p\n",
+ size, buf);
+
+ if (size <= RPC_BUFFER_MAXSIZE)
+ mempool_free(buf, rpc_buffer_mempool);
+ else
+ kfree(buf);
}
/*
if (tk_ops->rpc_call_prepare != NULL)
task->tk_action = rpc_prepare_task;
task->tk_calldata = calldata;
+ INIT_LIST_HEAD(&task->tk_task);
/* Initialize retry counters */
task->tk_garb_retry = 2;
task->tk_workqueue = rpciod_workqueue;
if (clnt) {
- atomic_inc(&clnt->cl_users);
+ kref_get(&clnt->cl_kref);
if (clnt->cl_softrtry)
task->tk_flags |= RPC_TASK_SOFT;
if (!clnt->cl_intr)
/* starting timestamp */
task->tk_start = jiffies;
- dprintk("RPC: %4d new task procpid %d\n", task->tk_pid,
+ dprintk("RPC: new task initialized, procpid %u\n",
current->pid);
}
static void rpc_free_task(struct rcu_head *rcu)
{
struct rpc_task *task = container_of(rcu, struct rpc_task, u.tk_rcu);
- dprintk("RPC: %4d freeing task\n", task->tk_pid);
+ dprintk("RPC: %5u freeing task\n", task->tk_pid);
mempool_free(task, rpc_task_mempool);
}
/*
- * Create a new task for the specified client. We have to
- * clean up after an allocation failure, as the client may
- * have specified "oneshot".
+ * Create a new task for the specified client.
*/
struct rpc_task *rpc_new_task(struct rpc_clnt *clnt, int flags, const struct rpc_call_ops *tk_ops, void *calldata)
{
task = rpc_alloc_task();
if (!task)
- goto cleanup;
+ goto out;
rpc_init_task(task, clnt, flags, tk_ops, calldata);
- dprintk("RPC: %4d allocated task\n", task->tk_pid);
+ dprintk("RPC: allocated task %p\n", task);
task->tk_flags |= RPC_TASK_DYNAMIC;
out:
return task;
-
-cleanup:
- /* Check whether to release the client */
- if (clnt) {
- printk("rpc_new_task: failed, users=%d, oneshot=%d\n",
- atomic_read(&clnt->cl_users), clnt->cl_oneshot);
- atomic_inc(&clnt->cl_users); /* pretend we were used ... */
- rpc_release_client(clnt);
- }
- goto out;
}
#ifdef RPC_DEBUG
BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID);
#endif
- dprintk("RPC: %4d release task\n", task->tk_pid);
-
- /* Remove from global task list */
- spin_lock(&rpc_sched_lock);
- list_del(&task->tk_task);
- spin_unlock(&rpc_sched_lock);
-
+ dprintk("RPC: %5u release task\n", task->tk_pid);
+
+ if (!list_empty(&task->tk_task)) {
+ struct rpc_clnt *clnt = task->tk_client;
+ /* Remove from client task list */
+ spin_lock(&clnt->cl_lock);
+ list_del(&task->tk_task);
+ spin_unlock(&clnt->cl_lock);
+ }
BUG_ON (RPC_IS_QUEUED(task));
/* Synchronously delete any running timer */
rpc_put_task(task);
}
-/**
- * rpc_run_task - Allocate a new RPC task, then run rpc_execute against it
- * @clnt: pointer to RPC client
- * @flags: RPC flags
- * @ops: RPC call ops
- * @data: user call data
- */
-struct rpc_task *rpc_run_task(struct rpc_clnt *clnt, int flags,
- const struct rpc_call_ops *ops,
- void *data)
-{
- struct rpc_task *task;
- task = rpc_new_task(clnt, flags, ops, data);
- if (task == NULL) {
- rpc_release_calldata(ops, data);
- return ERR_PTR(-ENOMEM);
- }
- atomic_inc(&task->tk_count);
- rpc_execute(task);
- return task;
-}
-EXPORT_SYMBOL(rpc_run_task);
-
/*
* Kill all tasks for the given client.
* XXX: kill their descendants as well?
void rpc_killall_tasks(struct rpc_clnt *clnt)
{
struct rpc_task *rovr;
- struct list_head *le;
- dprintk("RPC: killing all tasks for client %p\n", clnt);
+ if (list_empty(&clnt->cl_tasks))
+ return;
+ dprintk("RPC: killing all tasks for client %p\n", clnt);
/*
* Spin lock all_tasks to prevent changes...
*/
- spin_lock(&rpc_sched_lock);
- alltask_for_each(rovr, le, &all_tasks) {
+ spin_lock(&clnt->cl_lock);
+ list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
if (! RPC_IS_ACTIVATED(rovr))
continue;
- if (!clnt || rovr->tk_client == clnt) {
+ if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
rovr->tk_flags |= RPC_TASK_KILLED;
rpc_exit(rovr, -EIO);
rpc_wake_up_task(rovr);
}
}
- spin_unlock(&rpc_sched_lock);
-}
-
-static DECLARE_MUTEX_LOCKED(rpciod_running);
-
-static void rpciod_killall(void)
-{
- unsigned long flags;
-
- while (!list_empty(&all_tasks)) {
- clear_thread_flag(TIF_SIGPENDING);
- rpc_killall_tasks(NULL);
- flush_workqueue(rpciod_workqueue);
- if (!list_empty(&all_tasks)) {
- dprintk("rpciod_killall: waiting for tasks to exit\n");
- yield();
- }
- }
-
- spin_lock_irqsave(¤t->sighand->siglock, flags);
- recalc_sigpending();
- spin_unlock_irqrestore(¤t->sighand->siglock, flags);
+ spin_unlock(&clnt->cl_lock);
}
/*
struct workqueue_struct *wq;
int error = 0;
+ if (atomic_inc_not_zero(&rpciod_users))
+ return 0;
+
mutex_lock(&rpciod_mutex);
- dprintk("rpciod_up: users %d\n", rpciod_users);
- rpciod_users++;
- if (rpciod_workqueue)
- goto out;
- /*
- * If there's no pid, we should be the first user.
- */
- if (rpciod_users > 1)
- printk(KERN_WARNING "rpciod_up: no workqueue, %d users??\n", rpciod_users);
+
+ /* Guard against races with rpciod_down() */
+ if (rpciod_workqueue != NULL)
+ goto out_ok;
/*
* Create the rpciod thread and wait for it to start.
*/
+ dprintk("RPC: creating workqueue rpciod\n");
error = -ENOMEM;
wq = create_workqueue("rpciod");
- if (wq == NULL) {
- printk(KERN_WARNING "rpciod_up: create workqueue failed, error=%d\n", error);
- rpciod_users--;
+ if (wq == NULL)
goto out;
- }
+
rpciod_workqueue = wq;
error = 0;
+out_ok:
+ atomic_inc(&rpciod_users);
out:
mutex_unlock(&rpciod_mutex);
return error;
void
rpciod_down(void)
{
+ if (!atomic_dec_and_test(&rpciod_users))
+ return;
+
mutex_lock(&rpciod_mutex);
- dprintk("rpciod_down sema %d\n", rpciod_users);
- if (rpciod_users) {
- if (--rpciod_users)
- goto out;
- } else
- printk(KERN_WARNING "rpciod_down: no users??\n");
+ dprintk("RPC: destroying workqueue rpciod\n");
- if (!rpciod_workqueue) {
- dprintk("rpciod_down: Nothing to do!\n");
- goto out;
+ if (atomic_read(&rpciod_users) == 0 && rpciod_workqueue != NULL) {
+ destroy_workqueue(rpciod_workqueue);
+ rpciod_workqueue = NULL;
}
- rpciod_killall();
-
- destroy_workqueue(rpciod_workqueue);
- rpciod_workqueue = NULL;
- out:
mutex_unlock(&rpciod_mutex);
}
-#ifdef RPC_DEBUG
-void rpc_show_tasks(void)
-{
- struct list_head *le;
- struct rpc_task *t;
-
- spin_lock(&rpc_sched_lock);
- if (list_empty(&all_tasks)) {
- spin_unlock(&rpc_sched_lock);
- return;
- }
- printk("-pid- proc flgs status -client- -prog- --rqstp- -timeout "
- "-rpcwait -action- ---ops--\n");
- alltask_for_each(t, le, &all_tasks) {
- const char *rpc_waitq = "none";
-
- if (RPC_IS_QUEUED(t))
- rpc_waitq = rpc_qname(t->u.tk_wait.rpc_waitq);
-
- printk("%05d %04d %04x %06d %8p %6d %8p %08ld %8s %8p %8p\n",
- t->tk_pid,
- (t->tk_msg.rpc_proc ? t->tk_msg.rpc_proc->p_proc : -1),
- t->tk_flags, t->tk_status,
- t->tk_client,
- (t->tk_client ? t->tk_client->cl_prog : 0),
- t->tk_rqstp, t->tk_timeout,
- rpc_waitq,
- t->tk_action, t->tk_ops);
- }
- spin_unlock(&rpc_sched_lock);
-}
-#endif
-
void
rpc_destroy_mempool(void)
{
projects / linux-2.6 / blobdiff