[PATCH] io-accounting: core statistics

[linux-2.6] / kernel / workqueue.c

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 2945b094d871acdd2e0faa879080f75ae08fc750..db49886bfae17271ee19709a984c57c208c5065b 100644 (file)
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -30,6 +30,8 @@
 #include <linux/hardirq.h>
 #include <linux/mempolicy.h>
 #include <linux/freezer.h>
+#include <linux/kallsyms.h>
+#include <linux/debug_locks.h>
 
 /*
  * The per-CPU workqueue (if single thread, we always use the first
@@ -83,22 +85,19 @@ static inline int is_single_threaded(struct workqueue_struct *wq)
        return list_empty(&wq->list);
 }
 
+/*
+ * Set the workqueue on which a work item is to be run
+ * - Must *only* be called if the pending flag is set
+ */
 static inline void set_wq_data(struct work_struct *work, void *wq)
 {
-       unsigned long new, old, res;
+       unsigned long new;
+
+       BUG_ON(!work_pending(work));
 
-       /* assume the pending flag is already set and that the task has already
-        * been queued on this workqueue */
        new = (unsigned long) wq | (1UL << WORK_STRUCT_PENDING);
-       res = work->management;
-       if (res != new) {
-               do {
-                       old = res;
-                       new = (unsigned long) wq;
-                       new |= (old & WORK_STRUCT_FLAG_MASK);
-                       res = cmpxchg(&work->management, old, new);
-               } while (res != old);
-       }
+       new |= work->management & WORK_STRUCT_FLAG_MASK;
+       work->management = new;
 }
 
 static inline void *get_wq_data(struct work_struct *work)
@@ -106,6 +105,79 @@ static inline void *get_wq_data(struct work_struct *work)
        return (void *) (work->management & WORK_STRUCT_WQ_DATA_MASK);
 }
 
+static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work)
+{
+       int ret = 0;
+       unsigned long flags;
+
+       spin_lock_irqsave(&cwq->lock, flags);
+       /*
+        * We need to re-validate the work info after we've gotten
+        * the cpu_workqueue lock. We can run the work now iff:
+        *
+        *  - the wq_data still matches the cpu_workqueue_struct
+        *  - AND the work is still marked pending
+        *  - AND the work is still on a list (which will be this
+        *    workqueue_struct list)
+        *
+        * All these conditions are important, because we
+        * need to protect against the work being run right
+        * now on another CPU (all but the last one might be
+        * true if it's currently running and has not been
+        * released yet, for example).
+        */
+       if (get_wq_data(work) == cwq
+           && work_pending(work)
+           && !list_empty(&work->entry)) {
+               work_func_t f = work->func;
+               list_del_init(&work->entry);
+               spin_unlock_irqrestore(&cwq->lock, flags);
+
+               if (!test_bit(WORK_STRUCT_NOAUTOREL, &work->management))
+                       work_release(work);
+               f(work);
+
+               spin_lock_irqsave(&cwq->lock, flags);
+               cwq->remove_sequence++;
+               wake_up(&cwq->work_done);
+               ret = 1;
+       }
+       spin_unlock_irqrestore(&cwq->lock, flags);
+       return ret;
+}
+
+/**
+ * run_scheduled_work - run scheduled work synchronously
+ * @work: work to run
+ *
+ * This checks if the work was pending, and runs it
+ * synchronously if so. It returns a boolean to indicate
+ * whether it had any scheduled work to run or not.
+ *
+ * NOTE! This _only_ works for normal work_structs. You
+ * CANNOT use this for delayed work, because the wq data
+ * for delayed work will not point properly to the per-
+ * CPU workqueue struct, but will change!
+ */
+int fastcall run_scheduled_work(struct work_struct *work)
+{
+       for (;;) {
+               struct cpu_workqueue_struct *cwq;
+
+               if (!work_pending(work))
+                       return 0;
+               if (list_empty(&work->entry))
+                       return 0;
+               /* NOTE! This depends intimately on __queue_work! */
+               cwq = get_wq_data(work);
+               if (!cwq)
+                       return 0;
+               if (__run_work(cwq, work))
+                       return 1;
+       }
+}
+EXPORT_SYMBOL(run_scheduled_work);
+
 /* Preempt must be disabled. */
 static void __queue_work(struct cpu_workqueue_struct *cwq,
                         struct work_struct *work)
@@ -253,6 +325,17 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq)
                        work_release(work);
                f(work);
 
+               if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
+                       printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
+                                       "%s/0x%08x/%d\n",
+                                       current->comm, preempt_count(),
+                                       current->pid);
+                       printk(KERN_ERR "    last function: ");
+                       print_symbol("%s\n", (unsigned long)f);
+                       debug_show_held_locks(current);
+                       dump_stack();
+               }
+
                spin_lock_irqsave(&cwq->lock, flags);
                cwq->remove_sequence++;
                wake_up(&cwq->work_done);
@@ -642,7 +725,6 @@ int current_is_keventd(void)
 
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
 /* Take the work from this (downed) CPU. */
 static void take_over_work(struct workqueue_struct *wq, unsigned int cpu)
 {
@@ -725,7 +807,6 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
 
        return NOTIFY_OK;
 }
-#endif
 
 void init_workqueues(void)
 {