sched: wakeup preemption fix

[linux-2.6] / kernel / sched_fair.c

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 01859f662ab7c834826f9197e93ebe9d44ad548a..a3badf52bba2352e489c94bc768c9617bc4cb707 100644 (file)
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -22,7 +22,7 @@
 
 /*
  * Targeted preemption latency for CPU-bound tasks:
- * (default: 20ms, units: nanoseconds)
+ * (default: 20ms * ilog(ncpus), units: nanoseconds)
  *
  * NOTE: this latency value is not the same as the concept of
  * 'timeslice length' - timeslices in CFS are of variable length
@@ -32,19 +32,24 @@
  * (to see the precise effective timeslice length of your workload,
  *  run vmstat and monitor the context-switches (cs) field)
  */
-const_debug unsigned int sysctl_sched_latency = 20000000ULL;
+unsigned int sysctl_sched_latency = 20000000ULL;
 
 /*
- * After fork, child runs first. (default) If set to 0 then
- * parent will (try to) run first.
+ * Minimal preemption granularity for CPU-bound tasks:
+ * (default: 1 msec * ilog(ncpus), units: nanoseconds)
  */
-const_debug unsigned int sysctl_sched_child_runs_first = 1;
+unsigned int sysctl_sched_min_granularity = 1000000ULL;
 
 /*
- * Minimal preemption granularity for CPU-bound tasks:
- * (default: 2 msec, units: nanoseconds)
+ * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
+ */
+unsigned int sched_nr_latency = 20;
+
+/*
+ * After fork, child runs first. (default) If set to 0 then
+ * parent will (try to) run first.
  */
-const_debug unsigned int sysctl_sched_nr_latency = 20;
+const_debug unsigned int sysctl_sched_child_runs_first = 1;
 
 /*
  * sys_sched_yield() compat mode
@@ -56,23 +61,23 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
 
 /*
  * SCHED_BATCH wake-up granularity.
- * (default: 10 msec, units: nanoseconds)
+ * (default: 10 msec * ilog(ncpus), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-const_debug unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL;
+unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL;
 
 /*
  * SCHED_OTHER wake-up granularity.
- * (default: 10 msec, units: nanoseconds)
+ * (default: 10 msec * ilog(ncpus), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-const_debug unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
+unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -212,6 +217,22 @@ static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
  * Scheduling class statistics methods:
  */
 
+#ifdef CONFIG_SCHED_DEBUG
+int sched_nr_latency_handler(struct ctl_table *table, int write,
+               struct file *filp, void __user *buffer, size_t *lenp,
+               loff_t *ppos)
+{
+       int ret = proc_dointvec_minmax(table, write, filp, buffer, lenp, ppos);
+
+       if (ret || !write)
+               return ret;
+
+       sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
+                                       sysctl_sched_min_granularity);
+
+       return 0;
+}
+#endif
 
 /*
  * The idea is to set a period in which each task runs once.
@@ -224,7 +245,7 @@ static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
 static u64 __sched_period(unsigned long nr_running)
 {
        u64 period = sysctl_sched_latency;
-       unsigned long nr_latency = sysctl_sched_nr_latency;
+       unsigned long nr_latency = sched_nr_latency;
 
        if (unlikely(nr_running > nr_latency)) {
                period *= nr_running;
@@ -259,6 +280,7 @@ static u64 __sched_vslice(unsigned long rq_weight, unsigned long nr_running)
 {
        u64 vslice = __sched_period(nr_running);
 
+       vslice *= NICE_0_LOAD;
        do_div(vslice, rq_weight);
 
        return vslice;
@@ -472,19 +494,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
        } else if (sched_feat(APPROX_AVG) && cfs_rq->nr_running)
                vruntime += sched_vslice(cfs_rq)/2;
 
+       /*
+        * The 'current' period is already promised to the current tasks,
+        * however the extra weight of the new task will slow them down a
+        * little, place the new task so that it fits in the slot that
+        * stays open at the end.
+        */
        if (initial && sched_feat(START_DEBIT))
                vruntime += sched_vslice_add(cfs_rq, se);
 
        if (!initial) {
+               /* sleeps upto a single latency don't count. */
                if (sched_feat(NEW_FAIR_SLEEPERS) && entity_is_task(se) &&
                                task_of(se)->policy != SCHED_BATCH)
                        vruntime -= sysctl_sched_latency;
 
-               vruntime = max_t(s64, vruntime, se->vruntime);
+               /* ensure we never gain time by being placed backwards. */
+               vruntime = max_vruntime(se->vruntime, vruntime);
        }
 
        se->vruntime = vruntime;
-
 }
 
 static void
@@ -517,7 +546,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
 
        update_stats_dequeue(cfs_rq, se);
        if (sleep) {
-               se->peer_preempt = 0;
 #ifdef CONFIG_SCHEDSTATS
                if (entity_is_task(se)) {
                        struct task_struct *tsk = task_of(se);
@@ -545,10 +573,8 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
 
        ideal_runtime = sched_slice(cfs_rq, curr);
        delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
-       if (delta_exec > ideal_runtime ||
-                       (sched_feat(PREEMPT_RESTRICT) && curr->peer_preempt))
+       if (delta_exec > ideal_runtime)
                resched_task(rq_of(cfs_rq)->curr);
-       curr->peer_preempt = 0;
 }
 
 static void
@@ -837,12 +863,8 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
                if (unlikely(se->load.weight != NICE_0_LOAD))
                        gran = calc_delta_fair(gran, &se->load);
 
-               if (delta > gran) {
-                       int now = !sched_feat(PREEMPT_RESTRICT);
-
-                       if (now || p->prio < curr->prio || !se->peer_preempt++)
-                               resched_task(curr);
-               }
+               if (delta > gran)
+                       resched_task(curr);
        }
 }
 
@@ -1054,7 +1076,6 @@ static void task_new_fair(struct rq *rq, struct task_struct *p)
                swap(curr->vruntime, se->vruntime);
        }
 
-       se->peer_preempt = 0;
        enqueue_task_fair(rq, p, 0);
        resched_task(rq->curr);
 }