X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=block%2Fcfq-iosched.c;h=f4e1006c253d3ade81cc9b39ce80da6612b277a6;hb=7cece14acd063dd1c4e8933461d44ec6a5a5517b;hp=f28d1fb30608cd5667b6b6f9c67e545bdb4802de;hpb=75659ca0c10992dcb39258518368a0f6f56e935d;p=linux-2.6 diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index f28d1fb306..f4e1006c25 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -15,11 +15,13 @@ /* * tunables */ -static const int cfq_quantum = 4; /* max queue in one round of service */ +/* max queue in one round of service */ +static const int cfq_quantum = 4; static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 }; -static const int cfq_back_max = 16 * 1024; /* maximum backwards seek, in KiB */ -static const int cfq_back_penalty = 2; /* penalty of a backwards seek */ - +/* maximum backwards seek, in KiB */ +static const int cfq_back_max = 16 * 1024; +/* penalty of a backwards seek */ +static const int cfq_back_penalty = 2; static const int cfq_slice_sync = HZ / 10; static int cfq_slice_async = HZ / 25; static const int cfq_slice_async_rq = 2; @@ -37,7 +39,8 @@ static int cfq_slice_idle = HZ / 125; #define CFQ_SLICE_SCALE (5) -#define RQ_CIC(rq) ((struct cfq_io_context*)(rq)->elevator_private) +#define RQ_CIC(rq) \ + ((struct cfq_io_context *) (rq)->elevator_private) #define RQ_CFQQ(rq) ((rq)->elevator_private2) static struct kmem_cache *cfq_pool; @@ -171,15 +174,15 @@ enum cfqq_state_flags { #define CFQ_CFQQ_FNS(name) \ static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq) \ { \ - cfqq->flags |= (1 << CFQ_CFQQ_FLAG_##name); \ + (cfqq)->flags |= (1 << CFQ_CFQQ_FLAG_##name); \ } \ static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq) \ { \ - cfqq->flags &= ~(1 << CFQ_CFQQ_FLAG_##name); \ + (cfqq)->flags &= ~(1 << CFQ_CFQQ_FLAG_##name); \ } \ static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq) \ { \ - return (cfqq->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0; \ + return ((cfqq)->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0; \ } CFQ_CFQQ_FNS(on_rr); @@ -1005,7 +1008,8 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq, /* * follow expired path, else get first next available */ - if ((rq = cfq_check_fifo(cfqq)) == NULL) + rq = cfq_check_fifo(cfqq); + if (rq == NULL) rq = cfqq->next_rq; /* @@ -1139,43 +1143,37 @@ static void cfq_put_queue(struct cfq_queue *cfqq) } /* - * Call func for each cic attached to this ioc. Returns number of cic's seen. + * Call func for each cic attached to this ioc. */ -#define CIC_GANG_NR 16 -static unsigned int +static void call_for_each_cic(struct io_context *ioc, void (*func)(struct io_context *, struct cfq_io_context *)) { - struct cfq_io_context *cics[CIC_GANG_NR]; - unsigned long index = 0; - unsigned int called = 0; - int nr; + struct cfq_io_context *cic; + struct hlist_node *n; rcu_read_lock(); + hlist_for_each_entry_rcu(cic, n, &ioc->cic_list, cic_list) + func(ioc, cic); + rcu_read_unlock(); +} - do { - int i; - - /* - * Perhaps there's a better way - this just gang lookups from - * 0 to the end, restarting after each CIC_GANG_NR from the - * last key + 1. - */ - nr = radix_tree_gang_lookup(&ioc->radix_root, (void **) cics, - index, CIC_GANG_NR); - if (!nr) - break; +static void cfq_cic_free_rcu(struct rcu_head *head) +{ + struct cfq_io_context *cic; - called += nr; - index = 1 + (unsigned long) cics[nr - 1]->key; + cic = container_of(head, struct cfq_io_context, rcu_head); - for (i = 0; i < nr; i++) - func(ioc, cics[i]); - } while (nr == CIC_GANG_NR); + kmem_cache_free(cfq_ioc_pool, cic); + elv_ioc_count_dec(ioc_count); - rcu_read_unlock(); + if (ioc_gone && !elv_ioc_count_read(ioc_count)) + complete(ioc_gone); +} - return called; +static void cfq_cic_free(struct cfq_io_context *cic) +{ + call_rcu(&cic->rcu_head, cfq_cic_free_rcu); } static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic) @@ -1186,26 +1184,21 @@ static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic) spin_lock_irqsave(&ioc->lock, flags); radix_tree_delete(&ioc->radix_root, cic->dead_key); + hlist_del_rcu(&cic->cic_list); spin_unlock_irqrestore(&ioc->lock, flags); - kmem_cache_free(cfq_ioc_pool, cic); + cfq_cic_free(cic); } static void cfq_free_io_context(struct io_context *ioc) { - int freed; - /* - * ioc->refcount is zero here, so no more cic's are allowed to be - * linked into this ioc. So it should be ok to iterate over the known - * list, we will see all cic's since no new ones are added. + * ioc->refcount is zero here, or we are called from elv_unregister(), + * so no more cic's are allowed to be linked into this ioc. So it + * should be ok to iterate over the known list, we will see all cic's + * since no new ones are added. */ - freed = call_for_each_cic(ioc, cic_free_func); - - elv_ioc_count_mod(ioc_count, -freed); - - if (ioc_gone && !elv_ioc_count_read(ioc_count)) - complete(ioc_gone); + call_for_each_cic(ioc, cic_free_func); } static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq) @@ -1221,6 +1214,8 @@ static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq) static void __cfq_exit_single_io_context(struct cfq_data *cfqd, struct cfq_io_context *cic) { + struct io_context *ioc = cic->ioc; + list_del_init(&cic->queue_list); /* @@ -1230,6 +1225,9 @@ static void __cfq_exit_single_io_context(struct cfq_data *cfqd, cic->dead_key = (unsigned long) cic->key; cic->key = NULL; + if (ioc->ioc_data == cic) + rcu_assign_pointer(ioc->ioc_data, NULL); + if (cic->cfqq[ASYNC]) { cfq_exit_cfqq(cfqd, cic->cfqq[ASYNC]); cic->cfqq[ASYNC] = NULL; @@ -1262,7 +1260,6 @@ static void cfq_exit_single_io_context(struct io_context *ioc, */ static void cfq_exit_io_context(struct io_context *ioc) { - rcu_assign_pointer(ioc->ioc_data, NULL); call_for_each_cic(ioc, cfq_exit_single_io_context); } @@ -1276,6 +1273,7 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask) if (cic) { cic->last_end_request = jiffies; INIT_LIST_HEAD(&cic->queue_list); + INIT_HLIST_NODE(&cic->cic_list); cic->dtor = cfq_free_io_context; cic->exit = cfq_exit_io_context; elv_ioc_count_inc(ioc_count); @@ -1294,28 +1292,28 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc) ioprio_class = IOPRIO_PRIO_CLASS(ioc->ioprio); switch (ioprio_class) { - default: - printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class); - case IOPRIO_CLASS_NONE: - /* - * no prio set, place us in the middle of the BE classes - */ - cfqq->ioprio = task_nice_ioprio(tsk); - cfqq->ioprio_class = IOPRIO_CLASS_BE; - break; - case IOPRIO_CLASS_RT: - cfqq->ioprio = task_ioprio(ioc); - cfqq->ioprio_class = IOPRIO_CLASS_RT; - break; - case IOPRIO_CLASS_BE: - cfqq->ioprio = task_ioprio(ioc); - cfqq->ioprio_class = IOPRIO_CLASS_BE; - break; - case IOPRIO_CLASS_IDLE: - cfqq->ioprio_class = IOPRIO_CLASS_IDLE; - cfqq->ioprio = 7; - cfq_clear_cfqq_idle_window(cfqq); - break; + default: + printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class); + case IOPRIO_CLASS_NONE: + /* + * no prio set, place us in the middle of the BE classes + */ + cfqq->ioprio = task_nice_ioprio(tsk); + cfqq->ioprio_class = IOPRIO_CLASS_BE; + break; + case IOPRIO_CLASS_RT: + cfqq->ioprio = task_ioprio(ioc); + cfqq->ioprio_class = IOPRIO_CLASS_RT; + break; + case IOPRIO_CLASS_BE: + cfqq->ioprio = task_ioprio(ioc); + cfqq->ioprio_class = IOPRIO_CLASS_BE; + break; + case IOPRIO_CLASS_IDLE: + cfqq->ioprio_class = IOPRIO_CLASS_IDLE; + cfqq->ioprio = 7; + cfq_clear_cfqq_idle_window(cfqq); + break; } /* @@ -1427,7 +1425,7 @@ out: static struct cfq_queue ** cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio) { - switch(ioprio_class) { + switch (ioprio_class) { case IOPRIO_CLASS_RT: return &cfqd->async_cfqq[0][ioprio]; case IOPRIO_CLASS_BE: @@ -1471,15 +1469,6 @@ cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct io_context *ioc, return cfqq; } -static void cfq_cic_free(struct cfq_io_context *cic) -{ - kmem_cache_free(cfq_ioc_pool, cic); - elv_ioc_count_dec(ioc_count); - - if (ioc_gone && !elv_ioc_count_read(ioc_count)) - complete(ioc_gone); -} - /* * We drop cfq io contexts lazily, so we may find a dead one. */ @@ -1493,10 +1482,10 @@ cfq_drop_dead_cic(struct cfq_data *cfqd, struct io_context *ioc, spin_lock_irqsave(&ioc->lock, flags); - if (ioc->ioc_data == cic) - rcu_assign_pointer(ioc->ioc_data, NULL); + BUG_ON(ioc->ioc_data == cic); radix_tree_delete(&ioc->radix_root, (unsigned long) cfqd); + hlist_del_rcu(&cic->cic_list); spin_unlock_irqrestore(&ioc->lock, flags); cfq_cic_free(cic); @@ -1557,6 +1546,8 @@ static int cfq_cic_link(struct cfq_data *cfqd, struct io_context *ioc, spin_lock_irqsave(&ioc->lock, flags); ret = radix_tree_insert(&ioc->radix_root, (unsigned long) cfqd, cic); + if (!ret) + hlist_add_head_rcu(&cic->cic_list, &ioc->cic_list); spin_unlock_irqrestore(&ioc->lock, flags); radix_tree_preload_end(); @@ -2018,7 +2009,8 @@ static void cfq_idle_slice_timer(unsigned long data) spin_lock_irqsave(cfqd->queue->queue_lock, flags); - if ((cfqq = cfqd->active_queue) != NULL) { + cfqq = cfqd->active_queue; + if (cfqq) { timed_out = 0; /* @@ -2147,7 +2139,7 @@ static int __init cfq_slab_setup(void) if (!cfq_pool) goto fail; - cfq_ioc_pool = KMEM_CACHE(cfq_io_context, SLAB_DESTROY_BY_RCU); + cfq_ioc_pool = KMEM_CACHE(cfq_io_context, 0); if (!cfq_ioc_pool) goto fail; @@ -2212,14 +2204,18 @@ static ssize_t __FUNC(elevator_t *e, const char *page, size_t count) \ return ret; \ } STORE_FUNCTION(cfq_quantum_store, &cfqd->cfq_quantum, 1, UINT_MAX, 0); -STORE_FUNCTION(cfq_fifo_expire_sync_store, &cfqd->cfq_fifo_expire[1], 1, UINT_MAX, 1); -STORE_FUNCTION(cfq_fifo_expire_async_store, &cfqd->cfq_fifo_expire[0], 1, UINT_MAX, 1); +STORE_FUNCTION(cfq_fifo_expire_sync_store, &cfqd->cfq_fifo_expire[1], 1, + UINT_MAX, 1); +STORE_FUNCTION(cfq_fifo_expire_async_store, &cfqd->cfq_fifo_expire[0], 1, + UINT_MAX, 1); STORE_FUNCTION(cfq_back_seek_max_store, &cfqd->cfq_back_max, 0, UINT_MAX, 0); -STORE_FUNCTION(cfq_back_seek_penalty_store, &cfqd->cfq_back_penalty, 1, UINT_MAX, 0); +STORE_FUNCTION(cfq_back_seek_penalty_store, &cfqd->cfq_back_penalty, 1, + UINT_MAX, 0); STORE_FUNCTION(cfq_slice_idle_store, &cfqd->cfq_slice_idle, 0, UINT_MAX, 1); STORE_FUNCTION(cfq_slice_sync_store, &cfqd->cfq_slice[1], 1, UINT_MAX, 1); STORE_FUNCTION(cfq_slice_async_store, &cfqd->cfq_slice[0], 1, UINT_MAX, 1); -STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1, UINT_MAX, 0); +STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1, + UINT_MAX, 0); #undef STORE_FUNCTION #define CFQ_ATTR(name) \ @@ -2291,7 +2287,6 @@ static void __exit cfq_exit(void) smp_wmb(); if (elv_ioc_count_read(ioc_count)) wait_for_completion(ioc_gone); - synchronize_rcu(); cfq_slab_kill(); }