/*
* For the allocated request tables
*/
-struct kmem_cache *request_cachep;
+static struct kmem_cache *request_cachep;
/*
* For queue allocation
return;
if (!new_io) {
- __disk_stat_inc(rq->rq_disk, merges[rw]);
+ __all_stat_inc(rq->rq_disk, merges[rw], rq->sector);
} else {
+ struct hd_struct *part = get_part(rq->rq_disk, rq->sector);
disk_round_stats(rq->rq_disk);
rq->rq_disk->in_flight++;
+ if (part) {
+ part_round_stats(part);
+ part->in_flight++;
+ }
}
}
}
EXPORT_SYMBOL(blk_get_backing_dev_info);
-void rq_init(struct request_queue *q, struct request *rq)
+void blk_rq_init(struct request_queue *q, struct request *rq)
{
+ memset(rq, 0, sizeof(*rq));
+
INIT_LIST_HEAD(&rq->queuelist);
INIT_LIST_HEAD(&rq->donelist);
-
- rq->errors = 0;
- rq->bio = rq->biotail = NULL;
+ rq->q = q;
+ rq->sector = rq->hard_sector = (sector_t) -1;
INIT_HLIST_NODE(&rq->hash);
RB_CLEAR_NODE(&rq->rb_node);
- rq->ioprio = 0;
- rq->buffer = NULL;
+ rq->cmd = rq->__cmd;
+ rq->tag = -1;
rq->ref_count = 1;
- rq->q = q;
- rq->special = NULL;
- rq->data_len = 0;
- rq->data = NULL;
- rq->nr_phys_segments = 0;
- rq->sense = NULL;
- rq->end_io = NULL;
- rq->end_io_data = NULL;
- rq->completion_data = NULL;
- rq->next_rq = NULL;
}
+EXPORT_SYMBOL(blk_rq_init);
static void req_bio_endio(struct request *rq, struct bio *bio,
unsigned int nbytes, int error)
if (blk_pc_request(rq)) {
printk(KERN_INFO " cdb: ");
- for (bit = 0; bit < sizeof(rq->cmd); bit++)
+ for (bit = 0; bit < BLK_MAX_CDB; bit++)
printk("%02x ", rq->cmd[bit]);
printk("\n");
}
if (blk_queue_stopped(q))
return;
- if (!test_and_set_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags)) {
+ if (!test_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags)) {
+ __set_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags);
mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
blk_add_trace_generic(q, NULL, 0, BLK_TA_PLUG);
}
{
WARN_ON(!irqs_disabled());
- if (!test_and_clear_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags))
+ if (!test_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags))
return 0;
+ queue_flag_clear(QUEUE_FLAG_PLUGGED, q);
del_timer(&q->unplug_timer);
return 1;
}
{
WARN_ON(!irqs_disabled());
- clear_bit(QUEUE_FLAG_STOPPED, &q->queue_flags);
+ queue_flag_clear(QUEUE_FLAG_STOPPED, q);
/*
* one level of recursion is ok and is much faster than kicking
* the unplug handling
*/
- if (!test_and_set_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+ if (!test_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+ queue_flag_set(QUEUE_FLAG_REENTER, q);
q->request_fn(q);
- clear_bit(QUEUE_FLAG_REENTER, &q->queue_flags);
+ queue_flag_clear(QUEUE_FLAG_REENTER, q);
} else {
blk_plug_device(q);
kblockd_schedule_work(&q->unplug_work);
void blk_stop_queue(struct request_queue *q)
{
blk_remove_plug(q);
- set_bit(QUEUE_FLAG_STOPPED, &q->queue_flags);
+ queue_flag_set(QUEUE_FLAG_STOPPED, q);
}
EXPORT_SYMBOL(blk_stop_queue);
* blk_run_queue - run a single device queue
* @q: The queue to run
*/
-void blk_run_queue(struct request_queue *q)
+void __blk_run_queue(struct request_queue *q)
{
- unsigned long flags;
-
- spin_lock_irqsave(q->queue_lock, flags);
blk_remove_plug(q);
/*
* handling reinvoke the handler shortly if we already got there.
*/
if (!elv_queue_empty(q)) {
- if (!test_and_set_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+ if (!test_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+ queue_flag_set(QUEUE_FLAG_REENTER, q);
q->request_fn(q);
- clear_bit(QUEUE_FLAG_REENTER, &q->queue_flags);
+ queue_flag_clear(QUEUE_FLAG_REENTER, q);
} else {
blk_plug_device(q);
kblockd_schedule_work(&q->unplug_work);
}
}
+}
+EXPORT_SYMBOL(__blk_run_queue);
+
+/**
+ * blk_run_queue - run a single device queue
+ * @q: The queue to run
+ */
+void blk_run_queue(struct request_queue *q)
+{
+ unsigned long flags;
+ spin_lock_irqsave(q->queue_lock, flags);
+ __blk_run_queue(q);
spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_run_queue);
{
kobject_put(&q->kobj);
}
-EXPORT_SYMBOL(blk_put_queue);
void blk_cleanup_queue(struct request_queue *q)
{
mutex_lock(&q->sysfs_lock);
- set_bit(QUEUE_FLAG_DEAD, &q->queue_flags);
+ queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
mutex_unlock(&q->sysfs_lock);
if (q->elevator)
return 1;
}
-EXPORT_SYMBOL(blk_get_queue);
static inline void blk_free_request(struct request_queue *q, struct request *rq)
{
if (!rq)
return NULL;
+ blk_rq_init(q, rq);
+
/*
* first three bits are identical in rq->cmd_flags and bio->bi_rw,
* see bio.h and blkdev.h
if (ioc_batching(q, ioc))
ioc->nr_batch_requests--;
- rq_init(q, rq);
-
blk_add_trace_generic(q, bio, rw, BLK_TA_GETRQ);
out:
return rq;
}
EXPORT_SYMBOL_GPL(disk_round_stats);
+void part_round_stats(struct hd_struct *part)
+{
+ unsigned long now = jiffies;
+
+ if (now == part->stamp)
+ return;
+
+ if (part->in_flight) {
+ __part_stat_add(part, time_in_queue,
+ part->in_flight * (now - part->stamp));
+ __part_stat_add(part, io_ticks, (now - part->stamp));
+ }
+ part->stamp = now;
+}
+
/*
* queue lock must be held
*/
if (bio_sectors(bio) && bdev != bdev->bd_contains) {
struct hd_struct *p = bdev->bd_part;
- const int rw = bio_data_dir(bio);
-
- p->sectors[rw] += bio_sectors(bio);
- p->ios[rw]++;
bio->bi_sector += p->start_sect;
bio->bi_bdev = bdev->bd_contains;
if (blk_fs_request(req) && req->rq_disk) {
const int rw = rq_data_dir(req);
- disk_stat_add(req->rq_disk, sectors[rw], nr_bytes >> 9);
+ all_stat_add(req->rq_disk, sectors[rw],
+ nr_bytes >> 9, req->sector);
}
total_bytes = bio_nbytes = 0;
if (disk && blk_fs_request(req) && req != &req->q->bar_rq) {
unsigned long duration = jiffies - req->start_time;
const int rw = rq_data_dir(req);
+ struct hd_struct *part = get_part(disk, req->sector);
- __disk_stat_inc(disk, ios[rw]);
- __disk_stat_add(disk, ticks[rw], duration);
+ __all_stat_inc(disk, ios[rw], req->sector);
+ __all_stat_add(disk, ticks[rw], duration, req->sector);
disk_round_stats(disk);
disk->in_flight--;
+ if (part) {
+ part_round_stats(part);
+ part->in_flight--;
+ }
}
if (req->end_io)
/**
* blk_rq_bytes - Returns bytes left to complete in the entire request
+ * @rq: the request being processed
**/
unsigned int blk_rq_bytes(struct request *rq)
{
/**
* blk_rq_cur_bytes - Returns bytes left to complete in the current segment
+ * @rq: the request being processed
**/
unsigned int blk_rq_cur_bytes(struct request *rq)
{
projects / linux-2.6 / blobdiff