*/
#include <linux/bio.h>
+#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/completion.h>
#include <linux/kernel.h>
#define SG_MEMPOOL_NR ARRAY_SIZE(scsi_sg_pools)
#define SG_MEMPOOL_SIZE 2
-/*
- * The maximum number of SG segments that we will put inside a scatterlist
- * (unless chaining is used). Should ideally fit inside a single page, to
- * avoid a higher order allocation.
- */
-#define SCSI_MAX_SG_SEGMENTS 128
-
struct scsi_host_sg_pool {
size_t size;
char *name;
mempool_t *pool;
};
-#define SP(x) { x, "sgpool-" #x }
+#define SP(x) { x, "sgpool-" __stringify(x) }
+#if (SCSI_MAX_SG_SEGMENTS < 32)
+#error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
+#endif
static struct scsi_host_sg_pool scsi_sg_pools[] = {
SP(8),
SP(16),
-#if (SCSI_MAX_SG_SEGMENTS > 16)
- SP(32),
#if (SCSI_MAX_SG_SEGMENTS > 32)
- SP(64),
+ SP(32),
#if (SCSI_MAX_SG_SEGMENTS > 64)
+ SP(64),
+#if (SCSI_MAX_SG_SEGMENTS > 128)
SP(128),
+#if (SCSI_MAX_SG_SEGMENTS > 256)
+#error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
+#endif
#endif
#endif
#endif
+ SP(SCSI_MAX_SG_SEGMENTS)
};
#undef SP
+static struct kmem_cache *scsi_bidi_sdb_cache;
+
static void scsi_run_queue(struct request_queue *q);
/*
page = sg_page(sg);
off = sg->offset;
len = sg->length;
- data_len += len;
while (len > 0 && data_len > 0) {
/*
return NULL;
}
-/*
- * Like SCSI_MAX_SG_SEGMENTS, but for archs that have sg chaining. This limit
- * is totally arbitrary, a setting of 2048 will get you at least 8mb ios.
- */
-#define SCSI_MAX_SG_CHAIN_SEGMENTS 2048
-
static inline unsigned int scsi_sgtable_index(unsigned short nents)
{
unsigned int index;
- switch (nents) {
- case 1 ... 8:
+ BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
+
+ if (nents <= 8)
index = 0;
- break;
- case 9 ... 16:
- index = 1;
- break;
-#if (SCSI_MAX_SG_SEGMENTS > 16)
- case 17 ... 32:
- index = 2;
- break;
-#if (SCSI_MAX_SG_SEGMENTS > 32)
- case 33 ... 64:
- index = 3;
- break;
-#if (SCSI_MAX_SG_SEGMENTS > 64)
- case 65 ... 128:
- index = 4;
- break;
-#endif
-#endif
-#endif
- default:
- printk(KERN_ERR "scsi: bad segment count=%d\n", nents);
- BUG();
- }
+ else
+ index = get_count_order(nents) - 3;
return index;
}
scsi_free_sgtable(&cmd->sdb);
memset(&cmd->sdb, 0, sizeof(cmd->sdb));
+
+ if (scsi_bidi_cmnd(cmd)) {
+ struct scsi_data_buffer *bidi_sdb =
+ cmd->request->next_rq->special;
+ scsi_free_sgtable(bidi_sdb);
+ kmem_cache_free(scsi_bidi_sdb_cache, bidi_sdb);
+ cmd->request->next_rq->special = NULL;
+ }
}
EXPORT_SYMBOL(scsi_release_buffers);
+/*
+ * Bidi commands Must be complete as a whole, both sides at once.
+ * If part of the bytes were written and lld returned
+ * scsi_in()->resid and/or scsi_out()->resid this information will be left
+ * in req->data_len and req->next_rq->data_len. The upper-layer driver can
+ * decide what to do with this information.
+ */
+void scsi_end_bidi_request(struct scsi_cmnd *cmd)
+{
+ struct request *req = cmd->request;
+ unsigned int dlen = req->data_len;
+ unsigned int next_dlen = req->next_rq->data_len;
+
+ req->data_len = scsi_out(cmd)->resid;
+ req->next_rq->data_len = scsi_in(cmd)->resid;
+
+ /* The req and req->next_rq have not been completed */
+ BUG_ON(blk_end_bidi_request(req, 0, dlen, next_dlen));
+
+ scsi_release_buffers(cmd);
+
+ /*
+ * This will goose the queue request function at the end, so we don't
+ * need to worry about launching another command.
+ */
+ scsi_next_command(cmd);
+}
+
/*
* Function: scsi_io_completion()
*
req->sense_len = len;
}
}
+ if (scsi_bidi_cmnd(cmd)) {
+ /* will also release_buffers */
+ scsi_end_bidi_request(cmd);
+ return;
+ }
req->data_len = scsi_get_resid(cmd);
}
+ BUG_ON(blk_bidi_rq(req)); /* bidi not support for !blk_pc_request yet */
scsi_release_buffers(cmd);
/*
scsi_end_request(cmd, -EIO, this_count, !result);
}
-/*
- * Function: scsi_init_io()
- *
- * Purpose: SCSI I/O initialize function.
- *
- * Arguments: cmd - Command descriptor we wish to initialize
- *
- * Returns: 0 on success
- * BLKPREP_DEFER if the failure is retryable
- */
-int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
+static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
+ gfp_t gfp_mask)
{
- struct request *req = cmd->request;
- int count;
- struct scsi_data_buffer *sdb = &cmd->sdb;
+ int count;
/*
* If sg table allocation fails, requeue request later.
*/
if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
gfp_mask))) {
- scsi_unprep_request(req);
return BLKPREP_DEFER;
}
sdb->table.nents = count;
return BLKPREP_OK;
}
+
+/*
+ * Function: scsi_init_io()
+ *
+ * Purpose: SCSI I/O initialize function.
+ *
+ * Arguments: cmd - Command descriptor we wish to initialize
+ *
+ * Returns: 0 on success
+ * BLKPREP_DEFER if the failure is retryable
+ * BLKPREP_KILL if the failure is fatal
+ */
+int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
+{
+ int error = scsi_init_sgtable(cmd->request, &cmd->sdb, gfp_mask);
+ if (error)
+ goto err_exit;
+
+ if (blk_bidi_rq(cmd->request)) {
+ struct scsi_data_buffer *bidi_sdb = kmem_cache_zalloc(
+ scsi_bidi_sdb_cache, GFP_ATOMIC);
+ if (!bidi_sdb) {
+ error = BLKPREP_DEFER;
+ goto err_exit;
+ }
+
+ cmd->request->next_rq->special = bidi_sdb;
+ error = scsi_init_sgtable(cmd->request->next_rq, bidi_sdb,
+ GFP_ATOMIC);
+ if (error)
+ goto err_exit;
+ }
+
+ return BLKPREP_OK ;
+
+err_exit:
+ scsi_release_buffers(cmd);
+ if (error == BLKPREP_KILL)
+ scsi_put_command(cmd);
+ else /* BLKPREP_DEFER */
+ scsi_unprep_request(cmd->request);
+
+ return error;
+}
EXPORT_SYMBOL(scsi_init_io);
static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
request_fn_proc *request_fn)
{
struct request_queue *q;
+ struct device *dev = shost->shost_gendev.parent;
q = blk_init_queue(request_fn, NULL);
if (!q)
* this limit is imposed by hardware restrictions
*/
blk_queue_max_hw_segments(q, shost->sg_tablesize);
-
- /*
- * In the future, sg chaining support will be mandatory and this
- * ifdef can then go away. Right now we don't have all archs
- * converted, so better keep it safe.
- */
-#ifdef ARCH_HAS_SG_CHAIN
- if (shost->use_sg_chaining)
- blk_queue_max_phys_segments(q, SCSI_MAX_SG_CHAIN_SEGMENTS);
- else
- blk_queue_max_phys_segments(q, SCSI_MAX_SG_SEGMENTS);
-#else
- blk_queue_max_phys_segments(q, SCSI_MAX_SG_SEGMENTS);
-#endif
+ blk_queue_max_phys_segments(q, SCSI_MAX_SG_CHAIN_SEGMENTS);
blk_queue_max_sectors(q, shost->max_sectors);
blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
blk_queue_segment_boundary(q, shost->dma_boundary);
+ dma_set_seg_boundary(dev, shost->dma_boundary);
+
+ blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
if (!shost->use_clustering)
clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
return -ENOMEM;
}
+ scsi_bidi_sdb_cache = kmem_cache_create("scsi_bidi_sdb",
+ sizeof(struct scsi_data_buffer),
+ 0, 0, NULL);
+ if (!scsi_bidi_sdb_cache) {
+ printk(KERN_ERR "SCSI: can't init scsi bidi sdb cache\n");
+ goto cleanup_io_context;
+ }
+
for (i = 0; i < SG_MEMPOOL_NR; i++) {
struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
int size = sgp->size * sizeof(struct scatterlist);
if (!sgp->slab) {
printk(KERN_ERR "SCSI: can't init sg slab %s\n",
sgp->name);
+ goto cleanup_bidi_sdb;
}
sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
if (!sgp->pool) {
printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
sgp->name);
+ goto cleanup_bidi_sdb;
}
}
return 0;
+
+cleanup_bidi_sdb:
+ for (i = 0; i < SG_MEMPOOL_NR; i++) {
+ struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
+ if (sgp->pool)
+ mempool_destroy(sgp->pool);
+ if (sgp->slab)
+ kmem_cache_destroy(sgp->slab);
+ }
+ kmem_cache_destroy(scsi_bidi_sdb_cache);
+cleanup_io_context:
+ kmem_cache_destroy(scsi_io_context_cache);
+
+ return -ENOMEM;
}
void scsi_exit_queue(void)
int i;
kmem_cache_destroy(scsi_io_context_cache);
+ kmem_cache_destroy(scsi_bidi_sdb_cache);
for (i = 0; i < SG_MEMPOOL_NR; i++) {
struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
projects / linux-2.6 / blobdiff