* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband:
IB/ehca: Support small QP queues
IB/ehca: Make internal_create/destroy_qp() static
IB/ehca: Move ehca2ib_return_code() out of line
IB/ehca: Generate async event when SRQ limit reached
IB/ehca: Support large page MRs
IB/mlx4: Fix error path in create_qp_common()
mlx4_core: Change command token on timeout
IB/mthca: Change command token on timeout
IB/ipath: Remove ipath_layer dead code
IB/mlx4: Fix leaks in __mlx4_ib_modify_qp
#ifndef __EHCA_CLASSES_H__
#define __EHCA_CLASSES_H__
-
struct ehca_module;
struct ehca_qp;
struct ehca_cq;
struct ehca_sma_attr saved_attr;
};
+#define HCA_CAP_MR_PGSIZE_4K 1
+#define HCA_CAP_MR_PGSIZE_64K 2
+#define HCA_CAP_MR_PGSIZE_1M 4
+#define HCA_CAP_MR_PGSIZE_16M 8
+
struct ehca_shca {
struct ib_device ib_device;
struct ibmebus_dev *ibmebus_dev;
struct h_galpas galpas;
struct mutex modify_mutex;
u64 hca_cap;
+ /* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
+ u32 hca_cap_mr_pgsize;
int max_mtu;
};
struct ib_pd ib_pd;
struct ipz_pd fw_pd;
u32 ownpid;
+ /* small queue mgmt */
+ struct mutex lock;
+ struct list_head free[2];
+ struct list_head full[2];
};
enum ehca_ext_qp_type {
enum ehca_mr_flag flags;
u32 num_kpages; /* number of kernel pages */
u32 num_hwpages; /* number of hw pages to form MR */
+ u64 hwpage_size; /* hw page size used for this MR */
int acl; /* ACL (stored here for usage in reregister) */
u64 *start; /* virtual start address (stored here for */
/* usage in reregister) */
enum ehca_mr_pgi_type type;
u64 num_kpages;
u64 kpage_cnt;
+ u64 hwpage_size; /* hw page size used for this MR */
u64 num_hwpages; /* number of hw pages */
u64 hwpage_cnt; /* counter for hw pages */
u64 next_hwpage; /* next hw page in buffer/chunk/listelem */
void ehca_cleanup_av_cache(void);
int ehca_init_mrmw_cache(void);
void ehca_cleanup_mrmw_cache(void);
+int ehca_init_small_qp_cache(void);
+void ehca_cleanup_small_qp_cache(void);
extern rwlock_t ehca_qp_idr_lock;
extern rwlock_t ehca_cq_idr_lock;
u32 queue_length; /* queue length allocated in bytes */
u32 pagesize;
u32 toggle_state;
- u32 dummy; /* padding for 8 byte alignment */
+ u32 offset; /* save offset within a page for small_qp */
};
struct ehca_create_cq_resp {
LLQP_COMP_MASK = 0x60,
};
+struct ehca_alloc_queue_parms {
+ /* input parameters */
+ int max_wr;
+ int max_sge;
+ int page_size;
+ int is_small;
+
+ /* output parameters */
+ u16 act_nr_wqes;
+ u8 act_nr_sges;
+ u32 queue_size; /* bytes for small queues, pages otherwise */
+};
+
struct ehca_alloc_qp_parms {
-/* input parameters */
+ struct ehca_alloc_queue_parms squeue;
+ struct ehca_alloc_queue_parms rqueue;
+
+ /* input parameters */
enum ehca_service_type servicetype;
+ int qp_storage;
int sigtype;
enum ehca_ext_qp_type ext_type;
enum ehca_ll_comp_flags ll_comp_flags;
-
- int max_send_wr, max_recv_wr;
- int max_send_sge, max_recv_sge;
int ud_av_l_key_ctl;
u32 token;
u32 srq_qpn, srq_token, srq_limit;
-/* output parameters */
+ /* output parameters */
u32 real_qp_num;
struct ipz_qp_handle qp_handle;
struct h_galpas galpas;
-
- u16 act_nr_send_wqes;
- u16 act_nr_recv_wqes;
- u8 act_nr_recv_sges;
- u8 act_nr_send_sges;
-
- u32 nr_rq_pages;
- u32 nr_sq_pages;
};
int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
goto create_cq_exit2;
}
- ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages,
- EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0);
+ ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
+ EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
if (!ipz_rc) {
ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p",
ipz_rc, device);
return cq;
create_cq_exit4:
- ipz_queue_dtor(&my_cq->ipz_queue);
+ ipz_queue_dtor(NULL, &my_cq->ipz_queue);
create_cq_exit3:
h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
"ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
return ehca2ib_return_code(h_ret);
}
- ipz_queue_dtor(&my_cq->ipz_queue);
+ ipz_queue_dtor(NULL, &my_cq->ipz_queue);
kmem_cache_free(cq_cache, my_cq);
return 0;
return -EINVAL;
}
- ret = ipz_queue_ctor(&eq->ipz_queue, nr_pages,
- EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0);
+ ret = ipz_queue_ctor(NULL, &eq->ipz_queue, nr_pages,
+ EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0, 0);
if (!ret) {
ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq);
goto create_eq_exit1;
return 0;
create_eq_exit2:
- ipz_queue_dtor(&eq->ipz_queue);
+ ipz_queue_dtor(NULL, &eq->ipz_queue);
create_eq_exit1:
hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
ehca_err(&shca->ib_device, "Can't free EQ resources.");
return -EINVAL;
}
- ipz_queue_dtor(&eq->ipz_queue);
+ ipz_queue_dtor(NULL, &eq->ipz_queue);
return 0;
}
}
-static void qp_event_callback(struct ehca_shca *shca,
- u64 eqe,
- enum ib_event_type event_type)
+static void qp_event_callback(struct ehca_shca *shca, u64 eqe,
+ enum ib_event_type event_type, int fatal)
{
struct ib_event event;
struct ehca_qp *qp;
if (!qp)
return;
- ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
+ if (fatal)
+ ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
- if (!qp->ib_qp.event_handler)
- return;
+ event.device = &shca->ib_device;
- event.device = &shca->ib_device;
- event.event = event_type;
- event.element.qp = &qp->ib_qp;
+ if (qp->ext_type == EQPT_SRQ) {
+ if (!qp->ib_srq.event_handler)
+ return;
- qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+ event.event = fatal ? IB_EVENT_SRQ_ERR : event_type;
+ event.element.srq = &qp->ib_srq;
+ qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context);
+ } else {
+ if (!qp->ib_qp.event_handler)
+ return;
+
+ event.event = event_type;
+ event.element.qp = &qp->ib_qp;
+ qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+ }
return;
}
switch (identifier) {
case 0x02: /* path migrated */
- qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG);
+ qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0);
break;
case 0x03: /* communication established */
- qp_event_callback(shca, eqe, IB_EVENT_COMM_EST);
+ qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0);
break;
case 0x04: /* send queue drained */
- qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED);
+ qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0);
break;
case 0x05: /* QP error */
case 0x06: /* QP error */
- qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL);
+ qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1);
break;
case 0x07: /* CQ error */
case 0x08: /* CQ error */
ehca_err(&shca->ib_device, "Interface trace stopped.");
break;
case 0x14: /* first error capture info available */
+ ehca_info(&shca->ib_device, "First error capture available");
+ break;
+ case 0x15: /* SRQ limit reached */
+ qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0);
+ break;
default:
ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.",
identifier, shca->ib_device.name);
int ehca_poll_all_eqs = 1;
int ehca_static_rate = -1;
int ehca_scaling_code = 0;
+int ehca_mr_largepage = 0;
module_param_named(open_aqp1, ehca_open_aqp1, int, 0);
module_param_named(debug_level, ehca_debug_level, int, 0);
module_param_named(port_act_time, ehca_port_act_time, int, 0);
module_param_named(poll_all_eqs, ehca_poll_all_eqs, int, 0);
module_param_named(static_rate, ehca_static_rate, int, 0);
-module_param_named(scaling_code, ehca_scaling_code, int, 0);
+module_param_named(scaling_code, ehca_scaling_code, int, 0);
+module_param_named(mr_largepage, ehca_mr_largepage, int, 0);
MODULE_PARM_DESC(open_aqp1,
"AQP1 on startup (0: no (default), 1: yes)");
"set permanent static rate (default: disabled)");
MODULE_PARM_DESC(scaling_code,
"set scaling code (0: disabled/default, 1: enabled)");
+MODULE_PARM_DESC(mr_largepage,
+ "use large page for MR (0: use PAGE_SIZE (default), "
+ "1: use large page depending on MR size");
DEFINE_RWLOCK(ehca_qp_idr_lock);
DEFINE_RWLOCK(ehca_cq_idr_lock);
}
#endif
+int ehca2ib_return_code(u64 ehca_rc)
+{
+ switch (ehca_rc) {
+ case H_SUCCESS:
+ return 0;
+ case H_RESOURCE: /* Resource in use */
+ case H_BUSY:
+ return -EBUSY;
+ case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
+ case H_CONSTRAINED: /* resource constraint */
+ case H_NO_MEM:
+ return -ENOMEM;
+ default:
+ return -EINVAL;
+ }
+}
+
static int ehca_create_slab_caches(void)
{
int ret;
goto create_slab_caches5;
}
+ ret = ehca_init_small_qp_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create small queue SLAB cache.");
+ goto create_slab_caches6;
+ }
+
#ifdef CONFIG_PPC_64K_PAGES
ctblk_cache = kmem_cache_create("ehca_cache_ctblk",
EHCA_PAGESIZE, H_CB_ALIGNMENT,
NULL);
if (!ctblk_cache) {
ehca_gen_err("Cannot create ctblk SLAB cache.");
- ehca_cleanup_mrmw_cache();
- goto create_slab_caches5;
+ ehca_cleanup_small_qp_cache();
+ goto create_slab_caches6;
}
#endif
return 0;
+create_slab_caches6:
+ ehca_cleanup_mrmw_cache();
+
create_slab_caches5:
ehca_cleanup_av_cache();
static void ehca_destroy_slab_caches(void)
{
+ ehca_cleanup_small_qp_cache();
ehca_cleanup_mrmw_cache();
ehca_cleanup_av_cache();
ehca_cleanup_qp_cache();
if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap))
ehca_gen_dbg(" %s", hca_cap_descr[i].descr);
+ shca->hca_cap_mr_pgsize = rblock->memory_page_size_supported;
+
port = (struct hipz_query_port *)rblock;
h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
if (h_ret != H_SUCCESS) {
}
static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
+static ssize_t ehca_show_mr_largepage(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return sprintf(buf, "%d\n", ehca_mr_largepage);
+}
+static DEVICE_ATTR(mr_largepage, S_IRUGO, ehca_show_mr_largepage, NULL);
+
static struct attribute *ehca_dev_attrs[] = {
&dev_attr_adapter_handle.attr,
&dev_attr_num_ports.attr,
&dev_attr_cur_mw.attr,
&dev_attr_max_pd.attr,
&dev_attr_max_ah.attr,
+ &dev_attr_mr_largepage.attr,
NULL
};
*
* Authors: Dietmar Decker <ddecker@de.ibm.com>
* Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
*
* Copyright (c) 2005 IBM Corporation
*
static struct kmem_cache *mr_cache;
static struct kmem_cache *mw_cache;
+enum ehca_mr_pgsize {
+ EHCA_MR_PGSIZE4K = 0x1000L,
+ EHCA_MR_PGSIZE64K = 0x10000L,
+ EHCA_MR_PGSIZE1M = 0x100000L,
+ EHCA_MR_PGSIZE16M = 0x1000000L
+};
+
+extern int ehca_mr_largepage;
+
+static u32 ehca_encode_hwpage_size(u32 pgsize)
+{
+ u32 idx = 0;
+ pgsize >>= 12;
+ /*
+ * map mr page size into hw code:
+ * 0, 1, 2, 3 for 4K, 64K, 1M, 64M
+ */
+ while (!(pgsize & 1)) {
+ idx++;
+ pgsize >>= 4;
+ }
+ return idx;
+}
+
+static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca)
+{
+ if (shca->hca_cap_mr_pgsize & HCA_CAP_MR_PGSIZE_16M)
+ return EHCA_MR_PGSIZE16M;
+ return EHCA_MR_PGSIZE4K;
+}
+
static struct ehca_mr *ehca_mr_new(void)
{
struct ehca_mr *me;
struct ehca_mr_pginfo pginfo;
u32 num_kpages;
u32 num_hwpages;
+ u64 hw_pgsize;
num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
PAGE_SIZE);
- num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) +
- size, EHCA_PAGESIZE);
+ /* for kernel space we try most possible pgsize */
+ hw_pgsize = ehca_get_max_hwpage_size(shca);
+ num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size,
+ hw_pgsize);
memset(&pginfo, 0, sizeof(pginfo));
pginfo.type = EHCA_MR_PGI_PHYS;
pginfo.num_kpages = num_kpages;
+ pginfo.hwpage_size = hw_pgsize;
pginfo.num_hwpages = num_hwpages;
pginfo.u.phy.num_phys_buf = num_phys_buf;
pginfo.u.phy.phys_buf_array = phys_buf_array;
- pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
- EHCA_PAGESIZE);
+ pginfo.next_hwpage =
+ ((u64)iova_start & ~(hw_pgsize - 1)) / hw_pgsize;
ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
int ret;
u32 num_kpages;
u32 num_hwpages;
+ u64 hwpage_size;
if (!pd) {
ehca_gen_err("bad pd=%p", pd);
/* determine number of MR pages */
num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
- num_hwpages = NUM_CHUNKS((virt % EHCA_PAGESIZE) + length,
- EHCA_PAGESIZE);
+ /* select proper hw_pgsize */
+ if (ehca_mr_largepage &&
+ (shca->hca_cap_mr_pgsize & HCA_CAP_MR_PGSIZE_16M)) {
+ if (length <= EHCA_MR_PGSIZE4K
+ && PAGE_SIZE == EHCA_MR_PGSIZE4K)
+ hwpage_size = EHCA_MR_PGSIZE4K;
+ else if (length <= EHCA_MR_PGSIZE64K)
+ hwpage_size = EHCA_MR_PGSIZE64K;
+ else if (length <= EHCA_MR_PGSIZE1M)
+ hwpage_size = EHCA_MR_PGSIZE1M;
+ else
+ hwpage_size = EHCA_MR_PGSIZE16M;
+ } else
+ hwpage_size = EHCA_MR_PGSIZE4K;
+ ehca_dbg(pd->device, "hwpage_size=%lx", hwpage_size);
+reg_user_mr_fallback:
+ num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size);
/* register MR on HCA */
memset(&pginfo, 0, sizeof(pginfo));
pginfo.type = EHCA_MR_PGI_USER;
+ pginfo.hwpage_size = hwpage_size;
pginfo.num_kpages = num_kpages;
pginfo.num_hwpages = num_hwpages;
pginfo.u.usr.region = e_mr->umem;
- pginfo.next_hwpage = e_mr->umem->offset / EHCA_PAGESIZE;
+ pginfo.next_hwpage = e_mr->umem->offset / hwpage_size;
pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk,
(&e_mr->umem->chunk_list),
list);
ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
&e_mr->ib.ib_mr.rkey);
+ if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) {
+ ehca_warn(pd->device, "failed to register mr "
+ "with hwpage_size=%lx", hwpage_size);
+ ehca_info(pd->device, "try to register mr with "
+ "kpage_size=%lx", PAGE_SIZE);
+ /*
+ * this means kpages are not contiguous for a hw page
+ * try kernel page size as fallback solution
+ */
+ hwpage_size = PAGE_SIZE;
+ goto reg_user_mr_fallback;
+ }
if (ret) {
ib_mr = ERR_PTR(ret);
goto reg_user_mr_exit2;
new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
if (mr_rereg_mask & IB_MR_REREG_TRANS) {
+ u64 hw_pgsize = ehca_get_max_hwpage_size(shca);
+
new_start = iova_start; /* change address */
/* check physical buffer list and calculate size */
ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
}
num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
new_size, PAGE_SIZE);
- num_hwpages = NUM_CHUNKS(((u64)new_start % EHCA_PAGESIZE) +
- new_size, EHCA_PAGESIZE);
+ num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) +
+ new_size, hw_pgsize);
memset(&pginfo, 0, sizeof(pginfo));
pginfo.type = EHCA_MR_PGI_PHYS;
pginfo.num_kpages = num_kpages;
+ pginfo.hwpage_size = hw_pgsize;
pginfo.num_hwpages = num_hwpages;
pginfo.u.phy.num_phys_buf = num_phys_buf;
pginfo.u.phy.phys_buf_array = phys_buf_array;
- pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
- EHCA_PAGESIZE);
+ pginfo.next_hwpage =
+ ((u64)iova_start & ~(hw_pgsize - 1)) / hw_pgsize;
}
if (mr_rereg_mask & IB_MR_REREG_ACCESS)
new_acl = mr_access_flags;
int ret;
u32 tmp_lkey, tmp_rkey;
struct ehca_mr_pginfo pginfo;
+ u64 hw_pgsize;
/* check other parameters */
if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
ib_fmr = ERR_PTR(-EINVAL);
goto alloc_fmr_exit0;
}
- if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) &&
- ((1 << fmr_attr->page_shift) != PAGE_SIZE)) {
+ hw_pgsize = ehca_get_max_hwpage_size(shca);
+ if ((1 << fmr_attr->page_shift) != hw_pgsize) {
ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
fmr_attr->page_shift);
ib_fmr = ERR_PTR(-EINVAL);
/* register MR on HCA */
memset(&pginfo, 0, sizeof(pginfo));
+ /*
+ * pginfo.num_hwpages==0, ie register_rpages() will not be called
+ * but deferred to map_phys_fmr()
+ */
ret = ehca_reg_mr(shca, e_fmr, NULL,
fmr_attr->max_pages * (1 << fmr_attr->page_shift),
mr_access_flags, e_pd, &pginfo,
}
/* successful */
+ e_fmr->hwpage_size = hw_pgsize;
e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
e_fmr->fmr_max_pages = fmr_attr->max_pages;
e_fmr->fmr_max_maps = fmr_attr->max_maps;
memset(&pginfo, 0, sizeof(pginfo));
pginfo.type = EHCA_MR_PGI_FMR;
pginfo.num_kpages = list_len;
- pginfo.num_hwpages = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE);
+ pginfo.hwpage_size = e_fmr->hwpage_size;
+ pginfo.num_hwpages =
+ list_len * e_fmr->fmr_page_size / pginfo.hwpage_size;
pginfo.u.fmr.page_list = page_list;
- pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) /
- EHCA_PAGESIZE);
+ pginfo.next_hwpage =
+ (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size;
pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
struct ehca_mr_hipzout_parms hipzout;
ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
if (ehca_use_hp_mr == 1)
hipz_acl |= 0x00000001;
/* successful registration */
e_mr->num_kpages = pginfo->num_kpages;
e_mr->num_hwpages = pginfo->num_hwpages;
+ e_mr->hwpage_size = pginfo->hwpage_size;
e_mr->start = iova_start;
e_mr->size = size;
e_mr->acl = acl;
u32 i;
u64 *kpage;
+ if (!pginfo->num_hwpages) /* in case of fmr */
+ return 0;
+
kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
if (!kpage) {
ehca_err(&shca->ib_device, "kpage alloc failed");
goto ehca_reg_mr_rpages_exit0;
}
- /* max 512 pages per shot */
+ /* max MAX_RPAGES ehca mr pages per register call */
for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
ret = ehca_set_pagebuf(pginfo, rnum, kpage);
if (ret) {
ehca_err(&shca->ib_device, "ehca_set_pagebuf "
- "bad rc, ret=%x rnum=%x kpage=%p",
- ret, rnum, kpage);
+ "bad rc, ret=%x rnum=%x kpage=%p",
+ ret, rnum, kpage);
goto ehca_reg_mr_rpages_exit1;
}
} else
rpage = *kpage;
- h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr,
- 0, /* pagesize 4k */
- 0, rpage, rnum);
+ h_ret = hipz_h_register_rpage_mr(
+ shca->ipz_hca_handle, e_mr,
+ ehca_encode_hwpage_size(pginfo->hwpage_size),
+ 0, rpage, rnum);
if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
/*
struct ehca_mr_hipzout_parms hipzout;
ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
if (!kpage) {
*/
e_mr->num_kpages = pginfo->num_kpages;
e_mr->num_hwpages = pginfo->num_hwpages;
+ e_mr->hwpage_size = pginfo->hwpage_size;
e_mr->start = iova_start;
e_mr->size = size;
e_mr->acl = acl;
/* set some MR values */
e_mr->flags = save_mr.flags;
+ e_mr->hwpage_size = save_mr.hwpage_size;
e_mr->fmr_page_size = save_mr.fmr_page_size;
e_mr->fmr_max_pages = save_mr.fmr_max_pages;
e_mr->fmr_max_maps = save_mr.fmr_max_maps;
e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
- e_pd, pginfo, lkey, rkey);
+ e_pd, pginfo, lkey, rkey);
if (ret) {
u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
memcpy(&e_mr->flags, &(save_mr.flags),
/* set some MR values */
e_fmr->flags = save_fmr.flags;
+ e_fmr->hwpage_size = save_fmr.hwpage_size;
e_fmr->fmr_page_size = save_fmr.fmr_page_size;
e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
memset(&pginfo, 0, sizeof(pginfo));
pginfo.type = EHCA_MR_PGI_FMR;
- pginfo.num_kpages = 0;
- pginfo.num_hwpages = 0;
ret = ehca_reg_mr(shca, e_fmr, NULL,
(e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
memcpy(&e_fmr->flags, &(save_mr.flags),
sizeof(struct ehca_mr) - offset);
- goto ehca_unmap_one_fmr_exit0;
}
ehca_unmap_one_fmr_exit0:
struct ehca_mr_hipzout_parms hipzout;
ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
(u64)iova_start, hipz_acl, e_pd->fw_pd,
/* successful registration */
e_newmr->num_kpages = e_origmr->num_kpages;
e_newmr->num_hwpages = e_origmr->num_hwpages;
+ e_newmr->hwpage_size = e_origmr->hwpage_size;
e_newmr->start = iova_start;
e_newmr->size = e_origmr->size;
e_newmr->acl = acl;
struct ib_phys_buf ib_pbuf;
u32 num_kpages;
u32 num_hwpages;
+ u64 hw_pgsize;
e_mr = ehca_mr_new();
if (!e_mr) {
ib_pbuf.size = size_maxmr;
num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
PAGE_SIZE);
- num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + size_maxmr,
- EHCA_PAGESIZE);
+ hw_pgsize = ehca_get_max_hwpage_size(shca);
+ num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr,
+ hw_pgsize);
memset(&pginfo, 0, sizeof(pginfo));
pginfo.type = EHCA_MR_PGI_PHYS;
pginfo.num_kpages = num_kpages;
pginfo.num_hwpages = num_hwpages;
+ pginfo.hwpage_size = hw_pgsize;
pginfo.u.phy.num_phys_buf = 1;
pginfo.u.phy.phys_buf_array = &ib_pbuf;
struct ehca_mr_hipzout_parms hipzout;
ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
(u64)iova_start, hipz_acl, e_pd->fw_pd,
/* successful registration */
e_newmr->num_kpages = e_origmr->num_kpages;
e_newmr->num_hwpages = e_origmr->num_hwpages;
+ e_newmr->hwpage_size = e_origmr->hwpage_size;
e_newmr->start = iova_start;
e_newmr->size = e_origmr->size;
e_newmr->acl = acl;
u64 pgaddr;
u32 i = 0;
u32 j = 0;
+ int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
/* loop over desired chunk entries */
chunk = pginfo->u.usr.next_chunk;
<< PAGE_SHIFT ;
*kpage = phys_to_abs(pgaddr +
(pginfo->next_hwpage *
- EHCA_PAGESIZE));
+ pginfo->hwpage_size));
if ( !(*kpage) ) {
ehca_gen_err("pgaddr=%lx "
"chunk->page_list[i]=%lx "
(pginfo->hwpage_cnt)++;
(pginfo->next_hwpage)++;
kpage++;
- if (pginfo->next_hwpage %
- (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
+ if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
(pginfo->kpage_cnt)++;
(pginfo->u.usr.next_nmap)++;
pginfo->next_hwpage = 0;
return ret;
}
+/*
+ * check given pages for contiguous layout
+ * last page addr is returned in prev_pgaddr for further check
+ */
+static int ehca_check_kpages_per_ate(struct scatterlist *page_list,
+ int start_idx, int end_idx,
+ u64 *prev_pgaddr)
+{
+ int t;
+ for (t = start_idx; t <= end_idx; t++) {
+ u64 pgaddr = page_to_pfn(page_list[t].page) << PAGE_SHIFT;
+ ehca_gen_dbg("chunk_page=%lx value=%016lx", pgaddr,
+ *(u64 *)abs_to_virt(phys_to_abs(pgaddr)));
+ if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
+ ehca_gen_err("uncontiguous page found pgaddr=%lx "
+ "prev_pgaddr=%lx page_list_i=%x",
+ pgaddr, *prev_pgaddr, t);
+ return -EINVAL;
+ }
+ *prev_pgaddr = pgaddr;
+ }
+ return 0;
+}
+
+/* PAGE_SIZE < pginfo->hwpage_size */
+static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage)
+{
+ int ret = 0;
+ struct ib_umem_chunk *prev_chunk;
+ struct ib_umem_chunk *chunk;
+ u64 pgaddr, prev_pgaddr;
+ u32 i = 0;
+ u32 j = 0;
+ int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
+ int nr_kpages = kpages_per_hwpage;
+
+ /* loop over desired chunk entries */
+ chunk = pginfo->u.usr.next_chunk;
+ prev_chunk = pginfo->u.usr.next_chunk;
+ list_for_each_entry_continue(
+ chunk, (&(pginfo->u.usr.region->chunk_list)), list) {
+ for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) {
+ if (nr_kpages == kpages_per_hwpage) {
+ pgaddr = ( page_to_pfn(chunk->page_list[i].page)
+ << PAGE_SHIFT );
+ *kpage = phys_to_abs(pgaddr);
+ if ( !(*kpage) ) {
+ ehca_gen_err("pgaddr=%lx i=%x",
+ pgaddr, i);
+ ret = -EFAULT;
+ return ret;
+ }
+ /*
+ * The first page in a hwpage must be aligned;
+ * the first MR page is exempt from this rule.
+ */
+ if (pgaddr & (pginfo->hwpage_size - 1)) {
+ if (pginfo->hwpage_cnt) {
+ ehca_gen_err(
+ "invalid alignment "
+ "pgaddr=%lx i=%x "
+ "mr_pgsize=%lx",
+ pgaddr, i,
+ pginfo->hwpage_size);
+ ret = -EFAULT;
+ return ret;
+ }
+ /* first MR page */
+ pginfo->kpage_cnt =
+ (pgaddr &
+ (pginfo->hwpage_size - 1)) >>
+ PAGE_SHIFT;
+ nr_kpages -= pginfo->kpage_cnt;
+ *kpage = phys_to_abs(
+ pgaddr &
+ ~(pginfo->hwpage_size - 1));
+ }
+ ehca_gen_dbg("kpage=%lx chunk_page=%lx "
+ "value=%016lx", *kpage, pgaddr,
+ *(u64 *)abs_to_virt(
+ phys_to_abs(pgaddr)));
+ prev_pgaddr = pgaddr;
+ i++;
+ pginfo->kpage_cnt++;
+ pginfo->u.usr.next_nmap++;
+ nr_kpages--;
+ if (!nr_kpages)
+ goto next_kpage;
+ continue;
+ }
+ if (i + nr_kpages > chunk->nmap) {
+ ret = ehca_check_kpages_per_ate(
+ chunk->page_list, i,
+ chunk->nmap - 1, &prev_pgaddr);
+ if (ret) return ret;
+ pginfo->kpage_cnt += chunk->nmap - i;
+ pginfo->u.usr.next_nmap += chunk->nmap - i;
+ nr_kpages -= chunk->nmap - i;
+ break;
+ }
+
+ ret = ehca_check_kpages_per_ate(chunk->page_list, i,
+ i + nr_kpages - 1,
+ &prev_pgaddr);
+ if (ret) return ret;
+ i += nr_kpages;
+ pginfo->kpage_cnt += nr_kpages;
+ pginfo->u.usr.next_nmap += nr_kpages;
+next_kpage:
+ nr_kpages = kpages_per_hwpage;
+ (pginfo->hwpage_cnt)++;
+ kpage++;
+ j++;
+ if (j >= number) break;
+ }
+ if ((pginfo->u.usr.next_nmap >= chunk->nmap) &&
+ (j >= number)) {
+ pginfo->u.usr.next_nmap = 0;
+ prev_chunk = chunk;
+ break;
+ } else if (pginfo->u.usr.next_nmap >= chunk->nmap) {
+ pginfo->u.usr.next_nmap = 0;
+ prev_chunk = chunk;
+ } else if (j >= number)
+ break;
+ else
+ prev_chunk = chunk;
+ }
+ pginfo->u.usr.next_chunk =
+ list_prepare_entry(prev_chunk,
+ (&(pginfo->u.usr.region->chunk_list)),
+ list);
+ return ret;
+}
+
int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
u32 number,
u64 *kpage)
/* loop over desired phys_buf_array entries */
while (i < number) {
pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
- num_hw = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) +
- pbuf->size, EHCA_PAGESIZE);
- offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
+ num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +
+ pbuf->size, pginfo->hwpage_size);
+ offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /
+ pginfo->hwpage_size;
while (pginfo->next_hwpage < offs_hw + num_hw) {
/* sanity check */
if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
return -EFAULT;
}
*kpage = phys_to_abs(
- (pbuf->addr & EHCA_PAGEMASK)
- + (pginfo->next_hwpage * EHCA_PAGESIZE));
+ (pbuf->addr & ~(pginfo->hwpage_size - 1)) +
+ (pginfo->next_hwpage * pginfo->hwpage_size));
if ( !(*kpage) && pbuf->addr ) {
- ehca_gen_err("pbuf->addr=%lx "
- "pbuf->size=%lx "
+ ehca_gen_err("pbuf->addr=%lx pbuf->size=%lx "
"next_hwpage=%lx", pbuf->addr,
- pbuf->size,
- pginfo->next_hwpage);
+ pbuf->size, pginfo->next_hwpage);
return -EFAULT;
}
(pginfo->hwpage_cnt)++;
(pginfo->next_hwpage)++;
- if (pginfo->next_hwpage %
- (PAGE_SIZE / EHCA_PAGESIZE) == 0)
- (pginfo->kpage_cnt)++;
+ if (PAGE_SIZE >= pginfo->hwpage_size) {
+ if (pginfo->next_hwpage %
+ (PAGE_SIZE / pginfo->hwpage_size) == 0)
+ (pginfo->kpage_cnt)++;
+ } else
+ pginfo->kpage_cnt += pginfo->hwpage_size /
+ PAGE_SIZE;
kpage++;
i++;
if (i >= number) break;
/* loop over desired page_list entries */
fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
for (i = 0; i < number; i++) {
- *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
- pginfo->next_hwpage * EHCA_PAGESIZE);
+ *kpage = phys_to_abs((*fmrlist & ~(pginfo->hwpage_size - 1)) +
+ pginfo->next_hwpage * pginfo->hwpage_size);
if ( !(*kpage) ) {
ehca_gen_err("*fmrlist=%lx fmrlist=%p "
"next_listelem=%lx next_hwpage=%lx",
return -EFAULT;
}
(pginfo->hwpage_cnt)++;
- (pginfo->next_hwpage)++;
- kpage++;
- if (pginfo->next_hwpage %
- (pginfo->u.fmr.fmr_pgsize / EHCA_PAGESIZE) == 0) {
- (pginfo->kpage_cnt)++;
- (pginfo->u.fmr.next_listelem)++;
- fmrlist++;
- pginfo->next_hwpage = 0;
+ if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) {
+ if (pginfo->next_hwpage %
+ (pginfo->u.fmr.fmr_pgsize /
+ pginfo->hwpage_size) == 0) {
+ (pginfo->kpage_cnt)++;
+ (pginfo->u.fmr.next_listelem)++;
+ fmrlist++;
+ pginfo->next_hwpage = 0;
+ } else
+ (pginfo->next_hwpage)++;
+ } else {
+ unsigned int cnt_per_hwpage = pginfo->hwpage_size /
+ pginfo->u.fmr.fmr_pgsize;
+ unsigned int j;
+ u64 prev = *kpage;
+ /* check if adrs are contiguous */
+ for (j = 1; j < cnt_per_hwpage; j++) {
+ u64 p = phys_to_abs(fmrlist[j] &
+ ~(pginfo->hwpage_size - 1));
+ if (prev + pginfo->u.fmr.fmr_pgsize != p) {
+ ehca_gen_err("uncontiguous fmr pages "
+ "found prev=%lx p=%lx "
+ "idx=%x", prev, p, i + j);
+ return -EINVAL;
+ }
+ prev = p;
+ }
+ pginfo->kpage_cnt += cnt_per_hwpage;
+ pginfo->u.fmr.next_listelem += cnt_per_hwpage;
+ fmrlist += cnt_per_hwpage;
}
+ kpage++;
}
return ret;
}
ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
break;
case EHCA_MR_PGI_USER:
- ret = ehca_set_pagebuf_user1(pginfo, number, kpage);
+ ret = PAGE_SIZE >= pginfo->hwpage_size ?
+ ehca_set_pagebuf_user1(pginfo, number, kpage) :
+ ehca_set_pagebuf_user2(pginfo, number, kpage);
break;
case EHCA_MR_PGI_FMR:
ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
/*----------------------------------------------------------------------*/
/* sets page size in hipz access control for MR/MW. */
-void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/
+void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/
{
- return; /* HCA supports only 4k */
+ *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24);
} /* end ehca_mrmw_set_pgsize_hipz_acl() */
/*----------------------------------------------------------------------*/
void ehca_mrmw_map_acl(int ib_acl,
u32 *hipz_acl);
-void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl);
+void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl);
void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
int *ib_acl);
struct ib_ucontext *context, struct ib_udata *udata)
{
struct ehca_pd *pd;
+ int i;
pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL);
if (!pd) {
}
pd->ownpid = current->tgid;
+ for (i = 0; i < 2; i++) {
+ INIT_LIST_HEAD(&pd->free[i]);
+ INIT_LIST_HEAD(&pd->full[i]);
+ }
+ mutex_init(&pd->lock);
/*
* Kernel PD: when device = -1, 0
{
u32 cur_pid = current->tgid;
struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
+ int i, leftovers = 0;
+ extern struct kmem_cache *small_qp_cache;
+ struct ipz_small_queue_page *page, *tmp;
if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
my_pd->ownpid != cur_pid) {
return -EINVAL;
}
- kmem_cache_free(pd_cache,
- container_of(pd, struct ehca_pd, ib_pd));
+ for (i = 0; i < 2; i++) {
+ list_splice(&my_pd->full[i], &my_pd->free[i]);
+ list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) {
+ leftovers = 1;
+ free_page(page->page);
+ kmem_cache_free(small_qp_cache, page);
+ }
+ }
+
+ if (leftovers)
+ ehca_warn(pd->device,
+ "Some small queue pages were not freed");
+
+ kmem_cache_free(pd_cache, my_pd);
return 0;
}
resp->toggle_state = queue->toggle_state;
}
-static inline int ll_qp_msg_size(int nr_sge)
-{
- return 128 << nr_sge;
-}
-
/*
* init_qp_queue initializes/constructs r/squeue and registers queue pages.
*/
static inline int init_qp_queue(struct ehca_shca *shca,
+ struct ehca_pd *pd,
struct ehca_qp *my_qp,
struct ipz_queue *queue,
int q_type,
u64 expected_hret,
- int nr_q_pages,
- int wqe_size,
- int nr_sges)
+ struct ehca_alloc_queue_parms *parms,
+ int wqe_size)
{
- int ret, cnt, ipz_rc;
+ int ret, cnt, ipz_rc, nr_q_pages;
void *vpage;
u64 rpage, h_ret;
struct ib_device *ib_dev = &shca->ib_device;
struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle;
- if (!nr_q_pages)
+ if (!parms->queue_size)
return 0;
- ipz_rc = ipz_queue_ctor(queue, nr_q_pages, EHCA_PAGESIZE,
- wqe_size, nr_sges);
+ if (parms->is_small) {
+ nr_q_pages = 1;
+ ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
+ 128 << parms->page_size,
+ wqe_size, parms->act_nr_sges, 1);
+ } else {
+ nr_q_pages = parms->queue_size;
+ ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
+ EHCA_PAGESIZE, wqe_size,
+ parms->act_nr_sges, 0);
+ }
+
if (!ipz_rc) {
ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%x",
ipz_rc);
h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
my_qp->ipz_qp_handle,
NULL, 0, q_type,
- rpage, 1,
+ rpage, parms->is_small ? 0 : 1,
my_qp->galpas.kernel);
if (cnt == (nr_q_pages - 1)) { /* last page! */
if (h_ret != expected_hret) {
return 0;
init_qp_queue1:
- ipz_queue_dtor(queue);
+ ipz_queue_dtor(pd, queue);
return ret;
}
+static inline int ehca_calc_wqe_size(int act_nr_sge, int is_llqp)
+{
+ if (is_llqp)
+ return 128 << act_nr_sge;
+ else
+ return offsetof(struct ehca_wqe,
+ u.nud.sg_list[act_nr_sge]);
+}
+
+static void ehca_determine_small_queue(struct ehca_alloc_queue_parms *queue,
+ int req_nr_sge, int is_llqp)
+{
+ u32 wqe_size, q_size;
+ int act_nr_sge = req_nr_sge;
+
+ if (!is_llqp)
+ /* round up #SGEs so WQE size is a power of 2 */
+ for (act_nr_sge = 4; act_nr_sge <= 252;
+ act_nr_sge = 4 + 2 * act_nr_sge)
+ if (act_nr_sge >= req_nr_sge)
+ break;
+
+ wqe_size = ehca_calc_wqe_size(act_nr_sge, is_llqp);
+ q_size = wqe_size * (queue->max_wr + 1);
+
+ if (q_size <= 512)
+ queue->page_size = 2;
+ else if (q_size <= 1024)
+ queue->page_size = 3;
+ else
+ queue->page_size = 0;
+
+ queue->is_small = (queue->page_size != 0);
+}
+
/*
* Create an ib_qp struct that is either a QP or an SRQ, depending on
* the value of the is_srq parameter. If init_attr and srq_init_attr share
* fields, the field out of init_attr is used.
*/
-struct ehca_qp *internal_create_qp(struct ib_pd *pd,
- struct ib_qp_init_attr *init_attr,
- struct ib_srq_init_attr *srq_init_attr,
- struct ib_udata *udata, int is_srq)
+static struct ehca_qp *internal_create_qp(
+ struct ib_pd *pd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata, int is_srq)
{
struct ehca_qp *my_qp;
struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
if (my_qp->recv_cq)
parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle;
- parms.max_send_wr = init_attr->cap.max_send_wr;
- parms.max_recv_wr = init_attr->cap.max_recv_wr;
- parms.max_send_sge = max_send_sge;
- parms.max_recv_sge = max_recv_sge;
+ parms.squeue.max_wr = init_attr->cap.max_send_wr;
+ parms.rqueue.max_wr = init_attr->cap.max_recv_wr;
+ parms.squeue.max_sge = max_send_sge;
+ parms.rqueue.max_sge = max_recv_sge;
+
+ if (EHCA_BMASK_GET(HCA_CAP_MINI_QP, shca->hca_cap)
+ && !(context && udata)) { /* no small QP support in userspace ATM */
+ ehca_determine_small_queue(
+ &parms.squeue, max_send_sge, is_llqp);
+ ehca_determine_small_queue(
+ &parms.rqueue, max_recv_sge, is_llqp);
+ parms.qp_storage =
+ (parms.squeue.is_small || parms.rqueue.is_small);
+ }
h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms);
if (h_ret != H_SUCCESS) {
my_qp->ipz_qp_handle = parms.qp_handle;
my_qp->galpas = parms.galpas;
+ swqe_size = ehca_calc_wqe_size(parms.squeue.act_nr_sges, is_llqp);
+ rwqe_size = ehca_calc_wqe_size(parms.rqueue.act_nr_sges, is_llqp);
+
switch (qp_type) {
case IB_QPT_RC:
- if (!is_llqp) {
- swqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[
- (parms.act_nr_send_sges)]);
- rwqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[
- (parms.act_nr_recv_sges)]);
- } else { /* for LLQP we need to use msg size, not wqe size */
- swqe_size = ll_qp_msg_size(max_send_sge);
- rwqe_size = ll_qp_msg_size(max_recv_sge);
- parms.act_nr_send_sges = 1;
- parms.act_nr_recv_sges = 1;
+ if (is_llqp) {
+ parms.squeue.act_nr_sges = 1;
+ parms.rqueue.act_nr_sges = 1;
}
break;
- case IB_QPT_UC:
- swqe_size = offsetof(struct ehca_wqe,
- u.nud.sg_list[parms.act_nr_send_sges]);
- rwqe_size = offsetof(struct ehca_wqe,
- u.nud.sg_list[parms.act_nr_recv_sges]);
- break;
-
case IB_QPT_UD:
case IB_QPT_GSI:
case IB_QPT_SMI:
+ /* UD circumvention */
if (is_llqp) {
- swqe_size = ll_qp_msg_size(parms.act_nr_send_sges);
- rwqe_size = ll_qp_msg_size(parms.act_nr_recv_sges);
- parms.act_nr_send_sges = 1;
- parms.act_nr_recv_sges = 1;
+ parms.squeue.act_nr_sges = 1;
+ parms.rqueue.act_nr_sges = 1;
} else {
- /* UD circumvention */
- parms.act_nr_send_sges -= 2;
- parms.act_nr_recv_sges -= 2;
- swqe_size = offsetof(struct ehca_wqe, u.ud_av.sg_list[
- parms.act_nr_send_sges]);
- rwqe_size = offsetof(struct ehca_wqe, u.ud_av.sg_list[
- parms.act_nr_recv_sges]);
+ parms.squeue.act_nr_sges -= 2;
+ parms.rqueue.act_nr_sges -= 2;
}
if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) {
- parms.act_nr_send_wqes = init_attr->cap.max_send_wr;
- parms.act_nr_recv_wqes = init_attr->cap.max_recv_wr;
- parms.act_nr_send_sges = init_attr->cap.max_send_sge;
- parms.act_nr_recv_sges = init_attr->cap.max_recv_sge;
+ parms.squeue.act_nr_wqes = init_attr->cap.max_send_wr;
+ parms.rqueue.act_nr_wqes = init_attr->cap.max_recv_wr;
+ parms.squeue.act_nr_sges = init_attr->cap.max_send_sge;
+ parms.rqueue.act_nr_sges = init_attr->cap.max_recv_sge;
ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1;
}
/* initialize r/squeue and register queue pages */
if (HAS_SQ(my_qp)) {
ret = init_qp_queue(
- shca, my_qp, &my_qp->ipz_squeue, 0,
+ shca, my_pd, my_qp, &my_qp->ipz_squeue, 0,
HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS,
- parms.nr_sq_pages, swqe_size,
- parms.act_nr_send_sges);
+ &parms.squeue, swqe_size);
if (ret) {
ehca_err(pd->device, "Couldn't initialize squeue "
"and pages ret=%x", ret);
if (HAS_RQ(my_qp)) {
ret = init_qp_queue(
- shca, my_qp, &my_qp->ipz_rqueue, 1,
- H_SUCCESS, parms.nr_rq_pages, rwqe_size,
- parms.act_nr_recv_sges);
+ shca, my_pd, my_qp, &my_qp->ipz_rqueue, 1,
+ H_SUCCESS, &parms.rqueue, rwqe_size);
if (ret) {
ehca_err(pd->device, "Couldn't initialize rqueue "
"and pages ret=%x", ret);
}
init_attr->cap.max_inline_data = 0; /* not supported yet */
- init_attr->cap.max_recv_sge = parms.act_nr_recv_sges;
- init_attr->cap.max_recv_wr = parms.act_nr_recv_wqes;
- init_attr->cap.max_send_sge = parms.act_nr_send_sges;
- init_attr->cap.max_send_wr = parms.act_nr_send_wqes;
+ init_attr->cap.max_recv_sge = parms.rqueue.act_nr_sges;
+ init_attr->cap.max_recv_wr = parms.rqueue.act_nr_wqes;
+ init_attr->cap.max_send_sge = parms.squeue.act_nr_sges;
+ init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes;
my_qp->init_attr = *init_attr;
/* NOTE: define_apq0() not supported yet */
resp.ext_type = my_qp->ext_type;
resp.qkey = my_qp->qkey;
resp.real_qp_num = my_qp->real_qp_num;
+ resp.ipz_rqueue.offset = my_qp->ipz_rqueue.offset;
+ resp.ipz_squeue.offset = my_qp->ipz_squeue.offset;
if (HAS_SQ(my_qp))
queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue);
if (HAS_RQ(my_qp))
create_qp_exit4:
if (HAS_RQ(my_qp))
- ipz_queue_dtor(&my_qp->ipz_rqueue);
+ ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
create_qp_exit3:
if (HAS_SQ(my_qp))
- ipz_queue_dtor(&my_qp->ipz_squeue);
+ ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
create_qp_exit2:
hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp;
}
-int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
- struct ib_uobject *uobject);
+static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
+ struct ib_uobject *uobject);
struct ib_srq *ehca_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *srq_init_attr,
return ret;
}
-int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
- struct ib_uobject *uobject)
+static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
+ struct ib_uobject *uobject)
{
struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device);
struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
}
if (HAS_RQ(my_qp))
- ipz_queue_dtor(&my_qp->ipz_rqueue);
+ ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
if (HAS_SQ(my_qp))
- ipz_queue_dtor(&my_qp->ipz_squeue);
+ ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
kmem_cache_free(qp_cache, my_qp);
return 0;
}
#define EHCA_BMASK_GET(mask, value) \
(EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask)))
-
/* Converts ehca to ib return code */
-static inline int ehca2ib_return_code(u64 ehca_rc)
-{
- switch (ehca_rc) {
- case H_SUCCESS:
- return 0;
- case H_RESOURCE: /* Resource in use */
- case H_BUSY:
- return -EBUSY;
- case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
- case H_CONSTRAINED: /* resource constraint */
- case H_NO_MEM:
- return -ENOMEM;
- default:
- return -EINVAL;
- }
-}
-
+int ehca2ib_return_code(u64 ehca_rc);
#endif /* EHCA_TOOLS_H */
ehca_gen_err("vm_insert_page() failed rc=%x", ret);
return ret;
}
- start += PAGE_SIZE;
+ start += PAGE_SIZE;
}
vma->vm_private_data = mm_count;
(*mm_count)++;
#define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11)
#define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12)
#define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15)
+#define H_ALL_RES_QP_STORAGE EHCA_BMASK_IBM(16, 17)
#define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18)
#define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21)
#define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23)
#define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31)
+#define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35)
+#define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39)
#define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63)
#define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15)
| EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
| EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
| EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE,
+ parms->squeue.page_size)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE,
+ parms->rqueue.page_size)
| EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
!!(parms->ll_comp_flags & LLQP_RECV_COMP))
| EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
max_r10_reg =
EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
- parms->max_send_wr + 1)
+ parms->squeue.max_wr + 1)
| EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
- parms->max_recv_wr + 1)
+ parms->rqueue.max_wr + 1)
| EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
- parms->max_send_sge)
+ parms->squeue.max_sge)
| EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
- parms->max_recv_sge);
+ parms->rqueue.max_sge);
r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token);
parms->qp_handle.handle = outs[0];
parms->real_qp_num = (u32)outs[1];
- parms->act_nr_send_wqes =
+ parms->squeue.act_nr_wqes =
(u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
- parms->act_nr_recv_wqes =
+ parms->rqueue.act_nr_wqes =
(u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
- parms->act_nr_send_sges =
+ parms->squeue.act_nr_sges =
(u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
- parms->act_nr_recv_sges =
+ parms->rqueue.act_nr_sges =
(u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
- parms->nr_sq_pages =
+ parms->squeue.queue_size =
(u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
- parms->nr_rq_pages =
+ parms->rqueue.queue_size =
(u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
if (ret == H_SUCCESS)
{
return ehca_plpar_hcall_norets(H_REGISTER_RPAGES,
adapter_handle.handle, /* r4 */
- queue_type | pagesize << 8, /* r5 */
+ (u64)queue_type | ((u64)pagesize) << 8,
+ /* r5 */
resource_handle, /* r6 */
logical_address_of_page, /* r7 */
count, /* r8 */
const u64 count,
const struct h_galpa galpa)
{
- if (count != 1) {
+ if (count > 1) {
ehca_gen_err("Page counter=%lx", count);
return H_PARAMETER;
}
u64 ret;
u64 outs[PLPAR_HCALL9_BUFSIZE];
+ ehca_gen_dbg("kernel PAGE_SIZE=%x access_ctrl=%016x "
+ "vaddr=%lx length=%lx",
+ (u32)PAGE_SIZE, access_ctrl, vaddr, length);
ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
adapter_handle.handle, /* r4 */
5, /* r5 */
const u64 logical_address_of_page,
const u64 count)
{
+ extern int ehca_debug_level;
u64 ret;
+ if (unlikely(ehca_debug_level >= 2)) {
+ if (count > 1) {
+ u64 *kpage;
+ int i;
+ kpage = (u64 *)abs_to_virt(logical_address_of_page);
+ for (i = 0; i < count; i++)
+ ehca_gen_dbg("kpage[%d]=%p",
+ i, (void *)kpage[i]);
+ } else
+ ehca_gen_dbg("kpage=%p",
+ (void *)logical_address_of_page);
+ }
+
if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
ehca_gen_err("logical_address_of_page not on a 4k boundary "
"adapter_handle=%lx mr=%p mr_handle=%lx "
#include "ehca_tools.h"
#include "ipz_pt_fn.h"
+#include "ehca_classes.h"
+
+#define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT)
+
+struct kmem_cache *small_qp_cache;
void *ipz_qpageit_get_inc(struct ipz_queue *queue)
{
queue->current_q_offset -= queue->pagesize;
ret = NULL;
}
- if (((u64)ret) % EHCA_PAGESIZE) {
+ if (((u64)ret) % queue->pagesize) {
ehca_gen_err("ERROR!! not at PAGE-Boundary");
return NULL;
}
return -EINVAL;
}
-int ipz_queue_ctor(struct ipz_queue *queue,
- const u32 nr_of_pages,
- const u32 pagesize, const u32 qe_size, const u32 nr_of_sg)
+#if PAGE_SHIFT < EHCA_PAGESHIFT
+#error Kernel pages must be at least as large than eHCA pages (4K) !
+#endif
+
+/*
+ * allocate pages for queue:
+ * outer loop allocates whole kernel pages (page aligned) and
+ * inner loop divides a kernel page into smaller hca queue pages
+ */
+static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages)
{
- int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT;
- int f;
+ int k, f = 0;
+ u8 *kpage;
- if (pagesize > PAGE_SIZE) {
- ehca_gen_err("FATAL ERROR: pagesize=%x is greater "
- "than kernel page size", pagesize);
- return 0;
- }
- if (!pages_per_kpage) {
- ehca_gen_err("FATAL ERROR: invalid kernel page size. "
- "pages_per_kpage=%x", pages_per_kpage);
- return 0;
- }
- queue->queue_length = nr_of_pages * pagesize;
- queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *));
- if (!queue->queue_pages) {
- ehca_gen_err("ERROR!! didn't get the memory");
- return 0;
- }
- memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *));
- /*
- * allocate pages for queue:
- * outer loop allocates whole kernel pages (page aligned) and
- * inner loop divides a kernel page into smaller hca queue pages
- */
- f = 0;
while (f < nr_of_pages) {
- u8 *kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
- int k;
+ kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
if (!kpage)
- goto ipz_queue_ctor_exit0; /*NOMEM*/
- for (k = 0; k < pages_per_kpage && f < nr_of_pages; k++) {
- (queue->queue_pages)[f] = (struct ipz_page *)kpage;
+ goto out;
+
+ for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) {
+ queue->queue_pages[f] = (struct ipz_page *)kpage;
kpage += EHCA_PAGESIZE;
f++;
}
}
+ return 1;
- queue->current_q_offset = 0;
+out:
+ for (f = 0; f < nr_of_pages && queue->queue_pages[f];
+ f += PAGES_PER_KPAGE)
+ free_page((unsigned long)(queue->queue_pages)[f]);
+ return 0;
+}
+
+static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
+{
+ int order = ilog2(queue->pagesize) - 9;
+ struct ipz_small_queue_page *page;
+ unsigned long bit;
+
+ mutex_lock(&pd->lock);
+
+ if (!list_empty(&pd->free[order]))
+ page = list_entry(pd->free[order].next,
+ struct ipz_small_queue_page, list);
+ else {
+ page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL);
+ if (!page)
+ goto out;
+
+ page->page = get_zeroed_page(GFP_KERNEL);
+ if (!page->page) {
+ kmem_cache_free(small_qp_cache, page);
+ goto out;
+ }
+
+ list_add(&page->list, &pd->free[order]);
+ }
+
+ bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order);
+ __set_bit(bit, page->bitmap);
+ page->fill++;
+
+ if (page->fill == IPZ_SPAGE_PER_KPAGE >> order)
+ list_move(&page->list, &pd->full[order]);
+
+ mutex_unlock(&pd->lock);
+
+ queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9)));
+ queue->small_page = page;
+ return 1;
+
+out:
+ ehca_err(pd->ib_pd.device, "failed to allocate small queue page");
+ return 0;
+}
+
+static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
+{
+ int order = ilog2(queue->pagesize) - 9;
+ struct ipz_small_queue_page *page = queue->small_page;
+ unsigned long bit;
+ int free_page = 0;
+
+ bit = ((unsigned long)queue->queue_pages[0] & PAGE_MASK)
+ >> (order + 9);
+
+ mutex_lock(&pd->lock);
+
+ __clear_bit(bit, page->bitmap);
+ page->fill--;
+
+ if (page->fill == 0) {
+ list_del(&page->list);
+ free_page = 1;
+ }
+
+ if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1)
+ /* the page was full until we freed the chunk */
+ list_move_tail(&page->list, &pd->free[order]);
+
+ mutex_unlock(&pd->lock);
+
+ if (free_page) {
+ free_page(page->page);
+ kmem_cache_free(small_qp_cache, page);
+ }
+}
+
+int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
+ const u32 nr_of_pages, const u32 pagesize,
+ const u32 qe_size, const u32 nr_of_sg,
+ int is_small)
+{
+ if (pagesize > PAGE_SIZE) {
+ ehca_gen_err("FATAL ERROR: pagesize=%x "
+ "is greater than kernel page size", pagesize);
+ return 0;
+ }
+
+ /* init queue fields */
+ queue->queue_length = nr_of_pages * pagesize;
+ queue->pagesize = pagesize;
queue->qe_size = qe_size;
queue->act_nr_of_sg = nr_of_sg;
- queue->pagesize = pagesize;
+ queue->current_q_offset = 0;
queue->toggle_state = 1;
- return 1;
+ queue->small_page = NULL;
- ipz_queue_ctor_exit0:
- ehca_gen_err("Couldn't get alloc pages queue=%p f=%x nr_of_pages=%x",
- queue, f, nr_of_pages);
- for (f = 0; f < nr_of_pages; f += pages_per_kpage) {
- if (!(queue->queue_pages)[f])
- break;
- free_page((unsigned long)(queue->queue_pages)[f]);
+ /* allocate queue page pointers */
+ queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *));
+ if (!queue->queue_pages) {
+ ehca_gen_err("Couldn't allocate queue page list");
+ return 0;
}
+ memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *));
+
+ /* allocate actual queue pages */
+ if (is_small) {
+ if (!alloc_small_queue_page(queue, pd))
+ goto ipz_queue_ctor_exit0;
+ } else
+ if (!alloc_queue_pages(queue, nr_of_pages))
+ goto ipz_queue_ctor_exit0;
+
+ return 1;
+
+ipz_queue_ctor_exit0:
+ ehca_gen_err("Couldn't alloc pages queue=%p "
+ "nr_of_pages=%x", queue, nr_of_pages);
+ vfree(queue->queue_pages);
+
return 0;
}
-int ipz_queue_dtor(struct ipz_queue *queue)
+int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue)
{
- int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT;
- int g;
- int nr_pages;
+ int i, nr_pages;
if (!queue || !queue->queue_pages) {
ehca_gen_dbg("queue or queue_pages is NULL");
return 0;
}
- nr_pages = queue->queue_length / queue->pagesize;
- for (g = 0; g < nr_pages; g += pages_per_kpage)
- free_page((unsigned long)(queue->queue_pages)[g]);
+
+ if (queue->small_page)
+ free_small_queue_page(queue, pd);
+ else {
+ nr_pages = queue->queue_length / queue->pagesize;
+ for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE)
+ free_page((unsigned long)queue->queue_pages[i]);
+ }
+
vfree(queue->queue_pages);
return 1;
}
+
+int ehca_init_small_qp_cache(void)
+{
+ small_qp_cache = kmem_cache_create("ehca_cache_small_qp",
+ sizeof(struct ipz_small_queue_page),
+ 0, SLAB_HWCACHE_ALIGN, NULL);
+ if (!small_qp_cache)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void ehca_cleanup_small_qp_cache(void)
+{
+ kmem_cache_destroy(small_qp_cache);
+}
#include "ehca_tools.h"
#include "ehca_qes.h"
+struct ehca_pd;
+struct ipz_small_queue_page;
+
/* struct generic ehca page */
struct ipz_page {
u8 entries[EHCA_PAGESIZE];
};
+#define IPZ_SPAGE_PER_KPAGE (PAGE_SIZE / 512)
+
+struct ipz_small_queue_page {
+ unsigned long page;
+ unsigned long bitmap[IPZ_SPAGE_PER_KPAGE / BITS_PER_LONG];
+ int fill;
+ void *mapped_addr;
+ u32 mmap_count;
+ struct list_head list;
+};
+
/* struct generic queue in linux kernel virtual memory (kv) */
struct ipz_queue {
u64 current_q_offset; /* current queue entry */
u32 queue_length; /* queue length allocated in bytes */
u32 pagesize;
u32 toggle_state; /* toggle flag - per page */
- u32 dummy3; /* 64 bit alignment */
+ u32 offset; /* save offset within page for small_qp */
+ struct ipz_small_queue_page *small_page;
};
/*
* see ipz_qpt_ctor()
* returns true if ok, false if out of memory
*/
-int ipz_queue_ctor(struct ipz_queue *queue, const u32 nr_of_pages,
- const u32 pagesize, const u32 qe_size,
- const u32 nr_of_sg);
+int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
+ const u32 nr_of_pages, const u32 pagesize,
+ const u32 qe_size, const u32 nr_of_sg,
+ int is_small);
/*
* destructor for a ipz_queue_t
* see ipz_queue_ctor()
* returns true if ok, false if queue was NULL-ptr of free failed
*/
-int ipz_queue_dtor(struct ipz_queue *queue);
+int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue);
/*
* constructor for a ipz_qpt_t,
ipath_init_chip.o \
ipath_intr.o \
ipath_keys.o \
- ipath_layer.o \
ipath_mad.o \
ipath_mmap.o \
ipath_mr.o \
+++ /dev/null
-/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
- * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/*
- * These are the routines used by layered drivers, currently just the
- * layered ethernet driver and verbs layer.
- */
-
-#include <linux/io.h>
-#include <asm/byteorder.h>
-
-#include "ipath_kernel.h"
-#include "ipath_layer.h"
-#include "ipath_verbs.h"
-#include "ipath_common.h"
-
-/* Acquire before ipath_devs_lock. */
-static DEFINE_MUTEX(ipath_layer_mutex);
-
-u16 ipath_layer_rcv_opcode;
-
-static int (*layer_intr)(void *, u32);
-static int (*layer_rcv)(void *, void *, struct sk_buff *);
-static int (*layer_rcv_lid)(void *, void *);
-
-static void *(*layer_add_one)(int, struct ipath_devdata *);
-static void (*layer_remove_one)(void *);
-
-int __ipath_layer_intr(struct ipath_devdata *dd, u32 arg)
-{
- int ret = -ENODEV;
-
- if (dd->ipath_layer.l_arg && layer_intr)
- ret = layer_intr(dd->ipath_layer.l_arg, arg);
-
- return ret;
-}
-
-int ipath_layer_intr(struct ipath_devdata *dd, u32 arg)
-{
- int ret;
-
- mutex_lock(&ipath_layer_mutex);
-
- ret = __ipath_layer_intr(dd, arg);
-
- mutex_unlock(&ipath_layer_mutex);
-
- return ret;
-}
-
-int __ipath_layer_rcv(struct ipath_devdata *dd, void *hdr,
- struct sk_buff *skb)
-{
- int ret = -ENODEV;
-
- if (dd->ipath_layer.l_arg && layer_rcv)
- ret = layer_rcv(dd->ipath_layer.l_arg, hdr, skb);
-
- return ret;
-}
-
-int __ipath_layer_rcv_lid(struct ipath_devdata *dd, void *hdr)
-{
- int ret = -ENODEV;
-
- if (dd->ipath_layer.l_arg && layer_rcv_lid)
- ret = layer_rcv_lid(dd->ipath_layer.l_arg, hdr);
-
- return ret;
-}
-
-void ipath_layer_lid_changed(struct ipath_devdata *dd)
-{
- mutex_lock(&ipath_layer_mutex);
-
- if (dd->ipath_layer.l_arg && layer_intr)
- layer_intr(dd->ipath_layer.l_arg, IPATH_LAYER_INT_LID);
-
- mutex_unlock(&ipath_layer_mutex);
-}
-
-void ipath_layer_add(struct ipath_devdata *dd)
-{
- mutex_lock(&ipath_layer_mutex);
-
- if (layer_add_one)
- dd->ipath_layer.l_arg =
- layer_add_one(dd->ipath_unit, dd);
-
- mutex_unlock(&ipath_layer_mutex);
-}
-
-void ipath_layer_remove(struct ipath_devdata *dd)
-{
- mutex_lock(&ipath_layer_mutex);
-
- if (dd->ipath_layer.l_arg && layer_remove_one) {
- layer_remove_one(dd->ipath_layer.l_arg);
- dd->ipath_layer.l_arg = NULL;
- }
-
- mutex_unlock(&ipath_layer_mutex);
-}
-
-int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *),
- void (*l_remove)(void *),
- int (*l_intr)(void *, u32),
- int (*l_rcv)(void *, void *, struct sk_buff *),
- u16 l_rcv_opcode,
- int (*l_rcv_lid)(void *, void *))
-{
- struct ipath_devdata *dd, *tmp;
- unsigned long flags;
-
- mutex_lock(&ipath_layer_mutex);
-
- layer_add_one = l_add;
- layer_remove_one = l_remove;
- layer_intr = l_intr;
- layer_rcv = l_rcv;
- layer_rcv_lid = l_rcv_lid;
- ipath_layer_rcv_opcode = l_rcv_opcode;
-
- spin_lock_irqsave(&ipath_devs_lock, flags);
-
- list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
- if (!(dd->ipath_flags & IPATH_INITTED))
- continue;
-
- if (dd->ipath_layer.l_arg)
- continue;
-
- spin_unlock_irqrestore(&ipath_devs_lock, flags);
- dd->ipath_layer.l_arg = l_add(dd->ipath_unit, dd);
- spin_lock_irqsave(&ipath_devs_lock, flags);
- }
-
- spin_unlock_irqrestore(&ipath_devs_lock, flags);
- mutex_unlock(&ipath_layer_mutex);
-
- return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_register);
-
-void ipath_layer_unregister(void)
-{
- struct ipath_devdata *dd, *tmp;
- unsigned long flags;
-
- mutex_lock(&ipath_layer_mutex);
- spin_lock_irqsave(&ipath_devs_lock, flags);
-
- list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
- if (dd->ipath_layer.l_arg && layer_remove_one) {
- spin_unlock_irqrestore(&ipath_devs_lock, flags);
- layer_remove_one(dd->ipath_layer.l_arg);
- spin_lock_irqsave(&ipath_devs_lock, flags);
- dd->ipath_layer.l_arg = NULL;
- }
- }
-
- spin_unlock_irqrestore(&ipath_devs_lock, flags);
-
- layer_add_one = NULL;
- layer_remove_one = NULL;
- layer_intr = NULL;
- layer_rcv = NULL;
- layer_rcv_lid = NULL;
-
- mutex_unlock(&ipath_layer_mutex);
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_unregister);
-
-int ipath_layer_open(struct ipath_devdata *dd, u32 * pktmax)
-{
- int ret;
- u32 intval = 0;
-
- mutex_lock(&ipath_layer_mutex);
-
- if (!dd->ipath_layer.l_arg) {
- ret = -EINVAL;
- goto bail;
- }
-
- ret = ipath_setrcvhdrsize(dd, IPATH_HEADER_QUEUE_WORDS);
-
- if (ret < 0)
- goto bail;
-
- *pktmax = dd->ipath_ibmaxlen;
-
- if (*dd->ipath_statusp & IPATH_STATUS_IB_READY)
- intval |= IPATH_LAYER_INT_IF_UP;
- if (dd->ipath_lid)
- intval |= IPATH_LAYER_INT_LID;
- if (dd->ipath_mlid)
- intval |= IPATH_LAYER_INT_BCAST;
- /*
- * do this on open, in case low level is already up and
- * just layered driver was reloaded, etc.
- */
- if (intval)
- layer_intr(dd->ipath_layer.l_arg, intval);
-
- ret = 0;
-bail:
- mutex_unlock(&ipath_layer_mutex);
-
- return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_open);
-
-u16 ipath_layer_get_lid(struct ipath_devdata *dd)
-{
- return dd->ipath_lid;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_lid);
-
-/**
- * ipath_layer_get_mac - get the MAC address
- * @dd: the infinipath device
- * @mac: the MAC is put here
- *
- * This is the EUID-64 OUI octets (top 3), then
- * skip the next 2 (which should both be zero or 0xff).
- * The returned MAC is in network order
- * mac points to at least 6 bytes of buffer
- * We assume that by the time the LID is set, that the GUID is as valid
- * as it's ever going to be, rather than adding yet another status bit.
- */
-
-int ipath_layer_get_mac(struct ipath_devdata *dd, u8 * mac)
-{
- u8 *guid;
-
- guid = (u8 *) &dd->ipath_guid;
-
- mac[0] = guid[0];
- mac[1] = guid[1];
- mac[2] = guid[2];
- mac[3] = guid[5];
- mac[4] = guid[6];
- mac[5] = guid[7];
- if ((guid[3] || guid[4]) && !(guid[3] == 0xff && guid[4] == 0xff))
- ipath_dbg("Warning, guid bytes 3 and 4 not 0 or 0xffff: "
- "%x %x\n", guid[3], guid[4]);
- return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_mac);
-
-u16 ipath_layer_get_bcast(struct ipath_devdata *dd)
-{
- return dd->ipath_mlid;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_bcast);
-
-int ipath_layer_send_hdr(struct ipath_devdata *dd, struct ether_header *hdr)
-{
- int ret = 0;
- u32 __iomem *piobuf;
- u32 plen, *uhdr;
- size_t count;
- __be16 vlsllnh;
-
- if (!(dd->ipath_flags & IPATH_RCVHDRSZ_SET)) {
- ipath_dbg("send while not open\n");
- ret = -EINVAL;
- } else
- if ((dd->ipath_flags & (IPATH_LINKUNK | IPATH_LINKDOWN)) ||
- dd->ipath_lid == 0) {
- /*
- * lid check is for when sma hasn't yet configured
- */
- ret = -ENETDOWN;
- ipath_cdbg(VERBOSE, "send while not ready, "
- "mylid=%u, flags=0x%x\n",
- dd->ipath_lid, dd->ipath_flags);
- }
-
- vlsllnh = *((__be16 *) hdr);
- if (vlsllnh != htons(IPATH_LRH_BTH)) {
- ipath_dbg("Warning: lrh[0] wrong (%x, not %x); "
- "not sending\n", be16_to_cpu(vlsllnh),
- IPATH_LRH_BTH);
- ret = -EINVAL;
- }
- if (ret)
- goto done;
-
- /* Get a PIO buffer to use. */
- piobuf = ipath_getpiobuf(dd, NULL);
- if (piobuf == NULL) {
- ret = -EBUSY;
- goto done;
- }
-
- plen = (sizeof(*hdr) >> 2); /* actual length */
- ipath_cdbg(EPKT, "0x%x+1w pio %p\n", plen, piobuf);
-
- writeq(plen+1, piobuf); /* len (+1 for pad) to pbc, no flags */
- ipath_flush_wc();
- piobuf += 2;
- uhdr = (u32 *)hdr;
- count = plen-1; /* amount we can copy before trigger word */
- __iowrite32_copy(piobuf, uhdr, count);
- ipath_flush_wc();
- __raw_writel(uhdr[count], piobuf + count);
- ipath_flush_wc(); /* ensure it's sent, now */
-
- ipath_stats.sps_ether_spkts++; /* ether packet sent */
-
-done:
- return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_send_hdr);
-
-int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd)
-{
- set_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl);
-
- ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
- dd->ipath_sendctrl);
- return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_piointbufavail_int);
+++ /dev/null
-/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
- * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _IPATH_LAYER_H
-#define _IPATH_LAYER_H
-
-/*
- * This header file is for symbols shared between the infinipath driver
- * and drivers layered upon it (such as ipath).
- */
-
-struct sk_buff;
-struct ipath_devdata;
-struct ether_header;
-
-int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *),
- void (*l_remove)(void *),
- int (*l_intr)(void *, u32),
- int (*l_rcv)(void *, void *,
- struct sk_buff *),
- u16 rcv_opcode,
- int (*l_rcv_lid)(void *, void *));
-void ipath_layer_unregister(void);
-int ipath_layer_open(struct ipath_devdata *, u32 * pktmax);
-u16 ipath_layer_get_lid(struct ipath_devdata *dd);
-int ipath_layer_get_mac(struct ipath_devdata *dd, u8 *);
-u16 ipath_layer_get_bcast(struct ipath_devdata *dd);
-int ipath_layer_send_hdr(struct ipath_devdata *dd,
- struct ether_header *hdr);
-int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd);
-
-/* ipath_ether interrupt values */
-#define IPATH_LAYER_INT_IF_UP 0x2
-#define IPATH_LAYER_INT_IF_DOWN 0x4
-#define IPATH_LAYER_INT_LID 0x8
-#define IPATH_LAYER_INT_SEND_CONTINUE 0x10
-#define IPATH_LAYER_INT_BCAST 0x40
-
-extern unsigned ipath_debug; /* debugging bit mask */
-
-#endif /* _IPATH_LAYER_H */
#include <rdma/ib_pack.h>
#include <rdma/ib_user_verbs.h>
-#include "ipath_layer.h"
-
#define IPATH_MAX_RDMA_ATOMIC 4
#define QPN_MAX (1 << 24)
return 0;
err_wrid:
- if (pd->uobject && !init_attr->srq)
- mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
- else {
+ if (pd->uobject) {
+ if (!init_attr->srq)
+ mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context),
+ &qp->db);
+ } else {
kfree(qp->sq.wrid);
kfree(qp->rq.wrid);
}
if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
printk(KERN_ERR "path MTU (%u) is invalid\n",
attr->path_mtu);
- return -EINVAL;
+ goto out;
}
context->mtu_msgmax = (attr->path_mtu << 5) | 31;
}
if (attr_mask & IB_QP_AV) {
if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,
- attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) {
- err = -EINVAL;
+ attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
goto out;
- }
optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
MLX4_QP_OPTPAR_SCHED_QUEUE);
if (attr_mask & IB_QP_ALT_PATH) {
if (attr->alt_port_num == 0 ||
attr->alt_port_num > dev->dev->caps.num_ports)
- return -EINVAL;
+ goto out;
if (attr->alt_pkey_index >=
dev->dev->caps.pkey_table_len[attr->alt_port_num])
- return -EINVAL;
+ goto out;
if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
attr->alt_port_num))
- return -EINVAL;
+ goto out;
context->alt_path.pkey_index = attr->alt_pkey_index;
context->alt_path.ackto = attr->alt_timeout << 3;
context->status = status;
context->out_param = out_param;
- context->token += dev->cmd.token_mask + 1;
-
complete(&context->done);
}
spin_lock(&dev->cmd.context_lock);
BUG_ON(dev->cmd.free_head < 0);
context = &dev->cmd.context[dev->cmd.free_head];
+ context->token += dev->cmd.token_mask + 1;
dev->cmd.free_head = context->next;
spin_unlock(&dev->cmd.context_lock);
context->result = mlx4_status_to_errno(status);
context->out_param = out_param;
- context->token += priv->cmd.token_mask + 1;
-
complete(&context->done);
}
spin_lock(&cmd->context_lock);
BUG_ON(cmd->free_head < 0);
context = &cmd->context[cmd->free_head];
+ context->token += cmd->token_mask + 1;
cmd->free_head = context->next;
spin_unlock(&cmd->context_lock);
projects / linux-2.6 / commitdiff