/*
- * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
+ * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
#include <linux/cdev.h>
#include <linux/swap.h>
#include <linux/vmalloc.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/jiffies.h>
#include <asm/pgtable.h>
#include "ipath_kernel.h"
#include "ipath_common.h"
+#include "ipath_user_sdma.h"
static int ipath_open(struct inode *, struct file *);
static int ipath_close(struct inode *, struct file *);
static ssize_t ipath_write(struct file *, const char __user *, size_t,
loff_t *);
+static ssize_t ipath_writev(struct kiocb *, const struct iovec *,
+ unsigned long , loff_t);
static unsigned int ipath_poll(struct file *, struct poll_table_struct *);
static int ipath_mmap(struct file *, struct vm_area_struct *);
static const struct file_operations ipath_file_ops = {
.owner = THIS_MODULE,
.write = ipath_write,
+ .aio_write = ipath_writev,
.open = ipath_open,
.release = ipath_close,
.poll = ipath_poll,
kinfo->spi_piocnt = dd->ipath_pbufsport;
kinfo->spi_piobufbase = (u64) pd->port_piobufs;
kinfo->__spi_uregbase = (u64) dd->ipath_uregbase +
- dd->ipath_palign * pd->port_port;
+ dd->ipath_ureg_align * pd->port_port;
} else if (master) {
kinfo->spi_piocnt = (dd->ipath_pbufsport / subport_cnt) +
(dd->ipath_pbufsport % subport_cnt);
kinfo->spi_piobufbase = (u64) pd->port_piobufs +
dd->ipath_palign * kinfo->spi_piocnt * slave;
}
+
+ /*
+ * Set the PIO avail update threshold to no larger
+ * than the number of buffers per process. Note that
+ * we decrease it here, but won't ever increase it.
+ */
+ if (dd->ipath_pioupd_thresh &&
+ kinfo->spi_piocnt < dd->ipath_pioupd_thresh) {
+ unsigned long flags;
+
+ dd->ipath_pioupd_thresh = kinfo->spi_piocnt;
+ ipath_dbg("Decreased pio update threshold to %u\n",
+ dd->ipath_pioupd_thresh);
+ spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
+ dd->ipath_sendctrl &= ~(INFINIPATH_S_UPDTHRESH_MASK
+ << INFINIPATH_S_UPDTHRESH_SHIFT);
+ dd->ipath_sendctrl |= dd->ipath_pioupd_thresh
+ << INFINIPATH_S_UPDTHRESH_SHIFT;
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
+ dd->ipath_sendctrl);
+ spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
+ }
+
if (shared) {
kinfo->spi_port_uregbase = (u64) dd->ipath_uregbase +
- dd->ipath_palign * pd->port_port;
+ dd->ipath_ureg_align * pd->port_port;
kinfo->spi_port_rcvegrbuf = kinfo->spi_rcv_egrbufs;
kinfo->spi_port_rcvhdr_base = kinfo->spi_rcvhdr_base;
kinfo->spi_port_rcvhdr_tailaddr = kinfo->spi_rcvhdr_tailaddr;
kinfo->spi_pioalign = dd->ipath_palign;
kinfo->spi_qpair = IPATH_KD_QP;
- kinfo->spi_piosize = dd->ipath_ibmaxlen;
+ /*
+ * user mode PIO buffers are always 2KB, even when 4KB can
+ * be received, and sent via the kernel; this is ibmaxlen
+ * for 2K MTU.
+ */
+ kinfo->spi_piosize = dd->ipath_piosize2k - 2 * sizeof(u32);
kinfo->spi_mtu = dd->ipath_ibmaxlen; /* maxlen, not ibmtu */
kinfo->spi_port = pd->port_port;
kinfo->spi_subport = subport_fp(fp);
continue;
cnt++;
if (dd->ipath_pageshadow[porttid + tid]) {
+ struct page *p;
+ p = dd->ipath_pageshadow[porttid + tid];
+ dd->ipath_pageshadow[porttid + tid] = NULL;
ipath_cdbg(VERBOSE, "PID %u freeing TID %u\n",
pd->port_pid, tid);
dd->ipath_f_put_tid(dd, &tidbase[tid],
pci_unmap_page(dd->pcidev,
dd->ipath_physshadow[porttid + tid],
PAGE_SIZE, PCI_DMA_FROMDEVICE);
- ipath_release_user_pages(
- &dd->ipath_pageshadow[porttid + tid], 1);
- dd->ipath_pageshadow[porttid + tid] = NULL;
+ ipath_release_user_pages(&p, 1);
ipath_stats.sps_pageunlocks++;
} else
ipath_dbg("Unused tid %u, ignoring\n", tid);
* updated and correct itself, even in the face of software
* bugs.
*/
- *(volatile u64 *)pd->port_rcvhdrtail_kvaddr = 0;
- set_bit(INFINIPATH_R_PORTENABLE_SHIFT + pd->port_port,
+ if (pd->port_rcvhdrtail_kvaddr)
+ ipath_clear_rcvhdrtail(pd);
+ set_bit(dd->ipath_r_portenable_shift + pd->port_port,
&dd->ipath_rcvctrl);
} else
- clear_bit(INFINIPATH_R_PORTENABLE_SHIFT + pd->port_port,
+ clear_bit(dd->ipath_r_portenable_shift + pd->port_port,
&dd->ipath_rcvctrl);
ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
dd->ipath_rcvctrl);
egrcnt = dd->ipath_rcvegrcnt;
/* TID number offset for this port */
- egroff = pd->port_port * egrcnt;
+ egroff = (pd->port_port - 1) * egrcnt + dd->ipath_p0_rcvegrcnt;
egrsize = dd->ipath_rcvegrbufsize;
ipath_cdbg(VERBOSE, "Allocating %d egr buffers, at egrtid "
"offset %x, egrsize %u\n", egrcnt, egroff, egrsize);
phys = dd->ipath_physaddr + piobufs;
- /*
- * Don't mark this as non-cached, or we don't get the
- * write combining behavior we want on the PIO buffers!
- */
-
#if defined(__powerpc__)
/* There isn't a generic way to specify writethrough mappings */
pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE;
}
/*
- * ipath_file_vma_nopage - handle a VMA page fault.
+ * ipath_file_vma_fault - handle a VMA page fault.
*/
-static struct page *ipath_file_vma_nopage(struct vm_area_struct *vma,
- unsigned long address, int *type)
+static int ipath_file_vma_fault(struct vm_area_struct *vma,
+ struct vm_fault *vmf)
{
- unsigned long offset = address - vma->vm_start;
- struct page *page = NOPAGE_SIGBUS;
- void *pageptr;
+ struct page *page;
- /*
- * Convert the vmalloc address into a struct page.
- */
- pageptr = (void *)(offset + (vma->vm_pgoff << PAGE_SHIFT));
- page = vmalloc_to_page(pageptr);
+ page = vmalloc_to_page((void *)(vmf->pgoff << PAGE_SHIFT));
if (!page)
- goto out;
-
- /* Increment the reference count. */
+ return VM_FAULT_SIGBUS;
get_page(page);
- if (type)
- *type = VM_FAULT_MINOR;
-out:
- return page;
+ vmf->page = page;
+
+ return 0;
}
static struct vm_operations_struct ipath_file_vm_ops = {
- .nopage = ipath_file_vma_nopage,
+ .fault = ipath_file_vma_fault,
};
static int mmap_kvaddr(struct vm_area_struct *vma, u64 pgaddr,
goto bail;
}
- ureg = dd->ipath_uregbase + dd->ipath_palign * pd->port_port;
+ ureg = dd->ipath_uregbase + dd->ipath_ureg_align * pd->port_port;
if (!pd->port_subport_cnt) {
/* port is not shared */
piocnt = dd->ipath_pbufsport;
return ret;
}
+static unsigned ipath_poll_hdrqfull(struct ipath_portdata *pd)
+{
+ unsigned pollflag = 0;
+
+ if ((pd->poll_type & IPATH_POLL_TYPE_OVERFLOW) &&
+ pd->port_hdrqfull != pd->port_hdrqfull_poll) {
+ pollflag |= POLLIN | POLLRDNORM;
+ pd->port_hdrqfull_poll = pd->port_hdrqfull;
+ }
+
+ return pollflag;
+}
+
static unsigned int ipath_poll_urgent(struct ipath_portdata *pd,
struct file *fp,
struct poll_table_struct *pt)
dd = pd->port_dd;
- if (test_bit(IPATH_PORT_WAITING_OVERFLOW, &pd->int_flag)) {
- pollflag |= POLLERR;
- clear_bit(IPATH_PORT_WAITING_OVERFLOW, &pd->int_flag);
- }
+ /* variable access in ipath_poll_hdrqfull() needs this */
+ rmb();
+ pollflag = ipath_poll_hdrqfull(pd);
- if (test_bit(IPATH_PORT_WAITING_URG, &pd->int_flag)) {
+ if (pd->port_urgent != pd->port_urgent_poll) {
pollflag |= POLLIN | POLLRDNORM;
- clear_bit(IPATH_PORT_WAITING_URG, &pd->int_flag);
+ pd->port_urgent_poll = pd->port_urgent;
}
if (!pollflag) {
+ /* this saves a spin_lock/unlock in interrupt handler... */
set_bit(IPATH_PORT_WAITING_URG, &pd->port_flag);
- if (pd->poll_type & IPATH_POLL_TYPE_OVERFLOW)
- set_bit(IPATH_PORT_WAITING_OVERFLOW,
- &pd->port_flag);
-
+ /* flush waiting flag so don't miss an event... */
+ wmb();
poll_wait(fp, &pd->port_wait, pt);
}
struct file *fp,
struct poll_table_struct *pt)
{
- u32 head, tail;
+ u32 head;
+ u32 tail;
unsigned pollflag = 0;
struct ipath_devdata *dd;
dd = pd->port_dd;
- head = ipath_read_ureg32(dd, ur_rcvhdrhead, pd->port_port);
- tail = *(volatile u64 *)pd->port_rcvhdrtail_kvaddr;
+ /* variable access in ipath_poll_hdrqfull() needs this */
+ rmb();
+ pollflag = ipath_poll_hdrqfull(pd);
- if (test_bit(IPATH_PORT_WAITING_OVERFLOW, &pd->int_flag)) {
- pollflag |= POLLERR;
- clear_bit(IPATH_PORT_WAITING_OVERFLOW, &pd->int_flag);
- }
+ head = ipath_read_ureg32(dd, ur_rcvhdrhead, pd->port_port);
+ if (pd->port_rcvhdrtail_kvaddr)
+ tail = ipath_get_rcvhdrtail(pd);
+ else
+ tail = ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
- if (tail != head ||
- test_bit(IPATH_PORT_WAITING_RCV, &pd->int_flag)) {
+ if (head != tail)
pollflag |= POLLIN | POLLRDNORM;
- clear_bit(IPATH_PORT_WAITING_RCV, &pd->int_flag);
- }
-
- if (!pollflag) {
+ else {
+ /* this saves a spin_lock/unlock in interrupt handler */
set_bit(IPATH_PORT_WAITING_RCV, &pd->port_flag);
- if (pd->poll_type & IPATH_POLL_TYPE_OVERFLOW)
- set_bit(IPATH_PORT_WAITING_OVERFLOW,
- &pd->port_flag);
+ /* flush waiting flag so we don't miss an event */
+ wmb();
- set_bit(pd->port_port + INFINIPATH_R_INTRAVAIL_SHIFT,
+ set_bit(pd->port_port + dd->ipath_r_intravail_shift,
&dd->ipath_rcvctrl);
ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
port_fp(fp) = pd;
pd->port_pid = current->pid;
strncpy(pd->port_comm, current->comm, sizeof(pd->port_comm));
+ ipath_chg_pioavailkernel(dd,
+ dd->ipath_pbufsport * (pd->port_port - 1),
+ dd->ipath_pbufsport, 0);
ipath_stats.sps_ports++;
ret = 0;
} else
for (ndev = 0; ndev < devmax; ndev++) {
struct ipath_devdata *dd = ipath_lookup(ndev);
- if (!dd)
+ if (!usable(dd))
continue;
for (i = 1; i < dd->ipath_cfgports; i++) {
struct ipath_portdata *pd = dd->ipath_pd[i];
}
port_fp(fp) = pd;
subport_fp(fp) = pd->port_cnt++;
+ pd->port_subpid[subport_fp(fp)] = current->pid;
tidcursor_fp(fp) = 0;
pd->active_slaves |= 1 << subport_fp(fp);
ipath_cdbg(PROC,
if (ipath_compatible_subports(swmajor, swminor) &&
uinfo->spu_subport_cnt &&
(ret = find_shared_port(fp, uinfo))) {
- mutex_unlock(&ipath_mutex);
if (ret > 0)
ret = 0;
- goto done;
+ goto done_chk_sdma;
}
i_minor = iminor(fp->f_path.dentry->d_inode) - IPATH_USER_MINOR_BASE;
else
ret = find_best_unit(fp, uinfo);
+done_chk_sdma:
+ if (!ret) {
+ struct ipath_filedata *fd = fp->private_data;
+ const struct ipath_portdata *pd = fd->pd;
+ const struct ipath_devdata *dd = pd->port_dd;
+
+ fd->pq = ipath_user_sdma_queue_create(&dd->pcidev->dev,
+ dd->ipath_unit,
+ pd->port_port,
+ fd->subport);
+
+ if (!fd->pq)
+ ret = -ENOMEM;
+ }
+
mutex_unlock(&ipath_mutex);
done:
*/
head32 = ipath_read_ureg32(dd, ur_rcvegrindextail, pd->port_port);
ipath_write_ureg(dd, ur_rcvegrindexhead, head32, pd->port_port);
- dd->ipath_lastegrheads[pd->port_port] = -1;
- dd->ipath_lastrcvhdrqtails[pd->port_port] = -1;
+ pd->port_lastrcvhdrqtail = -1;
ipath_cdbg(VERBOSE, "Wrote port%d egrhead %x from tail regs\n",
pd->port_port, head32);
pd->port_tidcursor = 0; /* start at beginning after open */
+
+ /* initialize poll variables... */
+ pd->port_urgent = 0;
+ pd->port_urgent_poll = 0;
+ pd->port_hdrqfull_poll = pd->port_hdrqfull;
+
/*
- * now enable the port; the tail registers will be written to memory
- * by the chip as soon as it sees the write to
- * dd->ipath_kregs->kr_rcvctrl. The update only happens on
- * transition from 0 to 1, so clear it first, then set it as part of
- * enabling the port. This will (very briefly) affect any other
- * open ports, but it shouldn't be long enough to be an issue.
- * We explictly set the in-memory copy to 0 beforehand, so we don't
- * have to wait to be sure the DMA update has happened.
+ * Now enable the port for receive.
+ * For chips that are set to DMA the tail register to memory
+ * when they change (and when the update bit transitions from
+ * 0 to 1. So for those chips, we turn it off and then back on.
+ * This will (very briefly) affect any other open ports, but the
+ * duration is very short, and therefore isn't an issue. We
+ * explictly set the in-memory tail copy to 0 beforehand, so we
+ * don't have to wait to be sure the DMA update has happened
+ * (chip resets head/tail to 0 on transition to enable).
*/
- *(volatile u64 *)pd->port_rcvhdrtail_kvaddr = 0ULL;
- set_bit(INFINIPATH_R_PORTENABLE_SHIFT + pd->port_port,
+ set_bit(dd->ipath_r_portenable_shift + pd->port_port,
&dd->ipath_rcvctrl);
- ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
- dd->ipath_rcvctrl & ~INFINIPATH_R_TAILUPD);
+ if (!(dd->ipath_flags & IPATH_NODMA_RTAIL)) {
+ if (pd->port_rcvhdrtail_kvaddr)
+ ipath_clear_rcvhdrtail(pd);
+ ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
+ dd->ipath_rcvctrl &
+ ~(1ULL << dd->ipath_r_tailupd_shift));
+ }
ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
dd->ipath_rcvctrl);
/* Notify any waiting slaves */
ipath_cdbg(VERBOSE, "Port %u unlocking any locked expTID pages\n",
pd->port_port);
for (i = port_tidbase; i < maxtid; i++) {
- if (!dd->ipath_pageshadow[i])
+ struct page *ps = dd->ipath_pageshadow[i];
+
+ if (!ps)
continue;
+ dd->ipath_pageshadow[i] = NULL;
pci_unmap_page(dd->pcidev, dd->ipath_physshadow[i],
PAGE_SIZE, PCI_DMA_FROMDEVICE);
- ipath_release_user_pages_on_close(&dd->ipath_pageshadow[i],
- 1);
- dd->ipath_pageshadow[i] = NULL;
+ ipath_release_user_pages_on_close(&ps, 1);
cnt++;
ipath_stats.sps_pageunlocks++;
}
mutex_unlock(&ipath_mutex);
goto bail;
}
+
+ dd = pd->port_dd;
+
+ /* drain user sdma queue */
+ ipath_user_sdma_queue_drain(dd, fd->pq);
+ ipath_user_sdma_queue_destroy(fd->pq);
+
if (--pd->port_cnt) {
/*
* XXX If the master closes the port before the slave(s),
* the slave(s) don't wait for receive data forever.
*/
pd->active_slaves &= ~(1 << fd->subport);
+ pd->port_subpid[fd->subport] = 0;
mutex_unlock(&ipath_mutex);
goto bail;
}
port = pd->port_port;
- dd = pd->port_dd;
if (pd->port_hdrqfull) {
ipath_cdbg(PROC, "%s[%u] had %u rcvhdrqfull errors "
pd->port_rcvnowait = pd->port_pionowait = 0;
}
if (pd->port_flag) {
- ipath_dbg("port %u port_flag still set to 0x%lx\n",
+ ipath_cdbg(PROC, "port %u port_flag set: 0x%lx\n",
pd->port_port, pd->port_flag);
pd->port_flag = 0;
}
if (dd->ipath_kregbase) {
int i;
- /* atomically clear receive enable port. */
- clear_bit(INFINIPATH_R_PORTENABLE_SHIFT + port,
+ /* atomically clear receive enable port and intr avail. */
+ clear_bit(dd->ipath_r_portenable_shift + port,
+ &dd->ipath_rcvctrl);
+ clear_bit(pd->port_port + dd->ipath_r_intravail_shift,
&dd->ipath_rcvctrl);
ipath_write_kreg( dd, dd->ipath_kregs->kr_rcvctrl,
dd->ipath_rcvctrl);
i = dd->ipath_pbufsport * (port - 1);
ipath_disarm_piobufs(dd, i, dd->ipath_pbufsport);
+ ipath_chg_pioavailkernel(dd, i, dd->ipath_pbufsport, 1);
dd->ipath_f_clear_tids(dd, pd->port_port);
return ret;
}
-static int ipath_force_pio_avail_update(struct ipath_devdata *dd)
+static int ipath_sdma_get_inflight(struct ipath_user_sdma_queue *pq,
+ u32 __user *inflightp)
{
- u64 reg = dd->ipath_sendctrl;
+ const u32 val = ipath_user_sdma_inflight_counter(pq);
- clear_bit(IPATH_S_PIOBUFAVAILUPD, ®);
- ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, reg);
- ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
+ if (put_user(val, inflightp))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int ipath_sdma_get_complete(struct ipath_devdata *dd,
+ struct ipath_user_sdma_queue *pq,
+ u32 __user *completep)
+{
+ u32 val;
+ int err;
+
+ err = ipath_user_sdma_make_progress(dd, pq);
+ if (err < 0)
+ return err;
+
+ val = ipath_user_sdma_complete_counter(pq);
+ if (put_user(val, completep))
+ return -EFAULT;
return 0;
}
dest = &cmd.cmd.poll_type;
src = &ucmd->cmd.poll_type;
break;
+ case IPATH_CMD_ARMLAUNCH_CTRL:
+ copy = sizeof(cmd.cmd.armlaunch_ctrl);
+ dest = &cmd.cmd.armlaunch_ctrl;
+ src = &ucmd->cmd.armlaunch_ctrl;
+ break;
+ case IPATH_CMD_SDMA_INFLIGHT:
+ copy = sizeof(cmd.cmd.sdma_inflight);
+ dest = &cmd.cmd.sdma_inflight;
+ src = &ucmd->cmd.sdma_inflight;
+ break;
+ case IPATH_CMD_SDMA_COMPLETE:
+ copy = sizeof(cmd.cmd.sdma_complete);
+ dest = &cmd.cmd.sdma_complete;
+ src = &ucmd->cmd.sdma_complete;
+ break;
default:
ret = -EINVAL;
goto bail;
cmd.cmd.slave_mask_addr);
break;
case IPATH_CMD_PIOAVAILUPD:
- ret = ipath_force_pio_avail_update(pd->port_dd);
+ ipath_force_pio_avail_update(pd->port_dd);
break;
case IPATH_CMD_POLL_TYPE:
pd->poll_type = cmd.cmd.poll_type;
break;
+ case IPATH_CMD_ARMLAUNCH_CTRL:
+ if (cmd.cmd.armlaunch_ctrl)
+ ipath_enable_armlaunch(pd->port_dd);
+ else
+ ipath_disable_armlaunch(pd->port_dd);
+ break;
+ case IPATH_CMD_SDMA_INFLIGHT:
+ ret = ipath_sdma_get_inflight(user_sdma_queue_fp(fp),
+ (u32 __user *) (unsigned long)
+ cmd.cmd.sdma_inflight);
+ break;
+ case IPATH_CMD_SDMA_COMPLETE:
+ ret = ipath_sdma_get_complete(pd->port_dd,
+ user_sdma_queue_fp(fp),
+ (u32 __user *) (unsigned long)
+ cmd.cmd.sdma_complete);
+ break;
}
if (ret >= 0)
return ret;
}
+static ssize_t ipath_writev(struct kiocb *iocb, const struct iovec *iov,
+ unsigned long dim, loff_t off)
+{
+ struct file *filp = iocb->ki_filp;
+ struct ipath_filedata *fp = filp->private_data;
+ struct ipath_portdata *pd = port_fp(filp);
+ struct ipath_user_sdma_queue *pq = fp->pq;
+
+ if (!dim)
+ return -EINVAL;
+
+ return ipath_user_sdma_writev(pd->port_dd, pq, iov, dim);
+}
+
static struct class *ipath_class;
static int init_cdev(int minor, char *name, const struct file_operations *fops,