2 * Copyright (c) 2006 QLogic, Inc. All rights reserved.
3 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/spinlock.h>
35 #include <linux/idr.h>
36 #include <linux/pci.h>
37 #include <linux/delay.h>
38 #include <linux/netdevice.h>
39 #include <linux/vmalloc.h>
41 #include "ipath_kernel.h"
42 #include "ips_common.h"
43 #include "ipath_layer.h"
45 static void ipath_update_pio_bufs(struct ipath_devdata *);
47 const char *ipath_get_unit_name(int unit)
49 static char iname[16];
50 snprintf(iname, sizeof iname, "infinipath%u", unit);
54 EXPORT_SYMBOL_GPL(ipath_get_unit_name);
56 #define DRIVER_LOAD_MSG "QLogic " IPATH_DRV_NAME " loaded: "
57 #define PFX IPATH_DRV_NAME ": "
60 * The size has to be longer than this string, so we can append
61 * board/chip information to it in the init code.
63 const char ipath_core_version[] = IPATH_IDSTR "\n";
65 static struct idr unit_table;
66 DEFINE_SPINLOCK(ipath_devs_lock);
67 LIST_HEAD(ipath_dev_list);
69 wait_queue_head_t ipath_sma_state_wait;
71 unsigned ipath_debug = __IPATH_INFO;
73 module_param_named(debug, ipath_debug, uint, S_IWUSR | S_IRUGO);
74 MODULE_PARM_DESC(debug, "mask for debug prints");
75 EXPORT_SYMBOL_GPL(ipath_debug);
77 MODULE_LICENSE("GPL");
78 MODULE_AUTHOR("QLogic <support@pathscale.com>");
79 MODULE_DESCRIPTION("QLogic InfiniPath driver");
81 const char *ipath_ibcstatus_str[] = {
88 "LState6", /* unused */
89 "LState7", /* unused */
95 "LState0xD", /* unused */
101 * These variables are initialized in the chip-specific files
102 * but are defined here.
104 u16 ipath_gpio_sda_num, ipath_gpio_scl_num;
105 u64 ipath_gpio_sda, ipath_gpio_scl;
106 u64 infinipath_i_bitsextant;
107 ipath_err_t infinipath_e_bitsextant, infinipath_hwe_bitsextant;
108 u32 infinipath_i_rcvavail_mask, infinipath_i_rcvurg_mask;
110 static void __devexit ipath_remove_one(struct pci_dev *);
111 static int __devinit ipath_init_one(struct pci_dev *,
112 const struct pci_device_id *);
114 /* Only needed for registration, nothing else needs this info */
115 #define PCI_VENDOR_ID_PATHSCALE 0x1fc1
116 #define PCI_DEVICE_ID_INFINIPATH_HT 0xd
117 #define PCI_DEVICE_ID_INFINIPATH_PE800 0x10
119 static const struct pci_device_id ipath_pci_tbl[] = {
120 { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_INFINIPATH_HT) },
121 { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_INFINIPATH_PE800) },
125 MODULE_DEVICE_TABLE(pci, ipath_pci_tbl);
127 static struct pci_driver ipath_driver = {
128 .name = IPATH_DRV_NAME,
129 .probe = ipath_init_one,
130 .remove = __devexit_p(ipath_remove_one),
131 .id_table = ipath_pci_tbl,
135 static inline void read_bars(struct ipath_devdata *dd, struct pci_dev *dev,
136 u32 *bar0, u32 *bar1)
140 ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, bar0);
142 ipath_dev_err(dd, "failed to read bar0 before enable: "
145 ret = pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, bar1);
147 ipath_dev_err(dd, "failed to read bar1 before enable: "
150 ipath_dbg("Read bar0 %x bar1 %x\n", *bar0, *bar1);
153 static void ipath_free_devdata(struct pci_dev *pdev,
154 struct ipath_devdata *dd)
158 pci_set_drvdata(pdev, NULL);
160 if (dd->ipath_unit != -1) {
161 spin_lock_irqsave(&ipath_devs_lock, flags);
162 idr_remove(&unit_table, dd->ipath_unit);
163 list_del(&dd->ipath_list);
164 spin_unlock_irqrestore(&ipath_devs_lock, flags);
169 static struct ipath_devdata *ipath_alloc_devdata(struct pci_dev *pdev)
172 struct ipath_devdata *dd;
175 if (!idr_pre_get(&unit_table, GFP_KERNEL)) {
176 dd = ERR_PTR(-ENOMEM);
180 dd = vmalloc(sizeof(*dd));
182 dd = ERR_PTR(-ENOMEM);
185 memset(dd, 0, sizeof(*dd));
188 spin_lock_irqsave(&ipath_devs_lock, flags);
190 ret = idr_get_new(&unit_table, dd, &dd->ipath_unit);
192 printk(KERN_ERR IPATH_DRV_NAME
193 ": Could not allocate unit ID: error %d\n", -ret);
194 ipath_free_devdata(pdev, dd);
200 pci_set_drvdata(pdev, dd);
202 list_add(&dd->ipath_list, &ipath_dev_list);
205 spin_unlock_irqrestore(&ipath_devs_lock, flags);
211 static inline struct ipath_devdata *__ipath_lookup(int unit)
213 return idr_find(&unit_table, unit);
216 struct ipath_devdata *ipath_lookup(int unit)
218 struct ipath_devdata *dd;
221 spin_lock_irqsave(&ipath_devs_lock, flags);
222 dd = __ipath_lookup(unit);
223 spin_unlock_irqrestore(&ipath_devs_lock, flags);
228 int ipath_count_units(int *npresentp, int *nupp, u32 *maxportsp)
230 int nunits, npresent, nup;
231 struct ipath_devdata *dd;
235 nunits = npresent = nup = maxports = 0;
237 spin_lock_irqsave(&ipath_devs_lock, flags);
239 list_for_each_entry(dd, &ipath_dev_list, ipath_list) {
241 if ((dd->ipath_flags & IPATH_PRESENT) && dd->ipath_kregbase)
244 !(dd->ipath_flags & (IPATH_DISABLED | IPATH_LINKDOWN
247 if (dd->ipath_cfgports > maxports)
248 maxports = dd->ipath_cfgports;
251 spin_unlock_irqrestore(&ipath_devs_lock, flags);
254 *npresentp = npresent;
258 *maxportsp = maxports;
264 * These next two routines are placeholders in case we don't have per-arch
265 * code for controlling write combining. If explicit control of write
266 * combining is not available, performance will probably be awful.
269 int __attribute__((weak)) ipath_enable_wc(struct ipath_devdata *dd)
274 void __attribute__((weak)) ipath_disable_wc(struct ipath_devdata *dd)
278 static int __devinit ipath_init_one(struct pci_dev *pdev,
279 const struct pci_device_id *ent)
282 struct ipath_devdata *dd;
283 unsigned long long addr;
284 u32 bar0 = 0, bar1 = 0;
287 dd = ipath_alloc_devdata(pdev);
290 printk(KERN_ERR IPATH_DRV_NAME
291 ": Could not allocate devdata: error %d\n", -ret);
295 ipath_cdbg(VERBOSE, "initializing unit #%u\n", dd->ipath_unit);
297 read_bars(dd, pdev, &bar0, &bar1);
299 ret = pci_enable_device(pdev);
301 /* This can happen iff:
303 * We did a chip reset, and then failed to reprogram the
304 * BAR, or the chip reset due to an internal error. We then
305 * unloaded the driver and reloaded it.
307 * Both reset cases set the BAR back to initial state. For
308 * the latter case, the AER sticky error bit at offset 0x718
309 * should be set, but the Linux kernel doesn't yet know
310 * about that, it appears. If the original BAR was retained
311 * in the kernel data structures, this may be OK.
313 ipath_dev_err(dd, "enable unit %d failed: error %d\n",
314 dd->ipath_unit, -ret);
317 addr = pci_resource_start(pdev, 0);
318 len = pci_resource_len(pdev, 0);
319 ipath_cdbg(VERBOSE, "regbase (0) %llx len %d irq %x, vend %x/%x "
320 "driver_data %lx\n", addr, len, pdev->irq, ent->vendor,
321 ent->device, ent->driver_data);
323 read_bars(dd, pdev, &bar0, &bar1);
325 if (!bar1 && !(bar0 & ~0xf)) {
327 dev_info(&pdev->dev, "BAR is 0 (probable RESET), "
328 "rewriting as %llx\n", addr);
329 ret = pci_write_config_dword(
330 pdev, PCI_BASE_ADDRESS_0, addr);
332 ipath_dev_err(dd, "rewrite of BAR0 "
333 "failed: err %d\n", -ret);
336 ret = pci_write_config_dword(
337 pdev, PCI_BASE_ADDRESS_1, addr >> 32);
339 ipath_dev_err(dd, "rewrite of BAR1 "
340 "failed: err %d\n", -ret);
344 ipath_dev_err(dd, "BAR is 0 (probable RESET), "
345 "not usable until reboot\n");
351 ret = pci_request_regions(pdev, IPATH_DRV_NAME);
353 dev_info(&pdev->dev, "pci_request_regions unit %u fails: "
354 "err %d\n", dd->ipath_unit, -ret);
358 ret = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
361 * if the 64 bit setup fails, try 32 bit. Some systems
362 * do not setup 64 bit maps on systems with 2GB or less
365 ret = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
368 "Unable to set DMA mask for unit %u: %d\n",
369 dd->ipath_unit, ret);
373 ipath_dbg("No 64bit DMA mask, used 32 bit mask\n");
374 ret = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
377 "Unable to set DMA consistent mask "
379 dd->ipath_unit, ret);
384 ret = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
387 "Unable to set DMA consistent mask "
389 dd->ipath_unit, ret);
392 pci_set_master(pdev);
395 * Save BARs to rewrite after device reset. Save all 64 bits of
398 dd->ipath_pcibar0 = addr;
399 dd->ipath_pcibar1 = addr >> 32;
400 dd->ipath_deviceid = ent->device; /* save for later use */
401 dd->ipath_vendorid = ent->vendor;
403 /* setup the chip-specific functions, as early as possible. */
404 switch (ent->device) {
405 case PCI_DEVICE_ID_INFINIPATH_HT:
406 ipath_init_ht400_funcs(dd);
408 case PCI_DEVICE_ID_INFINIPATH_PE800:
409 ipath_init_pe800_funcs(dd);
412 ipath_dev_err(dd, "Found unknown QLogic deviceid 0x%x, "
413 "failing\n", ent->device);
417 for (j = 0; j < 6; j++) {
418 if (!pdev->resource[j].start)
420 ipath_cdbg(VERBOSE, "BAR %d start %llx, end %llx, len %llx\n",
421 j, (unsigned long long)pdev->resource[j].start,
422 (unsigned long long)pdev->resource[j].end,
423 (unsigned long long)pci_resource_len(pdev, j));
427 ipath_dev_err(dd, "No valid address in BAR 0!\n");
432 dd->ipath_deviceid = ent->device; /* save for later use */
433 dd->ipath_vendorid = ent->vendor;
435 ret = pci_read_config_byte(pdev, PCI_REVISION_ID, &rev);
437 ipath_dev_err(dd, "Failed to read PCI revision ID unit "
438 "%u: err %d\n", dd->ipath_unit, -ret);
439 goto bail_regions; /* shouldn't ever happen */
441 dd->ipath_pcirev = rev;
443 dd->ipath_kregbase = ioremap_nocache(addr, len);
445 if (!dd->ipath_kregbase) {
446 ipath_dbg("Unable to map io addr %llx to kvirt, failing\n",
451 dd->ipath_kregend = (u64 __iomem *)
452 ((void __iomem *)dd->ipath_kregbase + len);
453 dd->ipath_physaddr = addr; /* used for io_remap, etc. */
455 ipath_cdbg(VERBOSE, "mapped io addr %llx to kregbase %p\n",
456 addr, dd->ipath_kregbase);
459 * clear ipath_flags here instead of in ipath_init_chip as it is set
460 * by ipath_setup_htconfig.
464 if (dd->ipath_f_bus(dd, pdev))
465 ipath_dev_err(dd, "Failed to setup config space; "
466 "continuing anyway\n");
469 * set up our interrupt handler; SA_SHIRQ probably not needed,
470 * since MSI interrupts shouldn't be shared but won't hurt for now.
471 * check 0 irq after we return from chip-specific bus setup, since
472 * that can affect this due to setup
475 ipath_dev_err(dd, "irq is 0, BIOS error? Interrupts won't "
478 ret = request_irq(pdev->irq, ipath_intr, SA_SHIRQ,
481 ipath_dev_err(dd, "Couldn't setup irq handler, "
482 "irq=%u: %d\n", pdev->irq, ret);
487 ret = ipath_init_chip(dd, 0); /* do the chip-specific init */
491 ret = ipath_enable_wc(dd);
494 ipath_dev_err(dd, "Write combining not enabled "
495 "(err %d): performance may be poor\n",
500 ipath_device_create_group(&pdev->dev, dd);
501 ipathfs_add_device(dd);
509 iounmap((volatile void __iomem *) dd->ipath_kregbase);
512 pci_release_regions(pdev);
515 pci_disable_device(pdev);
518 ipath_free_devdata(pdev, dd);
524 static void __devexit ipath_remove_one(struct pci_dev *pdev)
526 struct ipath_devdata *dd;
528 ipath_cdbg(VERBOSE, "removing, pdev=%p\n", pdev);
532 dd = pci_get_drvdata(pdev);
533 ipath_layer_remove(dd);
534 ipath_diag_remove(dd);
535 ipath_user_remove(dd);
536 ipathfs_remove_device(dd);
537 ipath_device_remove_group(&pdev->dev, dd);
538 ipath_cdbg(VERBOSE, "Releasing pci memory regions, dd %p, "
539 "unit %u\n", dd, (u32) dd->ipath_unit);
540 if (dd->ipath_kregbase) {
541 ipath_cdbg(VERBOSE, "Unmapping kregbase %p\n",
543 iounmap((volatile void __iomem *) dd->ipath_kregbase);
544 dd->ipath_kregbase = NULL;
546 pci_release_regions(pdev);
547 ipath_cdbg(VERBOSE, "calling pci_disable_device\n");
548 pci_disable_device(pdev);
550 ipath_free_devdata(pdev, dd);
553 /* general driver use */
554 DEFINE_MUTEX(ipath_mutex);
556 static DEFINE_SPINLOCK(ipath_pioavail_lock);
559 * ipath_disarm_piobufs - cancel a range of PIO buffers
560 * @dd: the infinipath device
561 * @first: the first PIO buffer to cancel
562 * @cnt: the number of PIO buffers to cancel
564 * cancel a range of PIO buffers, used when they might be armed, but
565 * not triggered. Used at init to ensure buffer state, and also user
566 * process close, in case it died while writing to a PIO buffer
569 void ipath_disarm_piobufs(struct ipath_devdata *dd, unsigned first,
572 unsigned i, last = first + cnt;
573 u64 sendctrl, sendorig;
575 ipath_cdbg(PKT, "disarm %u PIObufs first=%u\n", cnt, first);
576 sendorig = dd->ipath_sendctrl | INFINIPATH_S_DISARM;
577 for (i = first; i < last; i++) {
578 sendctrl = sendorig |
579 (i << INFINIPATH_S_DISARMPIOBUF_SHIFT);
580 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
585 * Write it again with current value, in case ipath_sendctrl changed
586 * while we were looping; no critical bits that would require
589 * Write a 0, and then the original value, reading scratch in
590 * between. This seems to avoid a chip timing race that causes
591 * pioavail updates to memory to stop.
593 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
595 sendorig = ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
596 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
601 * ipath_wait_linkstate - wait for an IB link state change to occur
602 * @dd: the infinipath device
603 * @state: the state to wait for
604 * @msecs: the number of milliseconds to wait
606 * wait up to msecs milliseconds for IB link state change to occur for
607 * now, take the easy polling route. Currently used only by
608 * ipath_layer_set_linkstate. Returns 0 if state reached, otherwise
609 * -ETIMEDOUT state can have multiple states set, for any of several
612 int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state, int msecs)
614 dd->ipath_sma_state_wanted = state;
615 wait_event_interruptible_timeout(ipath_sma_state_wait,
616 (dd->ipath_flags & state),
617 msecs_to_jiffies(msecs));
618 dd->ipath_sma_state_wanted = 0;
620 if (!(dd->ipath_flags & state)) {
622 ipath_cdbg(SMA, "Didn't reach linkstate %s within %u ms\n",
623 /* test INIT ahead of DOWN, both can be set */
624 (state & IPATH_LINKINIT) ? "INIT" :
625 ((state & IPATH_LINKDOWN) ? "DOWN" :
626 ((state & IPATH_LINKARMED) ? "ARM" : "ACTIVE")),
628 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
629 ipath_cdbg(VERBOSE, "ibcc=%llx ibcstatus=%llx (%s)\n",
630 (unsigned long long) ipath_read_kreg64(
631 dd, dd->ipath_kregs->kr_ibcctrl),
632 (unsigned long long) val,
633 ipath_ibcstatus_str[val & 0xf]);
635 return (dd->ipath_flags & state) ? 0 : -ETIMEDOUT;
638 void ipath_decode_err(char *buf, size_t blen, ipath_err_t err)
641 if (err & INFINIPATH_E_RHDRLEN)
642 strlcat(buf, "rhdrlen ", blen);
643 if (err & INFINIPATH_E_RBADTID)
644 strlcat(buf, "rbadtid ", blen);
645 if (err & INFINIPATH_E_RBADVERSION)
646 strlcat(buf, "rbadversion ", blen);
647 if (err & INFINIPATH_E_RHDR)
648 strlcat(buf, "rhdr ", blen);
649 if (err & INFINIPATH_E_RLONGPKTLEN)
650 strlcat(buf, "rlongpktlen ", blen);
651 if (err & INFINIPATH_E_RSHORTPKTLEN)
652 strlcat(buf, "rshortpktlen ", blen);
653 if (err & INFINIPATH_E_RMAXPKTLEN)
654 strlcat(buf, "rmaxpktlen ", blen);
655 if (err & INFINIPATH_E_RMINPKTLEN)
656 strlcat(buf, "rminpktlen ", blen);
657 if (err & INFINIPATH_E_RFORMATERR)
658 strlcat(buf, "rformaterr ", blen);
659 if (err & INFINIPATH_E_RUNSUPVL)
660 strlcat(buf, "runsupvl ", blen);
661 if (err & INFINIPATH_E_RUNEXPCHAR)
662 strlcat(buf, "runexpchar ", blen);
663 if (err & INFINIPATH_E_RIBFLOW)
664 strlcat(buf, "ribflow ", blen);
665 if (err & INFINIPATH_E_REBP)
666 strlcat(buf, "EBP ", blen);
667 if (err & INFINIPATH_E_SUNDERRUN)
668 strlcat(buf, "sunderrun ", blen);
669 if (err & INFINIPATH_E_SPIOARMLAUNCH)
670 strlcat(buf, "spioarmlaunch ", blen);
671 if (err & INFINIPATH_E_SUNEXPERRPKTNUM)
672 strlcat(buf, "sunexperrpktnum ", blen);
673 if (err & INFINIPATH_E_SDROPPEDDATAPKT)
674 strlcat(buf, "sdroppeddatapkt ", blen);
675 if (err & INFINIPATH_E_SDROPPEDSMPPKT)
676 strlcat(buf, "sdroppedsmppkt ", blen);
677 if (err & INFINIPATH_E_SMAXPKTLEN)
678 strlcat(buf, "smaxpktlen ", blen);
679 if (err & INFINIPATH_E_SMINPKTLEN)
680 strlcat(buf, "sminpktlen ", blen);
681 if (err & INFINIPATH_E_SUNSUPVL)
682 strlcat(buf, "sunsupVL ", blen);
683 if (err & INFINIPATH_E_SPKTLEN)
684 strlcat(buf, "spktlen ", blen);
685 if (err & INFINIPATH_E_INVALIDADDR)
686 strlcat(buf, "invalidaddr ", blen);
687 if (err & INFINIPATH_E_RICRC)
688 strlcat(buf, "CRC ", blen);
689 if (err & INFINIPATH_E_RVCRC)
690 strlcat(buf, "VCRC ", blen);
691 if (err & INFINIPATH_E_RRCVEGRFULL)
692 strlcat(buf, "rcvegrfull ", blen);
693 if (err & INFINIPATH_E_RRCVHDRFULL)
694 strlcat(buf, "rcvhdrfull ", blen);
695 if (err & INFINIPATH_E_IBSTATUSCHANGED)
696 strlcat(buf, "ibcstatuschg ", blen);
697 if (err & INFINIPATH_E_RIBLOSTLINK)
698 strlcat(buf, "riblostlink ", blen);
699 if (err & INFINIPATH_E_HARDWARE)
700 strlcat(buf, "hardware ", blen);
701 if (err & INFINIPATH_E_RESET)
702 strlcat(buf, "reset ", blen);
706 * get_rhf_errstring - decode RHF errors
707 * @err: the err number
708 * @msg: the output buffer
709 * @len: the length of the output buffer
711 * only used one place now, may want more later
713 static void get_rhf_errstring(u32 err, char *msg, size_t len)
715 /* if no errors, and so don't need to check what's first */
718 if (err & INFINIPATH_RHF_H_ICRCERR)
719 strlcat(msg, "icrcerr ", len);
720 if (err & INFINIPATH_RHF_H_VCRCERR)
721 strlcat(msg, "vcrcerr ", len);
722 if (err & INFINIPATH_RHF_H_PARITYERR)
723 strlcat(msg, "parityerr ", len);
724 if (err & INFINIPATH_RHF_H_LENERR)
725 strlcat(msg, "lenerr ", len);
726 if (err & INFINIPATH_RHF_H_MTUERR)
727 strlcat(msg, "mtuerr ", len);
728 if (err & INFINIPATH_RHF_H_IHDRERR)
729 /* infinipath hdr checksum error */
730 strlcat(msg, "ipathhdrerr ", len);
731 if (err & INFINIPATH_RHF_H_TIDERR)
732 strlcat(msg, "tiderr ", len);
733 if (err & INFINIPATH_RHF_H_MKERR)
734 /* bad port, offset, etc. */
735 strlcat(msg, "invalid ipathhdr ", len);
736 if (err & INFINIPATH_RHF_H_IBERR)
737 strlcat(msg, "iberr ", len);
738 if (err & INFINIPATH_RHF_L_SWA)
739 strlcat(msg, "swA ", len);
740 if (err & INFINIPATH_RHF_L_SWB)
741 strlcat(msg, "swB ", len);
745 * ipath_get_egrbuf - get an eager buffer
746 * @dd: the infinipath device
747 * @bufnum: the eager buffer to get
750 * must only be called if ipath_pd[port] is known to be allocated
752 static inline void *ipath_get_egrbuf(struct ipath_devdata *dd, u32 bufnum,
755 return dd->ipath_port0_skbs ?
756 (void *)dd->ipath_port0_skbs[bufnum]->data : NULL;
760 * ipath_alloc_skb - allocate an skb and buffer with possible constraints
761 * @dd: the infinipath device
762 * @gfp_mask: the sk_buff SFP mask
764 struct sk_buff *ipath_alloc_skb(struct ipath_devdata *dd,
771 * Only fully supported way to handle this is to allocate lots
772 * extra, align as needed, and then do skb_reserve(). That wastes
773 * a lot of memory... I'll have to hack this into infinipath_copy
778 * We need 4 extra bytes for unaligned transfer copying
780 if (dd->ipath_flags & IPATH_4BYTE_TID) {
781 /* we need a 4KB multiple alignment, and there is no way
782 * to do it except to allocate extra and then skb_reserve
783 * enough to bring it up to the right alignment.
785 len = dd->ipath_ibmaxlen + 4 + (1 << 11) - 1;
788 len = dd->ipath_ibmaxlen + 4;
789 skb = __dev_alloc_skb(len, gfp_mask);
791 ipath_dev_err(dd, "Failed to allocate skbuff, length %u\n",
795 if (dd->ipath_flags & IPATH_4BYTE_TID) {
796 u32 una = ((1 << 11) - 1) & (unsigned long)(skb->data + 4);
798 skb_reserve(skb, 4 + (1 << 11) - una);
809 * ipath_rcv_layer - receive a packet for the layered (ethernet) driver
810 * @dd: the infinipath device
811 * @etail: the sk_buff number
812 * @tlen: the total packet length
813 * @hdr: the ethernet header
815 * Separate routine for better overall optimization
817 static void ipath_rcv_layer(struct ipath_devdata *dd, u32 etail,
818 u32 tlen, struct ether_header *hdr)
822 struct sk_buff *skb, *nskb;
824 if (dd->ipath_port0_skbs && hdr->sub_opcode == OPCODE_ENCAP) {
826 * Allocate a new sk_buff to replace the one we give
827 * to the network stack.
829 nskb = ipath_alloc_skb(dd, GFP_ATOMIC);
831 /* count OK packets that we drop */
832 ipath_stats.sps_krdrops++;
836 bthbytes = (u8 *) hdr->bth;
837 pad = (bthbytes[1] >> 4) & 3;
839 elen = tlen - (sizeof(*hdr) + pad + sizeof(u32));
841 skb = dd->ipath_port0_skbs[etail];
842 dd->ipath_port0_skbs[etail] = nskb;
845 dd->ipath_f_put_tid(dd, etail + (u64 __iomem *)
846 ((char __iomem *) dd->ipath_kregbase
847 + dd->ipath_rcvegrbase), 0,
848 virt_to_phys(nskb->data));
850 __ipath_layer_rcv(dd, hdr, skb);
852 /* another ether packet received */
853 ipath_stats.sps_ether_rpkts++;
855 else if (hdr->sub_opcode == OPCODE_LID_ARP)
856 __ipath_layer_rcv_lid(dd, hdr);
860 * ipath_kreceive - receive a packet
861 * @dd: the infinipath device
863 * called from interrupt handler for errors or receive interrupt
865 void ipath_kreceive(struct ipath_devdata *dd)
869 const u32 rsize = dd->ipath_rcvhdrentsize; /* words */
870 const u32 maxcnt = dd->ipath_rcvhdrcnt * rsize; /* words */
871 u32 etail = -1, l, hdrqtail;
872 struct ips_message_header *hdr;
873 u32 eflags, i, etype, tlen, pkttot = 0;
874 static u64 totcalls; /* stats, may eventually remove */
877 if (!dd->ipath_hdrqtailptr) {
879 "hdrqtailptr not set, can't do receives\n");
883 /* There is already a thread processing this queue. */
884 if (test_and_set_bit(0, &dd->ipath_rcv_pending))
887 if (dd->ipath_port0head ==
888 (u32)le64_to_cpu(*dd->ipath_hdrqtailptr))
891 /* read only once at start for performance */
892 hdrqtail = (u32)le64_to_cpu(*dd->ipath_hdrqtailptr);
894 for (i = 0, l = dd->ipath_port0head; l != hdrqtail; i++) {
898 rc = (u64 *) (dd->ipath_pd[0]->port_rcvhdrq + (l << 2));
899 hdr = (struct ips_message_header *)&rc[1];
901 * could make a network order version of IPATH_KD_QP, and
902 * do the obvious shift before masking to speed this up.
904 qp = ntohl(hdr->bth[1]) & 0xffffff;
905 bthbytes = (u8 *) hdr->bth;
907 eflags = ips_get_hdr_err_flags((__le32 *) rc);
908 etype = ips_get_rcv_type((__le32 *) rc);
910 tlen = ips_get_length_in_bytes((__le32 *) rc);
912 if (etype != RCVHQ_RCV_TYPE_EXPECTED) {
914 * it turns out that the chips uses an eager buffer
915 * for all non-expected packets, whether it "needs"
916 * one or not. So always get the index, but don't
917 * set ebuf (so we try to copy data) unless the
918 * length requires it.
920 etail = ips_get_index((__le32 *) rc);
921 if (tlen > sizeof(*hdr) ||
922 etype == RCVHQ_RCV_TYPE_NON_KD)
923 ebuf = ipath_get_egrbuf(dd, etail, 0);
927 * both tiderr and ipathhdrerr are set for all plain IB
928 * packets; only ipathhdrerr should be set.
931 if (etype != RCVHQ_RCV_TYPE_NON_KD && etype !=
932 RCVHQ_RCV_TYPE_ERROR && ips_get_ipath_ver(
933 hdr->iph.ver_port_tid_offset) !=
935 ipath_cdbg(PKT, "Bad InfiniPath protocol version "
939 if (eflags & ~(INFINIPATH_RHF_H_TIDERR |
940 INFINIPATH_RHF_H_IHDRERR)) {
941 get_rhf_errstring(eflags, emsg, sizeof emsg);
942 ipath_cdbg(PKT, "RHFerrs %x hdrqtail=%x typ=%u "
943 "tlen=%x opcode=%x egridx=%x: %s\n",
944 eflags, l, etype, tlen, bthbytes[0],
945 ips_get_index((__le32 *) rc), emsg);
946 } else if (etype == RCVHQ_RCV_TYPE_NON_KD) {
947 int ret = __ipath_verbs_rcv(dd, rc + 1,
951 "received IB packet, "
952 "not SMA (QP=%x)\n", qp);
953 } else if (etype == RCVHQ_RCV_TYPE_EAGER) {
954 if (qp == IPATH_KD_QP &&
955 bthbytes[0] == ipath_layer_rcv_opcode &&
957 ipath_rcv_layer(dd, etail, tlen,
958 (struct ether_header *)hdr);
960 ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
961 "qp=%x), len %x; ignored\n",
962 etype, bthbytes[0], qp, tlen);
964 else if (etype == RCVHQ_RCV_TYPE_EXPECTED)
965 ipath_dbg("Bug: Expected TID, opcode %x; ignored\n",
966 be32_to_cpu(hdr->bth[0]) & 0xff);
967 else if (eflags & (INFINIPATH_RHF_H_TIDERR |
968 INFINIPATH_RHF_H_IHDRERR)) {
970 * This is a type 3 packet, only the LRH is in the
971 * rcvhdrq, the rest of the header is in the eager
976 bthbytes = (u8 *) ebuf;
981 get_rhf_errstring(eflags, emsg, sizeof emsg);
982 ipath_dbg("Err %x (%s), opcode %x, egrbuf %x, "
983 "len %x\n", eflags, emsg, opcode, etail,
987 * error packet, type of error unknown.
988 * Probably type 3, but we don't know, so don't
989 * even try to print the opcode, etc.
991 ipath_dbg("Error Pkt, but no eflags! egrbuf %x, "
992 "len %x\nhdrq@%lx;hdrq+%x rhf: %llx; "
993 "hdr %llx %llx %llx %llx %llx\n",
994 etail, tlen, (unsigned long) rc, l,
995 (unsigned long long) rc[0],
996 (unsigned long long) rc[1],
997 (unsigned long long) rc[2],
998 (unsigned long long) rc[3],
999 (unsigned long long) rc[4],
1000 (unsigned long long) rc[5]);
1006 * update for each packet, to help prevent overflows if we
1007 * have lots of packets.
1009 (void)ipath_write_ureg(dd, ur_rcvhdrhead,
1010 dd->ipath_rhdrhead_intr_off | l, 0);
1011 if (etype != RCVHQ_RCV_TYPE_EXPECTED)
1012 (void)ipath_write_ureg(dd, ur_rcvegrindexhead,
1018 dd->ipath_port0head = l;
1020 if (pkttot > ipath_stats.sps_maxpkts_call)
1021 ipath_stats.sps_maxpkts_call = pkttot;
1022 ipath_stats.sps_port0pkts += pkttot;
1023 ipath_stats.sps_avgpkts_call =
1024 ipath_stats.sps_port0pkts / ++totcalls;
1027 clear_bit(0, &dd->ipath_rcv_pending);
1028 smp_mb__after_clear_bit();
1034 * ipath_update_pio_bufs - update shadow copy of the PIO availability map
1035 * @dd: the infinipath device
1037 * called whenever our local copy indicates we have run out of send buffers
1038 * NOTE: This can be called from interrupt context by some code
1039 * and from non-interrupt context by ipath_getpiobuf().
1042 static void ipath_update_pio_bufs(struct ipath_devdata *dd)
1044 unsigned long flags;
1046 const unsigned piobregs = (unsigned)dd->ipath_pioavregs;
1048 /* If the generation (check) bits have changed, then we update the
1049 * busy bit for the corresponding PIO buffer. This algorithm will
1050 * modify positions to the value they already have in some cases
1051 * (i.e., no change), but it's faster than changing only the bits
1052 * that have changed.
1054 * We would like to do this atomicly, to avoid spinlocks in the
1055 * critical send path, but that's not really possible, given the
1056 * type of changes, and that this routine could be called on
1057 * multiple cpu's simultaneously, so we lock in this routine only,
1058 * to avoid conflicting updates; all we change is the shadow, and
1059 * it's a single 64 bit memory location, so by definition the update
1060 * is atomic in terms of what other cpu's can see in testing the
1061 * bits. The spin_lock overhead isn't too bad, since it only
1062 * happens when all buffers are in use, so only cpu overhead, not
1063 * latency or bandwidth is affected.
1065 #define _IPATH_ALL_CHECKBITS 0x5555555555555555ULL
1066 if (!dd->ipath_pioavailregs_dma) {
1067 ipath_dbg("Update shadow pioavail, but regs_dma NULL!\n");
1070 if (ipath_debug & __IPATH_VERBDBG) {
1071 /* only if packet debug and verbose */
1072 volatile __le64 *dma = dd->ipath_pioavailregs_dma;
1073 unsigned long *shadow = dd->ipath_pioavailshadow;
1075 ipath_cdbg(PKT, "Refill avail, dma0=%llx shad0=%lx, "
1076 "d1=%llx s1=%lx, d2=%llx s2=%lx, d3=%llx "
1078 (unsigned long long) le64_to_cpu(dma[0]),
1080 (unsigned long long) le64_to_cpu(dma[1]),
1082 (unsigned long long) le64_to_cpu(dma[2]),
1084 (unsigned long long) le64_to_cpu(dma[3]),
1088 PKT, "2nd group, dma4=%llx shad4=%lx, "
1089 "d5=%llx s5=%lx, d6=%llx s6=%lx, "
1091 (unsigned long long) le64_to_cpu(dma[4]),
1093 (unsigned long long) le64_to_cpu(dma[5]),
1095 (unsigned long long) le64_to_cpu(dma[6]),
1097 (unsigned long long) le64_to_cpu(dma[7]),
1100 spin_lock_irqsave(&ipath_pioavail_lock, flags);
1101 for (i = 0; i < piobregs; i++) {
1102 u64 pchbusy, pchg, piov, pnew;
1104 * Chip Errata: bug 6641; even and odd qwords>3 are swapped
1109 dd->ipath_pioavailregs_dma[i - 1]);
1112 dd->ipath_pioavailregs_dma[i + 1]);
1114 piov = le64_to_cpu(dd->ipath_pioavailregs_dma[i]);
1115 pchg = _IPATH_ALL_CHECKBITS &
1116 ~(dd->ipath_pioavailshadow[i] ^ piov);
1117 pchbusy = pchg << INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT;
1118 if (pchg && (pchbusy & dd->ipath_pioavailshadow[i])) {
1119 pnew = dd->ipath_pioavailshadow[i] & ~pchbusy;
1120 pnew |= piov & pchbusy;
1121 dd->ipath_pioavailshadow[i] = pnew;
1124 spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1128 * ipath_setrcvhdrsize - set the receive header size
1129 * @dd: the infinipath device
1130 * @rhdrsize: the receive header size
1132 * called from user init code, and also layered driver init
1134 int ipath_setrcvhdrsize(struct ipath_devdata *dd, unsigned rhdrsize)
1138 if (dd->ipath_flags & IPATH_RCVHDRSZ_SET) {
1139 if (dd->ipath_rcvhdrsize != rhdrsize) {
1140 dev_info(&dd->pcidev->dev,
1141 "Error: can't set protocol header "
1142 "size %u, already %u\n",
1143 rhdrsize, dd->ipath_rcvhdrsize);
1146 ipath_cdbg(VERBOSE, "Reuse same protocol header "
1147 "size %u\n", dd->ipath_rcvhdrsize);
1148 } else if (rhdrsize > (dd->ipath_rcvhdrentsize -
1149 (sizeof(u64) / sizeof(u32)))) {
1150 ipath_dbg("Error: can't set protocol header size %u "
1151 "(> max %u)\n", rhdrsize,
1152 dd->ipath_rcvhdrentsize -
1153 (u32) (sizeof(u64) / sizeof(u32)));
1156 dd->ipath_flags |= IPATH_RCVHDRSZ_SET;
1157 dd->ipath_rcvhdrsize = rhdrsize;
1158 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvhdrsize,
1159 dd->ipath_rcvhdrsize);
1160 ipath_cdbg(VERBOSE, "Set protocol header size to %u\n",
1161 dd->ipath_rcvhdrsize);
1167 * ipath_getpiobuf - find an available pio buffer
1168 * @dd: the infinipath device
1169 * @pbufnum: the buffer number is placed here
1171 * do appropriate marking as busy, etc.
1172 * returns buffer number if one found (>=0), negative number is error.
1173 * Used by ipath_sma_send_pkt and ipath_layer_send
1175 u32 __iomem *ipath_getpiobuf(struct ipath_devdata *dd, u32 * pbufnum)
1177 int i, j, starti, updated = 0;
1178 unsigned piobcnt, iter;
1179 unsigned long flags;
1180 unsigned long *shadow = dd->ipath_pioavailshadow;
1183 piobcnt = (unsigned)(dd->ipath_piobcnt2k
1184 + dd->ipath_piobcnt4k);
1185 starti = dd->ipath_lastport_piobuf;
1186 iter = piobcnt - starti;
1187 if (dd->ipath_upd_pio_shadow) {
1189 * Minor optimization. If we had no buffers on last call,
1190 * start out by doing the update; continue and do scan even
1191 * if no buffers were updated, to be paranoid
1193 ipath_update_pio_bufs(dd);
1194 /* we scanned here, don't do it at end of scan */
1198 i = dd->ipath_lastpioindex;
1202 * while test_and_set_bit() is atomic, we do that and then the
1203 * change_bit(), and the pair is not. See if this is the cause
1204 * of the remaining armlaunch errors.
1206 spin_lock_irqsave(&ipath_pioavail_lock, flags);
1207 for (j = 0; j < iter; j++, i++) {
1211 * To avoid bus lock overhead, we first find a candidate
1212 * buffer, then do the test and set, and continue if that
1215 if (test_bit((2 * i) + 1, shadow) ||
1216 test_and_set_bit((2 * i) + 1, shadow))
1218 /* flip generation bit */
1219 change_bit(2 * i, shadow);
1222 spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
1225 volatile __le64 *dma = dd->ipath_pioavailregs_dma;
1228 * first time through; shadow exhausted, but may be real
1229 * buffers available, so go see; if any updated, rescan
1233 ipath_update_pio_bufs(dd);
1238 dd->ipath_upd_pio_shadow = 1;
1240 * not atomic, but if we lose one once in a while, that's OK
1242 ipath_stats.sps_nopiobufs++;
1243 if (!(++dd->ipath_consec_nopiobuf % 100000)) {
1245 "%u pio sends with no bufavail; dmacopy: "
1246 "%llx %llx %llx %llx; shadow: "
1247 "%lx %lx %lx %lx\n",
1248 dd->ipath_consec_nopiobuf,
1249 (unsigned long long) le64_to_cpu(dma[0]),
1250 (unsigned long long) le64_to_cpu(dma[1]),
1251 (unsigned long long) le64_to_cpu(dma[2]),
1252 (unsigned long long) le64_to_cpu(dma[3]),
1253 shadow[0], shadow[1], shadow[2],
1256 * 4 buffers per byte, 4 registers above, cover rest
1259 if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) >
1260 (sizeof(shadow[0]) * 4 * 4))
1261 ipath_dbg("2nd group: dmacopy: %llx %llx "
1262 "%llx %llx; shadow: %lx %lx "
1264 (unsigned long long)
1265 le64_to_cpu(dma[4]),
1266 (unsigned long long)
1267 le64_to_cpu(dma[5]),
1268 (unsigned long long)
1269 le64_to_cpu(dma[6]),
1270 (unsigned long long)
1271 le64_to_cpu(dma[7]),
1272 shadow[4], shadow[5],
1273 shadow[6], shadow[7]);
1281 * ran out of bufs, now some (at least this one we just
1282 * got) are now available, so tell the layered driver.
1284 __ipath_layer_intr(dd, IPATH_LAYER_INT_SEND_CONTINUE);
1287 * set next starting place. Since it's just an optimization,
1288 * it doesn't matter who wins on this, so no locking
1290 dd->ipath_lastpioindex = i + 1;
1291 if (dd->ipath_upd_pio_shadow)
1292 dd->ipath_upd_pio_shadow = 0;
1293 if (dd->ipath_consec_nopiobuf)
1294 dd->ipath_consec_nopiobuf = 0;
1295 if (i < dd->ipath_piobcnt2k)
1296 buf = (u32 __iomem *) (dd->ipath_pio2kbase +
1297 i * dd->ipath_palign);
1299 buf = (u32 __iomem *)
1300 (dd->ipath_pio4kbase +
1301 (i - dd->ipath_piobcnt2k) * dd->ipath_4kalign);
1302 ipath_cdbg(VERBOSE, "Return piobuf%u %uk @ %p\n",
1303 i, (i < dd->ipath_piobcnt2k) ? 2 : 4, buf);
1312 * ipath_create_rcvhdrq - create a receive header queue
1313 * @dd: the infinipath device
1314 * @pd: the port data
1316 * this must be contiguous memory (from an i/o perspective), and must be
1317 * DMA'able (which means for some systems, it will go through an IOMMU,
1318 * or be forced into a low address range).
1320 int ipath_create_rcvhdrq(struct ipath_devdata *dd,
1321 struct ipath_portdata *pd)
1325 if (!pd->port_rcvhdrq) {
1326 dma_addr_t phys_hdrqtail;
1327 gfp_t gfp_flags = GFP_USER | __GFP_COMP;
1328 int amt = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
1329 sizeof(u32), PAGE_SIZE);
1331 pd->port_rcvhdrq = dma_alloc_coherent(
1332 &dd->pcidev->dev, amt, &pd->port_rcvhdrq_phys,
1335 if (!pd->port_rcvhdrq) {
1336 ipath_dev_err(dd, "attempt to allocate %d bytes "
1337 "for port %u rcvhdrq failed\n",
1338 amt, pd->port_port);
1342 pd->port_rcvhdrtail_kvaddr = dma_alloc_coherent(
1343 &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail, GFP_KERNEL);
1344 if (!pd->port_rcvhdrtail_kvaddr) {
1345 ipath_dev_err(dd, "attempt to allocate 1 page "
1346 "for port %u rcvhdrqtailaddr failed\n",
1351 pd->port_rcvhdrqtailaddr_phys = phys_hdrqtail;
1353 pd->port_rcvhdrq_size = amt;
1355 ipath_cdbg(VERBOSE, "%d pages at %p (phys %lx) size=%lu "
1356 "for port %u rcvhdr Q\n",
1357 amt >> PAGE_SHIFT, pd->port_rcvhdrq,
1358 (unsigned long) pd->port_rcvhdrq_phys,
1359 (unsigned long) pd->port_rcvhdrq_size,
1362 ipath_cdbg(VERBOSE, "port %d hdrtailaddr, %llx physical\n",
1364 (unsigned long long) phys_hdrqtail);
1367 ipath_cdbg(VERBOSE, "reuse port %d rcvhdrq @%p %llx phys; "
1368 "hdrtailaddr@%p %llx physical\n",
1369 pd->port_port, pd->port_rcvhdrq,
1370 pd->port_rcvhdrq_phys, pd->port_rcvhdrtail_kvaddr,
1371 (unsigned long long)pd->port_rcvhdrqtailaddr_phys);
1373 /* clear for security and sanity on each use */
1374 memset(pd->port_rcvhdrq, 0, pd->port_rcvhdrq_size);
1375 memset((void *)pd->port_rcvhdrtail_kvaddr, 0, PAGE_SIZE);
1378 * tell chip each time we init it, even if we are re-using previous
1379 * memory (we zero the register at process close)
1381 ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdrtailaddr,
1382 pd->port_port, pd->port_rcvhdrqtailaddr_phys);
1383 ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
1384 pd->port_port, pd->port_rcvhdrq_phys);
1391 int ipath_waitfor_complete(struct ipath_devdata *dd, ipath_kreg reg_id,
1392 u64 bits_to_wait_for, u64 * valp)
1394 unsigned long timeout;
1398 lastval = ipath_read_kreg64(dd, reg_id);
1399 /* wait a ridiculously long time */
1400 timeout = jiffies + msecs_to_jiffies(5);
1402 val = ipath_read_kreg64(dd, reg_id);
1403 /* set so they have something, even on failures. */
1405 if ((val & bits_to_wait_for) == bits_to_wait_for) {
1410 ipath_cdbg(VERBOSE, "Changed from %llx to %llx, "
1411 "waiting for %llx bits\n",
1412 (unsigned long long) lastval,
1413 (unsigned long long) val,
1414 (unsigned long long) bits_to_wait_for);
1416 if (time_after(jiffies, timeout)) {
1417 ipath_dbg("Didn't get bits %llx in register 0x%x, "
1419 (unsigned long long) bits_to_wait_for,
1420 reg_id, (unsigned long long) *valp);
1430 * ipath_waitfor_mdio_cmdready - wait for last command to complete
1431 * @dd: the infinipath device
1433 * Like ipath_waitfor_complete(), but we wait for the CMDVALID bit to go
1434 * away indicating the last command has completed. It doesn't return data
1436 int ipath_waitfor_mdio_cmdready(struct ipath_devdata *dd)
1438 unsigned long timeout;
1442 /* wait a ridiculously long time */
1443 timeout = jiffies + msecs_to_jiffies(5);
1445 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_mdio);
1446 if (!(val & IPATH_MDIO_CMDVALID)) {
1451 if (time_after(jiffies, timeout)) {
1452 ipath_dbg("CMDVALID stuck in mdio reg? (%llx)\n",
1453 (unsigned long long) val);
1462 void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
1464 static const char *what[4] = {
1466 [INFINIPATH_IBCC_LINKCMD_INIT] = "INIT",
1467 [INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED",
1468 [INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE"
1470 int linkcmd = (which >> INFINIPATH_IBCC_LINKCMD_SHIFT) &
1471 INFINIPATH_IBCC_LINKCMD_MASK;
1473 ipath_cdbg(SMA, "Trying to move unit %u to %s, current ltstate "
1474 "is %s\n", dd->ipath_unit,
1476 ipath_ibcstatus_str[
1478 (dd, dd->ipath_kregs->kr_ibcstatus) >>
1479 INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) &
1480 INFINIPATH_IBCS_LINKTRAININGSTATE_MASK]);
1481 /* flush all queued sends when going to DOWN or INIT, to be sure that
1482 * they don't block SMA and other MAD packets */
1483 if (!linkcmd || linkcmd == INFINIPATH_IBCC_LINKCMD_INIT) {
1484 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1485 INFINIPATH_S_ABORT);
1486 ipath_disarm_piobufs(dd, dd->ipath_lastport_piobuf,
1487 (unsigned)(dd->ipath_piobcnt2k +
1488 dd->ipath_piobcnt4k) -
1489 dd->ipath_lastport_piobuf);
1492 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
1493 dd->ipath_ibcctrl | which);
1497 * ipath_read_kreg64_port - read a device's per-port 64-bit kernel register
1498 * @dd: the infinipath device
1499 * @regno: the register number to read
1500 * @port: the port containing the register
1502 * Registers that vary with the chip implementation constants (port)
1505 u64 ipath_read_kreg64_port(const struct ipath_devdata *dd, ipath_kreg regno,
1510 if (port < dd->ipath_portcnt &&
1511 (regno == dd->ipath_kregs->kr_rcvhdraddr ||
1512 regno == dd->ipath_kregs->kr_rcvhdrtailaddr))
1513 where = regno + port;
1517 return ipath_read_kreg64(dd, where);
1521 * ipath_write_kreg_port - write a device's per-port 64-bit kernel register
1522 * @dd: the infinipath device
1523 * @regno: the register number to write
1524 * @port: the port containing the register
1525 * @value: the value to write
1527 * Registers that vary with the chip implementation constants (port)
1530 void ipath_write_kreg_port(const struct ipath_devdata *dd, ipath_kreg regno,
1531 unsigned port, u64 value)
1535 if (port < dd->ipath_portcnt &&
1536 (regno == dd->ipath_kregs->kr_rcvhdraddr ||
1537 regno == dd->ipath_kregs->kr_rcvhdrtailaddr))
1538 where = regno + port;
1542 ipath_write_kreg(dd, where, value);
1546 * ipath_shutdown_device - shut down a device
1547 * @dd: the infinipath device
1549 * This is called to make the device quiet when we are about to
1550 * unload the driver, and also when the device is administratively
1551 * disabled. It does not free any data structures.
1552 * Everything it does has to be setup again by ipath_init_chip(dd,1)
1554 void ipath_shutdown_device(struct ipath_devdata *dd)
1558 ipath_dbg("Shutting down the device\n");
1560 dd->ipath_flags |= IPATH_LINKUNK;
1561 dd->ipath_flags &= ~(IPATH_INITTED | IPATH_LINKDOWN |
1562 IPATH_LINKINIT | IPATH_LINKARMED |
1564 *dd->ipath_statusp &= ~(IPATH_STATUS_IB_CONF |
1565 IPATH_STATUS_IB_READY);
1567 /* mask interrupts, but not errors */
1568 ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
1570 dd->ipath_rcvctrl = 0;
1571 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
1575 * gracefully stop all sends allowing any in progress to trickle out
1578 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, 0ULL);
1580 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
1582 * enough for anything that's going to trickle out to have actually
1588 * abort any armed or launched PIO buffers that didn't go. (self
1589 * clearing). Will cause any packet currently being transmitted to
1590 * go out with an EBP, and may also cause a short packet error on
1593 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
1594 INFINIPATH_S_ABORT);
1596 ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
1597 INFINIPATH_IBCC_LINKINITCMD_SHIFT);
1600 * we are shutting down, so tell the layered driver. We don't do
1601 * this on just a link state change, much like ethernet, a cable
1602 * unplug, etc. doesn't change driver state
1604 ipath_layer_intr(dd, IPATH_LAYER_INT_IF_DOWN);
1607 dd->ipath_control &= ~INFINIPATH_C_LINKENABLE;
1608 ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
1609 dd->ipath_control | INFINIPATH_C_FREEZEMODE);
1612 * clear SerdesEnable and turn the leds off; do this here because
1613 * we are unloading, so don't count on interrupts to move along
1614 * Turn the LEDs off explictly for the same reason.
1616 dd->ipath_f_quiet_serdes(dd);
1617 dd->ipath_f_setextled(dd, 0, 0);
1619 if (dd->ipath_stats_timer_active) {
1620 del_timer_sync(&dd->ipath_stats_timer);
1621 dd->ipath_stats_timer_active = 0;
1625 * clear all interrupts and errors, so that the next time the driver
1626 * is loaded or device is enabled, we know that whatever is set
1627 * happened while we were unloaded
1629 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear,
1630 ~0ULL & ~INFINIPATH_HWE_MEMBISTFAILED);
1631 ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, -1LL);
1632 ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, -1LL);
1636 * ipath_free_pddata - free a port's allocated data
1637 * @dd: the infinipath device
1638 * @pd: the portdata structure
1640 * free up any allocated data for a port
1641 * This should not touch anything that would affect a simultaneous
1642 * re-allocation of port data, because it is called after ipath_mutex
1643 * is released (and can be called from reinit as well).
1644 * It should never change any chip state, or global driver state.
1645 * (The only exception to global state is freeing the port0 port0_skbs.)
1647 void ipath_free_pddata(struct ipath_devdata *dd, struct ipath_portdata *pd)
1652 if (pd->port_rcvhdrq) {
1653 ipath_cdbg(VERBOSE, "free closed port %d rcvhdrq @ %p "
1654 "(size=%lu)\n", pd->port_port, pd->port_rcvhdrq,
1655 (unsigned long) pd->port_rcvhdrq_size);
1656 dma_free_coherent(&dd->pcidev->dev, pd->port_rcvhdrq_size,
1657 pd->port_rcvhdrq, pd->port_rcvhdrq_phys);
1658 pd->port_rcvhdrq = NULL;
1659 if (pd->port_rcvhdrtail_kvaddr) {
1660 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
1661 (void *)pd->port_rcvhdrtail_kvaddr,
1662 pd->port_rcvhdrqtailaddr_phys);
1663 pd->port_rcvhdrtail_kvaddr = NULL;
1666 if (pd->port_port && pd->port_rcvegrbuf) {
1669 for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
1670 void *base = pd->port_rcvegrbuf[e];
1671 size_t size = pd->port_rcvegrbuf_size;
1673 ipath_cdbg(VERBOSE, "egrbuf free(%p, %lu), "
1674 "chunk %u/%u\n", base,
1675 (unsigned long) size,
1676 e, pd->port_rcvegrbuf_chunks);
1677 dma_free_coherent(&dd->pcidev->dev, size,
1678 base, pd->port_rcvegrbuf_phys[e]);
1680 vfree(pd->port_rcvegrbuf);
1681 pd->port_rcvegrbuf = NULL;
1682 vfree(pd->port_rcvegrbuf_phys);
1683 pd->port_rcvegrbuf_phys = NULL;
1684 pd->port_rcvegrbuf_chunks = 0;
1685 } else if (pd->port_port == 0 && dd->ipath_port0_skbs) {
1687 struct sk_buff **skbs = dd->ipath_port0_skbs;
1689 dd->ipath_port0_skbs = NULL;
1690 ipath_cdbg(VERBOSE, "free closed port %d ipath_port0_skbs "
1691 "@ %p\n", pd->port_port, skbs);
1692 for (e = 0; e < dd->ipath_rcvegrcnt; e++)
1694 dev_kfree_skb(skbs[e]);
1697 kfree(pd->port_tid_pg_list);
1701 static int __init infinipath_init(void)
1705 ipath_dbg(KERN_INFO DRIVER_LOAD_MSG "%s", ipath_core_version);
1708 * These must be called before the driver is registered with
1709 * the PCI subsystem.
1711 idr_init(&unit_table);
1712 if (!idr_pre_get(&unit_table, GFP_KERNEL)) {
1717 ret = pci_register_driver(&ipath_driver);
1719 printk(KERN_ERR IPATH_DRV_NAME
1720 ": Unable to register driver: error %d\n", -ret);
1724 ret = ipath_driver_create_group(&ipath_driver.driver);
1726 printk(KERN_ERR IPATH_DRV_NAME ": Unable to create driver "
1727 "sysfs entries: error %d\n", -ret);
1731 ret = ipath_init_ipathfs();
1733 printk(KERN_ERR IPATH_DRV_NAME ": Unable to create "
1734 "ipathfs: error %d\n", -ret);
1741 ipath_driver_remove_group(&ipath_driver.driver);
1744 pci_unregister_driver(&ipath_driver);
1747 idr_destroy(&unit_table);
1753 static void cleanup_device(struct ipath_devdata *dd)
1757 ipath_shutdown_device(dd);
1759 if (*dd->ipath_statusp & IPATH_STATUS_CHIP_PRESENT) {
1760 /* can't do anything more with chip; needs re-init */
1761 *dd->ipath_statusp &= ~IPATH_STATUS_CHIP_PRESENT;
1762 if (dd->ipath_kregbase) {
1764 * if we haven't already cleaned up before these are
1765 * to ensure any register reads/writes "fail" until
1768 dd->ipath_kregbase = NULL;
1769 dd->ipath_uregbase = 0;
1770 dd->ipath_sregbase = 0;
1771 dd->ipath_cregbase = 0;
1772 dd->ipath_kregsize = 0;
1774 ipath_disable_wc(dd);
1777 if (dd->ipath_pioavailregs_dma) {
1778 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
1779 (void *) dd->ipath_pioavailregs_dma,
1780 dd->ipath_pioavailregs_phys);
1781 dd->ipath_pioavailregs_dma = NULL;
1784 if (dd->ipath_pageshadow) {
1785 struct page **tmpp = dd->ipath_pageshadow;
1788 ipath_cdbg(VERBOSE, "Unlocking any expTID pages still "
1790 for (port = 0; port < dd->ipath_cfgports; port++) {
1791 int port_tidbase = port * dd->ipath_rcvtidcnt;
1792 int maxtid = port_tidbase + dd->ipath_rcvtidcnt;
1793 for (i = port_tidbase; i < maxtid; i++) {
1796 ipath_release_user_pages(&tmpp[i], 1);
1802 ipath_stats.sps_pageunlocks += cnt;
1803 ipath_cdbg(VERBOSE, "There were still %u expTID "
1804 "entries locked\n", cnt);
1806 if (ipath_stats.sps_pagelocks ||
1807 ipath_stats.sps_pageunlocks)
1808 ipath_cdbg(VERBOSE, "%llu pages locked, %llu "
1809 "unlocked via ipath_m{un}lock\n",
1810 (unsigned long long)
1811 ipath_stats.sps_pagelocks,
1812 (unsigned long long)
1813 ipath_stats.sps_pageunlocks);
1815 ipath_cdbg(VERBOSE, "Free shadow page tid array at %p\n",
1816 dd->ipath_pageshadow);
1817 vfree(dd->ipath_pageshadow);
1818 dd->ipath_pageshadow = NULL;
1822 * free any resources still in use (usually just kernel ports)
1823 * at unload; we do for portcnt, not cfgports, because cfgports
1824 * could have changed while we were loaded.
1826 for (port = 0; port < dd->ipath_portcnt; port++) {
1827 struct ipath_portdata *pd = dd->ipath_pd[port];
1828 dd->ipath_pd[port] = NULL;
1829 ipath_free_pddata(dd, pd);
1831 kfree(dd->ipath_pd);
1833 * debuggability, in case some cleanup path tries to use it
1836 dd->ipath_pd = NULL;
1839 static void __exit infinipath_cleanup(void)
1841 struct ipath_devdata *dd, *tmp;
1842 unsigned long flags;
1844 ipath_exit_ipathfs();
1846 ipath_driver_remove_group(&ipath_driver.driver);
1848 spin_lock_irqsave(&ipath_devs_lock, flags);
1851 * turn off rcv, send, and interrupts for all ports, all drivers
1852 * should also hard reset the chip here?
1853 * free up port 0 (kernel) rcvhdr, egr bufs, and eventually tid bufs
1854 * for all versions of the driver, if they were allocated
1856 list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
1857 spin_unlock_irqrestore(&ipath_devs_lock, flags);
1859 if (dd->ipath_kregbase)
1863 if (dd->pcidev->irq) {
1865 "unit %u free_irq of irq %x\n",
1866 dd->ipath_unit, dd->pcidev->irq);
1867 free_irq(dd->pcidev->irq, dd);
1869 ipath_dbg("irq is 0, not doing free_irq "
1870 "for unit %u\n", dd->ipath_unit);
1873 * we check for NULL here, because it's outside
1874 * the kregbase check, and we need to call it
1875 * after the free_irq. Thus it's possible that
1876 * the function pointers were never initialized.
1878 if (dd->ipath_f_cleanup)
1879 /* clean up chip-specific stuff */
1880 dd->ipath_f_cleanup(dd);
1884 spin_lock_irqsave(&ipath_devs_lock, flags);
1887 spin_unlock_irqrestore(&ipath_devs_lock, flags);
1889 ipath_cdbg(VERBOSE, "Unregistering pci driver\n");
1890 pci_unregister_driver(&ipath_driver);
1892 idr_destroy(&unit_table);
1896 * ipath_reset_device - reset the chip if possible
1897 * @unit: the device to reset
1899 * Whether or not reset is successful, we attempt to re-initialize the chip
1900 * (that is, much like a driver unload/reload). We clear the INITTED flag
1901 * so that the various entry points will fail until we reinitialize. For
1902 * now, we only allow this if no user ports are open that use chip resources
1904 int ipath_reset_device(int unit)
1907 struct ipath_devdata *dd = ipath_lookup(unit);
1914 dev_info(&dd->pcidev->dev, "Reset on unit %u requested\n", unit);
1916 if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_PRESENT)) {
1917 dev_info(&dd->pcidev->dev, "Invalid unit number %u or "
1918 "not initialized or not present\n", unit);
1924 for (i = 1; i < dd->ipath_cfgports; i++) {
1925 if (dd->ipath_pd[i] && dd->ipath_pd[i]->port_cnt) {
1926 ipath_dbg("unit %u port %d is in use "
1927 "(PID %u cmd %s), can't reset\n",
1929 dd->ipath_pd[i]->port_pid,
1930 dd->ipath_pd[i]->port_comm);
1936 dd->ipath_flags &= ~IPATH_INITTED;
1937 ret = dd->ipath_f_reset(dd);
1939 ipath_dbg("reset was not successful\n");
1940 ipath_dbg("Trying to reinitialize unit %u after reset attempt\n",
1942 ret = ipath_init_chip(dd, 1);
1944 ipath_dev_err(dd, "Reinitialize unit %u after "
1945 "reset failed with %d\n", unit, ret);
1947 dev_info(&dd->pcidev->dev, "Reinitialized unit %u after "
1948 "resetting\n", unit);
1954 module_init(infinipath_init);
1955 module_exit(infinipath_cleanup);