err.no Git - linux-2.6/blob - drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2007 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mii.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, ssid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, ssid, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 1, 0),  /* PE9000 */
  84         CH_DEVICE(0x21, 1, 1),  /* T302E */
  85         CH_DEVICE(0x22, 1, 2),  /* T310E */
  86         CH_DEVICE(0x23, 1, 3),  /* T320X */
  87         CH_DEVICE(0x24, 1, 1),  /* T302X */
  88         CH_DEVICE(0x25, 1, 3),  /* T320E */
  89         CH_DEVICE(0x26, 1, 2),  /* T310X */
  90         CH_DEVICE(0x30, 1, 2),  /* T3B10 */
  91         CH_DEVICE(0x31, 1, 3),  /* T3B20 */
  92         CH_DEVICE(0x32, 1, 1),  /* T3B02 */
  93         {0,}
  94 };
  95
  96 MODULE_DESCRIPTION(DRV_DESC);
  97 MODULE_AUTHOR("Chelsio Communications");
  98 MODULE_LICENSE("Dual BSD/GPL");
  99 MODULE_VERSION(DRV_VERSION);
 100 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 101
 102 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 103
 104 module_param(dflt_msg_enable, int, 0644);
 105 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 106
 107 /*
 108  * The driver uses the best interrupt scheme available on a platform in the
 109  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 110  * of these schemes the driver may consider as follows:
 111  *
 112  * msi = 2: choose from among all three options
 113  * msi = 1: only consider MSI and pin interrupts
 114  * msi = 0: force pin interrupts
 115  */
 116 static int msi = 2;
 117
 118 module_param(msi, int, 0644);
 119 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 120
 121 /*
 122  * The driver enables offload as a default.
 123  * To disable it, use ofld_disable = 1.
 124  */
 125
 126 static int ofld_disable = 0;
 127
 128 module_param(ofld_disable, int, 0644);
 129 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 130
 131 /*
 132  * We have work elements that we need to cancel when an interface is taken
 133  * down.  Normally the work elements would be executed by keventd but that
 134  * can deadlock because of linkwatch.  If our close method takes the rtnl
 135  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 136  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 137  * for our work to complete.  Get our own work queue to solve this.
 138  */
 139 static struct workqueue_struct *cxgb3_wq;
 140
 141 /**
 142  *      link_report - show link status and link speed/duplex
 143  *      @p: the port whose settings are to be reported
 144  *
 145  *      Shows the link status, speed, and duplex of a port.
 146  */
 147 static void link_report(struct net_device *dev)
 148 {
 149         if (!netif_carrier_ok(dev))
 150                 printk(KERN_INFO "%s: link down\n", dev->name);
 151         else {
 152                 const char *s = "10Mbps";
 153                 const struct port_info *p = netdev_priv(dev);
 154
 155                 switch (p->link_config.speed) {
 156                 case SPEED_10000:
 157                         s = "10Gbps";
 158                         break;
 159                 case SPEED_1000:
 160                         s = "1000Mbps";
 161                         break;
 162                 case SPEED_100:
 163                         s = "100Mbps";
 164                         break;
 165                 }
 166
 167                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 168                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 169         }
 170 }
 171
 172 /**
 173  *      t3_os_link_changed - handle link status changes
 174  *      @adapter: the adapter associated with the link change
 175  *      @port_id: the port index whose limk status has changed
 176  *      @link_stat: the new status of the link
 177  *      @speed: the new speed setting
 178  *      @duplex: the new duplex setting
 179  *      @pause: the new flow-control setting
 180  *
 181  *      This is the OS-dependent handler for link status changes.  The OS
 182  *      neutral handler takes care of most of the processing for these events,
 183  *      then calls this handler for any OS-specific processing.
 184  */
 185 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 186                         int speed, int duplex, int pause)
 187 {
 188         struct net_device *dev = adapter->port[port_id];
 189         struct port_info *pi = netdev_priv(dev);
 190         struct cmac *mac = &pi->mac;
 191
 192         /* Skip changes from disabled ports. */
 193         if (!netif_running(dev))
 194                 return;
 195
 196         if (link_stat != netif_carrier_ok(dev)) {
 197                 if (link_stat) {
 198                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 199                         netif_carrier_on(dev);
 200                 } else {
 201                         netif_carrier_off(dev);
 202                         pi->phy.ops->power_down(&pi->phy, 1);
 203                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 204                         t3_link_start(&pi->phy, mac, &pi->link_config);
 205                 }
 206
 207                 link_report(dev);
 208         }
 209 }
 210
 211 static void cxgb_set_rxmode(struct net_device *dev)
 212 {
 213         struct t3_rx_mode rm;
 214         struct port_info *pi = netdev_priv(dev);
 215
 216         init_rx_mode(&rm, dev, dev->mc_list);
 217         t3_mac_set_rx_mode(&pi->mac, &rm);
 218 }
 219
 220 /**
 221  *      link_start - enable a port
 222  *      @dev: the device to enable
 223  *
 224  *      Performs the MAC and PHY actions needed to enable a port.
 225  */
 226 static void link_start(struct net_device *dev)
 227 {
 228         struct t3_rx_mode rm;
 229         struct port_info *pi = netdev_priv(dev);
 230         struct cmac *mac = &pi->mac;
 231
 232         init_rx_mode(&rm, dev, dev->mc_list);
 233         t3_mac_reset(mac);
 234         t3_mac_set_mtu(mac, dev->mtu);
 235         t3_mac_set_address(mac, 0, dev->dev_addr);
 236         t3_mac_set_rx_mode(mac, &rm);
 237         t3_link_start(&pi->phy, mac, &pi->link_config);
 238         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 239 }
 240
 241 static inline void cxgb_disable_msi(struct adapter *adapter)
 242 {
 243         if (adapter->flags & USING_MSIX) {
 244                 pci_disable_msix(adapter->pdev);
 245                 adapter->flags &= ~USING_MSIX;
 246         } else if (adapter->flags & USING_MSI) {
 247                 pci_disable_msi(adapter->pdev);
 248                 adapter->flags &= ~USING_MSI;
 249         }
 250 }
 251
 252 /*
 253  * Interrupt handler for asynchronous events used with MSI-X.
 254  */
 255 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 256 {
 257         t3_slow_intr_handler(cookie);
 258         return IRQ_HANDLED;
 259 }
 260
 261 /*
 262  * Name the MSI-X interrupts.
 263  */
 264 static void name_msix_vecs(struct adapter *adap)
 265 {
 266         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 267
 268         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 269         adap->msix_info[0].desc[n] = 0;
 270
 271         for_each_port(adap, j) {
 272                 struct net_device *d = adap->port[j];
 273                 const struct port_info *pi = netdev_priv(d);
 274
 275                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 276                         snprintf(adap->msix_info[msi_idx].desc, n,
 277                                  "%s (queue %d)", d->name, i);
 278                         adap->msix_info[msi_idx].desc[n] = 0;
 279                 }
 280         }
 281 }
 282
 283 static int request_msix_data_irqs(struct adapter *adap)
 284 {
 285         int i, j, err, qidx = 0;
 286
 287         for_each_port(adap, i) {
 288                 int nqsets = adap2pinfo(adap, i)->nqsets;
 289
 290                 for (j = 0; j < nqsets; ++j) {
 291                         err = request_irq(adap->msix_info[qidx + 1].vec,
 292                                           t3_intr_handler(adap,
 293                                                           adap->sge.qs[qidx].
 294                                                           rspq.polling), 0,
 295                                           adap->msix_info[qidx + 1].desc,
 296                                           &adap->sge.qs[qidx]);
 297                         if (err) {
 298                                 while (--qidx >= 0)
 299                                         free_irq(adap->msix_info[qidx + 1].vec,
 300                                                  &adap->sge.qs[qidx]);
 301                                 return err;
 302                         }
 303                         qidx++;
 304                 }
 305         }
 306         return 0;
 307 }
 308
 309 /**
 310  *      setup_rss - configure RSS
 311  *      @adap: the adapter
 312  *
 313  *      Sets up RSS to distribute packets to multiple receive queues.  We
 314  *      configure the RSS CPU lookup table to distribute to the number of HW
 315  *      receive queues, and the response queue lookup table to narrow that
 316  *      down to the response queues actually configured for each port.
 317  *      We always configure the RSS mapping for two ports since the mapping
 318  *      table has plenty of entries.
 319  */
 320 static void setup_rss(struct adapter *adap)
 321 {
 322         int i;
 323         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 324         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 325         u8 cpus[SGE_QSETS + 1];
 326         u16 rspq_map[RSS_TABLE_SIZE];
 327
 328         for (i = 0; i < SGE_QSETS; ++i)
 329                 cpus[i] = i;
 330         cpus[SGE_QSETS] = 0xff; /* terminator */
 331
 332         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 333                 rspq_map[i] = i % nq0;
 334                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 335         }
 336
 337         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 338                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 339                       V_RRCPLCPUSIZE(6), cpus, rspq_map);
 340 }
 341
 342 static void init_napi(struct adapter *adap)
 343 {
 344         int i;
 345
 346         for (i = 0; i < SGE_QSETS; i++) {
 347                 struct sge_qset *qs = &adap->sge.qs[i];
 348
 349                 if (qs->adap)
 350                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 351                                        64);
 352         }
 353 }
 354
 355 /*
 356  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 357  * both netdevices representing interfaces and the dummy ones for the extra
 358  * queues.
 359  */
 360 static void quiesce_rx(struct adapter *adap)
 361 {
 362         int i;
 363
 364         for (i = 0; i < SGE_QSETS; i++)
 365                 if (adap->sge.qs[i].adap)
 366                         napi_disable(&adap->sge.qs[i].napi);
 367 }
 368
 369 static void enable_all_napi(struct adapter *adap)
 370 {
 371         int i;
 372         for (i = 0; i < SGE_QSETS; i++)
 373                 if (adap->sge.qs[i].adap)
 374                         napi_enable(&adap->sge.qs[i].napi);
 375 }
 376
 377 /**
 378  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 379  *      @adap: the adapter
 380  *
 381  *      Determines how many sets of SGE queues to use and initializes them.
 382  *      We support multiple queue sets per port if we have MSI-X, otherwise
 383  *      just one queue set per port.
 384  */
 385 static int setup_sge_qsets(struct adapter *adap)
 386 {
 387         int i, j, err, irq_idx = 0, qset_idx = 0;
 388         unsigned int ntxq = SGE_TXQ_PER_SET;
 389
 390         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 391                 irq_idx = -1;
 392
 393         for_each_port(adap, i) {
 394                 struct net_device *dev = adap->port[i];
 395                 struct port_info *pi = netdev_priv(dev);
 396
 397                 pi->qs = &adap->sge.qs[pi->first_qset];
 398                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 399                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 400                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 401                                                              irq_idx,
 402                                 &adap->params.sge.qset[qset_idx], ntxq, dev);
 403                         if (err) {
 404                                 t3_free_sge_resources(adap);
 405                                 return err;
 406                         }
 407                 }
 408         }
 409
 410         return 0;
 411 }
 412
 413 static ssize_t attr_show(struct device *d, struct device_attribute *attr,
 414                          char *buf,
 415                          ssize_t(*format) (struct net_device *, char *))
 416 {
 417         ssize_t len;
 418
 419         /* Synchronize with ioctls that may shut down the device */
 420         rtnl_lock();
 421         len = (*format) (to_net_dev(d), buf);
 422         rtnl_unlock();
 423         return len;
 424 }
 425
 426 static ssize_t attr_store(struct device *d, struct device_attribute *attr,
 427                           const char *buf, size_t len,
 428                           ssize_t(*set) (struct net_device *, unsigned int),
 429                           unsigned int min_val, unsigned int max_val)
 430 {
 431         char *endp;
 432         ssize_t ret;
 433         unsigned int val;
 434
 435         if (!capable(CAP_NET_ADMIN))
 436                 return -EPERM;
 437
 438         val = simple_strtoul(buf, &endp, 0);
 439         if (endp == buf || val < min_val || val > max_val)
 440                 return -EINVAL;
 441
 442         rtnl_lock();
 443         ret = (*set) (to_net_dev(d), val);
 444         if (!ret)
 445                 ret = len;
 446         rtnl_unlock();
 447         return ret;
 448 }
 449
 450 #define CXGB3_SHOW(name, val_expr) \
 451 static ssize_t format_##name(struct net_device *dev, char *buf) \
 452 { \
 453         struct port_info *pi = netdev_priv(dev); \
 454         struct adapter *adap = pi->adapter; \
 455         return sprintf(buf, "%u\n", val_expr); \
 456 } \
 457 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 458                            char *buf) \
 459 { \
 460         return attr_show(d, attr, buf, format_##name); \
 461 }
 462
 463 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 464 {
 465         struct port_info *pi = netdev_priv(dev);
 466         struct adapter *adap = pi->adapter;
 467         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 468
 469         if (adap->flags & FULL_INIT_DONE)
 470                 return -EBUSY;
 471         if (val && adap->params.rev == 0)
 472                 return -EINVAL;
 473         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 474             min_tids)
 475                 return -EINVAL;
 476         adap->params.mc5.nfilters = val;
 477         return 0;
 478 }
 479
 480 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 481                               const char *buf, size_t len)
 482 {
 483         return attr_store(d, attr, buf, len, set_nfilters, 0, ~0);
 484 }
 485
 486 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 487 {
 488         struct port_info *pi = netdev_priv(dev);
 489         struct adapter *adap = pi->adapter;
 490
 491         if (adap->flags & FULL_INIT_DONE)
 492                 return -EBUSY;
 493         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 494             MC5_MIN_TIDS)
 495                 return -EINVAL;
 496         adap->params.mc5.nservers = val;
 497         return 0;
 498 }
 499
 500 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 501                               const char *buf, size_t len)
 502 {
 503         return attr_store(d, attr, buf, len, set_nservers, 0, ~0);
 504 }
 505
 506 #define CXGB3_ATTR_R(name, val_expr) \
 507 CXGB3_SHOW(name, val_expr) \
 508 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 509
 510 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 511 CXGB3_SHOW(name, val_expr) \
 512 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 513
 514 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 515 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 516 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 517
 518 static struct attribute *cxgb3_attrs[] = {
 519         &dev_attr_cam_size.attr,
 520         &dev_attr_nfilters.attr,
 521         &dev_attr_nservers.attr,
 522         NULL
 523 };
 524
 525 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 526
 527 static ssize_t tm_attr_show(struct device *d, struct device_attribute *attr,
 528                             char *buf, int sched)
 529 {
 530         struct port_info *pi = netdev_priv(to_net_dev(d));
 531         struct adapter *adap = pi->adapter;
 532         unsigned int v, addr, bpt, cpt;
 533         ssize_t len;
 534
 535         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 536         rtnl_lock();
 537         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 538         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 539         if (sched & 1)
 540                 v >>= 16;
 541         bpt = (v >> 8) & 0xff;
 542         cpt = v & 0xff;
 543         if (!cpt)
 544                 len = sprintf(buf, "disabled\n");
 545         else {
 546                 v = (adap->params.vpd.cclk * 1000) / cpt;
 547                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 548         }
 549         rtnl_unlock();
 550         return len;
 551 }
 552
 553 static ssize_t tm_attr_store(struct device *d, struct device_attribute *attr,
 554                              const char *buf, size_t len, int sched)
 555 {
 556         struct port_info *pi = netdev_priv(to_net_dev(d));
 557         struct adapter *adap = pi->adapter;
 558         unsigned int val;
 559         char *endp;
 560         ssize_t ret;
 561
 562         if (!capable(CAP_NET_ADMIN))
 563                 return -EPERM;
 564
 565         val = simple_strtoul(buf, &endp, 0);
 566         if (endp == buf || val > 10000000)
 567                 return -EINVAL;
 568
 569         rtnl_lock();
 570         ret = t3_config_sched(adap, val, sched);
 571         if (!ret)
 572                 ret = len;
 573         rtnl_unlock();
 574         return ret;
 575 }
 576
 577 #define TM_ATTR(name, sched) \
 578 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 579                            char *buf) \
 580 { \
 581         return tm_attr_show(d, attr, buf, sched); \
 582 } \
 583 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 584                             const char *buf, size_t len) \
 585 { \
 586         return tm_attr_store(d, attr, buf, len, sched); \
 587 } \
 588 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 589
 590 TM_ATTR(sched0, 0);
 591 TM_ATTR(sched1, 1);
 592 TM_ATTR(sched2, 2);
 593 TM_ATTR(sched3, 3);
 594 TM_ATTR(sched4, 4);
 595 TM_ATTR(sched5, 5);
 596 TM_ATTR(sched6, 6);
 597 TM_ATTR(sched7, 7);
 598
 599 static struct attribute *offload_attrs[] = {
 600         &dev_attr_sched0.attr,
 601         &dev_attr_sched1.attr,
 602         &dev_attr_sched2.attr,
 603         &dev_attr_sched3.attr,
 604         &dev_attr_sched4.attr,
 605         &dev_attr_sched5.attr,
 606         &dev_attr_sched6.attr,
 607         &dev_attr_sched7.attr,
 608         NULL
 609 };
 610
 611 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 612
 613 /*
 614  * Sends an sk_buff to an offload queue driver
 615  * after dealing with any active network taps.
 616  */
 617 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 618 {
 619         int ret;
 620
 621         local_bh_disable();
 622         ret = t3_offload_tx(tdev, skb);
 623         local_bh_enable();
 624         return ret;
 625 }
 626
 627 static int write_smt_entry(struct adapter *adapter, int idx)
 628 {
 629         struct cpl_smt_write_req *req;
 630         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 631
 632         if (!skb)
 633                 return -ENOMEM;
 634
 635         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 636         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 637         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 638         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 639         req->iff = idx;
 640         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 641         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 642         skb->priority = 1;
 643         offload_tx(&adapter->tdev, skb);
 644         return 0;
 645 }
 646
 647 static int init_smt(struct adapter *adapter)
 648 {
 649         int i;
 650
 651         for_each_port(adapter, i)
 652             write_smt_entry(adapter, i);
 653         return 0;
 654 }
 655
 656 static void init_port_mtus(struct adapter *adapter)
 657 {
 658         unsigned int mtus = adapter->port[0]->mtu;
 659
 660         if (adapter->port[1])
 661                 mtus |= adapter->port[1]->mtu << 16;
 662         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 663 }
 664
 665 static void send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 666                               int hi, int port)
 667 {
 668         struct sk_buff *skb;
 669         struct mngt_pktsched_wr *req;
 670
 671         skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 672         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 673         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 674         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 675         req->sched = sched;
 676         req->idx = qidx;
 677         req->min = lo;
 678         req->max = hi;
 679         req->binding = port;
 680         t3_mgmt_tx(adap, skb);
 681 }
 682
 683 static void bind_qsets(struct adapter *adap)
 684 {
 685         int i, j;
 686
 687         for_each_port(adap, i) {
 688                 const struct port_info *pi = adap2pinfo(adap, i);
 689
 690                 for (j = 0; j < pi->nqsets; ++j)
 691                         send_pktsched_cmd(adap, 1, pi->first_qset + j, -1,
 692                                           -1, i);
 693         }
 694 }
 695
 696 #define FW_FNAME "t3fw-%d.%d.%d.bin"
 697 #define TPSRAM_NAME "t3%c_protocol_sram-%d.%d.%d.bin"
 698
 699 static int upgrade_fw(struct adapter *adap)
 700 {
 701         int ret;
 702         char buf[64];
 703         const struct firmware *fw;
 704         struct device *dev = &adap->pdev->dev;
 705
 706         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
 707                  FW_VERSION_MINOR, FW_VERSION_MICRO);
 708         ret = request_firmware(&fw, buf, dev);
 709         if (ret < 0) {
 710                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
 711                         buf);
 712                 return ret;
 713         }
 714         ret = t3_load_fw(adap, fw->data, fw->size);
 715         release_firmware(fw);
 716
 717         if (ret == 0)
 718                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
 719                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 720         else
 721                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
 722                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 723
 724         return ret;
 725 }
 726
 727 static inline char t3rev2char(struct adapter *adapter)
 728 {
 729         char rev = 0;
 730
 731         switch(adapter->params.rev) {
 732         case T3_REV_B:
 733         case T3_REV_B2:
 734                 rev = 'b';
 735                 break;
 736         }
 737         return rev;
 738 }
 739
 740 int update_tpsram(struct adapter *adap)
 741 {
 742         const struct firmware *tpsram;
 743         char buf[64];
 744         struct device *dev = &adap->pdev->dev;
 745         int ret;
 746         char rev;
 747
 748         rev = t3rev2char(adap);
 749         if (!rev)
 750                 return 0;
 751
 752         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
 753                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 754
 755         ret = request_firmware(&tpsram, buf, dev);
 756         if (ret < 0) {
 757                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
 758                         buf);
 759                 return ret;
 760         }
 761
 762         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
 763         if (ret)
 764                 goto release_tpsram;
 765
 766         ret = t3_set_proto_sram(adap, tpsram->data);
 767         if (ret == 0)
 768                 dev_info(dev,
 769                          "successful update of protocol engine "
 770                          "to %d.%d.%d\n",
 771                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 772         else
 773                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
 774                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 775         if (ret)
 776                 dev_err(dev, "loading protocol SRAM failed\n");
 777
 778 release_tpsram:
 779         release_firmware(tpsram);
 780
 781         return ret;
 782 }
 783
 784 /**
 785  *      cxgb_up - enable the adapter
 786  *      @adapter: adapter being enabled
 787  *
 788  *      Called when the first port is enabled, this function performs the
 789  *      actions necessary to make an adapter operational, such as completing
 790  *      the initialization of HW modules, and enabling interrupts.
 791  *
 792  *      Must be called with the rtnl lock held.
 793  */
 794 static int cxgb_up(struct adapter *adap)
 795 {
 796         int err = 0;
 797         int must_load;
 798
 799         if (!(adap->flags & FULL_INIT_DONE)) {
 800                 err = t3_check_fw_version(adap);
 801                 if (err == -EINVAL)
 802                         err = upgrade_fw(adap);
 803                 if (err)
 804                         goto out;
 805
 806                 err = t3_check_tpsram_version(adap, &must_load);
 807                 if (err == -EINVAL) {
 808                         err = update_tpsram(adap);
 809                         if (err && must_load)
 810                                 goto out;
 811                 }
 812
 813                 err = t3_init_hw(adap, 0);
 814                 if (err)
 815                         goto out;
 816
 817                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 818
 819                 err = setup_sge_qsets(adap);
 820                 if (err)
 821                         goto out;
 822
 823                 setup_rss(adap);
 824                 init_napi(adap);
 825                 adap->flags |= FULL_INIT_DONE;
 826         }
 827
 828         t3_intr_clear(adap);
 829
 830         if (adap->flags & USING_MSIX) {
 831                 name_msix_vecs(adap);
 832                 err = request_irq(adap->msix_info[0].vec,
 833                                   t3_async_intr_handler, 0,
 834                                   adap->msix_info[0].desc, adap);
 835                 if (err)
 836                         goto irq_err;
 837
 838                 if (request_msix_data_irqs(adap)) {
 839                         free_irq(adap->msix_info[0].vec, adap);
 840                         goto irq_err;
 841                 }
 842         } else if ((err = request_irq(adap->pdev->irq,
 843                                       t3_intr_handler(adap,
 844                                                       adap->sge.qs[0].rspq.
 845                                                       polling),
 846                                       (adap->flags & USING_MSI) ?
 847                                        0 : IRQF_SHARED,
 848                                       adap->name, adap)))
 849                 goto irq_err;
 850
 851         enable_all_napi(adap);
 852         t3_sge_start(adap);
 853         t3_intr_enable(adap);
 854
 855         if ((adap->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
 856                 bind_qsets(adap);
 857         adap->flags |= QUEUES_BOUND;
 858
 859 out:
 860         return err;
 861 irq_err:
 862         CH_ERR(adap, "request_irq failed, err %d\n", err);
 863         goto out;
 864 }
 865
 866 /*
 867  * Release resources when all the ports and offloading have been stopped.
 868  */
 869 static void cxgb_down(struct adapter *adapter)
 870 {
 871         t3_sge_stop(adapter);
 872         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
 873         t3_intr_disable(adapter);
 874         spin_unlock_irq(&adapter->work_lock);
 875
 876         if (adapter->flags & USING_MSIX) {
 877                 int i, n = 0;
 878
 879                 free_irq(adapter->msix_info[0].vec, adapter);
 880                 for_each_port(adapter, i)
 881                     n += adap2pinfo(adapter, i)->nqsets;
 882
 883                 for (i = 0; i < n; ++i)
 884                         free_irq(adapter->msix_info[i + 1].vec,
 885                                  &adapter->sge.qs[i]);
 886         } else
 887                 free_irq(adapter->pdev->irq, adapter);
 888
 889         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
 890         quiesce_rx(adapter);
 891 }
 892
 893 static void schedule_chk_task(struct adapter *adap)
 894 {
 895         unsigned int timeo;
 896
 897         timeo = adap->params.linkpoll_period ?
 898             (HZ * adap->params.linkpoll_period) / 10 :
 899             adap->params.stats_update_period * HZ;
 900         if (timeo)
 901                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
 902 }
 903
 904 static int offload_open(struct net_device *dev)
 905 {
 906         struct port_info *pi = netdev_priv(dev);
 907         struct adapter *adapter = pi->adapter;
 908         struct t3cdev *tdev = dev2t3cdev(dev);
 909         int adap_up = adapter->open_device_map & PORT_MASK;
 910         int err = 0;
 911
 912         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
 913                 return 0;
 914
 915         if (!adap_up && (err = cxgb_up(adapter)) < 0)
 916                 return err;
 917
 918         t3_tp_set_offload_mode(adapter, 1);
 919         tdev->lldev = adapter->port[0];
 920         err = cxgb3_offload_activate(adapter);
 921         if (err)
 922                 goto out;
 923
 924         init_port_mtus(adapter);
 925         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
 926                      adapter->params.b_wnd,
 927                      adapter->params.rev == 0 ?
 928                      adapter->port[0]->mtu : 0xffff);
 929         init_smt(adapter);
 930
 931         /* Never mind if the next step fails */
 932         sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group);
 933
 934         /* Call back all registered clients */
 935         cxgb3_add_clients(tdev);
 936
 937 out:
 938         /* restore them in case the offload module has changed them */
 939         if (err) {
 940                 t3_tp_set_offload_mode(adapter, 0);
 941                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
 942                 cxgb3_set_dummy_ops(tdev);
 943         }
 944         return err;
 945 }
 946
 947 static int offload_close(struct t3cdev *tdev)
 948 {
 949         struct adapter *adapter = tdev2adap(tdev);
 950
 951         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
 952                 return 0;
 953
 954         /* Call back all registered clients */
 955         cxgb3_remove_clients(tdev);
 956
 957         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
 958
 959         tdev->lldev = NULL;
 960         cxgb3_set_dummy_ops(tdev);
 961         t3_tp_set_offload_mode(adapter, 0);
 962         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
 963
 964         if (!adapter->open_device_map)
 965                 cxgb_down(adapter);
 966
 967         cxgb3_offload_deactivate(adapter);
 968         return 0;
 969 }
 970
 971 static int cxgb_open(struct net_device *dev)
 972 {
 973         struct port_info *pi = netdev_priv(dev);
 974         struct adapter *adapter = pi->adapter;
 975         int other_ports = adapter->open_device_map & PORT_MASK;
 976         int err;
 977
 978         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0) {
 979                 quiesce_rx(adapter);
 980                 return err;
 981         }
 982
 983         set_bit(pi->port_id, &adapter->open_device_map);
 984         if (is_offload(adapter) && !ofld_disable) {
 985                 err = offload_open(dev);
 986                 if (err)
 987                         printk(KERN_WARNING
 988                                "Could not initialize offload capabilities\n");
 989         }
 990
 991         link_start(dev);
 992         t3_port_intr_enable(adapter, pi->port_id);
 993         netif_start_queue(dev);
 994         if (!other_ports)
 995                 schedule_chk_task(adapter);
 996
 997         return 0;
 998 }
 999
1000 static int cxgb_close(struct net_device *dev)
1001 {
1002         struct port_info *pi = netdev_priv(dev);
1003         struct adapter *adapter = pi->adapter;
1004
1005         t3_port_intr_disable(adapter, pi->port_id);
1006         netif_stop_queue(dev);
1007         pi->phy.ops->power_down(&pi->phy, 1);
1008         netif_carrier_off(dev);
1009         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1010
1011         spin_lock(&adapter->work_lock); /* sync with update task */
1012         clear_bit(pi->port_id, &adapter->open_device_map);
1013         spin_unlock(&adapter->work_lock);
1014
1015         if (!(adapter->open_device_map & PORT_MASK))
1016                 cancel_rearming_delayed_workqueue(cxgb3_wq,
1017                                                   &adapter->adap_check_task);
1018
1019         if (!adapter->open_device_map)
1020                 cxgb_down(adapter);
1021
1022         return 0;
1023 }
1024
1025 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1026 {
1027         struct port_info *pi = netdev_priv(dev);
1028         struct adapter *adapter = pi->adapter;
1029         struct net_device_stats *ns = &pi->netstats;
1030         const struct mac_stats *pstats;
1031
1032         spin_lock(&adapter->stats_lock);
1033         pstats = t3_mac_update_stats(&pi->mac);
1034         spin_unlock(&adapter->stats_lock);
1035
1036         ns->tx_bytes = pstats->tx_octets;
1037         ns->tx_packets = pstats->tx_frames;
1038         ns->rx_bytes = pstats->rx_octets;
1039         ns->rx_packets = pstats->rx_frames;
1040         ns->multicast = pstats->rx_mcast_frames;
1041
1042         ns->tx_errors = pstats->tx_underrun;
1043         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1044             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1045             pstats->rx_fifo_ovfl;
1046
1047         /* detailed rx_errors */
1048         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1049         ns->rx_over_errors = 0;
1050         ns->rx_crc_errors = pstats->rx_fcs_errs;
1051         ns->rx_frame_errors = pstats->rx_symbol_errs;
1052         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1053         ns->rx_missed_errors = pstats->rx_cong_drops;
1054
1055         /* detailed tx_errors */
1056         ns->tx_aborted_errors = 0;
1057         ns->tx_carrier_errors = 0;
1058         ns->tx_fifo_errors = pstats->tx_underrun;
1059         ns->tx_heartbeat_errors = 0;
1060         ns->tx_window_errors = 0;
1061         return ns;
1062 }
1063
1064 static u32 get_msglevel(struct net_device *dev)
1065 {
1066         struct port_info *pi = netdev_priv(dev);
1067         struct adapter *adapter = pi->adapter;
1068
1069         return adapter->msg_enable;
1070 }
1071
1072 static void set_msglevel(struct net_device *dev, u32 val)
1073 {
1074         struct port_info *pi = netdev_priv(dev);
1075         struct adapter *adapter = pi->adapter;
1076
1077         adapter->msg_enable = val;
1078 }
1079
1080 static char stats_strings[][ETH_GSTRING_LEN] = {
1081         "TxOctetsOK         ",
1082         "TxFramesOK         ",
1083         "TxMulticastFramesOK",
1084         "TxBroadcastFramesOK",
1085         "TxPauseFrames      ",
1086         "TxUnderrun         ",
1087         "TxExtUnderrun      ",
1088
1089         "TxFrames64         ",
1090         "TxFrames65To127    ",
1091         "TxFrames128To255   ",
1092         "TxFrames256To511   ",
1093         "TxFrames512To1023  ",
1094         "TxFrames1024To1518 ",
1095         "TxFrames1519ToMax  ",
1096
1097         "RxOctetsOK         ",
1098         "RxFramesOK         ",
1099         "RxMulticastFramesOK",
1100         "RxBroadcastFramesOK",
1101         "RxPauseFrames      ",
1102         "RxFCSErrors        ",
1103         "RxSymbolErrors     ",
1104         "RxShortErrors      ",
1105         "RxJabberErrors     ",
1106         "RxLengthErrors     ",
1107         "RxFIFOoverflow     ",
1108
1109         "RxFrames64         ",
1110         "RxFrames65To127    ",
1111         "RxFrames128To255   ",
1112         "RxFrames256To511   ",
1113         "RxFrames512To1023  ",
1114         "RxFrames1024To1518 ",
1115         "RxFrames1519ToMax  ",
1116
1117         "PhyFIFOErrors      ",
1118         "TSO                ",
1119         "VLANextractions    ",
1120         "VLANinsertions     ",
1121         "TxCsumOffload      ",
1122         "RxCsumGood         ",
1123         "RxDrops            ",
1124
1125         "CheckTXEnToggled   ",
1126         "CheckResets        ",
1127
1128 };
1129
1130 static int get_stats_count(struct net_device *dev)
1131 {
1132         return ARRAY_SIZE(stats_strings);
1133 }
1134
1135 #define T3_REGMAP_SIZE (3 * 1024)
1136
1137 static int get_regs_len(struct net_device *dev)
1138 {
1139         return T3_REGMAP_SIZE;
1140 }
1141
1142 static int get_eeprom_len(struct net_device *dev)
1143 {
1144         return EEPROMSIZE;
1145 }
1146
1147 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1148 {
1149         struct port_info *pi = netdev_priv(dev);
1150         struct adapter *adapter = pi->adapter;
1151         u32 fw_vers = 0;
1152         u32 tp_vers = 0;
1153
1154         t3_get_fw_version(adapter, &fw_vers);
1155         t3_get_tp_version(adapter, &tp_vers);
1156
1157         strcpy(info->driver, DRV_NAME);
1158         strcpy(info->version, DRV_VERSION);
1159         strcpy(info->bus_info, pci_name(adapter->pdev));
1160         if (!fw_vers)
1161                 strcpy(info->fw_version, "N/A");
1162         else {
1163                 snprintf(info->fw_version, sizeof(info->fw_version),
1164                          "%s %u.%u.%u TP %u.%u.%u",
1165                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1166                          G_FW_VERSION_MAJOR(fw_vers),
1167                          G_FW_VERSION_MINOR(fw_vers),
1168                          G_FW_VERSION_MICRO(fw_vers),
1169                          G_TP_VERSION_MAJOR(tp_vers),
1170                          G_TP_VERSION_MINOR(tp_vers),
1171                          G_TP_VERSION_MICRO(tp_vers));
1172         }
1173 }
1174
1175 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1176 {
1177         if (stringset == ETH_SS_STATS)
1178                 memcpy(data, stats_strings, sizeof(stats_strings));
1179 }
1180
1181 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1182                                             struct port_info *p, int idx)
1183 {
1184         int i;
1185         unsigned long tot = 0;
1186
1187         for (i = 0; i < p->nqsets; ++i)
1188                 tot += adapter->sge.qs[i + p->first_qset].port_stats[idx];
1189         return tot;
1190 }
1191
1192 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1193                       u64 *data)
1194 {
1195         struct port_info *pi = netdev_priv(dev);
1196         struct adapter *adapter = pi->adapter;
1197         const struct mac_stats *s;
1198
1199         spin_lock(&adapter->stats_lock);
1200         s = t3_mac_update_stats(&pi->mac);
1201         spin_unlock(&adapter->stats_lock);
1202
1203         *data++ = s->tx_octets;
1204         *data++ = s->tx_frames;
1205         *data++ = s->tx_mcast_frames;
1206         *data++ = s->tx_bcast_frames;
1207         *data++ = s->tx_pause;
1208         *data++ = s->tx_underrun;
1209         *data++ = s->tx_fifo_urun;
1210
1211         *data++ = s->tx_frames_64;
1212         *data++ = s->tx_frames_65_127;
1213         *data++ = s->tx_frames_128_255;
1214         *data++ = s->tx_frames_256_511;
1215         *data++ = s->tx_frames_512_1023;
1216         *data++ = s->tx_frames_1024_1518;
1217         *data++ = s->tx_frames_1519_max;
1218
1219         *data++ = s->rx_octets;
1220         *data++ = s->rx_frames;
1221         *data++ = s->rx_mcast_frames;
1222         *data++ = s->rx_bcast_frames;
1223         *data++ = s->rx_pause;
1224         *data++ = s->rx_fcs_errs;
1225         *data++ = s->rx_symbol_errs;
1226         *data++ = s->rx_short;
1227         *data++ = s->rx_jabber;
1228         *data++ = s->rx_too_long;
1229         *data++ = s->rx_fifo_ovfl;
1230
1231         *data++ = s->rx_frames_64;
1232         *data++ = s->rx_frames_65_127;
1233         *data++ = s->rx_frames_128_255;
1234         *data++ = s->rx_frames_256_511;
1235         *data++ = s->rx_frames_512_1023;
1236         *data++ = s->rx_frames_1024_1518;
1237         *data++ = s->rx_frames_1519_max;
1238
1239         *data++ = pi->phy.fifo_errors;
1240
1241         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1242         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1243         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1244         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1245         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1246         *data++ = s->rx_cong_drops;
1247
1248         *data++ = s->num_toggled;
1249         *data++ = s->num_resets;
1250 }
1251
1252 static inline void reg_block_dump(struct adapter *ap, void *buf,
1253                                   unsigned int start, unsigned int end)
1254 {
1255         u32 *p = buf + start;
1256
1257         for (; start <= end; start += sizeof(u32))
1258                 *p++ = t3_read_reg(ap, start);
1259 }
1260
1261 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1262                      void *buf)
1263 {
1264         struct port_info *pi = netdev_priv(dev);
1265         struct adapter *ap = pi->adapter;
1266
1267         /*
1268          * Version scheme:
1269          * bits 0..9: chip version
1270          * bits 10..15: chip revision
1271          * bit 31: set for PCIe cards
1272          */
1273         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1274
1275         /*
1276          * We skip the MAC statistics registers because they are clear-on-read.
1277          * Also reading multi-register stats would need to synchronize with the
1278          * periodic mac stats accumulation.  Hard to justify the complexity.
1279          */
1280         memset(buf, 0, T3_REGMAP_SIZE);
1281         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1282         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1283         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1284         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1285         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1286         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1287                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1288         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1289                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1290 }
1291
1292 static int restart_autoneg(struct net_device *dev)
1293 {
1294         struct port_info *p = netdev_priv(dev);
1295
1296         if (!netif_running(dev))
1297                 return -EAGAIN;
1298         if (p->link_config.autoneg != AUTONEG_ENABLE)
1299                 return -EINVAL;
1300         p->phy.ops->autoneg_restart(&p->phy);
1301         return 0;
1302 }
1303
1304 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1305 {
1306         struct port_info *pi = netdev_priv(dev);
1307         struct adapter *adapter = pi->adapter;
1308         int i;
1309
1310         if (data == 0)
1311                 data = 2;
1312
1313         for (i = 0; i < data * 2; i++) {
1314                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1315                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1316                 if (msleep_interruptible(500))
1317                         break;
1318         }
1319         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1320                          F_GPIO0_OUT_VAL);
1321         return 0;
1322 }
1323
1324 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1325 {
1326         struct port_info *p = netdev_priv(dev);
1327
1328         cmd->supported = p->link_config.supported;
1329         cmd->advertising = p->link_config.advertising;
1330
1331         if (netif_carrier_ok(dev)) {
1332                 cmd->speed = p->link_config.speed;
1333                 cmd->duplex = p->link_config.duplex;
1334         } else {
1335                 cmd->speed = -1;
1336                 cmd->duplex = -1;
1337         }
1338
1339         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1340         cmd->phy_address = p->phy.addr;
1341         cmd->transceiver = XCVR_EXTERNAL;
1342         cmd->autoneg = p->link_config.autoneg;
1343         cmd->maxtxpkt = 0;
1344         cmd->maxrxpkt = 0;
1345         return 0;
1346 }
1347
1348 static int speed_duplex_to_caps(int speed, int duplex)
1349 {
1350         int cap = 0;
1351
1352         switch (speed) {
1353         case SPEED_10:
1354                 if (duplex == DUPLEX_FULL)
1355                         cap = SUPPORTED_10baseT_Full;
1356                 else
1357                         cap = SUPPORTED_10baseT_Half;
1358                 break;
1359         case SPEED_100:
1360                 if (duplex == DUPLEX_FULL)
1361                         cap = SUPPORTED_100baseT_Full;
1362                 else
1363                         cap = SUPPORTED_100baseT_Half;
1364                 break;
1365         case SPEED_1000:
1366                 if (duplex == DUPLEX_FULL)
1367                         cap = SUPPORTED_1000baseT_Full;
1368                 else
1369                         cap = SUPPORTED_1000baseT_Half;
1370                 break;
1371         case SPEED_10000:
1372                 if (duplex == DUPLEX_FULL)
1373                         cap = SUPPORTED_10000baseT_Full;
1374         }
1375         return cap;
1376 }
1377
1378 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1379                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1380                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1381                       ADVERTISED_10000baseT_Full)
1382
1383 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1384 {
1385         struct port_info *p = netdev_priv(dev);
1386         struct link_config *lc = &p->link_config;
1387
1388         if (!(lc->supported & SUPPORTED_Autoneg))
1389                 return -EOPNOTSUPP;     /* can't change speed/duplex */
1390
1391         if (cmd->autoneg == AUTONEG_DISABLE) {
1392                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1393
1394                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1395                         return -EINVAL;
1396                 lc->requested_speed = cmd->speed;
1397                 lc->requested_duplex = cmd->duplex;
1398                 lc->advertising = 0;
1399         } else {
1400                 cmd->advertising &= ADVERTISED_MASK;
1401                 cmd->advertising &= lc->supported;
1402                 if (!cmd->advertising)
1403                         return -EINVAL;
1404                 lc->requested_speed = SPEED_INVALID;
1405                 lc->requested_duplex = DUPLEX_INVALID;
1406                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1407         }
1408         lc->autoneg = cmd->autoneg;
1409         if (netif_running(dev))
1410                 t3_link_start(&p->phy, &p->mac, lc);
1411         return 0;
1412 }
1413
1414 static void get_pauseparam(struct net_device *dev,
1415                            struct ethtool_pauseparam *epause)
1416 {
1417         struct port_info *p = netdev_priv(dev);
1418
1419         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1420         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1421         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1422 }
1423
1424 static int set_pauseparam(struct net_device *dev,
1425                           struct ethtool_pauseparam *epause)
1426 {
1427         struct port_info *p = netdev_priv(dev);
1428         struct link_config *lc = &p->link_config;
1429
1430         if (epause->autoneg == AUTONEG_DISABLE)
1431                 lc->requested_fc = 0;
1432         else if (lc->supported & SUPPORTED_Autoneg)
1433                 lc->requested_fc = PAUSE_AUTONEG;
1434         else
1435                 return -EINVAL;
1436
1437         if (epause->rx_pause)
1438                 lc->requested_fc |= PAUSE_RX;
1439         if (epause->tx_pause)
1440                 lc->requested_fc |= PAUSE_TX;
1441         if (lc->autoneg == AUTONEG_ENABLE) {
1442                 if (netif_running(dev))
1443                         t3_link_start(&p->phy, &p->mac, lc);
1444         } else {
1445                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1446                 if (netif_running(dev))
1447                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1448         }
1449         return 0;
1450 }
1451
1452 static u32 get_rx_csum(struct net_device *dev)
1453 {
1454         struct port_info *p = netdev_priv(dev);
1455
1456         return p->rx_csum_offload;
1457 }
1458
1459 static int set_rx_csum(struct net_device *dev, u32 data)
1460 {
1461         struct port_info *p = netdev_priv(dev);
1462
1463         p->rx_csum_offload = data;
1464         return 0;
1465 }
1466
1467 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1468 {
1469         struct port_info *pi = netdev_priv(dev);
1470         struct adapter *adapter = pi->adapter;
1471         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1472
1473         e->rx_max_pending = MAX_RX_BUFFERS;
1474         e->rx_mini_max_pending = 0;
1475         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1476         e->tx_max_pending = MAX_TXQ_ENTRIES;
1477
1478         e->rx_pending = q->fl_size;
1479         e->rx_mini_pending = q->rspq_size;
1480         e->rx_jumbo_pending = q->jumbo_size;
1481         e->tx_pending = q->txq_size[0];
1482 }
1483
1484 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1485 {
1486         struct port_info *pi = netdev_priv(dev);
1487         struct adapter *adapter = pi->adapter;
1488         struct qset_params *q;
1489         int i;
1490
1491         if (e->rx_pending > MAX_RX_BUFFERS ||
1492             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1493             e->tx_pending > MAX_TXQ_ENTRIES ||
1494             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1495             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1496             e->rx_pending < MIN_FL_ENTRIES ||
1497             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1498             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1499                 return -EINVAL;
1500
1501         if (adapter->flags & FULL_INIT_DONE)
1502                 return -EBUSY;
1503
1504         q = &adapter->params.sge.qset[pi->first_qset];
1505         for (i = 0; i < pi->nqsets; ++i, ++q) {
1506                 q->rspq_size = e->rx_mini_pending;
1507                 q->fl_size = e->rx_pending;
1508                 q->jumbo_size = e->rx_jumbo_pending;
1509                 q->txq_size[0] = e->tx_pending;
1510                 q->txq_size[1] = e->tx_pending;
1511                 q->txq_size[2] = e->tx_pending;
1512         }
1513         return 0;
1514 }
1515
1516 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1517 {
1518         struct port_info *pi = netdev_priv(dev);
1519         struct adapter *adapter = pi->adapter;
1520         struct qset_params *qsp = &adapter->params.sge.qset[0];
1521         struct sge_qset *qs = &adapter->sge.qs[0];
1522
1523         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1524                 return -EINVAL;
1525
1526         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1527         t3_update_qset_coalesce(qs, qsp);
1528         return 0;
1529 }
1530
1531 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1532 {
1533         struct port_info *pi = netdev_priv(dev);
1534         struct adapter *adapter = pi->adapter;
1535         struct qset_params *q = adapter->params.sge.qset;
1536
1537         c->rx_coalesce_usecs = q->coalesce_usecs;
1538         return 0;
1539 }
1540
1541 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1542                       u8 * data)
1543 {
1544         struct port_info *pi = netdev_priv(dev);
1545         struct adapter *adapter = pi->adapter;
1546         int i, err = 0;
1547
1548         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1549         if (!buf)
1550                 return -ENOMEM;
1551
1552         e->magic = EEPROM_MAGIC;
1553         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1554                 err = t3_seeprom_read(adapter, i, (u32 *) & buf[i]);
1555
1556         if (!err)
1557                 memcpy(data, buf + e->offset, e->len);
1558         kfree(buf);
1559         return err;
1560 }
1561
1562 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1563                       u8 * data)
1564 {
1565         struct port_info *pi = netdev_priv(dev);
1566         struct adapter *adapter = pi->adapter;
1567         u32 aligned_offset, aligned_len, *p;
1568         u8 *buf;
1569         int err = 0;
1570
1571         if (eeprom->magic != EEPROM_MAGIC)
1572                 return -EINVAL;
1573
1574         aligned_offset = eeprom->offset & ~3;
1575         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1576
1577         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1578                 buf = kmalloc(aligned_len, GFP_KERNEL);
1579                 if (!buf)
1580                         return -ENOMEM;
1581                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *) buf);
1582                 if (!err && aligned_len > 4)
1583                         err = t3_seeprom_read(adapter,
1584                                               aligned_offset + aligned_len - 4,
1585                                               (u32 *) & buf[aligned_len - 4]);
1586                 if (err)
1587                         goto out;
1588                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1589         } else
1590                 buf = data;
1591
1592         err = t3_seeprom_wp(adapter, 0);
1593         if (err)
1594                 goto out;
1595
1596         for (p = (u32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
1597                 err = t3_seeprom_write(adapter, aligned_offset, *p);
1598                 aligned_offset += 4;
1599         }
1600
1601         if (!err)
1602                 err = t3_seeprom_wp(adapter, 1);
1603 out:
1604         if (buf != data)
1605                 kfree(buf);
1606         return err;
1607 }
1608
1609 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1610 {
1611         wol->supported = 0;
1612         wol->wolopts = 0;
1613         memset(&wol->sopass, 0, sizeof(wol->sopass));
1614 }
1615
1616 static const struct ethtool_ops cxgb_ethtool_ops = {
1617         .get_settings = get_settings,
1618         .set_settings = set_settings,
1619         .get_drvinfo = get_drvinfo,
1620         .get_msglevel = get_msglevel,
1621         .set_msglevel = set_msglevel,
1622         .get_ringparam = get_sge_param,
1623         .set_ringparam = set_sge_param,
1624         .get_coalesce = get_coalesce,
1625         .set_coalesce = set_coalesce,
1626         .get_eeprom_len = get_eeprom_len,
1627         .get_eeprom = get_eeprom,
1628         .set_eeprom = set_eeprom,
1629         .get_pauseparam = get_pauseparam,
1630         .set_pauseparam = set_pauseparam,
1631         .get_rx_csum = get_rx_csum,
1632         .set_rx_csum = set_rx_csum,
1633         .get_tx_csum = ethtool_op_get_tx_csum,
1634         .set_tx_csum = ethtool_op_set_tx_csum,
1635         .get_sg = ethtool_op_get_sg,
1636         .set_sg = ethtool_op_set_sg,
1637         .get_link = ethtool_op_get_link,
1638         .get_strings = get_strings,
1639         .phys_id = cxgb3_phys_id,
1640         .nway_reset = restart_autoneg,
1641         .get_stats_count = get_stats_count,
1642         .get_ethtool_stats = get_stats,
1643         .get_regs_len = get_regs_len,
1644         .get_regs = get_regs,
1645         .get_wol = get_wol,
1646         .get_tso = ethtool_op_get_tso,
1647         .set_tso = ethtool_op_set_tso,
1648 };
1649
1650 static int in_range(int val, int lo, int hi)
1651 {
1652         return val < 0 || (val <= hi && val >= lo);
1653 }
1654
1655 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
1656 {
1657         struct port_info *pi = netdev_priv(dev);
1658         struct adapter *adapter = pi->adapter;
1659         u32 cmd;
1660         int ret;
1661
1662         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
1663                 return -EFAULT;
1664
1665         switch (cmd) {
1666         case CHELSIO_SET_QSET_PARAMS:{
1667                 int i;
1668                 struct qset_params *q;
1669                 struct ch_qset_params t;
1670
1671                 if (!capable(CAP_NET_ADMIN))
1672                         return -EPERM;
1673                 if (copy_from_user(&t, useraddr, sizeof(t)))
1674                         return -EFAULT;
1675                 if (t.qset_idx >= SGE_QSETS)
1676                         return -EINVAL;
1677                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
1678                         !in_range(t.cong_thres, 0, 255) ||
1679                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
1680                                 MAX_TXQ_ENTRIES) ||
1681                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
1682                                 MAX_TXQ_ENTRIES) ||
1683                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1684                                 MAX_CTRL_TXQ_ENTRIES) ||
1685                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
1686                                 MAX_RX_BUFFERS)
1687                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
1688                                         MAX_RX_JUMBO_BUFFERS)
1689                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
1690                                         MAX_RSPQ_ENTRIES))
1691                         return -EINVAL;
1692                 if ((adapter->flags & FULL_INIT_DONE) &&
1693                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
1694                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
1695                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
1696                         t.polling >= 0 || t.cong_thres >= 0))
1697                         return -EBUSY;
1698
1699                 q = &adapter->params.sge.qset[t.qset_idx];
1700
1701                 if (t.rspq_size >= 0)
1702                         q->rspq_size = t.rspq_size;
1703                 if (t.fl_size[0] >= 0)
1704                         q->fl_size = t.fl_size[0];
1705                 if (t.fl_size[1] >= 0)
1706                         q->jumbo_size = t.fl_size[1];
1707                 if (t.txq_size[0] >= 0)
1708                         q->txq_size[0] = t.txq_size[0];
1709                 if (t.txq_size[1] >= 0)
1710                         q->txq_size[1] = t.txq_size[1];
1711                 if (t.txq_size[2] >= 0)
1712                         q->txq_size[2] = t.txq_size[2];
1713                 if (t.cong_thres >= 0)
1714                         q->cong_thres = t.cong_thres;
1715                 if (t.intr_lat >= 0) {
1716                         struct sge_qset *qs =
1717                                 &adapter->sge.qs[t.qset_idx];
1718
1719                         q->coalesce_usecs = t.intr_lat;
1720                         t3_update_qset_coalesce(qs, q);
1721                 }
1722                 if (t.polling >= 0) {
1723                         if (adapter->flags & USING_MSIX)
1724                                 q->polling = t.polling;
1725                         else {
1726                                 /* No polling with INTx for T3A */
1727                                 if (adapter->params.rev == 0 &&
1728                                         !(adapter->flags & USING_MSI))
1729                                         t.polling = 0;
1730
1731                                 for (i = 0; i < SGE_QSETS; i++) {
1732                                         q = &adapter->params.sge.
1733                                                 qset[i];
1734                                         q->polling = t.polling;
1735                                 }
1736                         }
1737                 }
1738                 break;
1739         }
1740         case CHELSIO_GET_QSET_PARAMS:{
1741                 struct qset_params *q;
1742                 struct ch_qset_params t;
1743
1744                 if (copy_from_user(&t, useraddr, sizeof(t)))
1745                         return -EFAULT;
1746                 if (t.qset_idx >= SGE_QSETS)
1747                         return -EINVAL;
1748
1749                 q = &adapter->params.sge.qset[t.qset_idx];
1750                 t.rspq_size = q->rspq_size;
1751                 t.txq_size[0] = q->txq_size[0];
1752                 t.txq_size[1] = q->txq_size[1];
1753                 t.txq_size[2] = q->txq_size[2];
1754                 t.fl_size[0] = q->fl_size;
1755                 t.fl_size[1] = q->jumbo_size;
1756                 t.polling = q->polling;
1757                 t.intr_lat = q->coalesce_usecs;
1758                 t.cong_thres = q->cong_thres;
1759
1760                 if (copy_to_user(useraddr, &t, sizeof(t)))
1761                         return -EFAULT;
1762                 break;
1763         }
1764         case CHELSIO_SET_QSET_NUM:{
1765                 struct ch_reg edata;
1766                 struct port_info *pi = netdev_priv(dev);
1767                 unsigned int i, first_qset = 0, other_qsets = 0;
1768
1769                 if (!capable(CAP_NET_ADMIN))
1770                         return -EPERM;
1771                 if (adapter->flags & FULL_INIT_DONE)
1772                         return -EBUSY;
1773                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
1774                         return -EFAULT;
1775                 if (edata.val < 1 ||
1776                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
1777                         return -EINVAL;
1778
1779                 for_each_port(adapter, i)
1780                         if (adapter->port[i] && adapter->port[i] != dev)
1781                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
1782
1783                 if (edata.val + other_qsets > SGE_QSETS)
1784                         return -EINVAL;
1785
1786                 pi->nqsets = edata.val;
1787
1788                 for_each_port(adapter, i)
1789                         if (adapter->port[i]) {
1790                                 pi = adap2pinfo(adapter, i);
1791                                 pi->first_qset = first_qset;
1792                                 first_qset += pi->nqsets;
1793                         }
1794                 break;
1795         }
1796         case CHELSIO_GET_QSET_NUM:{
1797                 struct ch_reg edata;
1798                 struct port_info *pi = netdev_priv(dev);
1799
1800                 edata.cmd = CHELSIO_GET_QSET_NUM;
1801                 edata.val = pi->nqsets;
1802                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
1803                         return -EFAULT;
1804                 break;
1805         }
1806         case CHELSIO_LOAD_FW:{
1807                 u8 *fw_data;
1808                 struct ch_mem_range t;
1809
1810                 if (!capable(CAP_NET_ADMIN))
1811                         return -EPERM;
1812                 if (copy_from_user(&t, useraddr, sizeof(t)))
1813                         return -EFAULT;
1814
1815                 fw_data = kmalloc(t.len, GFP_KERNEL);
1816                 if (!fw_data)
1817                         return -ENOMEM;
1818
1819                 if (copy_from_user
1820                         (fw_data, useraddr + sizeof(t), t.len)) {
1821                         kfree(fw_data);
1822                         return -EFAULT;
1823                 }
1824
1825                 ret = t3_load_fw(adapter, fw_data, t.len);
1826                 kfree(fw_data);
1827                 if (ret)
1828                         return ret;
1829                 break;
1830         }
1831         case CHELSIO_SETMTUTAB:{
1832                 struct ch_mtus m;
1833                 int i;
1834
1835                 if (!is_offload(adapter))
1836                         return -EOPNOTSUPP;
1837                 if (!capable(CAP_NET_ADMIN))
1838                         return -EPERM;
1839                 if (offload_running(adapter))
1840                         return -EBUSY;
1841                 if (copy_from_user(&m, useraddr, sizeof(m)))
1842                         return -EFAULT;
1843                 if (m.nmtus != NMTUS)
1844                         return -EINVAL;
1845                 if (m.mtus[0] < 81)     /* accommodate SACK */
1846                         return -EINVAL;
1847
1848                 /* MTUs must be in ascending order */
1849                 for (i = 1; i < NMTUS; ++i)
1850                         if (m.mtus[i] < m.mtus[i - 1])
1851                                 return -EINVAL;
1852
1853                 memcpy(adapter->params.mtus, m.mtus,
1854                         sizeof(adapter->params.mtus));
1855                 break;
1856         }
1857         case CHELSIO_GET_PM:{
1858                 struct tp_params *p = &adapter->params.tp;
1859                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
1860
1861                 if (!is_offload(adapter))
1862                         return -EOPNOTSUPP;
1863                 m.tx_pg_sz = p->tx_pg_size;
1864                 m.tx_num_pg = p->tx_num_pgs;
1865                 m.rx_pg_sz = p->rx_pg_size;
1866                 m.rx_num_pg = p->rx_num_pgs;
1867                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
1868                 if (copy_to_user(useraddr, &m, sizeof(m)))
1869                         return -EFAULT;
1870                 break;
1871         }
1872         case CHELSIO_SET_PM:{
1873                 struct ch_pm m;
1874                 struct tp_params *p = &adapter->params.tp;
1875
1876                 if (!is_offload(adapter))
1877                         return -EOPNOTSUPP;
1878                 if (!capable(CAP_NET_ADMIN))
1879                         return -EPERM;
1880                 if (adapter->flags & FULL_INIT_DONE)
1881                         return -EBUSY;
1882                 if (copy_from_user(&m, useraddr, sizeof(m)))
1883                         return -EFAULT;
1884                 if (!is_power_of_2(m.rx_pg_sz) ||
1885                         !is_power_of_2(m.tx_pg_sz))
1886                         return -EINVAL; /* not power of 2 */
1887                 if (!(m.rx_pg_sz & 0x14000))
1888                         return -EINVAL; /* not 16KB or 64KB */
1889                 if (!(m.tx_pg_sz & 0x1554000))
1890                         return -EINVAL;
1891                 if (m.tx_num_pg == -1)
1892                         m.tx_num_pg = p->tx_num_pgs;
1893                 if (m.rx_num_pg == -1)
1894                         m.rx_num_pg = p->rx_num_pgs;
1895                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
1896                         return -EINVAL;
1897                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
1898                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
1899                         return -EINVAL;
1900                 p->rx_pg_size = m.rx_pg_sz;
1901                 p->tx_pg_size = m.tx_pg_sz;
1902                 p->rx_num_pgs = m.rx_num_pg;
1903                 p->tx_num_pgs = m.tx_num_pg;
1904                 break;
1905         }
1906         case CHELSIO_GET_MEM:{
1907                 struct ch_mem_range t;
1908                 struct mc7 *mem;
1909                 u64 buf[32];
1910
1911                 if (!is_offload(adapter))
1912                         return -EOPNOTSUPP;
1913                 if (!(adapter->flags & FULL_INIT_DONE))
1914                         return -EIO;    /* need the memory controllers */
1915                 if (copy_from_user(&t, useraddr, sizeof(t)))
1916                         return -EFAULT;
1917                 if ((t.addr & 7) || (t.len & 7))
1918                         return -EINVAL;
1919                 if (t.mem_id == MEM_CM)
1920                         mem = &adapter->cm;
1921                 else if (t.mem_id == MEM_PMRX)
1922                         mem = &adapter->pmrx;
1923                 else if (t.mem_id == MEM_PMTX)
1924                         mem = &adapter->pmtx;
1925                 else
1926                         return -EINVAL;
1927
1928                 /*
1929                  * Version scheme:
1930                  * bits 0..9: chip version
1931                  * bits 10..15: chip revision
1932                  */
1933                 t.version = 3 | (adapter->params.rev << 10);
1934                 if (copy_to_user(useraddr, &t, sizeof(t)))
1935                         return -EFAULT;
1936
1937                 /*
1938                  * Read 256 bytes at a time as len can be large and we don't
1939                  * want to use huge intermediate buffers.
1940                  */
1941                 useraddr += sizeof(t);  /* advance to start of buffer */
1942                 while (t.len) {
1943                         unsigned int chunk =
1944                                 min_t(unsigned int, t.len, sizeof(buf));
1945
1946                         ret =
1947                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
1948                                                 buf);
1949                         if (ret)
1950                                 return ret;
1951                         if (copy_to_user(useraddr, buf, chunk))
1952                                 return -EFAULT;
1953                         useraddr += chunk;
1954                         t.addr += chunk;
1955                         t.len -= chunk;
1956                 }
1957                 break;
1958         }
1959         case CHELSIO_SET_TRACE_FILTER:{
1960                 struct ch_trace t;
1961                 const struct trace_params *tp;
1962
1963                 if (!capable(CAP_NET_ADMIN))
1964                         return -EPERM;
1965                 if (!offload_running(adapter))
1966                         return -EAGAIN;
1967                 if (copy_from_user(&t, useraddr, sizeof(t)))
1968                         return -EFAULT;
1969
1970                 tp = (const struct trace_params *)&t.sip;
1971                 if (t.config_tx)
1972                         t3_config_trace_filter(adapter, tp, 0,
1973                                                 t.invert_match,
1974                                                 t.trace_tx);
1975                 if (t.config_rx)
1976                         t3_config_trace_filter(adapter, tp, 1,
1977                                                 t.invert_match,
1978                                                 t.trace_rx);
1979                 break;
1980         }
1981         default:
1982                 return -EOPNOTSUPP;
1983         }
1984         return 0;
1985 }
1986
1987 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
1988 {
1989         struct mii_ioctl_data *data = if_mii(req);
1990         struct port_info *pi = netdev_priv(dev);
1991         struct adapter *adapter = pi->adapter;
1992         int ret, mmd;
1993
1994         switch (cmd) {
1995         case SIOCGMIIPHY:
1996                 data->phy_id = pi->phy.addr;
1997                 /* FALLTHRU */
1998         case SIOCGMIIREG:{
1999                 u32 val;
2000                 struct cphy *phy = &pi->phy;
2001
2002                 if (!phy->mdio_read)
2003                         return -EOPNOTSUPP;
2004                 if (is_10G(adapter)) {
2005                         mmd = data->phy_id >> 8;
2006                         if (!mmd)
2007                                 mmd = MDIO_DEV_PCS;
2008                         else if (mmd > MDIO_DEV_XGXS)
2009                                 return -EINVAL;
2010
2011                         ret =
2012                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2013                                                 mmd, data->reg_num, &val);
2014                 } else
2015                         ret =
2016                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2017                                                 0, data->reg_num & 0x1f,
2018                                                 &val);
2019                 if (!ret)
2020                         data->val_out = val;
2021                 break;
2022         }
2023         case SIOCSMIIREG:{
2024                 struct cphy *phy = &pi->phy;
2025
2026                 if (!capable(CAP_NET_ADMIN))
2027                         return -EPERM;
2028                 if (!phy->mdio_write)
2029                         return -EOPNOTSUPP;
2030                 if (is_10G(adapter)) {
2031                         mmd = data->phy_id >> 8;
2032                         if (!mmd)
2033                                 mmd = MDIO_DEV_PCS;
2034                         else if (mmd > MDIO_DEV_XGXS)
2035                                 return -EINVAL;
2036
2037                         ret =
2038                                 phy->mdio_write(adapter,
2039                                                 data->phy_id & 0x1f, mmd,
2040                                                 data->reg_num,
2041                                                 data->val_in);
2042                 } else
2043                         ret =
2044                                 phy->mdio_write(adapter,
2045                                                 data->phy_id & 0x1f, 0,
2046                                                 data->reg_num & 0x1f,
2047                                                 data->val_in);
2048                 break;
2049         }
2050         case SIOCCHIOCTL:
2051                 return cxgb_extension_ioctl(dev, req->ifr_data);
2052         default:
2053                 return -EOPNOTSUPP;
2054         }
2055         return ret;
2056 }
2057
2058 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2059 {
2060         struct port_info *pi = netdev_priv(dev);
2061         struct adapter *adapter = pi->adapter;
2062         int ret;
2063
2064         if (new_mtu < 81)       /* accommodate SACK */
2065                 return -EINVAL;
2066         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2067                 return ret;
2068         dev->mtu = new_mtu;
2069         init_port_mtus(adapter);
2070         if (adapter->params.rev == 0 && offload_running(adapter))
2071                 t3_load_mtus(adapter, adapter->params.mtus,
2072                              adapter->params.a_wnd, adapter->params.b_wnd,
2073                              adapter->port[0]->mtu);
2074         return 0;
2075 }
2076
2077 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2078 {
2079         struct port_info *pi = netdev_priv(dev);
2080         struct adapter *adapter = pi->adapter;
2081         struct sockaddr *addr = p;
2082
2083         if (!is_valid_ether_addr(addr->sa_data))
2084                 return -EINVAL;
2085
2086         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2087         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2088         if (offload_running(adapter))
2089                 write_smt_entry(adapter, pi->port_id);
2090         return 0;
2091 }
2092
2093 /**
2094  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2095  * @adap: the adapter
2096  * @p: the port
2097  *
2098  * Ensures that current Rx processing on any of the queues associated with
2099  * the given port completes before returning.  We do this by acquiring and
2100  * releasing the locks of the response queues associated with the port.
2101  */
2102 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2103 {
2104         int i;
2105
2106         for (i = 0; i < p->nqsets; i++) {
2107                 struct sge_rspq *q = &adap->sge.qs[i + p->first_qset].rspq;
2108
2109                 spin_lock_irq(&q->lock);
2110                 spin_unlock_irq(&q->lock);
2111         }
2112 }
2113
2114 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2115 {
2116         struct port_info *pi = netdev_priv(dev);
2117         struct adapter *adapter = pi->adapter;
2118
2119         pi->vlan_grp = grp;
2120         if (adapter->params.rev > 0)
2121                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2122         else {
2123                 /* single control for all ports */
2124                 unsigned int i, have_vlans = 0;
2125                 for_each_port(adapter, i)
2126                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2127
2128                 t3_set_vlan_accel(adapter, 1, have_vlans);
2129         }
2130         t3_synchronize_rx(adapter, pi);
2131 }
2132
2133 #ifdef CONFIG_NET_POLL_CONTROLLER
2134 static void cxgb_netpoll(struct net_device *dev)
2135 {
2136         struct port_info *pi = netdev_priv(dev);
2137         struct adapter *adapter = pi->adapter;
2138         int qidx;
2139
2140         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2141                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2142                 void *source;
2143
2144                 if (adapter->flags & USING_MSIX)
2145                         source = qs;
2146                 else
2147                         source = adapter;
2148
2149                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2150         }
2151 }
2152 #endif
2153
2154 /*
2155  * Periodic accumulation of MAC statistics.
2156  */
2157 static void mac_stats_update(struct adapter *adapter)
2158 {
2159         int i;
2160
2161         for_each_port(adapter, i) {
2162                 struct net_device *dev = adapter->port[i];
2163                 struct port_info *p = netdev_priv(dev);
2164
2165                 if (netif_running(dev)) {
2166                         spin_lock(&adapter->stats_lock);
2167                         t3_mac_update_stats(&p->mac);
2168                         spin_unlock(&adapter->stats_lock);
2169                 }
2170         }
2171 }
2172
2173 static void check_link_status(struct adapter *adapter)
2174 {
2175         int i;
2176
2177         for_each_port(adapter, i) {
2178                 struct net_device *dev = adapter->port[i];
2179                 struct port_info *p = netdev_priv(dev);
2180
2181                 if (!(p->port_type->caps & SUPPORTED_IRQ) && netif_running(dev))
2182                         t3_link_changed(adapter, i);
2183         }
2184 }
2185
2186 static void check_t3b2_mac(struct adapter *adapter)
2187 {
2188         int i;
2189
2190         if (!rtnl_trylock())    /* synchronize with ifdown */
2191                 return;
2192
2193         for_each_port(adapter, i) {
2194                 struct net_device *dev = adapter->port[i];
2195                 struct port_info *p = netdev_priv(dev);
2196                 int status;
2197
2198                 if (!netif_running(dev))
2199                         continue;
2200
2201                 status = 0;
2202                 if (netif_running(dev) && netif_carrier_ok(dev))
2203                         status = t3b2_mac_watchdog_task(&p->mac);
2204                 if (status == 1)
2205                         p->mac.stats.num_toggled++;
2206                 else if (status == 2) {
2207                         struct cmac *mac = &p->mac;
2208
2209                         t3_mac_set_mtu(mac, dev->mtu);
2210                         t3_mac_set_address(mac, 0, dev->dev_addr);
2211                         cxgb_set_rxmode(dev);
2212                         t3_link_start(&p->phy, mac, &p->link_config);
2213                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2214                         t3_port_intr_enable(adapter, p->port_id);
2215                         p->mac.stats.num_resets++;
2216                 }
2217         }
2218         rtnl_unlock();
2219 }
2220
2221
2222 static void t3_adap_check_task(struct work_struct *work)
2223 {
2224         struct adapter *adapter = container_of(work, struct adapter,
2225                                                adap_check_task.work);
2226         const struct adapter_params *p = &adapter->params;
2227
2228         adapter->check_task_cnt++;
2229
2230         /* Check link status for PHYs without interrupts */
2231         if (p->linkpoll_period)
2232                 check_link_status(adapter);
2233
2234         /* Accumulate MAC stats if needed */
2235         if (!p->linkpoll_period ||
2236             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2237             p->stats_update_period) {
2238                 mac_stats_update(adapter);
2239                 adapter->check_task_cnt = 0;
2240         }
2241
2242         if (p->rev == T3_REV_B2)
2243                 check_t3b2_mac(adapter);
2244
2245         /* Schedule the next check update if any port is active. */
2246         spin_lock(&adapter->work_lock);
2247         if (adapter->open_device_map & PORT_MASK)
2248                 schedule_chk_task(adapter);
2249         spin_unlock(&adapter->work_lock);
2250 }
2251
2252 /*
2253  * Processes external (PHY) interrupts in process context.
2254  */
2255 static void ext_intr_task(struct work_struct *work)
2256 {
2257         struct adapter *adapter = container_of(work, struct adapter,
2258                                                ext_intr_handler_task);
2259
2260         t3_phy_intr_handler(adapter);
2261
2262         /* Now reenable external interrupts */
2263         spin_lock_irq(&adapter->work_lock);
2264         if (adapter->slow_intr_mask) {
2265                 adapter->slow_intr_mask |= F_T3DBG;
2266                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2267                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2268                              adapter->slow_intr_mask);
2269         }
2270         spin_unlock_irq(&adapter->work_lock);
2271 }
2272
2273 /*
2274  * Interrupt-context handler for external (PHY) interrupts.
2275  */
2276 void t3_os_ext_intr_handler(struct adapter *adapter)
2277 {
2278         /*
2279          * Schedule a task to handle external interrupts as they may be slow
2280          * and we use a mutex to protect MDIO registers.  We disable PHY
2281          * interrupts in the meantime and let the task reenable them when
2282          * it's done.
2283          */
2284         spin_lock(&adapter->work_lock);
2285         if (adapter->slow_intr_mask) {
2286                 adapter->slow_intr_mask &= ~F_T3DBG;
2287                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2288                              adapter->slow_intr_mask);
2289                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2290         }
2291         spin_unlock(&adapter->work_lock);
2292 }
2293
2294 void t3_fatal_err(struct adapter *adapter)
2295 {
2296         unsigned int fw_status[4];
2297
2298         if (adapter->flags & FULL_INIT_DONE) {
2299                 t3_sge_stop(adapter);
2300                 t3_intr_disable(adapter);
2301         }
2302         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2303         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2304                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2305                          fw_status[0], fw_status[1],
2306                          fw_status[2], fw_status[3]);
2307
2308 }
2309
2310 static int __devinit cxgb_enable_msix(struct adapter *adap)
2311 {
2312         struct msix_entry entries[SGE_QSETS + 1];
2313         int i, err;
2314
2315         for (i = 0; i < ARRAY_SIZE(entries); ++i)
2316                 entries[i].entry = i;
2317
2318         err = pci_enable_msix(adap->pdev, entries, ARRAY_SIZE(entries));
2319         if (!err) {
2320                 for (i = 0; i < ARRAY_SIZE(entries); ++i)
2321                         adap->msix_info[i].vec = entries[i].vector;
2322         } else if (err > 0)
2323                 dev_info(&adap->pdev->dev,
2324                        "only %d MSI-X vectors left, not using MSI-X\n", err);
2325         return err;
2326 }
2327
2328 static void __devinit print_port_info(struct adapter *adap,
2329                                       const struct adapter_info *ai)
2330 {
2331         static const char *pci_variant[] = {
2332                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
2333         };
2334
2335         int i;
2336         char buf[80];
2337
2338         if (is_pcie(adap))
2339                 snprintf(buf, sizeof(buf), "%s x%d",
2340                          pci_variant[adap->params.pci.variant],
2341                          adap->params.pci.width);
2342         else
2343                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
2344                          pci_variant[adap->params.pci.variant],
2345                          adap->params.pci.speed, adap->params.pci.width);
2346
2347         for_each_port(adap, i) {
2348                 struct net_device *dev = adap->port[i];
2349                 const struct port_info *pi = netdev_priv(dev);
2350
2351                 if (!test_bit(i, &adap->registered_device_map))
2352                         continue;
2353                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
2354                        dev->name, ai->desc, pi->port_type->desc,
2355                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
2356                        (adap->flags & USING_MSIX) ? " MSI-X" :
2357                        (adap->flags & USING_MSI) ? " MSI" : "");
2358                 if (adap->name == dev->name && adap->params.vpd.mclk)
2359                         printk(KERN_INFO "%s: %uMB CM, %uMB PMTX, %uMB PMRX\n",
2360                                adap->name, t3_mc7_size(&adap->cm) >> 20,
2361                                t3_mc7_size(&adap->pmtx) >> 20,
2362                                t3_mc7_size(&adap->pmrx) >> 20);
2363         }
2364 }
2365
2366 static int __devinit init_one(struct pci_dev *pdev,
2367                               const struct pci_device_id *ent)
2368 {
2369         static int version_printed;
2370
2371         int i, err, pci_using_dac = 0;
2372         unsigned long mmio_start, mmio_len;
2373         const struct adapter_info *ai;
2374         struct adapter *adapter = NULL;
2375         struct port_info *pi;
2376
2377         if (!version_printed) {
2378                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
2379                 ++version_printed;
2380         }
2381
2382         if (!cxgb3_wq) {
2383                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
2384                 if (!cxgb3_wq) {
2385                         printk(KERN_ERR DRV_NAME
2386                                ": cannot initialize work queue\n");
2387                         return -ENOMEM;
2388                 }
2389         }
2390
2391         err = pci_request_regions(pdev, DRV_NAME);
2392         if (err) {
2393                 /* Just info, some other driver may have claimed the device. */
2394                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
2395                 return err;
2396         }
2397
2398         err = pci_enable_device(pdev);
2399         if (err) {
2400                 dev_err(&pdev->dev, "cannot enable PCI device\n");
2401                 goto out_release_regions;
2402         }
2403
2404         if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
2405                 pci_using_dac = 1;
2406                 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2407                 if (err) {
2408                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
2409                                "coherent allocations\n");
2410                         goto out_disable_device;
2411                 }
2412         } else if ((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK)) != 0) {
2413                 dev_err(&pdev->dev, "no usable DMA configuration\n");
2414                 goto out_disable_device;
2415         }
2416
2417         pci_set_master(pdev);
2418
2419         mmio_start = pci_resource_start(pdev, 0);
2420         mmio_len = pci_resource_len(pdev, 0);
2421         ai = t3_get_adapter_info(ent->driver_data);
2422
2423         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2424         if (!adapter) {
2425                 err = -ENOMEM;
2426                 goto out_disable_device;
2427         }
2428
2429         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
2430         if (!adapter->regs) {
2431                 dev_err(&pdev->dev, "cannot map device registers\n");
2432                 err = -ENOMEM;
2433                 goto out_free_adapter;
2434         }
2435
2436         adapter->pdev = pdev;
2437         adapter->name = pci_name(pdev);
2438         adapter->msg_enable = dflt_msg_enable;
2439         adapter->mmio_len = mmio_len;
2440
2441         mutex_init(&adapter->mdio_lock);
2442         spin_lock_init(&adapter->work_lock);
2443         spin_lock_init(&adapter->stats_lock);
2444
2445         INIT_LIST_HEAD(&adapter->adapter_list);
2446         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
2447         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
2448
2449         for (i = 0; i < ai->nports; ++i) {
2450                 struct net_device *netdev;
2451
2452                 netdev = alloc_etherdev(sizeof(struct port_info));
2453                 if (!netdev) {
2454                         err = -ENOMEM;
2455                         goto out_free_dev;
2456                 }
2457
2458                 SET_MODULE_OWNER(netdev);
2459                 SET_NETDEV_DEV(netdev, &pdev->dev);
2460
2461                 adapter->port[i] = netdev;
2462                 pi = netdev_priv(netdev);
2463                 pi->adapter = adapter;
2464                 pi->rx_csum_offload = 1;
2465                 pi->nqsets = 1;
2466                 pi->first_qset = i;
2467                 pi->activity = 0;
2468                 pi->port_id = i;
2469                 netif_carrier_off(netdev);
2470                 netdev->irq = pdev->irq;
2471                 netdev->mem_start = mmio_start;
2472                 netdev->mem_end = mmio_start + mmio_len - 1;
2473                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
2474                 netdev->features |= NETIF_F_LLTX;
2475                 if (pci_using_dac)
2476                         netdev->features |= NETIF_F_HIGHDMA;
2477
2478                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
2479                 netdev->vlan_rx_register = vlan_rx_register;
2480
2481                 netdev->open = cxgb_open;
2482                 netdev->stop = cxgb_close;
2483                 netdev->hard_start_xmit = t3_eth_xmit;
2484                 netdev->get_stats = cxgb_get_stats;
2485                 netdev->set_multicast_list = cxgb_set_rxmode;
2486                 netdev->do_ioctl = cxgb_ioctl;
2487                 netdev->change_mtu = cxgb_change_mtu;
2488                 netdev->set_mac_address = cxgb_set_mac_addr;
2489 #ifdef CONFIG_NET_POLL_CONTROLLER
2490                 netdev->poll_controller = cxgb_netpoll;
2491 #endif
2492
2493                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
2494         }
2495
2496         pci_set_drvdata(pdev, adapter);
2497         if (t3_prep_adapter(adapter, ai, 1) < 0) {
2498                 err = -ENODEV;
2499                 goto out_free_dev;
2500         }
2501
2502         /*
2503          * The card is now ready to go.  If any errors occur during device
2504          * registration we do not fail the whole card but rather proceed only
2505          * with the ports we manage to register successfully.  However we must
2506          * register at least one net device.
2507          */
2508         for_each_port(adapter, i) {
2509                 err = register_netdev(adapter->port[i]);
2510                 if (err)
2511                         dev_warn(&pdev->dev,
2512                                  "cannot register net device %s, skipping\n",
2513                                  adapter->port[i]->name);
2514                 else {
2515                         /*
2516                          * Change the name we use for messages to the name of
2517                          * the first successfully registered interface.
2518                          */
2519                         if (!adapter->registered_device_map)
2520                                 adapter->name = adapter->port[i]->name;
2521
2522                         __set_bit(i, &adapter->registered_device_map);
2523                 }
2524         }
2525         if (!adapter->registered_device_map) {
2526                 dev_err(&pdev->dev, "could not register any net devices\n");
2527                 goto out_free_dev;
2528         }
2529
2530         /* Driver's ready. Reflect it on LEDs */
2531         t3_led_ready(adapter);
2532
2533         if (is_offload(adapter)) {
2534                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
2535                 cxgb3_adapter_ofld(adapter);
2536         }
2537
2538         /* See what interrupts we'll be using */
2539         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
2540                 adapter->flags |= USING_MSIX;
2541         else if (msi > 0 && pci_enable_msi(pdev) == 0)
2542                 adapter->flags |= USING_MSI;
2543
2544         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
2545                                  &cxgb3_attr_group);
2546
2547         print_port_info(adapter, ai);
2548         return 0;
2549
2550 out_free_dev:
2551         iounmap(adapter->regs);
2552         for (i = ai->nports - 1; i >= 0; --i)
2553                 if (adapter->port[i])
2554                         free_netdev(adapter->port[i]);
2555
2556 out_free_adapter:
2557         kfree(adapter);
2558
2559 out_disable_device:
2560         pci_disable_device(pdev);
2561 out_release_regions:
2562         pci_release_regions(pdev);
2563         pci_set_drvdata(pdev, NULL);
2564         return err;
2565 }
2566
2567 static void __devexit remove_one(struct pci_dev *pdev)
2568 {
2569         struct adapter *adapter = pci_get_drvdata(pdev);
2570
2571         if (adapter) {
2572                 int i;
2573
2574                 t3_sge_stop(adapter);
2575                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
2576                                    &cxgb3_attr_group);
2577
2578                 for_each_port(adapter, i)
2579                     if (test_bit(i, &adapter->registered_device_map))
2580                         unregister_netdev(adapter->port[i]);
2581
2582                 if (is_offload(adapter)) {
2583                         cxgb3_adapter_unofld(adapter);
2584                         if (test_bit(OFFLOAD_DEVMAP_BIT,
2585                                      &adapter->open_device_map))
2586                                 offload_close(&adapter->tdev);
2587                 }
2588
2589                 t3_free_sge_resources(adapter);
2590                 cxgb_disable_msi(adapter);
2591
2592                 for_each_port(adapter, i)
2593                         if (adapter->port[i])
2594                                 free_netdev(adapter->port[i]);
2595
2596                 iounmap(adapter->regs);
2597                 kfree(adapter);
2598                 pci_release_regions(pdev);
2599                 pci_disable_device(pdev);
2600                 pci_set_drvdata(pdev, NULL);
2601         }
2602 }
2603
2604 static struct pci_driver driver = {
2605         .name = DRV_NAME,
2606         .id_table = cxgb3_pci_tbl,
2607         .probe = init_one,
2608         .remove = __devexit_p(remove_one),
2609 };
2610
2611 static int __init cxgb3_init_module(void)
2612 {
2613         int ret;
2614
2615         cxgb3_offload_init();
2616
2617         ret = pci_register_driver(&driver);
2618         return ret;
2619 }
2620
2621 static void __exit cxgb3_cleanup_module(void)
2622 {
2623         pci_unregister_driver(&driver);
2624         if (cxgb3_wq)
2625                 destroy_workqueue(cxgb3_wq);
2626 }
2627
2628 module_init(cxgb3_init_module);
2629 module_exit(cxgb3_cleanup_module);