err.no Git - linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41
  42 #include "nes.h"
  43
  44 static u32 crit_err_count;
  45 u32 int_mod_timer_init;
  46 u32 int_mod_cq_depth_256;
  47 u32 int_mod_cq_depth_128;
  48 u32 int_mod_cq_depth_32;
  49 u32 int_mod_cq_depth_24;
  50 u32 int_mod_cq_depth_16;
  51 u32 int_mod_cq_depth_4;
  52 u32 int_mod_cq_depth_1;
  53
  54 #include "nes_cm.h"
  55
  56 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  57 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  58 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  59                            u8 OneG_Mode);
  60 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  61 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  62 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  63 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  64                                    struct nes_hw_aeqe *aeqe);
  65 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  66 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  67
  68 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  69 static unsigned char *nes_iwarp_state_str[] = {
  70         "Non-Existant",
  71         "Idle",
  72         "RTS",
  73         "Closing",
  74         "RSVD1",
  75         "Terminate",
  76         "Error",
  77         "RSVD2",
  78 };
  79
  80 static unsigned char *nes_tcp_state_str[] = {
  81         "Non-Existant",
  82         "Closed",
  83         "Listen",
  84         "SYN Sent",
  85         "SYN Rcvd",
  86         "Established",
  87         "Close Wait",
  88         "FIN Wait 1",
  89         "Closing",
  90         "Last Ack",
  91         "FIN Wait 2",
  92         "Time Wait",
  93         "RSVD1",
  94         "RSVD2",
  95         "RSVD3",
  96         "RSVD4",
  97 };
  98 #endif
  99
 100
 101 /**
 102  * nes_nic_init_timer_defaults
 103  */
 104 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 105 {
 106         unsigned long flags;
 107         struct nes_adapter *nesadapter = nesdev->nesadapter;
 108         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 109
 110         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 111
 112         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 113         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 114         if (jumbomode) {
 115                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 116                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 117                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 118         } else {
 119                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 120                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 121                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 122         }
 123
 124         /* todo use netdev->mtu to set thresholds */
 125         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 126 }
 127
 128
 129 /**
 130  * nes_nic_init_timer
 131  */
 132 static void  nes_nic_init_timer(struct nes_device *nesdev)
 133 {
 134         unsigned long flags;
 135         struct nes_adapter *nesadapter = nesdev->nesadapter;
 136         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 137
 138         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 139
 140         if (shared_timer->timer_in_use_old == 0) {
 141                 nesdev->deepcq_count = 0;
 142                 shared_timer->timer_direction_upward = 0;
 143                 shared_timer->timer_direction_downward = 0;
 144                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 145                 shared_timer->timer_in_use_old = 0;
 146
 147         }
 148         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 149                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 150                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 151                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 152         }
 153         /* todo use netdev->mtu to set thresholds */
 154         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 155 }
 156
 157
 158 /**
 159  * nes_nic_tune_timer
 160  */
 161 static void nes_nic_tune_timer(struct nes_device *nesdev)
 162 {
 163         unsigned long flags;
 164         struct nes_adapter *nesadapter = nesdev->nesadapter;
 165         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 166         u16 cq_count = nesdev->currcq_count;
 167
 168         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 169
 170         if (shared_timer->cq_count_old <= cq_count)
 171                 shared_timer->cq_direction_downward = 0;
 172         else
 173                 shared_timer->cq_direction_downward++;
 174         shared_timer->cq_count_old = cq_count;
 175         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 176                 if (cq_count <= shared_timer->threshold_low &&
 177                     shared_timer->threshold_low > 4) {
 178                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 179                         shared_timer->cq_direction_downward=0;
 180                         nesdev->currcq_count = 0;
 181                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 182                         return;
 183                 }
 184         }
 185
 186         if (cq_count > 1) {
 187                 nesdev->deepcq_count += cq_count;
 188                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 189                         shared_timer->timer_direction_upward++;
 190                         shared_timer->timer_direction_downward = 0;
 191                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 192                         shared_timer->timer_direction_upward = 0;
 193                         shared_timer->timer_direction_downward = 0;
 194                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 195                         shared_timer->timer_direction_downward++;
 196                         shared_timer->timer_direction_upward = 0;
 197                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 198                         shared_timer->timer_in_use -= 2;
 199                         shared_timer->timer_direction_upward = 0;
 200                         shared_timer->timer_direction_downward++;
 201                 } else {
 202                         shared_timer->timer_in_use -= 4;
 203                         shared_timer->timer_direction_upward = 0;
 204                         shared_timer->timer_direction_downward++;
 205                 }
 206
 207                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 208                         shared_timer->timer_in_use += 3;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward = 0;
 211                 }
 212                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 213                         shared_timer->timer_in_use -= 4 ;
 214                         shared_timer->timer_direction_downward = 0;
 215                         shared_timer->timer_direction_upward = 0;
 216                 }
 217         }
 218
 219         /* boundary checking */
 220         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 221                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 222         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 223                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 224         }
 225
 226         nesdev->currcq_count = 0;
 227
 228         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 229 }
 230
 231
 232 /**
 233  * nes_init_adapter - initialize adapter
 234  */
 235 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 236         struct nes_adapter *nesadapter = NULL;
 237         unsigned long num_pds;
 238         u32 u32temp;
 239         u32 port_count;
 240         u16 max_rq_wrs;
 241         u16 max_sq_wrs;
 242         u32 max_mr;
 243         u32 max_256pbl;
 244         u32 max_4kpbl;
 245         u32 max_qp;
 246         u32 max_irrq;
 247         u32 max_cq;
 248         u32 hte_index_mask;
 249         u32 adapter_size;
 250         u32 arp_table_size;
 251         u16 vendor_id;
 252         u8  OneG_Mode;
 253         u8  func_index;
 254
 255         /* search the list of existing adapters */
 256         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 257                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 258                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 259                                 nesdev->pcidev->devfn,
 260                                 PCI_SLOT(nesadapter->devfn),
 261                                 nesadapter->bus_number,
 262                                 PCI_SLOT(nesdev->pcidev->devfn),
 263                                 nesdev->pcidev->bus->number );
 264                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 265                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 266                         nesadapter->ref_count++;
 267                         return nesadapter;
 268                 }
 269         }
 270
 271         /* no adapter found */
 272         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 273         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 274                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 275                                 hw_rev);
 276                 return NULL;
 277         }
 278
 279         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 280                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 281                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 282                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 283                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 284
 285         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 286
 287
 288         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 289                 return NULL;
 290         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 291                 return NULL;
 292         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 293
 294         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 295         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 296
 297         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 298         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 299                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 300                                 max_qp, u32temp);
 301                 max_qp = (u32)1 << (u32temp & 0x001f);
 302         }
 303
 304         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 305         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 306                         max_qp, hte_index_mask);
 307
 308         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 309
 310         max_irrq = 1 << (u32temp & 0x001f);
 311
 312         if (max_qp > max_irrq) {
 313                 max_qp = max_irrq;
 314                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 315                                 max_qp);
 316         }
 317
 318         /* there should be no reason to allocate more pds than qps */
 319         if (num_pds > max_qp)
 320                 num_pds = max_qp;
 321
 322         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 323         max_mr = (u32)8192 << (u32temp & 0x7);
 324
 325         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 326         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 327         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 328         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 329
 330         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 331         arp_table_size = 1 << u32temp;
 332
 333         adapter_size = (sizeof(struct nes_adapter) +
 334                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 335         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 336         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 337         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 340         adapter_size += sizeof(struct nes_qp **) * max_qp;
 341
 342         /* allocate a new adapter struct */
 343         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 344         if (nesadapter == NULL) {
 345                 return NULL;
 346         }
 347
 348         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 349                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 350
 351         /* populate the new nesadapter */
 352         nesadapter->devfn = nesdev->pcidev->devfn;
 353         nesadapter->bus_number = nesdev->pcidev->bus->number;
 354         nesadapter->ref_count = 1;
 355         nesadapter->timer_int_req = 0xffff0000;
 356         nesadapter->OneG_Mode = OneG_Mode;
 357         nesadapter->doorbell_start = nesdev->doorbell_region;
 358
 359         /* nesadapter->tick_delta = clk_divisor; */
 360         nesadapter->hw_rev = hw_rev;
 361         nesadapter->port_count = port_count;
 362
 363         nesadapter->max_qp = max_qp;
 364         nesadapter->hte_index_mask = hte_index_mask;
 365         nesadapter->max_irrq = max_irrq;
 366         nesadapter->max_mr = max_mr;
 367         nesadapter->max_256pbl = max_256pbl - 1;
 368         nesadapter->max_4kpbl = max_4kpbl - 1;
 369         nesadapter->max_cq = max_cq;
 370         nesadapter->free_256pbl = max_256pbl - 1;
 371         nesadapter->free_4kpbl = max_4kpbl - 1;
 372         nesadapter->max_pd = num_pds;
 373         nesadapter->arp_table_size = arp_table_size;
 374
 375         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 376         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 377                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 378                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 379                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 380         } else {
 381                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 382                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 383                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 384                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 385         }
 386         /* Setup and enable the periodic timer */
 387         if (nesadapter->et_rx_coalesce_usecs_irq)
 388                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 389                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 390         else
 391                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 392
 393         nesadapter->base_pd = 1;
 394
 395         nesadapter->device_cap_flags =
 396                 IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
 397
 398         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 399                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 400         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 401         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 402         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 403         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 404         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 405
 406
 407         /* mark the usual suspect QPs and CQs as in use */
 408         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 409                 set_bit(u32temp, nesadapter->allocated_qps);
 410                 set_bit(u32temp, nesadapter->allocated_cqs);
 411         }
 412
 413         for (u32temp = 0; u32temp < 20; u32temp++)
 414                 set_bit(u32temp, nesadapter->allocated_pds);
 415         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 416
 417         max_rq_wrs = ((u32temp >> 8) & 3);
 418         switch (max_rq_wrs) {
 419                 case 0:
 420                         max_rq_wrs = 4;
 421                         break;
 422                 case 1:
 423                         max_rq_wrs = 16;
 424                         break;
 425                 case 2:
 426                         max_rq_wrs = 32;
 427                         break;
 428                 case 3:
 429                         max_rq_wrs = 512;
 430                         break;
 431         }
 432
 433         max_sq_wrs = (u32temp & 3);
 434         switch (max_sq_wrs) {
 435                 case 0:
 436                         max_sq_wrs = 4;
 437                         break;
 438                 case 1:
 439                         max_sq_wrs = 16;
 440                         break;
 441                 case 2:
 442                         max_sq_wrs = 32;
 443                         break;
 444                 case 3:
 445                         max_sq_wrs = 512;
 446                         break;
 447         }
 448         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 449         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 450
 451         nesadapter->max_sge = 4;
 452         nesadapter->max_cqe = 32767;
 453
 454         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 455                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 456                 kfree(nesadapter);
 457                 return NULL;
 458         }
 459
 460         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 461         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 462                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 463
 464         /* setup port configuration */
 465         if (nesadapter->port_count == 1) {
 466                 u32temp = 0x00000000;
 467                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 468                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 469                 else
 470                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 471         } else {
 472                 if (nesadapter->port_count == 2)
 473                         u32temp = 0x00000044;
 474                 else
 475                         u32temp = 0x000000e4;
 476                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 477         }
 478
 479         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 480         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 481                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 482
 483         spin_lock_init(&nesadapter->resource_lock);
 484         spin_lock_init(&nesadapter->phy_lock);
 485         spin_lock_init(&nesadapter->pbl_lock);
 486         spin_lock_init(&nesadapter->periodic_timer_lock);
 487
 488         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 489         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 490         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 491         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 492
 493         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 494                 u32 pcs_control_status0, pcs_control_status1;
 495                 u32 reset_value;
 496                 u32 i = 0;
 497                 u32 int_cnt = 0;
 498                 u32 ext_cnt = 0;
 499                 unsigned long flags;
 500                 u32 j = 0;
 501
 502                 pcs_control_status0 = nes_read_indexed(nesdev,
 503                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 504                 pcs_control_status1 = nes_read_indexed(nesdev,
 505                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 506
 507                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 508                         pcs_control_status0 = nes_read_indexed(nesdev,
 509                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 510                         pcs_control_status1 = nes_read_indexed(nesdev,
 511                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 512                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 513                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 514                                 int_cnt++;
 515                         msleep(1);
 516                 }
 517                 if (int_cnt > 1) {
 518                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 519                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 520                         mh_detected++;
 521                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 522                         reset_value |= 0x0000003d;
 523                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 524
 525                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 526                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 527                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 528
 529                         pcs_control_status0 = nes_read_indexed(nesdev,
 530                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 531                         pcs_control_status1 = nes_read_indexed(nesdev,
 532                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 533
 534                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 535                                 pcs_control_status0 = nes_read_indexed(nesdev,
 536                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 537                                 pcs_control_status1 = nes_read_indexed(nesdev,
 538                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 539                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 540                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 541                                         if (++ext_cnt > int_cnt) {
 542                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 543                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 544                                                                 0x0000F0C8);
 545                                                 mh_detected++;
 546                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 547                                                 reset_value |= 0x0000003d;
 548                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 549
 550                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 551                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 552                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 553                                                 break;
 554                                         }
 555                                 }
 556                                 msleep(1);
 557                         }
 558                 }
 559         }
 560
 561         if (nesadapter->hw_rev == NE020_REV) {
 562                 init_timer(&nesadapter->mh_timer);
 563                 nesadapter->mh_timer.function = nes_mh_fix;
 564                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 565                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 566                 add_timer(&nesadapter->mh_timer);
 567         } else {
 568                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 569         }
 570
 571         init_timer(&nesadapter->lc_timer);
 572         nesadapter->lc_timer.function = nes_clc;
 573         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 574         nesadapter->lc_timer.data = (unsigned long)nesdev;
 575         add_timer(&nesadapter->lc_timer);
 576
 577         list_add_tail(&nesadapter->list, &nes_adapter_list);
 578
 579         for (func_index = 0; func_index < 8; func_index++) {
 580                 pci_bus_read_config_word(nesdev->pcidev->bus,
 581                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 582                                         func_index), 0, &vendor_id);
 583                 if (vendor_id == 0xffff)
 584                         break;
 585         }
 586         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 587                 func_index, pci_name(nesdev->pcidev));
 588         nesadapter->adapter_fcn_count = func_index;
 589
 590         return nesadapter;
 591 }
 592
 593
 594 /**
 595  * nes_reset_adapter_ne020
 596  */
 597 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 598 {
 599         u32 port_count;
 600         u32 u32temp;
 601         u32 i;
 602
 603         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 604         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 605         /* TODO: assuming that both SERDES are set the same for now */
 606         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 607         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 608                         u32temp, port_count);
 609         if (*OneG_Mode)
 610                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 611         u32temp &= 0xff00ffc0;
 612         switch (port_count) {
 613                 case 1:
 614                         u32temp |= 0x00ee0000;
 615                         break;
 616                 case 2:
 617                         u32temp |= 0x00cc0000;
 618                         break;
 619                 case 4:
 620                         u32temp |= 0x00000000;
 621                         break;
 622                 default:
 623                         return 0;
 624                         break;
 625         }
 626
 627         /* check and do full reset if needed */
 628         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 629                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 630                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 631
 632                 i = 0;
 633                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 634                         mdelay(1);
 635                 if (i >= 10000) {
 636                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 637                         return 0;
 638                 }
 639         }
 640
 641         /* port reset */
 642         switch (port_count) {
 643                 case 1:
 644                         u32temp |= 0x00ee0010;
 645                         break;
 646                 case 2:
 647                         u32temp |= 0x00cc0030;
 648                         break;
 649                 case 4:
 650                         u32temp |= 0x00000030;
 651                         break;
 652         }
 653
 654         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 655         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 656
 657         i = 0;
 658         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 659                 mdelay(1);
 660         if (i >= 10000) {
 661                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 662                 return 0;
 663         }
 664
 665         /* serdes 0 */
 666         i = 0;
 667         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 668                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 669                 mdelay(1);
 670         if (i >= 5000) {
 671                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 672                 return 0;
 673         }
 674
 675         /* serdes 1 */
 676         if (port_count > 1) {
 677                 i = 0;
 678                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 679                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 680                         mdelay(1);
 681                 if (i >= 5000) {
 682                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 683                         return 0;
 684                 }
 685         }
 686
 687
 688
 689         i = 0;
 690         while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 691                 mdelay(1);
 692         if (i >= 10000) {
 693                 printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 694                                 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 695                 return 0;
 696         }
 697
 698         return port_count;
 699 }
 700
 701
 702 /**
 703  * nes_init_serdes
 704  */
 705 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 706                            u8 OneG_Mode)
 707 {
 708         int i;
 709         u32 u32temp;
 710
 711         if (hw_rev != NE020_REV) {
 712                 /* init serdes 0 */
 713
 714                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 715                 if (!OneG_Mode)
 716                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 717                 if (port_count > 1) {
 718                         /* init serdes 1 */
 719                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 720                         if (!OneG_Mode)
 721                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 722                         }
 723         } else {
 724                 /* init serdes 0 */
 725                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 726                 i = 0;
 727                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 728                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 729                         mdelay(1);
 730                 if (i >= 5000) {
 731                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 732                         return 1;
 733                 }
 734                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 735                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 736                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 737                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 738                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 739                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 740                 if (OneG_Mode)
 741                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 742                 else
 743                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 744
 745                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 746                 if (port_count > 1) {
 747                         /* init serdes 1 */
 748                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 749                         i = 0;
 750                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 751                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 752                                 mdelay(1);
 753                         if (i >= 5000) {
 754                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 755                                 /* return 1; */
 756                         }
 757                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 762                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 764                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 765                 }
 766         }
 767         return 0;
 768 }
 769
 770
 771 /**
 772  * nes_init_csr_ne020
 773  * Initialize registers for ne020 hardware
 774  */
 775 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 776 {
 777         u32 u32temp;
 778
 779         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 780
 781         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 782         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 783         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 784         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 785         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 786         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 787         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 788         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 789
 790         /* TODO: move these MAC register settings to NIC bringup */
 791         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 792         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 793         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 794         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 795         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 796         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 797         if (port_count > 1) {
 798                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 799                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 800                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 801                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 802                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 803                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 804                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 805         }
 806         if (port_count > 2) {
 807                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 808                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 809                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 810                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 811                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 812                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 813                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 814
 815                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 816                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 817                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 818                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 819                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 820                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 821                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 822         }
 823
 824         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 825         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 826         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 827         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 828         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 829         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 830         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 831         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 832
 833         /* TODO: move this to code, get from EEPROM */
 834         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 835         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 836         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 837                                                                                                                 //
 838         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 839         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 840
 841         if (hw_rev != NE020_REV) {
 842                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 843                 u32temp |= 0x80000000;
 844                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 845                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 846                 u32temp &= 0x7fffffff;
 847                 u32temp |= 0x7fff0010;
 848                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 849         }
 850 }
 851
 852
 853 /**
 854  * nes_destroy_adapter - destroy the adapter structure
 855  */
 856 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 857 {
 858         struct nes_adapter *tmp_adapter;
 859
 860         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 861                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 862                                 tmp_adapter);
 863         }
 864
 865         nesadapter->ref_count--;
 866         if (!nesadapter->ref_count) {
 867                 if (nesadapter->hw_rev == NE020_REV) {
 868                         del_timer(&nesadapter->mh_timer);
 869                 }
 870                 del_timer(&nesadapter->lc_timer);
 871
 872                 list_del(&nesadapter->list);
 873                 kfree(nesadapter);
 874         }
 875 }
 876
 877
 878 /**
 879  * nes_init_cqp
 880  */
 881 int nes_init_cqp(struct nes_device *nesdev)
 882 {
 883         struct nes_adapter *nesadapter = nesdev->nesadapter;
 884         struct nes_hw_cqp_qp_context *cqp_qp_context;
 885         struct nes_hw_cqp_wqe *cqp_wqe;
 886         struct nes_hw_ceq *ceq;
 887         struct nes_hw_ceq *nic_ceq;
 888         struct nes_hw_aeq *aeq;
 889         void *vmem;
 890         dma_addr_t pmem;
 891         u32 count=0;
 892         u32 cqp_head;
 893         u64 u64temp;
 894         u32 u32temp;
 895
 896         /* allocate CQP memory */
 897         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 898         /* SQ is 512 byte aligned, others are 256 byte aligned */
 899         nesdev->cqp_mem_size = 512 +
 900                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 901                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 902                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 903                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 904                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 905                         sizeof(struct nes_hw_cqp_qp_context);
 906
 907         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 908                         &nesdev->cqp_pbase);
 909         if (!nesdev->cqp_vbase) {
 910                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 911                 return -ENOMEM;
 912         }
 913         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 914
 915         /* Allocate a twice the number of CQP requests as the SQ size */
 916         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 917                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 918         if (nesdev->nes_cqp_requests == NULL) {
 919                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 920                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 921                                 nesdev->cqp.sq_pbase);
 922                 return -ENOMEM;
 923         }
 924
 925         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 926                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 927
 928         spin_lock_init(&nesdev->cqp.lock);
 929         init_waitqueue_head(&nesdev->cqp.waitq);
 930
 931         /* Setup Various Structures */
 932         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 933                         ~(unsigned long)(512 - 1));
 934         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 935                         ~(unsigned long long)(512 - 1));
 936
 937         nesdev->cqp.sq_vbase = vmem;
 938         nesdev->cqp.sq_pbase = pmem;
 939         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 940         nesdev->cqp.sq_head = 0;
 941         nesdev->cqp.sq_tail = 0;
 942         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 943
 944         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 945         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 946
 947         nesdev->ccq.cq_vbase = vmem;
 948         nesdev->ccq.cq_pbase = pmem;
 949         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 950         nesdev->ccq.cq_head = 0;
 951         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 952         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 953
 954         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 955         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 956
 957         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 958         ceq = &nesadapter->ceq[nesdev->ceq_index];
 959         ceq->ceq_vbase = vmem;
 960         ceq->ceq_pbase = pmem;
 961         ceq->ceq_size = NES_CCEQ_SIZE;
 962         ceq->ceq_head = 0;
 963
 964         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 965         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 966
 967         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 968         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 969         nic_ceq->ceq_vbase = vmem;
 970         nic_ceq->ceq_pbase = pmem;
 971         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 972         nic_ceq->ceq_head = 0;
 973
 974         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 975         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 976
 977         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 978         aeq->aeq_vbase = vmem;
 979         aeq->aeq_pbase = pmem;
 980         aeq->aeq_size = nesadapter->max_qp;
 981         aeq->aeq_head = 0;
 982
 983         /* Setup QP Context */
 984         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 985         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 986
 987         cqp_qp_context = vmem;
 988         cqp_qp_context->context_words[0] =
 989                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 990         cqp_qp_context->context_words[1] = 0;
 991         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 992         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 993
 994
 995         /* Write the address to Create CQP */
 996         if ((sizeof(dma_addr_t) > 4)) {
 997                 nes_write_indexed(nesdev,
 998                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 999                                 ((u64)pmem) >> 32);
1000         } else {
1001                 nes_write_indexed(nesdev,
1002                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1003         }
1004         nes_write_indexed(nesdev,
1005                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1006                         (u32)pmem);
1007
1008         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1009         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1010
1011         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1012                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1013                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1014         }
1015
1016         /* Write Create CCQ WQE */
1017         cqp_head = nesdev->cqp.sq_head++;
1018         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1019         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1020         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1021                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1022                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1023         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1024                             (nesdev->ccq.cq_number |
1025                              ((u32)nesdev->ceq_index << 16)));
1026         u64temp = (u64)nesdev->ccq.cq_pbase;
1027         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1028         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1029         u64temp = (unsigned long)&nesdev->ccq;
1030         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1031                         cpu_to_le32((u32)(u64temp >> 1));
1032         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1033                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1034         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1035
1036         /* Write Create CEQ WQE */
1037         cqp_head = nesdev->cqp.sq_head++;
1038         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1039         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1040         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1041                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1042         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1043         u64temp = (u64)ceq->ceq_pbase;
1044         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1045
1046         /* Write Create AEQ WQE */
1047         cqp_head = nesdev->cqp.sq_head++;
1048         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1049         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1050         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1051                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1052         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1053         u64temp = (u64)aeq->aeq_pbase;
1054         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1055
1056         /* Write Create NIC CEQ WQE */
1057         cqp_head = nesdev->cqp.sq_head++;
1058         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1059         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1060         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1061                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1062         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1063         u64temp = (u64)nic_ceq->ceq_pbase;
1064         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1065
1066         /* Poll until CCQP done */
1067         count = 0;
1068         do {
1069                 if (count++ > 1000) {
1070                         printk(KERN_ERR PFX "Error creating CQP\n");
1071                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1072                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1073                         return -1;
1074                 }
1075                 udelay(10);
1076         } while (!(nes_read_indexed(nesdev,
1077                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1078
1079         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1080                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1081
1082         u32temp = 0x04800000;
1083         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1084
1085         /* wait for the CCQ, CEQ, and AEQ to get created */
1086         count = 0;
1087         do {
1088                 if (count++ > 1000) {
1089                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1090                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1091                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1092                         return -1;
1093                 }
1094                 udelay(10);
1095         } while (((nes_read_indexed(nesdev,
1096                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1097
1098         /* dump the QP status value */
1099         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1100                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1101
1102         nesdev->cqp.sq_tail++;
1103
1104         return 0;
1105 }
1106
1107
1108 /**
1109  * nes_destroy_cqp
1110  */
1111 int nes_destroy_cqp(struct nes_device *nesdev)
1112 {
1113         struct nes_hw_cqp_wqe *cqp_wqe;
1114         u32 count = 0;
1115         u32 cqp_head;
1116         unsigned long flags;
1117
1118         do {
1119                 if (count++ > 1000)
1120                         break;
1121                 udelay(10);
1122         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1123
1124         /* Reset CCQ */
1125         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1126                         nesdev->ccq.cq_number);
1127
1128         /* Disable device interrupts */
1129         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1130
1131         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1132
1133         /* Destroy the AEQ */
1134         cqp_head = nesdev->cqp.sq_head++;
1135         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1136         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1137         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1138                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1139         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1140
1141         /* Destroy the NIC CEQ */
1142         cqp_head = nesdev->cqp.sq_head++;
1143         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1144         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1145         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1146                         ((u32)nesdev->nic_ceq_index << 8));
1147
1148         /* Destroy the CEQ */
1149         cqp_head = nesdev->cqp.sq_head++;
1150         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1151         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1152         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1153                         (nesdev->ceq_index << 8));
1154
1155         /* Destroy the CCQ */
1156         cqp_head = nesdev->cqp.sq_head++;
1157         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1158         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1159         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1160         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1161                         ((u32)nesdev->ceq_index << 16));
1162
1163         /* Destroy CQP */
1164         cqp_head = nesdev->cqp.sq_head++;
1165         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1166         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1167         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1168                         NES_CQP_QP_TYPE_CQP);
1169         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1170
1171         barrier();
1172         /* Ring doorbell (5 WQEs) */
1173         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1174
1175         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1176
1177         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1178         count = 0;
1179         do {
1180                 if (count++ > 1000) {
1181                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1182                                         PCI_FUNC(nesdev->pcidev->devfn));
1183                         break;
1184                 }
1185                 udelay(10);
1186         } while (((nes_read_indexed(nesdev,
1187                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1188
1189         /* dump the QP status value */
1190         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1191                         PCI_FUNC(nesdev->pcidev->devfn),
1192                         nes_read_indexed(nesdev,
1193                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1194
1195         kfree(nesdev->nes_cqp_requests);
1196
1197         /* Free the control structures */
1198         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1199                         nesdev->cqp.sq_pbase);
1200
1201         return 0;
1202 }
1203
1204
1205 /**
1206  * nes_init_phy
1207  */
1208 int nes_init_phy(struct nes_device *nesdev)
1209 {
1210         struct nes_adapter *nesadapter = nesdev->nesadapter;
1211         u32 counter = 0;
1212         u32 mac_index = nesdev->mac_index;
1213         u32 tx_config;
1214         u16 phy_data;
1215
1216         if (nesadapter->OneG_Mode) {
1217                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1218                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1219                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1220                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1221                         tx_config |= 0x04;
1222                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1223                 }
1224
1225                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1226                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1227                                 nesadapter->phy_index[mac_index], phy_data);
1228                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index],  0xb000);
1229
1230                 /* Reset the PHY */
1231                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1232                 udelay(100);
1233                 counter = 0;
1234                 do {
1235                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1236                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1237                         if (counter++ > 100) break;
1238                 } while (phy_data & 0x8000);
1239
1240                 /* Setting no phy loopback */
1241                 phy_data &= 0xbfff;
1242                 phy_data |= 0x1140;
1243                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1244                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1245                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1246
1247                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1248                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1249
1250                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1251                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1252
1253                 /* Setting the interrupt mask */
1254                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1255                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1256                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1257
1258                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1259                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1260
1261                 /* turning on flow control */
1262                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1263                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1264                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1265                                 (phy_data & ~(0x03E0)) | 0xc00);
1266                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1267                                 phy_data | 0xc00); */
1268                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1269                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1270
1271                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1272                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1273                 /* Clear Half duplex */
1274                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1275                                 phy_data & ~(0x0100));
1276                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1277                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1278
1279                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1280                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1281         } else {
1282                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
1283                         /* setup 10G MDIO operation */
1284                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1285                         tx_config |= 0x14;
1286                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1287                 }
1288         }
1289         return 0;
1290 }
1291
1292
1293 /**
1294  * nes_replenish_nic_rq
1295  */
1296 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1297 {
1298         unsigned long flags;
1299         dma_addr_t bus_address;
1300         struct sk_buff *skb;
1301         struct nes_hw_nic_rq_wqe *nic_rqe;
1302         struct nes_hw_nic *nesnic;
1303         struct nes_device *nesdev;
1304         u32 rx_wqes_posted = 0;
1305
1306         nesnic = &nesvnic->nic;
1307         nesdev = nesvnic->nesdev;
1308         spin_lock_irqsave(&nesnic->rq_lock, flags);
1309         if (nesnic->replenishing_rq !=0) {
1310                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1311                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1312                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1313                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1314                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1315                         add_timer(&nesvnic->rq_wqes_timer);
1316                 } else
1317                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1318                 return;
1319         }
1320         nesnic->replenishing_rq = 1;
1321         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1322         do {
1323                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1324                 if (skb) {
1325                         skb->dev = nesvnic->netdev;
1326
1327                         bus_address = pci_map_single(nesdev->pcidev,
1328                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1329
1330                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1331                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1332                                         cpu_to_le32(nesvnic->max_frame_size);
1333                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1334                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1335                                         cpu_to_le32((u32)bus_address);
1336                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1337                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1338                         nesnic->rx_skb[nesnic->rq_head] = skb;
1339                         nesnic->rq_head++;
1340                         nesnic->rq_head &= nesnic->rq_size - 1;
1341                         atomic_dec(&nesvnic->rx_skbs_needed);
1342                         barrier();
1343                         if (++rx_wqes_posted == 255) {
1344                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1345                                 rx_wqes_posted = 0;
1346                         }
1347                 } else {
1348                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1349                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1350                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1351                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1352                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1353                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1354                                 add_timer(&nesvnic->rq_wqes_timer);
1355                         } else
1356                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1357                         break;
1358                 }
1359         } while (atomic_read(&nesvnic->rx_skbs_needed));
1360         barrier();
1361         if (rx_wqes_posted)
1362                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1363         nesnic->replenishing_rq = 0;
1364 }
1365
1366
1367 /**
1368  * nes_rq_wqes_timeout
1369  */
1370 static void nes_rq_wqes_timeout(unsigned long parm)
1371 {
1372         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1373         printk("%s: Timer fired.\n", __func__);
1374         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1375         if (atomic_read(&nesvnic->rx_skbs_needed))
1376                 nes_replenish_nic_rq(nesvnic);
1377 }
1378
1379
1380 /**
1381  * nes_init_nic_qp
1382  */
1383 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1384 {
1385         struct nes_hw_cqp_wqe *cqp_wqe;
1386         struct nes_hw_nic_sq_wqe *nic_sqe;
1387         struct nes_hw_nic_qp_context *nic_context;
1388         struct sk_buff *skb;
1389         struct nes_hw_nic_rq_wqe *nic_rqe;
1390         struct nes_vnic *nesvnic = netdev_priv(netdev);
1391         unsigned long flags;
1392         void *vmem;
1393         dma_addr_t pmem;
1394         u64 u64temp;
1395         int ret;
1396         u32 cqp_head;
1397         u32 counter;
1398         u32 wqe_count;
1399         u8 jumbomode=0;
1400
1401         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1402         nesvnic->nic_mem_size = 256 +
1403                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1404                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1405                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1406                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1407                         sizeof(struct nes_hw_nic_qp_context);
1408
1409         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1410                         &nesvnic->nic_pbase);
1411         if (!nesvnic->nic_vbase) {
1412                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1413                 return -ENOMEM;
1414         }
1415         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1416         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1417                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1418
1419         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1420                         ~(unsigned long)(256 - 1));
1421         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1422                         ~(unsigned long long)(256 - 1));
1423
1424         /* Setup the first Fragment buffers */
1425         nesvnic->nic.first_frag_vbase = vmem;
1426
1427         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1428                 nesvnic->nic.frag_paddr[counter] = pmem;
1429                 pmem += sizeof(struct nes_first_frag);
1430         }
1431
1432         /* setup the SQ */
1433         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1434
1435         nesvnic->nic.sq_vbase = (void *)vmem;
1436         nesvnic->nic.sq_pbase = pmem;
1437         nesvnic->nic.sq_head = 0;
1438         nesvnic->nic.sq_tail = 0;
1439         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1440         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1441                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1442                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1443                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1444                                 NES_NIC_SQ_WQE_COMPLETION);
1445                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1446                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1447                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1448                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1449                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1450                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1451         }
1452
1453         nesvnic->get_cqp_request = nes_get_cqp_request;
1454         nesvnic->post_cqp_request = nes_post_cqp_request;
1455         nesvnic->mcrq_mcast_filter = NULL;
1456
1457         spin_lock_init(&nesvnic->nic.sq_lock);
1458         spin_lock_init(&nesvnic->nic.rq_lock);
1459
1460         /* setup the RQ */
1461         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1462         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1463
1464
1465         nesvnic->nic.rq_vbase = vmem;
1466         nesvnic->nic.rq_pbase = pmem;
1467         nesvnic->nic.rq_head = 0;
1468         nesvnic->nic.rq_tail = 0;
1469         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1470
1471         /* setup the CQ */
1472         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1473         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1474
1475         if (nesdev->nesadapter->netdev_count > 2)
1476                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1477         else
1478                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1479
1480         nesvnic->nic_cq.cq_vbase = vmem;
1481         nesvnic->nic_cq.cq_pbase = pmem;
1482         nesvnic->nic_cq.cq_head = 0;
1483         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1484
1485         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1486
1487         /* Send CreateCQ request to CQP */
1488         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1489         cqp_head = nesdev->cqp.sq_head;
1490
1491         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1492         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1493
1494         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1495                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1496                         ((u32)nesvnic->nic_cq.cq_size << 16));
1497         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1498                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1499         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1500         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1501         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1502         u64temp = (unsigned long)&nesvnic->nic_cq;
1503         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1504         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1505                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1506         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1507         if (++cqp_head >= nesdev->cqp.sq_size)
1508                 cqp_head = 0;
1509         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1510         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1511
1512         /* Send CreateQP request to CQP */
1513         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1514         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1515                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1516                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1517         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1518                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1519                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1520         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1521                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1522         }
1523
1524         u64temp = (u64)nesvnic->nic.sq_pbase;
1525         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1526         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1527         u64temp = (u64)nesvnic->nic.rq_pbase;
1528         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1529         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1530
1531         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1532                         NES_CQP_QP_TYPE_NIC);
1533         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1534         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1535                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1536         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1537
1538         if (++cqp_head >= nesdev->cqp.sq_size)
1539                 cqp_head = 0;
1540         nesdev->cqp.sq_head = cqp_head;
1541
1542         barrier();
1543
1544         /* Ring doorbell (2 WQEs) */
1545         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1546
1547         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1548         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1549                         nesvnic->nic.qp_id);
1550
1551         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1552                         NES_EVENT_TIMEOUT);
1553         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1554                         nesvnic->nic.qp_id, ret);
1555         if (!ret) {
1556                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1557                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1558                                 nesvnic->nic_pbase);
1559                 return -EIO;
1560         }
1561
1562         /* Populate the RQ */
1563         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1564                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1565                 if (!skb) {
1566                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1567
1568                         nes_destroy_nic_qp(nesvnic);
1569                         return -ENOMEM;
1570                 }
1571
1572                 skb->dev = netdev;
1573
1574                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1575                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1576
1577                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1578                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1579                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1580                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1581                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1582                 nesvnic->nic.rx_skb[counter] = skb;
1583         }
1584
1585         wqe_count = NES_NIC_WQ_SIZE - 1;
1586         nesvnic->nic.rq_head = wqe_count;
1587         barrier();
1588         do {
1589                 counter = min(wqe_count, ((u32)255));
1590                 wqe_count -= counter;
1591                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1592         } while (wqe_count);
1593         init_timer(&nesvnic->rq_wqes_timer);
1594         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1595         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1596         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1597
1598         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1599         {
1600                 nes_nic_init_timer(nesdev);
1601                 if (netdev->mtu > 1500)
1602                         jumbomode = 1;
1603                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1604         }
1605
1606         return 0;
1607 }
1608
1609
1610 /**
1611  * nes_destroy_nic_qp
1612  */
1613 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1614 {
1615         struct nes_device *nesdev = nesvnic->nesdev;
1616         struct nes_hw_cqp_wqe *cqp_wqe;
1617         struct nes_hw_nic_rq_wqe *nic_rqe;
1618         u64 wqe_frag;
1619         u32 cqp_head;
1620         unsigned long flags;
1621         int ret;
1622
1623         /* Free remaining NIC receive buffers */
1624         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1625                 nic_rqe = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1626                 wqe_frag = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1627                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1628                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1629                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1630                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1631                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1632         }
1633
1634         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1635
1636         /* Destroy NIC QP */
1637         cqp_head = nesdev->cqp.sq_head;
1638         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1639         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1640
1641         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1642                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1643         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1644                 nesvnic->nic.qp_id);
1645
1646         if (++cqp_head >= nesdev->cqp.sq_size)
1647                 cqp_head = 0;
1648
1649         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1650
1651         /* Destroy NIC CQ */
1652         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1653         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1654                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1655         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1656                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1657
1658         if (++cqp_head >= nesdev->cqp.sq_size)
1659                 cqp_head = 0;
1660
1661         nesdev->cqp.sq_head = cqp_head;
1662         barrier();
1663
1664         /* Ring doorbell (2 WQEs) */
1665         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1666
1667         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1668         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1669                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1670                         cqp_head, nesdev->cqp.sq_head,
1671                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1672
1673         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1674                         NES_EVENT_TIMEOUT);
1675
1676         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1677                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1678                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1679         if (!ret) {
1680                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1681                                 nesvnic->nic.qp_id);
1682         }
1683
1684         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1685                         nesvnic->nic_pbase);
1686 }
1687
1688 /**
1689  * nes_napi_isr
1690  */
1691 int nes_napi_isr(struct nes_device *nesdev)
1692 {
1693         struct nes_adapter *nesadapter = nesdev->nesadapter;
1694         u32 int_stat;
1695
1696         if (nesdev->napi_isr_ran) {
1697                 /* interrupt status has already been read in ISR */
1698                 int_stat = nesdev->int_stat;
1699         } else {
1700                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1701                 nesdev->int_stat = int_stat;
1702                 nesdev->napi_isr_ran = 1;
1703         }
1704
1705         int_stat &= nesdev->int_req;
1706         /* iff NIC, process here, else wait for DPC */
1707         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1708                 nesdev->napi_isr_ran = 0;
1709                 nes_write32(nesdev->regs+NES_INT_STAT,
1710                                 (int_stat &
1711                                 ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1712
1713                 /* Process the CEQs */
1714                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1715
1716                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1717                                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1718                                           ((nesadapter->et_use_adaptive_rx_coalesce) &&
1719                                            (nesdev->deepcq_count > nesadapter->et_pkt_rate_low)))) ) {
1720                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1721                                 /* Enable Periodic timer interrupts */
1722                                 nesdev->int_req |= NES_INT_TIMER;
1723                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1724                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1725                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1726                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1727                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1728                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1729                         }
1730
1731                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1732                         {
1733                                 nes_nic_init_timer(nesdev);
1734                         }
1735                         /* Enable interrupts, except CEQs */
1736                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1737                 } else {
1738                         /* Enable interrupts, make sure timer is off */
1739                         nesdev->int_req &= ~NES_INT_TIMER;
1740                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1741                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1742                 }
1743                 nesdev->deepcq_count = 0;
1744                 return 1;
1745         } else {
1746                 return 0;
1747         }
1748 }
1749
1750
1751 /**
1752  * nes_dpc
1753  */
1754 void nes_dpc(unsigned long param)
1755 {
1756         struct nes_device *nesdev = (struct nes_device *)param;
1757         struct nes_adapter *nesadapter = nesdev->nesadapter;
1758         u32 counter;
1759         u32 loop_counter = 0;
1760         u32 int_status_bit;
1761         u32 int_stat;
1762         u32 timer_stat;
1763         u32 temp_int_stat;
1764         u32 intf_int_stat;
1765         u32 debug_error;
1766         u32 processed_intf_int = 0;
1767         u16 processed_timer_int = 0;
1768         u16 completion_ints = 0;
1769         u16 timer_ints = 0;
1770
1771         /* nes_debug(NES_DBG_ISR, "\n"); */
1772
1773         do {
1774                 timer_stat = 0;
1775                 if (nesdev->napi_isr_ran) {
1776                         nesdev->napi_isr_ran = 0;
1777                         int_stat = nesdev->int_stat;
1778                 } else
1779                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1780                 if (processed_intf_int != 0)
1781                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1782                 else
1783                         int_stat &= nesdev->int_req;
1784                 if (processed_timer_int == 0) {
1785                         processed_timer_int = 1;
1786                         if (int_stat & NES_INT_TIMER) {
1787                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1788                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1789                                         int_stat &= ~NES_INT_TIMER;
1790                                 }
1791                         }
1792                 } else {
1793                         int_stat &= ~NES_INT_TIMER;
1794                 }
1795
1796                 if (int_stat) {
1797                         if (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1798                                         NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)) {
1799                                 /* Ack the interrupts */
1800                                 nes_write32(nesdev->regs+NES_INT_STAT,
1801                                                 (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1802                                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1803                         }
1804
1805                         temp_int_stat = int_stat;
1806                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1807                                 if (int_stat & int_status_bit) {
1808                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1809                                         temp_int_stat &= ~int_status_bit;
1810                                         completion_ints = 1;
1811                                 }
1812                                 if (!(temp_int_stat & 0x0000ffff))
1813                                         break;
1814                                 int_status_bit <<= 1;
1815                         }
1816
1817                         /* Process the AEQ for this pci function */
1818                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1819                         if (int_stat & int_status_bit) {
1820                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1821                         }
1822
1823                         /* Process the MAC interrupt for this pci function */
1824                         int_status_bit = 1 << (24 + nesdev->mac_index);
1825                         if (int_stat & int_status_bit) {
1826                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1827                         }
1828
1829                         if (int_stat & NES_INT_TIMER) {
1830                                 if (timer_stat & nesdev->timer_int_req) {
1831                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1832                                                         (timer_stat & nesdev->timer_int_req) |
1833                                                         ~(nesdev->nesadapter->timer_int_req));
1834                                         timer_ints = 1;
1835                                 }
1836                         }
1837
1838                         if (int_stat & NES_INT_INTF) {
1839                                 processed_intf_int = 1;
1840                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1841                                 intf_int_stat &= nesdev->intf_int_req;
1842                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1843                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1844                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1845                                                         (u16)debug_error);
1846                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1847                                                         0x01010000 | (debug_error & 0x0000ffff));
1848                                         /* BUG(); */
1849                                         if (crit_err_count++ > 10)
1850                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1851                                 }
1852                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1853                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1854                                         BUG();
1855                                 }
1856                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
1857                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
1858                                         BUG();
1859                                 }
1860                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
1861                         }
1862
1863                         if (int_stat & NES_INT_TSW) {
1864                         }
1865                 }
1866                 /* Don't use the interface interrupt bit stay in loop */
1867                 int_stat &= ~NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1868                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3;
1869         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
1870
1871         if (timer_ints == 1) {
1872                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
1873                         if (completion_ints == 0) {
1874                                 nesdev->timer_only_int_count++;
1875                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
1876                                         nesdev->timer_only_int_count = 0;
1877                                         nesdev->int_req &= ~NES_INT_TIMER;
1878                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1879                                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1880                                 } else {
1881                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1882                                 }
1883                         } else {
1884                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1885                                 {
1886                                         nes_nic_init_timer(nesdev);
1887                                 }
1888                                 nesdev->timer_only_int_count = 0;
1889                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1890                         }
1891                 } else {
1892                         nesdev->timer_only_int_count = 0;
1893                         nesdev->int_req &= ~NES_INT_TIMER;
1894                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1895                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1896                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1897                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1898                 }
1899         } else {
1900                 if ( (completion_ints == 1) &&
1901                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
1902                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1903                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
1904                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
1905                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
1906                         nesdev->timer_only_int_count = 0;
1907                         nesdev->int_req |= NES_INT_TIMER;
1908                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1909                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1910                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1911                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1912                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1913                 } else {
1914                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1915                 }
1916         }
1917         nesdev->deepcq_count = 0;
1918 }
1919
1920
1921 /**
1922  * nes_process_ceq
1923  */
1924 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
1925 {
1926         u64 u64temp;
1927         struct nes_hw_cq *cq;
1928         u32 head;
1929         u32 ceq_size;
1930
1931         /* nes_debug(NES_DBG_CQ, "\n"); */
1932         head = ceq->ceq_head;
1933         ceq_size = ceq->ceq_size;
1934
1935         do {
1936                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
1937                                 NES_CEQE_VALID) {
1938                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX])))<<32) |
1939                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
1940                         u64temp <<= 1;
1941                         cq = *((struct nes_hw_cq **)&u64temp);
1942                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
1943                         barrier();
1944                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
1945
1946                         /* call the event handler */
1947                         cq->ce_handler(nesdev, cq);
1948
1949                         if (++head >= ceq_size)
1950                                 head = 0;
1951                 } else {
1952                         break;
1953                 }
1954
1955         } while (1);
1956
1957         ceq->ceq_head = head;
1958 }
1959
1960
1961 /**
1962  * nes_process_aeq
1963  */
1964 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
1965 {
1966 //      u64 u64temp;
1967         u32 head;
1968         u32 aeq_size;
1969         u32 aeqe_misc;
1970         u32 aeqe_cq_id;
1971         struct nes_hw_aeqe volatile *aeqe;
1972
1973         head = aeq->aeq_head;
1974         aeq_size = aeq->aeq_size;
1975
1976         do {
1977                 aeqe = &aeq->aeq_vbase[head];
1978                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
1979                         break;
1980                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
1981                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
1982                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
1983                         if (aeqe_cq_id >= NES_FIRST_QPN) {
1984                                 /* dealing with an accelerated QP related AE */
1985 //                              u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])))<<32) |
1986 //                                      ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
1987                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
1988                         } else {
1989                                 /* TODO: dealing with a CQP related AE */
1990                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
1991                                                 (u16)(aeqe_misc >> 16));
1992                         }
1993                 }
1994
1995                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
1996
1997                 if (++head >= aeq_size)
1998                         head = 0;
1999         }
2000         while (1);
2001         aeq->aeq_head = head;
2002 }
2003
2004 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2005 {
2006         struct nes_adapter *nesadapter = nesdev->nesadapter;
2007         u32 reset_value;
2008         u32 i=0;
2009         u32 u32temp;
2010
2011         if (nesadapter->hw_rev == NE020_REV) {
2012                 return;
2013         }
2014         mh_detected++;
2015
2016         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2017
2018         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2019                 reset_value |= 0x0000001d;
2020         else
2021                 reset_value |= 0x0000002d;
2022
2023         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2024                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2025                         nesadapter->link_interrupt_count[0] = 0;
2026                         nesadapter->link_interrupt_count[1] = 0;
2027                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2028                         if (0x00000040 & u32temp)
2029                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2030                         else
2031                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2032
2033                         reset_value |= 0x0000003d;
2034                 }
2035                 nesadapter->link_interrupt_count[mac_index] = 0;
2036         }
2037
2038         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2039
2040         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2041                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2042
2043         if (0x0000003d == (reset_value & 0x0000003d)) {
2044                 u32 pcs_control_status0, pcs_control_status1;
2045
2046                 for (i = 0; i < 10; i++) {
2047                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2048                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2049                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2050                              && (pcs_control_status0 & 0x00100000))
2051                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2052                                 && (pcs_control_status1 & 0x00100000)))
2053                                 continue;
2054                         else
2055                                 break;
2056                 }
2057                 if (10 == i) {
2058                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2059                         if (0x00000040 & u32temp)
2060                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2061                         else
2062                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2063
2064                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2065
2066                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2067                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2068                 }
2069         }
2070 }
2071
2072 /**
2073  * nes_process_mac_intr
2074  */
2075 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2076 {
2077         unsigned long flags;
2078         u32 pcs_control_status;
2079         struct nes_adapter *nesadapter = nesdev->nesadapter;
2080         struct nes_vnic *nesvnic;
2081         u32 mac_status;
2082         u32 mac_index = nesdev->mac_index;
2083         u32 u32temp;
2084         u16 phy_data;
2085         u16 temp_phy_data;
2086
2087         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2088         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2089                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2090                 return;
2091         }
2092         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2093         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2094
2095         /* ack the MAC interrupt */
2096         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2097         /* Clear the interrupt */
2098         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2099
2100         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2101
2102         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2103                 nesdev->link_status_interrupts++;
2104                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2105                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2106                         nes_reset_link(nesdev, mac_index);
2107                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2108                 }
2109                 /* read the PHY interrupt status register */
2110                 if (nesadapter->OneG_Mode) {
2111                         do {
2112                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2113                                                 nesadapter->phy_index[mac_index], &phy_data);
2114                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2115                                                 nesadapter->phy_index[mac_index], phy_data);
2116                         } while (phy_data&0x8000);
2117
2118                         temp_phy_data = 0;
2119                         do {
2120                                 nes_read_1G_phy_reg(nesdev, 0x11,
2121                                                 nesadapter->phy_index[mac_index], &phy_data);
2122                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2123                                                 nesadapter->phy_index[mac_index], phy_data);
2124                                 if (temp_phy_data == phy_data)
2125                                         break;
2126                                 temp_phy_data = phy_data;
2127                         } while (1);
2128
2129                         nes_read_1G_phy_reg(nesdev, 0x1e,
2130                                         nesadapter->phy_index[mac_index], &phy_data);
2131                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2132                                         nesadapter->phy_index[mac_index], phy_data);
2133
2134                         nes_read_1G_phy_reg(nesdev, 1,
2135                                         nesadapter->phy_index[mac_index], &phy_data);
2136                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2137                                         nesadapter->phy_index[mac_index], phy_data);
2138
2139                         if (temp_phy_data & 0x1000) {
2140                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2141                                 phy_data = 4;
2142                         } else {
2143                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2144                         }
2145                 }
2146                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2147                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2148                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2149                 pcs_control_status = nes_read_indexed(nesdev,
2150                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2151                 pcs_control_status = nes_read_indexed(nesdev,
2152                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2153                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2154                                 mac_index, pcs_control_status);
2155                 if (nesadapter->OneG_Mode) {
2156                         u32temp = 0x01010000;
2157                         if (nesadapter->port_count > 2) {
2158                                 u32temp |= 0x02020000;
2159                         }
2160                         if ((pcs_control_status & u32temp)!= u32temp) {
2161                                 phy_data = 0;
2162                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2163                         }
2164                 } else if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
2165                         nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2166                         temp_phy_data = (u16)nes_read_indexed(nesdev,
2167                                                                 NES_IDX_MAC_MDIO_CONTROL);
2168                         u32temp = 20;
2169                         do {
2170                                 nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2171                                 phy_data = (u16)nes_read_indexed(nesdev,
2172                                                                 NES_IDX_MAC_MDIO_CONTROL);
2173                                 if ((phy_data == temp_phy_data) || (!(--u32temp)))
2174                                         break;
2175                                 temp_phy_data = phy_data;
2176                         } while (1);
2177                         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2178                                 __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2179
2180                 } else {
2181                         phy_data = (0x0f0f0000 == (pcs_control_status & 0x0f1f0000)) ? 4 : 0;
2182                 }
2183
2184                 if (phy_data & 0x0004) {
2185                         nesadapter->mac_link_down[mac_index] = 0;
2186                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2187                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2188                                                 nesvnic->linkup);
2189                                 if (nesvnic->linkup == 0) {
2190                                         printk(PFX "The Link is now up for port %u, netdev %p.\n",
2191                                                         mac_index, nesvnic->netdev);
2192                                         if (netif_queue_stopped(nesvnic->netdev))
2193                                                 netif_start_queue(nesvnic->netdev);
2194                                         nesvnic->linkup = 1;
2195                                         netif_carrier_on(nesvnic->netdev);
2196                                 }
2197                         }
2198                 } else {
2199                         nesadapter->mac_link_down[mac_index] = 1;
2200                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2201                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2202                                                 nesvnic->linkup);
2203                                 if (nesvnic->linkup == 1) {
2204                                         printk(PFX "The Link is now down for port %u, netdev %p.\n",
2205                                                         mac_index, nesvnic->netdev);
2206                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2207                                                 netif_stop_queue(nesvnic->netdev);
2208                                         nesvnic->linkup = 0;
2209                                         netif_carrier_off(nesvnic->netdev);
2210                                 }
2211                         }
2212                 }
2213         }
2214
2215         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2216 }
2217
2218
2219
2220 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2221 {
2222         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2223
2224         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2225 }
2226
2227
2228 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2229 * getting out of nic_ce_handler
2230 */
2231 #define MAX_RQES_TO_PROCESS     384
2232
2233 /**
2234  * nes_nic_ce_handler
2235  */
2236 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2237 {
2238         u64 u64temp;
2239         dma_addr_t bus_address;
2240         struct nes_hw_nic *nesnic;
2241         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2242         struct nes_adapter *nesadapter = nesdev->nesadapter;
2243         struct nes_hw_nic_rq_wqe *nic_rqe;
2244         struct nes_hw_nic_sq_wqe *nic_sqe;
2245         struct sk_buff *skb;
2246         struct sk_buff *rx_skb;
2247         __le16 *wqe_fragment_length;
2248         u32 head;
2249         u32 cq_size;
2250         u32 rx_pkt_size;
2251         u32 cqe_count=0;
2252         u32 cqe_errv;
2253         u32 cqe_misc;
2254         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2255         u16 vlan_tag;
2256         u16 pkt_type;
2257         u16 rqes_processed = 0;
2258         u8 sq_cqes = 0;
2259
2260         head = cq->cq_head;
2261         cq_size = cq->cq_size;
2262         cq->cqes_pending = 1;
2263         do {
2264                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2265                                 NES_NIC_CQE_VALID) {
2266                         nesnic = &nesvnic->nic;
2267                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2268                         if (cqe_misc & NES_NIC_CQE_SQ) {
2269                                 sq_cqes++;
2270                                 wqe_fragment_index = 1;
2271                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2272                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2273                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2274                                 /* bump past the vlan tag */
2275                                 wqe_fragment_length++;
2276                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2277                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2278                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2279                                         bus_address = (dma_addr_t)u64temp;
2280                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2281                                                 pci_unmap_single(nesdev->pcidev,
2282                                                                 bus_address,
2283                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2284                                                                 PCI_DMA_TODEVICE);
2285                                         }
2286                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2287                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2288                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2289                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2290                                                         bus_address = (dma_addr_t)u64temp;
2291                                                         pci_unmap_page(nesdev->pcidev,
2292                                                                         bus_address,
2293                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2294                                                                         PCI_DMA_TODEVICE);
2295                                                 } else
2296                                                         break;
2297                                         }
2298                                         if (skb)
2299                                                 dev_kfree_skb_any(skb);
2300                                 }
2301                                 nesnic->sq_tail++;
2302                                 nesnic->sq_tail &= nesnic->sq_size-1;
2303                                 if (sq_cqes > 128) {
2304                                         barrier();
2305                                 /* restart the queue if it had been stopped */
2306                                 if (netif_queue_stopped(nesvnic->netdev))
2307                                         netif_wake_queue(nesvnic->netdev);
2308                                         sq_cqes = 0;
2309                                 }
2310                         } else {
2311                                 rqes_processed ++;
2312
2313                                 cq->rx_cqes_completed++;
2314                                 cq->rx_pkts_indicated++;
2315                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2316                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2317                                 /* Get the skb */
2318                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2319                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2320                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2321                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2322                                 pci_unmap_single(nesdev->pcidev, bus_address,
2323                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2324                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2325                                 /* rx_skb->len = rx_pkt_size; */
2326                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2327                                 skb_put(rx_skb, rx_pkt_size);
2328                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2329                                 nesnic->rq_tail++;
2330                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2331
2332                                 atomic_inc(&nesvnic->rx_skbs_needed);
2333                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2334                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2335                                                         cq->cq_number | (cqe_count << 16));
2336 //                                      nesadapter->tune_timer.cq_count += cqe_count;
2337                                         nesdev->currcq_count += cqe_count;
2338                                         cqe_count = 0;
2339                                         nes_replenish_nic_rq(nesvnic);
2340                                 }
2341                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2342                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2343                                 rx_skb->ip_summed = CHECKSUM_NONE;
2344
2345                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2346                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2347                                         if ((cqe_errv &
2348                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2349                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2350                                                 if (nesvnic->rx_checksum_disabled == 0) {
2351                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2352                                                 }
2353                                         } else
2354                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2355                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2356                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2357
2358                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2359                                         if ((cqe_errv &
2360                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2361                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2362                                                 if (nesvnic->rx_checksum_disabled == 0) {
2363                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2364                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2365                                                                   nesvnic->netdev->name); */
2366                                                 }
2367                                         } else
2368                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2369                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2370                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2371                                         }
2372                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2373                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2374
2375                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2376                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2377                                 } else {
2378                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2379                                                 vlan_tag = (u16)(le32_to_cpu(
2380                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2381                                                                 >> 16);
2382                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2383                                                                 nesvnic->netdev->name, vlan_tag);
2384                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2385                                         } else {
2386                                                 nes_netif_rx(rx_skb);
2387                                         }
2388                                 }
2389
2390                                 nesvnic->netdev->last_rx = jiffies;
2391                                 /* nesvnic->netstats.rx_packets++; */
2392                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2393                         }
2394
2395                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2396                         /* Accounting... */
2397                         cqe_count++;
2398                         if (++head >= cq_size)
2399                                 head = 0;
2400                         if (cqe_count == 255) {
2401                                 /* Replenish Nic CQ */
2402                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2403                                                 cq->cq_number | (cqe_count << 16));
2404 //                              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2405                                 nesdev->currcq_count += cqe_count;
2406                                 cqe_count = 0;
2407                         }
2408
2409                         if (cq->rx_cqes_completed >= nesvnic->budget)
2410                                 break;
2411                 } else {
2412                         cq->cqes_pending = 0;
2413                         break;
2414                 }
2415
2416         } while (1);
2417
2418         if (sq_cqes) {
2419                 barrier();
2420                 /* restart the queue if it had been stopped */
2421                 if (netif_queue_stopped(nesvnic->netdev))
2422                         netif_wake_queue(nesvnic->netdev);
2423         }
2424
2425         cq->cq_head = head;
2426         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2427                         cq->cq_number, cqe_count, cq->cq_head); */
2428         cq->cqe_allocs_pending = cqe_count;
2429         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2430         {
2431 //              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2432                 nesdev->currcq_count += cqe_count;
2433                 nes_nic_tune_timer(nesdev);
2434         }
2435         if (atomic_read(&nesvnic->rx_skbs_needed))
2436                 nes_replenish_nic_rq(nesvnic);
2437         }
2438
2439
2440 /**
2441  * nes_cqp_ce_handler
2442  */
2443 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2444 {
2445         u64 u64temp;
2446         unsigned long flags;
2447         struct nes_hw_cqp *cqp = NULL;
2448         struct nes_cqp_request *cqp_request;
2449         struct nes_hw_cqp_wqe *cqp_wqe;
2450         u32 head;
2451         u32 cq_size;
2452         u32 cqe_count=0;
2453         u32 error_code;
2454         /* u32 counter; */
2455
2456         head = cq->cq_head;
2457         cq_size = cq->cq_size;
2458
2459         do {
2460                 /* process the CQE */
2461                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2462                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2463
2464                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2465                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2466                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
2467                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2468                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2469                         cqp = *((struct nes_hw_cqp **)&u64temp);
2470
2471                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2472                         if (error_code) {
2473                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2474                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2475                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2476                                                 (u16)(error_code >> 16),
2477                                                 (u16)error_code);
2478                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2479                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2480                         }
2481
2482                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2483                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX])))<<32) |
2484                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2485                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2486                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2487                         if (cqp_request) {
2488                                 if (cqp_request->waiting) {
2489                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2490                                         cqp_request->major_code = (u16)(error_code >> 16);
2491                                         cqp_request->minor_code = (u16)error_code;
2492                                         barrier();
2493                                         cqp_request->request_done = 1;
2494                                         wake_up(&cqp_request->waitq);
2495                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2496                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2497                                                                 cqp_request,
2498                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2499                                                 if (cqp_request->dynamic) {
2500                                                         kfree(cqp_request);
2501                                                 } else {
2502                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2503                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2504                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2505                                                 }
2506                                         }
2507                                 } else if (cqp_request->callback) {
2508                                         /* Envoke the callback routine */
2509                                         cqp_request->cqp_callback(nesdev, cqp_request);
2510                                         if (cqp_request->dynamic) {
2511                                                 kfree(cqp_request);
2512                                         } else {
2513                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2514                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2515                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2516                                         }
2517                                 } else {
2518                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2519                                                         cqp_request,
2520                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2521                                         if (cqp_request->dynamic) {
2522                                                 kfree(cqp_request);
2523                                         } else {
2524                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2525                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2526                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2527                                         }
2528                                 }
2529                         } else {
2530                                 wake_up(&nesdev->cqp.waitq);
2531                         }
2532
2533                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2534                         nes_write32(nesdev->regs+NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2535                         if (++cqp->sq_tail >= cqp->sq_size)
2536                                 cqp->sq_tail = 0;
2537
2538                         /* Accounting... */
2539                         cqe_count++;
2540                         if (++head >= cq_size)
2541                                 head = 0;
2542                 } else {
2543                         break;
2544                 }
2545         } while (1);
2546         cq->cq_head = head;
2547
2548         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2549         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2550                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2551                         (nesdev->cqp.sq_size - 1)) != 1)) {
2552                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2553                                 struct nes_cqp_request, list);
2554                 list_del_init(&cqp_request->list);
2555                 head = nesdev->cqp.sq_head++;
2556                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2557                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2558                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2559                 barrier();
2560                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2561                         cpu_to_le32((u32)((unsigned long)cqp_request));
2562                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2563                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2564                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2565                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2566                 /* Ring doorbell (1 WQEs) */
2567                 barrier();
2568                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2569         }
2570         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2571
2572         /* Arm the CCQ */
2573         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2574                         cq->cq_number);
2575         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2576 }
2577
2578
2579 /**
2580  * nes_process_iwarp_aeqe
2581  */
2582 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2583                                    struct nes_hw_aeqe *aeqe)
2584 {
2585         u64 context;
2586         u64 aeqe_context = 0;
2587         unsigned long flags;
2588         struct nes_qp *nesqp;
2589         int resource_allocated;
2590         /* struct iw_cm_id *cm_id; */
2591         struct nes_adapter *nesadapter = nesdev->nesadapter;
2592         struct ib_event ibevent;
2593         /* struct iw_cm_event cm_event; */
2594         u32 aeq_info;
2595         u32 next_iwarp_state = 0;
2596         u16 async_event_id;
2597         u8 tcp_state;
2598         u8 iwarp_state;
2599
2600         nes_debug(NES_DBG_AEQ, "\n");
2601         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2602         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2603                 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2604                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2605         } else {
2606                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2607                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2608                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2609                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2610                 BUG_ON(!context);
2611         }
2612
2613         async_event_id = (u16)aeq_info;
2614         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2615         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2616         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2617                         " Tcp state = %s, iWARP state = %s\n",
2618                         async_event_id,
2619                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2620                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2621
2622
2623         switch (async_event_id) {
2624                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2625                         nesqp = *((struct nes_qp **)&context);
2626                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2627                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2628                                 nes_add_ref(&nesqp->ibqp);
2629                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2630                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2631                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2632                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2633                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2634                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2635                                                 async_event_id, nesqp->last_aeq, tcp_state);
2636                         }
2637                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2638                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2639                                 /* FIN Received but tcp state or IB state moved on,
2640                                                 should expect a close complete */
2641                                 return;
2642                         }
2643                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2644                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2645                 case NES_AEQE_AEID_TERMINATE_SENT:
2646                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2647                 case NES_AEQE_AEID_RESET_SENT:
2648                         nesqp = *((struct nes_qp **)&context);
2649                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2650                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2651                         }
2652                         nes_add_ref(&nesqp->ibqp);
2653                         spin_lock_irqsave(&nesqp->lock, flags);
2654                         nesqp->hw_iwarp_state = iwarp_state;
2655                         nesqp->hw_tcp_state = tcp_state;
2656                         nesqp->last_aeq = async_event_id;
2657
2658                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2659                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2660                                 nesqp->hte_added = 0;
2661                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2662                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2663                                                 nesqp->hwqp.qp_id);
2664                                 nes_hw_modify_qp(nesdev, nesqp,
2665                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2666                                 spin_lock_irqsave(&nesqp->lock, flags);
2667                         }
2668
2669                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2670                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2671                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2672                                 switch (nesqp->hw_iwarp_state) {
2673                                         case NES_AEQE_IWARP_STATE_RTS:
2674                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2675                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2676                                                 break;
2677                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2678                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2679                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2680                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2681                                                         next_iwarp_state |= 0x02000000;
2682                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2683                                                 }
2684                                                 break;
2685                                         default:
2686                                                 next_iwarp_state = 0;
2687                                 }
2688                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2689                                 if (next_iwarp_state) {
2690                                         nes_add_ref(&nesqp->ibqp);
2691                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2692                                                         " also added another reference\n",
2693                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2694                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2695                                 }
2696                                 nes_cm_disconn(nesqp);
2697                         } else {
2698                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2699                                         /* FIN Received but ib state not RTS,
2700                                                         close complete will be on its way */
2701                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2702                                         nes_rem_ref(&nesqp->ibqp);
2703                                         return;
2704                                 }
2705                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2706                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2707                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2708                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2709                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2710                                                         " also added another reference\n",
2711                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2712                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2713                                 }
2714                                 nes_cm_disconn(nesqp);
2715                         }
2716                         break;
2717                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2718                         nesqp = *((struct nes_qp **)&context);
2719                         spin_lock_irqsave(&nesqp->lock, flags);
2720                         nesqp->hw_iwarp_state = iwarp_state;
2721                         nesqp->hw_tcp_state = tcp_state;
2722                         nesqp->last_aeq = async_event_id;
2723                         spin_unlock_irqrestore(&nesqp->lock, flags);
2724                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2725                                         " event on QP%u \n  Q2 Data:\n",
2726                                         nesqp->hwqp.qp_id);
2727                         if (nesqp->ibqp.event_handler) {
2728                                 ibevent.device = nesqp->ibqp.device;
2729                                 ibevent.element.qp = &nesqp->ibqp;
2730                                 ibevent.event = IB_EVENT_QP_FATAL;
2731                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2732                         }
2733                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2734                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2735                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2736                                 nes_add_ref(&nesqp->ibqp);
2737                                 nes_cm_disconn(nesqp);
2738                         } else {
2739                                 nesqp->in_disconnect = 0;
2740                                 wake_up(&nesqp->kick_waitq);
2741                         }
2742                         break;
2743                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2744                         nesqp = *((struct nes_qp **)&context);
2745                         nes_add_ref(&nesqp->ibqp);
2746                         spin_lock_irqsave(&nesqp->lock, flags);
2747                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2748                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2749                         nesqp->last_aeq = async_event_id;
2750                         if (nesqp->cm_id) {
2751                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2752                                                 " event on QP%u, remote IP = 0x%08X \n",
2753                                                 nesqp->hwqp.qp_id,
2754                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2755                         } else {
2756                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2757                                                 " event on QP%u \n",
2758                                                 nesqp->hwqp.qp_id);
2759                         }
2760                         spin_unlock_irqrestore(&nesqp->lock, flags);
2761                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2762                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2763                         if (nesqp->ibqp.event_handler) {
2764                                 ibevent.device = nesqp->ibqp.device;
2765                                 ibevent.element.qp = &nesqp->ibqp;
2766                                 ibevent.event = IB_EVENT_QP_FATAL;
2767                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2768                         }
2769                         break;
2770                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2771                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2772                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2773                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2774                         } else {
2775                                 /* TODO: get the actual WQE and mask off wqe index */
2776                                 context &= ~((u64)511);
2777                                 nesqp = *((struct nes_qp **)&context);
2778                         }
2779                         spin_lock_irqsave(&nesqp->lock, flags);
2780                         nesqp->hw_iwarp_state = iwarp_state;
2781                         nesqp->hw_tcp_state = tcp_state;
2782                         nesqp->last_aeq = async_event_id;
2783                         spin_unlock_irqrestore(&nesqp->lock, flags);
2784                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2785                                         nesqp->hwqp.qp_id);
2786                         if (nesqp->ibqp.event_handler) {
2787                                 ibevent.device = nesqp->ibqp.device;
2788                                 ibevent.element.qp = &nesqp->ibqp;
2789                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2790                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2791                         }
2792                         break;
2793                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
2794                         nesqp = *((struct nes_qp **)&context);
2795                         spin_lock_irqsave(&nesqp->lock, flags);
2796                         nesqp->hw_iwarp_state = iwarp_state;
2797                         nesqp->hw_tcp_state = tcp_state;
2798                         nesqp->last_aeq = async_event_id;
2799                         spin_unlock_irqrestore(&nesqp->lock, flags);
2800                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
2801                                         nesqp->hwqp.qp_id);
2802                         if (nesqp->ibqp.event_handler) {
2803                                 ibevent.device = nesqp->ibqp.device;
2804                                 ibevent.element.qp = &nesqp->ibqp;
2805                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2806                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2807                         }
2808                         break;
2809                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
2810                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
2811                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2812                         spin_lock_irqsave(&nesqp->lock, flags);
2813                         nesqp->hw_iwarp_state = iwarp_state;
2814                         nesqp->hw_tcp_state = tcp_state;
2815                         nesqp->last_aeq = async_event_id;
2816                         spin_unlock_irqrestore(&nesqp->lock, flags);
2817                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
2818                                         " nesqp = %p, AE reported %p\n",
2819                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
2820                         if (nesqp->ibqp.event_handler) {
2821                                 ibevent.device = nesqp->ibqp.device;
2822                                 ibevent.element.qp = &nesqp->ibqp;
2823                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2824                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2825                         }
2826                         break;
2827                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
2828                         context <<= 1;
2829                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
2830                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
2831                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
2832                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2833                         if (resource_allocated) {
2834                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
2835                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2836                         }
2837                         break;
2838                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
2839                         nesqp = nesadapter->qp_table[le32_to_cpu(
2840                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2841                         spin_lock_irqsave(&nesqp->lock, flags);
2842                         nesqp->hw_iwarp_state = iwarp_state;
2843                         nesqp->hw_tcp_state = tcp_state;
2844                         nesqp->last_aeq = async_event_id;
2845                         spin_unlock_irqrestore(&nesqp->lock, flags);
2846                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
2847                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
2848                                         nesqp->hwqp.qp_id);
2849                         if (nesqp->ibqp.event_handler) {
2850                                 ibevent.device = nesqp->ibqp.device;
2851                                 ibevent.element.qp = &nesqp->ibqp;
2852                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2853                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2854                         }
2855                         /* tell cm to disconnect, cm will queue work to thread */
2856                         nes_add_ref(&nesqp->ibqp);
2857                         nes_cm_disconn(nesqp);
2858                         break;
2859                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
2860                         nesqp = *((struct nes_qp **)&context);
2861                         spin_lock_irqsave(&nesqp->lock, flags);
2862                         nesqp->hw_iwarp_state = iwarp_state;
2863                         nesqp->hw_tcp_state = tcp_state;
2864                         nesqp->last_aeq = async_event_id;
2865                         spin_unlock_irqrestore(&nesqp->lock, flags);
2866                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
2867                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
2868                                         nesqp->hwqp.qp_id);
2869                         if (nesqp->ibqp.event_handler) {
2870                                 ibevent.device = nesqp->ibqp.device;
2871                                 ibevent.element.qp = &nesqp->ibqp;
2872                                 ibevent.event = IB_EVENT_QP_FATAL;
2873                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2874                         }
2875                         /* tell cm to disconnect, cm will queue work to thread */
2876                         nes_add_ref(&nesqp->ibqp);
2877                         nes_cm_disconn(nesqp);
2878                         break;
2879                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
2880                         nesqp = *((struct nes_qp **)&context);
2881                         spin_lock_irqsave(&nesqp->lock, flags);
2882                         nesqp->hw_iwarp_state = iwarp_state;
2883                         nesqp->hw_tcp_state = tcp_state;
2884                         nesqp->last_aeq = async_event_id;
2885                         spin_unlock_irqrestore(&nesqp->lock, flags);
2886                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
2887                                         " event on QP%u \n  Q2 Data:\n",
2888                                         nesqp->hwqp.qp_id);
2889                         if (nesqp->ibqp.event_handler) {
2890                                 ibevent.device = nesqp->ibqp.device;
2891                                 ibevent.element.qp = &nesqp->ibqp;
2892                                 ibevent.event = IB_EVENT_QP_FATAL;
2893                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2894                         }
2895                         /* tell cm to disconnect, cm will queue work to thread */
2896                         nes_add_ref(&nesqp->ibqp);
2897                         nes_cm_disconn(nesqp);
2898                         break;
2899                         /* TODO: additional AEs need to be here */
2900                 default:
2901                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
2902                                         async_event_id);
2903                         break;
2904         }
2905
2906 }
2907
2908
2909 /**
2910  * nes_iwarp_ce_handler
2911  */
2912 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
2913 {
2914         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
2915
2916         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
2917                         nescq->hw_cq.cq_number); */
2918         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
2919
2920         if (nescq->ibcq.comp_handler)
2921                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
2922
2923         return;
2924 }
2925
2926
2927 /**
2928  * nes_manage_apbvt()
2929  */
2930 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
2931                 u32 nic_index, u32 add_port)
2932 {
2933         struct nes_device *nesdev = nesvnic->nesdev;
2934         struct nes_hw_cqp_wqe *cqp_wqe;
2935         unsigned long flags;
2936         struct nes_cqp_request *cqp_request;
2937         int ret = 0;
2938         u16 major_code;
2939
2940         /* Send manage APBVT request to CQP */
2941         cqp_request = nes_get_cqp_request(nesdev);
2942         if (cqp_request == NULL) {
2943                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
2944                 return -ENOMEM;
2945         }
2946         cqp_request->waiting = 1;
2947         cqp_wqe = &cqp_request->cqp_wqe;
2948
2949         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
2950                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
2951                         accel_local_port, accel_local_port, nic_index);
2952
2953         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2954         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
2955                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
2956         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2957                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
2958
2959         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
2960
2961         atomic_set(&cqp_request->refcount, 2);
2962         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
2963
2964         if (add_port == NES_MANAGE_APBVT_ADD)
2965                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
2966                                 NES_EVENT_TIMEOUT);
2967         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
2968                         ret, cqp_request->major_code, cqp_request->minor_code);
2969         major_code = cqp_request->major_code;
2970         if (atomic_dec_and_test(&cqp_request->refcount)) {
2971                 if (cqp_request->dynamic) {
2972                         kfree(cqp_request);
2973                 } else {
2974                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2975                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2976                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2977                 }
2978         }
2979         if (!ret)
2980                 return -ETIME;
2981         else if (major_code)
2982                 return -EIO;
2983         else
2984                 return 0;
2985 }
2986
2987
2988 /**
2989  * nes_manage_arp_cache
2990  */
2991 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
2992                 u32 ip_addr, u32 action)
2993 {
2994         struct nes_hw_cqp_wqe *cqp_wqe;
2995         struct nes_vnic *nesvnic = netdev_priv(netdev);
2996         struct nes_device *nesdev;
2997         struct nes_cqp_request *cqp_request;
2998         int arp_index;
2999
3000         nesdev = nesvnic->nesdev;
3001         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3002         if (arp_index == -1) {
3003                 return;
3004         }
3005
3006         /* update the ARP entry */
3007         cqp_request = nes_get_cqp_request(nesdev);
3008         if (cqp_request == NULL) {
3009                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3010                 return;
3011         }
3012         cqp_request->waiting = 0;
3013         cqp_wqe = &cqp_request->cqp_wqe;
3014         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3015
3016         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3017                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3018         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3019                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3020         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3021
3022         if (action == NES_ARP_ADD) {
3023                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3024                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3025                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3026                                 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3027                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3028                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3029         } else {
3030                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3031                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3032         }
3033
3034         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3035                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3036
3037         atomic_set(&cqp_request->refcount, 1);
3038         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3039 }
3040
3041
3042 /**
3043  * flush_wqes
3044  */
3045 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3046                 u32 which_wq, u32 wait_completion)
3047 {
3048         unsigned long flags;
3049         struct nes_cqp_request *cqp_request;
3050         struct nes_hw_cqp_wqe *cqp_wqe;
3051         int ret;
3052
3053         cqp_request = nes_get_cqp_request(nesdev);
3054         if (cqp_request == NULL) {
3055                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3056                 return;
3057         }
3058         if (wait_completion) {
3059                 cqp_request->waiting = 1;
3060                 atomic_set(&cqp_request->refcount, 2);
3061         } else {
3062                 cqp_request->waiting = 0;
3063         }
3064         cqp_wqe = &cqp_request->cqp_wqe;
3065         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3066
3067         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3068                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3069         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3070
3071         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3072
3073         if (wait_completion) {
3074                 /* Wait for CQP */
3075                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3076                                 NES_EVENT_TIMEOUT);
3077                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3078                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3079                                 ret, cqp_request->major_code, cqp_request->minor_code);
3080                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3081                         if (cqp_request->dynamic) {
3082                                 kfree(cqp_request);
3083                         } else {
3084                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3085                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3086                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3087                         }
3088                 }
3089         }
3090 }