git.oblomov.eu Git - linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77
  78 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  79 static unsigned char *nes_iwarp_state_str[] = {
  80         "Non-Existant",
  81         "Idle",
  82         "RTS",
  83         "Closing",
  84         "RSVD1",
  85         "Terminate",
  86         "Error",
  87         "RSVD2",
  88 };
  89
  90 static unsigned char *nes_tcp_state_str[] = {
  91         "Non-Existant",
  92         "Closed",
  93         "Listen",
  94         "SYN Sent",
  95         "SYN Rcvd",
  96         "Established",
  97         "Close Wait",
  98         "FIN Wait 1",
  99         "Closing",
 100         "Last Ack",
 101         "FIN Wait 2",
 102         "Time Wait",
 103         "RSVD1",
 104         "RSVD2",
 105         "RSVD3",
 106         "RSVD4",
 107 };
 108 #endif
 109
 110
 111 /**
 112  * nes_nic_init_timer_defaults
 113  */
 114 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 115 {
 116         unsigned long flags;
 117         struct nes_adapter *nesadapter = nesdev->nesadapter;
 118         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 119
 120         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 121
 122         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 123         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 124         if (jumbomode) {
 125                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 128         } else {
 129                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 130                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 131                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 132         }
 133
 134         /* todo use netdev->mtu to set thresholds */
 135         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 136 }
 137
 138
 139 /**
 140  * nes_nic_init_timer
 141  */
 142 static void  nes_nic_init_timer(struct nes_device *nesdev)
 143 {
 144         unsigned long flags;
 145         struct nes_adapter *nesadapter = nesdev->nesadapter;
 146         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 147
 148         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 149
 150         if (shared_timer->timer_in_use_old == 0) {
 151                 nesdev->deepcq_count = 0;
 152                 shared_timer->timer_direction_upward = 0;
 153                 shared_timer->timer_direction_downward = 0;
 154                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 155                 shared_timer->timer_in_use_old = 0;
 156
 157         }
 158         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 159                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 160                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 161                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 162         }
 163         /* todo use netdev->mtu to set thresholds */
 164         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 165 }
 166
 167
 168 /**
 169  * nes_nic_tune_timer
 170  */
 171 static void nes_nic_tune_timer(struct nes_device *nesdev)
 172 {
 173         unsigned long flags;
 174         struct nes_adapter *nesadapter = nesdev->nesadapter;
 175         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 176         u16 cq_count = nesdev->currcq_count;
 177
 178         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 179
 180         if (shared_timer->cq_count_old <= cq_count)
 181                 shared_timer->cq_direction_downward = 0;
 182         else
 183                 shared_timer->cq_direction_downward++;
 184         shared_timer->cq_count_old = cq_count;
 185         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 186                 if (cq_count <= shared_timer->threshold_low &&
 187                     shared_timer->threshold_low > 4) {
 188                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 189                         shared_timer->cq_direction_downward=0;
 190                         nesdev->currcq_count = 0;
 191                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 192                         return;
 193                 }
 194         }
 195
 196         if (cq_count > 1) {
 197                 nesdev->deepcq_count += cq_count;
 198                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 199                         shared_timer->timer_direction_upward++;
 200                         shared_timer->timer_direction_downward = 0;
 201                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 202                         shared_timer->timer_direction_upward = 0;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 205                         shared_timer->timer_direction_downward++;
 206                         shared_timer->timer_direction_upward = 0;
 207                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 208                         shared_timer->timer_in_use -= 2;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 } else {
 212                         shared_timer->timer_in_use -= 4;
 213                         shared_timer->timer_direction_upward = 0;
 214                         shared_timer->timer_direction_downward++;
 215                 }
 216
 217                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 218                         shared_timer->timer_in_use += 3;
 219                         shared_timer->timer_direction_upward = 0;
 220                         shared_timer->timer_direction_downward = 0;
 221                 }
 222                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 223                         shared_timer->timer_in_use -= 4 ;
 224                         shared_timer->timer_direction_downward = 0;
 225                         shared_timer->timer_direction_upward = 0;
 226                 }
 227         }
 228
 229         /* boundary checking */
 230         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 231                 shared_timer->timer_in_use = shared_timer->threshold_high;
 232         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 233                 shared_timer->timer_in_use = shared_timer->threshold_low;
 234
 235         nesdev->currcq_count = 0;
 236
 237         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 238 }
 239
 240
 241 /**
 242  * nes_init_adapter - initialize adapter
 243  */
 244 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 245         struct nes_adapter *nesadapter = NULL;
 246         unsigned long num_pds;
 247         u32 u32temp;
 248         u32 port_count;
 249         u16 max_rq_wrs;
 250         u16 max_sq_wrs;
 251         u32 max_mr;
 252         u32 max_256pbl;
 253         u32 max_4kpbl;
 254         u32 max_qp;
 255         u32 max_irrq;
 256         u32 max_cq;
 257         u32 hte_index_mask;
 258         u32 adapter_size;
 259         u32 arp_table_size;
 260         u16 vendor_id;
 261         u16 device_id;
 262         u8  OneG_Mode;
 263         u8  func_index;
 264
 265         /* search the list of existing adapters */
 266         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 267                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 268                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 269                                 nesdev->pcidev->devfn,
 270                                 PCI_SLOT(nesadapter->devfn),
 271                                 nesadapter->bus_number,
 272                                 PCI_SLOT(nesdev->pcidev->devfn),
 273                                 nesdev->pcidev->bus->number );
 274                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 275                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 276                         nesadapter->ref_count++;
 277                         return nesadapter;
 278                 }
 279         }
 280
 281         /* no adapter found */
 282         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 283         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 284                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 285                                 hw_rev);
 286                 return NULL;
 287         }
 288
 289         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 290                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 291                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 292                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 294
 295         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 296
 297
 298         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 299                 return NULL;
 300
 301         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 302         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 303
 304         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 305         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 306                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 307                                 max_qp, u32temp);
 308                 max_qp = (u32)1 << (u32temp & 0x001f);
 309         }
 310
 311         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 312         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 313                         max_qp, hte_index_mask);
 314
 315         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 316
 317         max_irrq = 1 << (u32temp & 0x001f);
 318
 319         if (max_qp > max_irrq) {
 320                 max_qp = max_irrq;
 321                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 322                                 max_qp);
 323         }
 324
 325         /* there should be no reason to allocate more pds than qps */
 326         if (num_pds > max_qp)
 327                 num_pds = max_qp;
 328
 329         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 330         max_mr = (u32)8192 << (u32temp & 0x7);
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 333         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 334         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 335         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 336
 337         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 338         arp_table_size = 1 << u32temp;
 339
 340         adapter_size = (sizeof(struct nes_adapter) +
 341                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 342         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 343         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 347         adapter_size += sizeof(struct nes_qp **) * max_qp;
 348
 349         /* allocate a new adapter struct */
 350         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 351         if (nesadapter == NULL) {
 352                 return NULL;
 353         }
 354
 355         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 356                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 357
 358         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 359                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 360                 kfree(nesadapter);
 361                 return NULL;
 362         }
 363
 364         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 365                                 (nesadapter->mac_addr_low >> 24);
 366
 367         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 368                                  PCI_DEVICE_ID, &device_id);
 369         nesadapter->vendor_part_id = device_id;
 370
 371         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 372                                                         OneG_Mode)) {
 373                 kfree(nesadapter);
 374                 return NULL;
 375         }
 376         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 377
 378         memset(nesadapter->pft_mcast_map, 255,
 379                sizeof nesadapter->pft_mcast_map);
 380
 381         /* populate the new nesadapter */
 382         nesadapter->devfn = nesdev->pcidev->devfn;
 383         nesadapter->bus_number = nesdev->pcidev->bus->number;
 384         nesadapter->ref_count = 1;
 385         nesadapter->timer_int_req = 0xffff0000;
 386         nesadapter->OneG_Mode = OneG_Mode;
 387         nesadapter->doorbell_start = nesdev->doorbell_region;
 388
 389         /* nesadapter->tick_delta = clk_divisor; */
 390         nesadapter->hw_rev = hw_rev;
 391         nesadapter->port_count = port_count;
 392
 393         nesadapter->max_qp = max_qp;
 394         nesadapter->hte_index_mask = hte_index_mask;
 395         nesadapter->max_irrq = max_irrq;
 396         nesadapter->max_mr = max_mr;
 397         nesadapter->max_256pbl = max_256pbl - 1;
 398         nesadapter->max_4kpbl = max_4kpbl - 1;
 399         nesadapter->max_cq = max_cq;
 400         nesadapter->free_256pbl = max_256pbl - 1;
 401         nesadapter->free_4kpbl = max_4kpbl - 1;
 402         nesadapter->max_pd = num_pds;
 403         nesadapter->arp_table_size = arp_table_size;
 404
 405         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 406         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 407                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 408                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 409                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 410         } else {
 411                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 412                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 413                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 414                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 415         }
 416         /* Setup and enable the periodic timer */
 417         if (nesadapter->et_rx_coalesce_usecs_irq)
 418                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 419                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 420         else
 421                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 422
 423         nesadapter->base_pd = 1;
 424
 425         nesadapter->device_cap_flags =
 426                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 427
 428         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 429                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 430         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 431         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 432         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 433         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 434         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 435
 436
 437         /* mark the usual suspect QPs and CQs as in use */
 438         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 439                 set_bit(u32temp, nesadapter->allocated_qps);
 440                 set_bit(u32temp, nesadapter->allocated_cqs);
 441         }
 442
 443         for (u32temp = 0; u32temp < 20; u32temp++)
 444                 set_bit(u32temp, nesadapter->allocated_pds);
 445         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 446
 447         max_rq_wrs = ((u32temp >> 8) & 3);
 448         switch (max_rq_wrs) {
 449                 case 0:
 450                         max_rq_wrs = 4;
 451                         break;
 452                 case 1:
 453                         max_rq_wrs = 16;
 454                         break;
 455                 case 2:
 456                         max_rq_wrs = 32;
 457                         break;
 458                 case 3:
 459                         max_rq_wrs = 512;
 460                         break;
 461         }
 462
 463         max_sq_wrs = (u32temp & 3);
 464         switch (max_sq_wrs) {
 465                 case 0:
 466                         max_sq_wrs = 4;
 467                         break;
 468                 case 1:
 469                         max_sq_wrs = 16;
 470                         break;
 471                 case 2:
 472                         max_sq_wrs = 32;
 473                         break;
 474                 case 3:
 475                         max_sq_wrs = 512;
 476                         break;
 477         }
 478         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 479         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 480
 481         nesadapter->max_sge = 4;
 482         nesadapter->max_cqe = 32767;
 483
 484         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 485                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 486                 kfree(nesadapter);
 487                 return NULL;
 488         }
 489
 490         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 491         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 492                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 493
 494         /* setup port configuration */
 495         if (nesadapter->port_count == 1) {
 496                 nesadapter->log_port = 0x00000000;
 497                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 498                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 499                 else
 500                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 501         } else {
 502                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 503                         nesadapter->log_port = 0x000000D8;
 504                 } else {
 505                         if (nesadapter->port_count == 2)
 506                                 nesadapter->log_port = 0x00000044;
 507                         else
 508                                 nesadapter->log_port = 0x000000e4;
 509                 }
 510                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 511         }
 512
 513         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 514                                                 nesadapter->log_port);
 515         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 516                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 517
 518         spin_lock_init(&nesadapter->resource_lock);
 519         spin_lock_init(&nesadapter->phy_lock);
 520         spin_lock_init(&nesadapter->pbl_lock);
 521         spin_lock_init(&nesadapter->periodic_timer_lock);
 522
 523         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 524         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 525         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 527
 528         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 529                 u32 pcs_control_status0, pcs_control_status1;
 530                 u32 reset_value;
 531                 u32 i = 0;
 532                 u32 int_cnt = 0;
 533                 u32 ext_cnt = 0;
 534                 unsigned long flags;
 535                 u32 j = 0;
 536
 537                 pcs_control_status0 = nes_read_indexed(nesdev,
 538                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 539                 pcs_control_status1 = nes_read_indexed(nesdev,
 540                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 541
 542                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 543                         pcs_control_status0 = nes_read_indexed(nesdev,
 544                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 545                         pcs_control_status1 = nes_read_indexed(nesdev,
 546                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 547                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 548                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 549                                 int_cnt++;
 550                         msleep(1);
 551                 }
 552                 if (int_cnt > 1) {
 553                         u32 sds;
 554                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 555                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 556                         sds |= 0x00000040;
 557                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 558                         mh_detected++;
 559                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 560                         reset_value |= 0x0000003d;
 561                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 562
 563                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 564                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 565                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 566
 567                         pcs_control_status0 = nes_read_indexed(nesdev,
 568                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 569                         pcs_control_status1 = nes_read_indexed(nesdev,
 570                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 571
 572                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 573                                 pcs_control_status0 = nes_read_indexed(nesdev,
 574                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 575                                 pcs_control_status1 = nes_read_indexed(nesdev,
 576                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 577                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 578                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 579                                         if (++ext_cnt > int_cnt) {
 580                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 581                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 582                                                                 0x0000F0C8);
 583                                                 mh_detected++;
 584                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 585                                                 reset_value |= 0x0000003d;
 586                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 587
 588                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 589                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 590                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 591                                                 break;
 592                                         }
 593                                 }
 594                                 msleep(1);
 595                         }
 596                 }
 597         }
 598
 599         if (nesadapter->hw_rev == NE020_REV) {
 600                 init_timer(&nesadapter->mh_timer);
 601                 nesadapter->mh_timer.function = nes_mh_fix;
 602                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 603                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 604                 add_timer(&nesadapter->mh_timer);
 605         } else {
 606                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 607         }
 608
 609         init_timer(&nesadapter->lc_timer);
 610         nesadapter->lc_timer.function = nes_clc;
 611         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 612         nesadapter->lc_timer.data = (unsigned long)nesdev;
 613         add_timer(&nesadapter->lc_timer);
 614
 615         list_add_tail(&nesadapter->list, &nes_adapter_list);
 616
 617         for (func_index = 0; func_index < 8; func_index++) {
 618                 pci_bus_read_config_word(nesdev->pcidev->bus,
 619                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 620                                         func_index), 0, &vendor_id);
 621                 if (vendor_id == 0xffff)
 622                         break;
 623         }
 624         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 625                 func_index, pci_name(nesdev->pcidev));
 626         nesadapter->adapter_fcn_count = func_index;
 627
 628         return nesadapter;
 629 }
 630
 631
 632 /**
 633  * nes_reset_adapter_ne020
 634  */
 635 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 636 {
 637         u32 port_count;
 638         u32 u32temp;
 639         u32 i;
 640
 641         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 642         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 643         /* TODO: assuming that both SERDES are set the same for now */
 644         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 645         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 646                         u32temp, port_count);
 647         if (*OneG_Mode)
 648                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 649         u32temp &= 0xff00ffc0;
 650         switch (port_count) {
 651                 case 1:
 652                         u32temp |= 0x00ee0000;
 653                         break;
 654                 case 2:
 655                         u32temp |= 0x00cc0000;
 656                         break;
 657                 case 4:
 658                         u32temp |= 0x00000000;
 659                         break;
 660                 default:
 661                         return 0;
 662                         break;
 663         }
 664
 665         /* check and do full reset if needed */
 666         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 667                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 668                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 669
 670                 i = 0;
 671                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 672                         mdelay(1);
 673                 if (i >= 10000) {
 674                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 675                         return 0;
 676                 }
 677
 678                 i = 0;
 679                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 680                         mdelay(1);
 681                 if (i >= 10000) {
 682                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 683                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 684                         return 0;
 685                 }
 686         }
 687
 688         /* port reset */
 689         switch (port_count) {
 690                 case 1:
 691                         u32temp |= 0x00ee0010;
 692                         break;
 693                 case 2:
 694                         u32temp |= 0x00cc0030;
 695                         break;
 696                 case 4:
 697                         u32temp |= 0x00000030;
 698                         break;
 699         }
 700
 701         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 702         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 703
 704         i = 0;
 705         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 706                 mdelay(1);
 707         if (i >= 10000) {
 708                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 709                 return 0;
 710         }
 711
 712         /* serdes 0 */
 713         i = 0;
 714         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 715                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 716                 mdelay(1);
 717         if (i >= 5000) {
 718                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 719                 return 0;
 720         }
 721
 722         /* serdes 1 */
 723         if (port_count > 1) {
 724                 i = 0;
 725                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 726                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 727                         mdelay(1);
 728                 if (i >= 5000) {
 729                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 730                         return 0;
 731                 }
 732         }
 733
 734         return port_count;
 735 }
 736
 737
 738 /**
 739  * nes_init_serdes
 740  */
 741 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 742                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 743 {
 744         int i;
 745         u32 u32temp;
 746         u32 sds;
 747
 748         if (hw_rev != NE020_REV) {
 749                 /* init serdes 0 */
 750                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
 751                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 752                 else
 753                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 754
 755                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 756                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 757                         sds |= 0x00000100;
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 759                 }
 760                 if (!OneG_Mode)
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 762
 763                 if (port_count < 2)
 764                         return 0;
 765
 766                 /* init serdes 1 */
 767                 switch (nesadapter->phy_type[1]) {
 768                 case NES_PHY_TYPE_ARGUS:
 769                 case NES_PHY_TYPE_SFP_D:
 770                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 771                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 772                         break;
 773                 case NES_PHY_TYPE_CX4:
 774                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 775                         sds &= 0xFFFFFFBF;
 776                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 777                         if (wide_ppm_offset)
 778                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 779                         else
 780                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 781                         break;
 782                 case NES_PHY_TYPE_PUMA_1G:
 783                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 784                         sds |= 0x000000100;
 785                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 786                 }
 787                 if (!OneG_Mode)
 788                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 789         } else {
 790                 /* init serdes 0 */
 791                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 792                 i = 0;
 793                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 794                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 795                         mdelay(1);
 796                 if (i >= 5000) {
 797                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 798                         return 1;
 799                 }
 800                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 801                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 802                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 805                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 806                 if (OneG_Mode)
 807                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 808                 else
 809                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 810
 811                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 812                 if (port_count > 1) {
 813                         /* init serdes 1 */
 814                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 815                         i = 0;
 816                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 817                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 818                                 mdelay(1);
 819                         if (i >= 5000) {
 820                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 821                                 /* return 1; */
 822                         }
 823                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 824                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 830                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 831                 }
 832         }
 833         return 0;
 834 }
 835
 836
 837 /**
 838  * nes_init_csr_ne020
 839  * Initialize registers for ne020 hardware
 840  */
 841 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 842 {
 843         u32 u32temp;
 844
 845         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 846
 847         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 848         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 849         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 850         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 851         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 852         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 853         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 854         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 855
 856         /* TODO: move these MAC register settings to NIC bringup */
 857         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 858         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 859         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 860         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 861         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 862         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 863         if (port_count > 1) {
 864                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 865                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 866                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 867                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 868                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 869                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 870                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 871         }
 872         if (port_count > 2) {
 873                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 874                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 875                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 876                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 877                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 878                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 879                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 880
 881                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 882                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 883                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 884                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 885                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 886                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 887                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 888         }
 889
 890         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 891         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 892         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 893                                                          0x00000001);
 894         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 895         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 896         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 897         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 898         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 899
 900         /* TODO: move this to code, get from EEPROM */
 901         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 902         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 903         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 904
 905         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 906         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 907
 908         if (hw_rev != NE020_REV) {
 909                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 910                 u32temp |= 0x80000000;
 911                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 912                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 913                 u32temp &= 0x7fffffff;
 914                 u32temp |= 0x7fff0010;
 915                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 916         }
 917 }
 918
 919
 920 /**
 921  * nes_destroy_adapter - destroy the adapter structure
 922  */
 923 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 924 {
 925         struct nes_adapter *tmp_adapter;
 926
 927         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 928                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 929                                 tmp_adapter);
 930         }
 931
 932         nesadapter->ref_count--;
 933         if (!nesadapter->ref_count) {
 934                 if (nesadapter->hw_rev == NE020_REV) {
 935                         del_timer(&nesadapter->mh_timer);
 936                 }
 937                 del_timer(&nesadapter->lc_timer);
 938
 939                 list_del(&nesadapter->list);
 940                 kfree(nesadapter);
 941         }
 942 }
 943
 944
 945 /**
 946  * nes_init_cqp
 947  */
 948 int nes_init_cqp(struct nes_device *nesdev)
 949 {
 950         struct nes_adapter *nesadapter = nesdev->nesadapter;
 951         struct nes_hw_cqp_qp_context *cqp_qp_context;
 952         struct nes_hw_cqp_wqe *cqp_wqe;
 953         struct nes_hw_ceq *ceq;
 954         struct nes_hw_ceq *nic_ceq;
 955         struct nes_hw_aeq *aeq;
 956         void *vmem;
 957         dma_addr_t pmem;
 958         u32 count=0;
 959         u32 cqp_head;
 960         u64 u64temp;
 961         u32 u32temp;
 962
 963         /* allocate CQP memory */
 964         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 965         /* SQ is 512 byte aligned, others are 256 byte aligned */
 966         nesdev->cqp_mem_size = 512 +
 967                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 968                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 969                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 970                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 971                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 972                         sizeof(struct nes_hw_cqp_qp_context);
 973
 974         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 975                         &nesdev->cqp_pbase);
 976         if (!nesdev->cqp_vbase) {
 977                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 978                 return -ENOMEM;
 979         }
 980         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 981
 982         /* Allocate a twice the number of CQP requests as the SQ size */
 983         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 984                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 985         if (nesdev->nes_cqp_requests == NULL) {
 986                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 987                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 988                                 nesdev->cqp.sq_pbase);
 989                 return -ENOMEM;
 990         }
 991
 992         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 993                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 994
 995         spin_lock_init(&nesdev->cqp.lock);
 996         init_waitqueue_head(&nesdev->cqp.waitq);
 997
 998         /* Setup Various Structures */
 999         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1000                         ~(unsigned long)(512 - 1));
1001         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1002                         ~(unsigned long long)(512 - 1));
1003
1004         nesdev->cqp.sq_vbase = vmem;
1005         nesdev->cqp.sq_pbase = pmem;
1006         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1007         nesdev->cqp.sq_head = 0;
1008         nesdev->cqp.sq_tail = 0;
1009         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1010
1011         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1012         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1013
1014         nesdev->ccq.cq_vbase = vmem;
1015         nesdev->ccq.cq_pbase = pmem;
1016         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1017         nesdev->ccq.cq_head = 0;
1018         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1019         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1020
1021         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1022         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1023
1024         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1025         ceq = &nesadapter->ceq[nesdev->ceq_index];
1026         ceq->ceq_vbase = vmem;
1027         ceq->ceq_pbase = pmem;
1028         ceq->ceq_size = NES_CCEQ_SIZE;
1029         ceq->ceq_head = 0;
1030
1031         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1032         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1033
1034         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1035         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1036         nic_ceq->ceq_vbase = vmem;
1037         nic_ceq->ceq_pbase = pmem;
1038         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1039         nic_ceq->ceq_head = 0;
1040
1041         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1042         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1043
1044         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1045         aeq->aeq_vbase = vmem;
1046         aeq->aeq_pbase = pmem;
1047         aeq->aeq_size = nesadapter->max_qp;
1048         aeq->aeq_head = 0;
1049
1050         /* Setup QP Context */
1051         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1052         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1053
1054         cqp_qp_context = vmem;
1055         cqp_qp_context->context_words[0] =
1056                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1057         cqp_qp_context->context_words[1] = 0;
1058         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1059         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1060
1061
1062         /* Write the address to Create CQP */
1063         if ((sizeof(dma_addr_t) > 4)) {
1064                 nes_write_indexed(nesdev,
1065                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1066                                 ((u64)pmem) >> 32);
1067         } else {
1068                 nes_write_indexed(nesdev,
1069                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1070         }
1071         nes_write_indexed(nesdev,
1072                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1073                         (u32)pmem);
1074
1075         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1076         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1077
1078         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1079                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1080                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1081         }
1082
1083         /* Write Create CCQ WQE */
1084         cqp_head = nesdev->cqp.sq_head++;
1085         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1086         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1087         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1088                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1089                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1090         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1091                             (nesdev->ccq.cq_number |
1092                              ((u32)nesdev->ceq_index << 16)));
1093         u64temp = (u64)nesdev->ccq.cq_pbase;
1094         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1095         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1096         u64temp = (unsigned long)&nesdev->ccq;
1097         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1098                         cpu_to_le32((u32)(u64temp >> 1));
1099         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1100                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1101         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1102
1103         /* Write Create CEQ WQE */
1104         cqp_head = nesdev->cqp.sq_head++;
1105         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1106         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1107         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1108                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1109         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1110         u64temp = (u64)ceq->ceq_pbase;
1111         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1112
1113         /* Write Create AEQ WQE */
1114         cqp_head = nesdev->cqp.sq_head++;
1115         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1116         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1117         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1118                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1119         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1120         u64temp = (u64)aeq->aeq_pbase;
1121         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1122
1123         /* Write Create NIC CEQ WQE */
1124         cqp_head = nesdev->cqp.sq_head++;
1125         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1126         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1127         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1128                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1129         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1130         u64temp = (u64)nic_ceq->ceq_pbase;
1131         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1132
1133         /* Poll until CCQP done */
1134         count = 0;
1135         do {
1136                 if (count++ > 1000) {
1137                         printk(KERN_ERR PFX "Error creating CQP\n");
1138                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1139                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1140                         return -1;
1141                 }
1142                 udelay(10);
1143         } while (!(nes_read_indexed(nesdev,
1144                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1145
1146         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1147                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1148
1149         u32temp = 0x04800000;
1150         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1151
1152         /* wait for the CCQ, CEQ, and AEQ to get created */
1153         count = 0;
1154         do {
1155                 if (count++ > 1000) {
1156                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1157                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1158                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1159                         return -1;
1160                 }
1161                 udelay(10);
1162         } while (((nes_read_indexed(nesdev,
1163                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1164
1165         /* dump the QP status value */
1166         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1167                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1168
1169         nesdev->cqp.sq_tail++;
1170
1171         return 0;
1172 }
1173
1174
1175 /**
1176  * nes_destroy_cqp
1177  */
1178 int nes_destroy_cqp(struct nes_device *nesdev)
1179 {
1180         struct nes_hw_cqp_wqe *cqp_wqe;
1181         u32 count = 0;
1182         u32 cqp_head;
1183         unsigned long flags;
1184
1185         do {
1186                 if (count++ > 1000)
1187                         break;
1188                 udelay(10);
1189         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1190
1191         /* Reset CCQ */
1192         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1193                         nesdev->ccq.cq_number);
1194
1195         /* Disable device interrupts */
1196         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1197
1198         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1199
1200         /* Destroy the AEQ */
1201         cqp_head = nesdev->cqp.sq_head++;
1202         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1203         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1204         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1205                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1206         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1207
1208         /* Destroy the NIC CEQ */
1209         cqp_head = nesdev->cqp.sq_head++;
1210         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1211         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1212         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1213                         ((u32)nesdev->nic_ceq_index << 8));
1214
1215         /* Destroy the CEQ */
1216         cqp_head = nesdev->cqp.sq_head++;
1217         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1218         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1219         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1220                         (nesdev->ceq_index << 8));
1221
1222         /* Destroy the CCQ */
1223         cqp_head = nesdev->cqp.sq_head++;
1224         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1225         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1226         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1227         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1228                         ((u32)nesdev->ceq_index << 16));
1229
1230         /* Destroy CQP */
1231         cqp_head = nesdev->cqp.sq_head++;
1232         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1233         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1234         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1235                         NES_CQP_QP_TYPE_CQP);
1236         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1237
1238         barrier();
1239         /* Ring doorbell (5 WQEs) */
1240         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1241
1242         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1243
1244         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1245         count = 0;
1246         do {
1247                 if (count++ > 1000) {
1248                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1249                                         PCI_FUNC(nesdev->pcidev->devfn));
1250                         break;
1251                 }
1252                 udelay(10);
1253         } while (((nes_read_indexed(nesdev,
1254                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1255
1256         /* dump the QP status value */
1257         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1258                         PCI_FUNC(nesdev->pcidev->devfn),
1259                         nes_read_indexed(nesdev,
1260                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1261
1262         kfree(nesdev->nes_cqp_requests);
1263
1264         /* Free the control structures */
1265         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1266                         nesdev->cqp.sq_pbase);
1267
1268         return 0;
1269 }
1270
1271
1272 /**
1273  * nes_init_phy
1274  */
1275 int nes_init_phy(struct nes_device *nesdev)
1276 {
1277         struct nes_adapter *nesadapter = nesdev->nesadapter;
1278         u32 counter = 0;
1279         u32 sds;
1280         u32 mac_index = nesdev->mac_index;
1281         u32 tx_config = 0;
1282         u16 phy_data;
1283         u32 temp_phy_data = 0;
1284         u32 temp_phy_data2 = 0;
1285         u8  phy_type = nesadapter->phy_type[mac_index];
1286         u8  phy_index = nesadapter->phy_index[mac_index];
1287
1288         if ((nesadapter->OneG_Mode) &&
1289             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1290                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1291                 if (phy_type == NES_PHY_TYPE_1G) {
1292                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1293                         tx_config &= 0xFFFFFFE3;
1294                         tx_config |= 0x04;
1295                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1296                 }
1297
1298                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1299                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1300
1301                 /* Reset the PHY */
1302                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1303                 udelay(100);
1304                 counter = 0;
1305                 do {
1306                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1307                         if (counter++ > 100)
1308                                 break;
1309                 } while (phy_data & 0x8000);
1310
1311                 /* Setting no phy loopback */
1312                 phy_data &= 0xbfff;
1313                 phy_data |= 0x1140;
1314                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1315                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1316                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1317                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1318
1319                 /* Setting the interrupt mask */
1320                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1321                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1322                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1323
1324                 /* turning on flow control */
1325                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1326                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1327                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1328
1329                 /* Clear Half duplex */
1330                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1331                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1332                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1333
1334                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1335                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1336
1337                 return 0;
1338         }
1339
1340         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1341             (phy_type == NES_PHY_TYPE_ARGUS) ||
1342             (phy_type == NES_PHY_TYPE_SFP_D)) {
1343                 /* setup 10G MDIO operation */
1344                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1345                 tx_config &= 0xFFFFFFE3;
1346                 tx_config |= 0x15;
1347                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1348         }
1349         if ((phy_type == NES_PHY_TYPE_ARGUS) ||
1350             (phy_type == NES_PHY_TYPE_SFP_D)) {
1351                 /* Check firmware heartbeat */
1352                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1353                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1354                 udelay(1500);
1355                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1356                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1357
1358                 if (temp_phy_data != temp_phy_data2)
1359                         return 0;
1360
1361                 /* no heartbeat, configure the PHY */
1362                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1363                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1364                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1365                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1366                 if (phy_type == NES_PHY_TYPE_ARGUS) {
1367                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1368                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1369                 } else {
1370                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1371                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1372                 }
1373                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1374                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1375                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1376
1377                 /* setup LEDs */
1378                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1379                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1380                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1381
1382                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1383
1384                 /* Bring PHY out of reset */
1385                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1386
1387                 /* Check for heartbeat */
1388                 counter = 0;
1389                 mdelay(690);
1390                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1391                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1392                 do {
1393                         if (counter++ > 150) {
1394                                 nes_debug(NES_DBG_PHY, "No PHY heartbeat\n");
1395                                 break;
1396                         }
1397                         mdelay(1);
1398                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1399                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1400                 } while ((temp_phy_data2 == temp_phy_data));
1401
1402                 /* wait for tracking */
1403                 counter = 0;
1404                 do {
1405                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1406                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1407                         if (counter++ > 300) {
1408                                 nes_debug(NES_DBG_PHY, "PHY did not track\n");
1409                                 break;
1410                         }
1411                         mdelay(10);
1412                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1413
1414                 /* setup signal integrity */
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1420
1421                 /* reset serdes */
1422                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1423                                        mac_index * 0x200);
1424                 sds |= 0x1;
1425                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1426                                   mac_index * 0x200, sds);
1427                 sds &= 0xfffffffe;
1428                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1429                                   mac_index * 0x200, sds);
1430
1431                 counter = 0;
1432                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1433                                 && (counter++ < 5000))
1434                         ;
1435         }
1436         return 0;
1437 }
1438
1439
1440 /**
1441  * nes_replenish_nic_rq
1442  */
1443 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1444 {
1445         unsigned long flags;
1446         dma_addr_t bus_address;
1447         struct sk_buff *skb;
1448         struct nes_hw_nic_rq_wqe *nic_rqe;
1449         struct nes_hw_nic *nesnic;
1450         struct nes_device *nesdev;
1451         u32 rx_wqes_posted = 0;
1452
1453         nesnic = &nesvnic->nic;
1454         nesdev = nesvnic->nesdev;
1455         spin_lock_irqsave(&nesnic->rq_lock, flags);
1456         if (nesnic->replenishing_rq !=0) {
1457                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1458                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1459                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1460                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1461                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1462                         add_timer(&nesvnic->rq_wqes_timer);
1463                 } else
1464                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1465                 return;
1466         }
1467         nesnic->replenishing_rq = 1;
1468         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1469         do {
1470                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1471                 if (skb) {
1472                         skb->dev = nesvnic->netdev;
1473
1474                         bus_address = pci_map_single(nesdev->pcidev,
1475                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1476
1477                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1478                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1479                                         cpu_to_le32(nesvnic->max_frame_size);
1480                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1481                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1482                                         cpu_to_le32((u32)bus_address);
1483                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1484                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1485                         nesnic->rx_skb[nesnic->rq_head] = skb;
1486                         nesnic->rq_head++;
1487                         nesnic->rq_head &= nesnic->rq_size - 1;
1488                         atomic_dec(&nesvnic->rx_skbs_needed);
1489                         barrier();
1490                         if (++rx_wqes_posted == 255) {
1491                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1492                                 rx_wqes_posted = 0;
1493                         }
1494                 } else {
1495                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1496                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1497                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1498                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1499                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1500                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1501                                 add_timer(&nesvnic->rq_wqes_timer);
1502                         } else
1503                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1504                         break;
1505                 }
1506         } while (atomic_read(&nesvnic->rx_skbs_needed));
1507         barrier();
1508         if (rx_wqes_posted)
1509                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1510         nesnic->replenishing_rq = 0;
1511 }
1512
1513
1514 /**
1515  * nes_rq_wqes_timeout
1516  */
1517 static void nes_rq_wqes_timeout(unsigned long parm)
1518 {
1519         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1520         printk("%s: Timer fired.\n", __func__);
1521         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1522         if (atomic_read(&nesvnic->rx_skbs_needed))
1523                 nes_replenish_nic_rq(nesvnic);
1524 }
1525
1526
1527 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1528                                void **tcph, u64 *hdr_flags, void *priv)
1529 {
1530         unsigned int ip_len;
1531         struct iphdr *iph;
1532         skb_reset_network_header(skb);
1533         iph = ip_hdr(skb);
1534         if (iph->protocol != IPPROTO_TCP)
1535                 return -1;
1536         ip_len = ip_hdrlen(skb);
1537         skb_set_transport_header(skb, ip_len);
1538         *tcph = tcp_hdr(skb);
1539
1540         *hdr_flags = LRO_IPV4 | LRO_TCP;
1541         *iphdr = iph;
1542         return 0;
1543 }
1544
1545
1546 /**
1547  * nes_init_nic_qp
1548  */
1549 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1550 {
1551         struct nes_hw_cqp_wqe *cqp_wqe;
1552         struct nes_hw_nic_sq_wqe *nic_sqe;
1553         struct nes_hw_nic_qp_context *nic_context;
1554         struct sk_buff *skb;
1555         struct nes_hw_nic_rq_wqe *nic_rqe;
1556         struct nes_vnic *nesvnic = netdev_priv(netdev);
1557         unsigned long flags;
1558         void *vmem;
1559         dma_addr_t pmem;
1560         u64 u64temp;
1561         int ret;
1562         u32 cqp_head;
1563         u32 counter;
1564         u32 wqe_count;
1565         u8 jumbomode=0;
1566
1567         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1568         nesvnic->nic_mem_size = 256 +
1569                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1570                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1571                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1572                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1573                         sizeof(struct nes_hw_nic_qp_context);
1574
1575         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1576                         &nesvnic->nic_pbase);
1577         if (!nesvnic->nic_vbase) {
1578                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1579                 return -ENOMEM;
1580         }
1581         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1582         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1583                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1584
1585         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1586                         ~(unsigned long)(256 - 1));
1587         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1588                         ~(unsigned long long)(256 - 1));
1589
1590         /* Setup the first Fragment buffers */
1591         nesvnic->nic.first_frag_vbase = vmem;
1592
1593         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1594                 nesvnic->nic.frag_paddr[counter] = pmem;
1595                 pmem += sizeof(struct nes_first_frag);
1596         }
1597
1598         /* setup the SQ */
1599         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1600
1601         nesvnic->nic.sq_vbase = (void *)vmem;
1602         nesvnic->nic.sq_pbase = pmem;
1603         nesvnic->nic.sq_head = 0;
1604         nesvnic->nic.sq_tail = 0;
1605         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1606         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1607                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1608                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1609                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1610                                 NES_NIC_SQ_WQE_COMPLETION);
1611                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1612                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1613                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1614                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1615                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1616                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1617         }
1618
1619         nesvnic->get_cqp_request = nes_get_cqp_request;
1620         nesvnic->post_cqp_request = nes_post_cqp_request;
1621         nesvnic->mcrq_mcast_filter = NULL;
1622
1623         spin_lock_init(&nesvnic->nic.rq_lock);
1624
1625         /* setup the RQ */
1626         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1627         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1628
1629
1630         nesvnic->nic.rq_vbase = vmem;
1631         nesvnic->nic.rq_pbase = pmem;
1632         nesvnic->nic.rq_head = 0;
1633         nesvnic->nic.rq_tail = 0;
1634         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1635
1636         /* setup the CQ */
1637         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1638         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1639
1640         if (nesdev->nesadapter->netdev_count > 2)
1641                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1642         else
1643                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1644
1645         nesvnic->nic_cq.cq_vbase = vmem;
1646         nesvnic->nic_cq.cq_pbase = pmem;
1647         nesvnic->nic_cq.cq_head = 0;
1648         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1649
1650         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1651
1652         /* Send CreateCQ request to CQP */
1653         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1654         cqp_head = nesdev->cqp.sq_head;
1655
1656         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1657         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1658
1659         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1660                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1661                         ((u32)nesvnic->nic_cq.cq_size << 16));
1662         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1663                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1664         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1665         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1666         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1667         u64temp = (unsigned long)&nesvnic->nic_cq;
1668         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1669         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1670                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1671         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1672         if (++cqp_head >= nesdev->cqp.sq_size)
1673                 cqp_head = 0;
1674         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1675         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1676
1677         /* Send CreateQP request to CQP */
1678         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1679         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1680                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1681                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1682         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1683                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1684                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1685         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1686                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1687         }
1688
1689         u64temp = (u64)nesvnic->nic.sq_pbase;
1690         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1691         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1692         u64temp = (u64)nesvnic->nic.rq_pbase;
1693         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1694         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1695
1696         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1697                         NES_CQP_QP_TYPE_NIC);
1698         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1699         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1700                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1701         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1702
1703         if (++cqp_head >= nesdev->cqp.sq_size)
1704                 cqp_head = 0;
1705         nesdev->cqp.sq_head = cqp_head;
1706
1707         barrier();
1708
1709         /* Ring doorbell (2 WQEs) */
1710         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1711
1712         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1713         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1714                         nesvnic->nic.qp_id);
1715
1716         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1717                         NES_EVENT_TIMEOUT);
1718         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1719                         nesvnic->nic.qp_id, ret);
1720         if (!ret) {
1721                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1722                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1723                                 nesvnic->nic_pbase);
1724                 return -EIO;
1725         }
1726
1727         /* Populate the RQ */
1728         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1729                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1730                 if (!skb) {
1731                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1732
1733                         nes_destroy_nic_qp(nesvnic);
1734                         return -ENOMEM;
1735                 }
1736
1737                 skb->dev = netdev;
1738
1739                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1740                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1741
1742                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1743                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1744                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1745                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1746                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1747                 nesvnic->nic.rx_skb[counter] = skb;
1748         }
1749
1750         wqe_count = NES_NIC_WQ_SIZE - 1;
1751         nesvnic->nic.rq_head = wqe_count;
1752         barrier();
1753         do {
1754                 counter = min(wqe_count, ((u32)255));
1755                 wqe_count -= counter;
1756                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1757         } while (wqe_count);
1758         init_timer(&nesvnic->rq_wqes_timer);
1759         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1760         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1761         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1762         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1763         {
1764                 nes_nic_init_timer(nesdev);
1765                 if (netdev->mtu > 1500)
1766                         jumbomode = 1;
1767                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1768         }
1769         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1770         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1771         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1772         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1773         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1774         nesvnic->lro_mgr.dev            = netdev;
1775         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1776         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1777         return 0;
1778 }
1779
1780
1781 /**
1782  * nes_destroy_nic_qp
1783  */
1784 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1785 {
1786         u64 u64temp;
1787         dma_addr_t bus_address;
1788         struct nes_device *nesdev = nesvnic->nesdev;
1789         struct nes_hw_cqp_wqe *cqp_wqe;
1790         struct nes_hw_nic_sq_wqe *nic_sqe;
1791         struct nes_hw_nic_rq_wqe *nic_rqe;
1792         __le16 *wqe_fragment_length;
1793         u16  wqe_fragment_index;
1794         u64 wqe_frag;
1795         u32 cqp_head;
1796         unsigned long flags;
1797         int ret;
1798
1799         /* Free remaining NIC receive buffers */
1800         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1801                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1802                 wqe_frag  = (u64)le32_to_cpu(
1803                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1804                 wqe_frag |= ((u64)le32_to_cpu(
1805                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1806                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1807                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1808                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1809                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1810         }
1811
1812         /* Free remaining NIC transmit buffers */
1813         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1814                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1815                 wqe_fragment_index = 1;
1816                 wqe_fragment_length = (__le16 *)
1817                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1818                 /* bump past the vlan tag */
1819                 wqe_fragment_length++;
1820                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1821                         u64temp = (u64)le32_to_cpu(
1822                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1823                                 wqe_fragment_index*2]);
1824                         u64temp += ((u64)le32_to_cpu(
1825                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1826                                 + wqe_fragment_index*2]))<<32;
1827                         bus_address = (dma_addr_t)u64temp;
1828                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1829                                         nesvnic->nic.first_frag_overflow)) {
1830                                 pci_unmap_single(nesdev->pcidev,
1831                                                 bus_address,
1832                                                 le16_to_cpu(wqe_fragment_length[
1833                                                         wqe_fragment_index++]),
1834                                                 PCI_DMA_TODEVICE);
1835                         }
1836                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1837                                 if (wqe_fragment_length[wqe_fragment_index]) {
1838                                         u64temp = le32_to_cpu(
1839                                                 nic_sqe->wqe_words[
1840                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1841                                                 wqe_fragment_index*2]);
1842                                         u64temp += ((u64)le32_to_cpu(
1843                                                 nic_sqe->wqe_words[
1844                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1845                                                 wqe_fragment_index*2]))<<32;
1846                                         bus_address = (dma_addr_t)u64temp;
1847                                         pci_unmap_page(nesdev->pcidev,
1848                                                         bus_address,
1849                                                         le16_to_cpu(
1850                                                         wqe_fragment_length[
1851                                                         wqe_fragment_index]),
1852                                                         PCI_DMA_TODEVICE);
1853                                 } else
1854                                         break;
1855                         }
1856                 }
1857                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1858                         dev_kfree_skb(
1859                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1860
1861                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1862                                         & (nesvnic->nic.sq_size - 1);
1863         }
1864
1865         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1866
1867         /* Destroy NIC QP */
1868         cqp_head = nesdev->cqp.sq_head;
1869         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1870         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1871
1872         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1873                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1874         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1875                 nesvnic->nic.qp_id);
1876
1877         if (++cqp_head >= nesdev->cqp.sq_size)
1878                 cqp_head = 0;
1879
1880         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1881
1882         /* Destroy NIC CQ */
1883         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1884         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1885                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1886         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1887                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1888
1889         if (++cqp_head >= nesdev->cqp.sq_size)
1890                 cqp_head = 0;
1891
1892         nesdev->cqp.sq_head = cqp_head;
1893         barrier();
1894
1895         /* Ring doorbell (2 WQEs) */
1896         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1897
1898         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1899         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1900                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1901                         cqp_head, nesdev->cqp.sq_head,
1902                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1903
1904         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1905                         NES_EVENT_TIMEOUT);
1906
1907         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1908                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1909                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1910         if (!ret) {
1911                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1912                                 nesvnic->nic.qp_id);
1913         }
1914
1915         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1916                         nesvnic->nic_pbase);
1917 }
1918
1919 /**
1920  * nes_napi_isr
1921  */
1922 int nes_napi_isr(struct nes_device *nesdev)
1923 {
1924         struct nes_adapter *nesadapter = nesdev->nesadapter;
1925         u32 int_stat;
1926
1927         if (nesdev->napi_isr_ran) {
1928                 /* interrupt status has already been read in ISR */
1929                 int_stat = nesdev->int_stat;
1930         } else {
1931                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1932                 nesdev->int_stat = int_stat;
1933                 nesdev->napi_isr_ran = 1;
1934         }
1935
1936         int_stat &= nesdev->int_req;
1937         /* iff NIC, process here, else wait for DPC */
1938         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1939                 nesdev->napi_isr_ran = 0;
1940                 nes_write32(nesdev->regs + NES_INT_STAT,
1941                         (int_stat &
1942                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1943
1944                 /* Process the CEQs */
1945                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1946
1947                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1948                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1949                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1950                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1951                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1952                                 /* Enable Periodic timer interrupts */
1953                                 nesdev->int_req |= NES_INT_TIMER;
1954                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1955                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1956                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1957                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1958                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1959                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1960                         }
1961
1962                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1963                         {
1964                                 nes_nic_init_timer(nesdev);
1965                         }
1966                         /* Enable interrupts, except CEQs */
1967                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1968                 } else {
1969                         /* Enable interrupts, make sure timer is off */
1970                         nesdev->int_req &= ~NES_INT_TIMER;
1971                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1972                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1973                 }
1974                 nesdev->deepcq_count = 0;
1975                 return 1;
1976         } else {
1977                 return 0;
1978         }
1979 }
1980
1981 static void process_critical_error(struct nes_device *nesdev)
1982 {
1983         u32 debug_error;
1984         u32 nes_idx_debug_error_masks0 = 0;
1985         u16 error_module = 0;
1986
1987         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1988         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1989                         (u16)debug_error);
1990         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1991                         0x01010000 | (debug_error & 0x0000ffff));
1992         if (crit_err_count++ > 10)
1993                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1994         error_module = (u16) (debug_error & 0x1F00) >> 8;
1995         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
1996                         nes_max_critical_error_count) {
1997                 printk(KERN_ERR PFX "Masking off critical error for module "
1998                         "0x%02X\n", (u16)error_module);
1999                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2000                         NES_IDX_DEBUG_ERROR_MASKS0);
2001                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2002                         nes_idx_debug_error_masks0 | (1 << error_module));
2003         }
2004 }
2005 /**
2006  * nes_dpc
2007  */
2008 void nes_dpc(unsigned long param)
2009 {
2010         struct nes_device *nesdev = (struct nes_device *)param;
2011         struct nes_adapter *nesadapter = nesdev->nesadapter;
2012         u32 counter;
2013         u32 loop_counter = 0;
2014         u32 int_status_bit;
2015         u32 int_stat;
2016         u32 timer_stat;
2017         u32 temp_int_stat;
2018         u32 intf_int_stat;
2019         u32 processed_intf_int = 0;
2020         u16 processed_timer_int = 0;
2021         u16 completion_ints = 0;
2022         u16 timer_ints = 0;
2023
2024         /* nes_debug(NES_DBG_ISR, "\n"); */
2025
2026         do {
2027                 timer_stat = 0;
2028                 if (nesdev->napi_isr_ran) {
2029                         nesdev->napi_isr_ran = 0;
2030                         int_stat = nesdev->int_stat;
2031                 } else
2032                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2033                 if (processed_intf_int != 0)
2034                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2035                 else
2036                         int_stat &= nesdev->int_req;
2037                 if (processed_timer_int == 0) {
2038                         processed_timer_int = 1;
2039                         if (int_stat & NES_INT_TIMER) {
2040                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2041                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2042                                         int_stat &= ~NES_INT_TIMER;
2043                                 }
2044                         }
2045                 } else {
2046                         int_stat &= ~NES_INT_TIMER;
2047                 }
2048
2049                 if (int_stat) {
2050                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2051                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2052                                 /* Ack the interrupts */
2053                                 nes_write32(nesdev->regs+NES_INT_STAT,
2054                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2055                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2056                         }
2057
2058                         temp_int_stat = int_stat;
2059                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2060                                 if (int_stat & int_status_bit) {
2061                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2062                                         temp_int_stat &= ~int_status_bit;
2063                                         completion_ints = 1;
2064                                 }
2065                                 if (!(temp_int_stat & 0x0000ffff))
2066                                         break;
2067                                 int_status_bit <<= 1;
2068                         }
2069
2070                         /* Process the AEQ for this pci function */
2071                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2072                         if (int_stat & int_status_bit) {
2073                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2074                         }
2075
2076                         /* Process the MAC interrupt for this pci function */
2077                         int_status_bit = 1 << (24 + nesdev->mac_index);
2078                         if (int_stat & int_status_bit) {
2079                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2080                         }
2081
2082                         if (int_stat & NES_INT_TIMER) {
2083                                 if (timer_stat & nesdev->timer_int_req) {
2084                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2085                                                         (timer_stat & nesdev->timer_int_req) |
2086                                                         ~(nesdev->nesadapter->timer_int_req));
2087                                         timer_ints = 1;
2088                                 }
2089                         }
2090
2091                         if (int_stat & NES_INT_INTF) {
2092                                 processed_intf_int = 1;
2093                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2094                                 intf_int_stat &= nesdev->intf_int_req;
2095                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2096                                         process_critical_error(nesdev);
2097                                 }
2098                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2099                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2100                                         BUG();
2101                                 }
2102                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2103                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2104                                         BUG();
2105                                 }
2106                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2107                         }
2108
2109                         if (int_stat & NES_INT_TSW) {
2110                         }
2111                 }
2112                 /* Don't use the interface interrupt bit stay in loop */
2113                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2114                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2115         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2116
2117         if (timer_ints == 1) {
2118                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2119                         if (completion_ints == 0) {
2120                                 nesdev->timer_only_int_count++;
2121                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2122                                         nesdev->timer_only_int_count = 0;
2123                                         nesdev->int_req &= ~NES_INT_TIMER;
2124                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2125                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2126                                 } else {
2127                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2128                                 }
2129                         } else {
2130                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2131                                 {
2132                                         nes_nic_init_timer(nesdev);
2133                                 }
2134                                 nesdev->timer_only_int_count = 0;
2135                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2136                         }
2137                 } else {
2138                         nesdev->timer_only_int_count = 0;
2139                         nesdev->int_req &= ~NES_INT_TIMER;
2140                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2141                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2142                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2143                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2144                 }
2145         } else {
2146                 if ( (completion_ints == 1) &&
2147                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2148                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2149                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2150                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2151                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2152                         nesdev->timer_only_int_count = 0;
2153                         nesdev->int_req |= NES_INT_TIMER;
2154                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2155                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2156                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2157                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2158                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2159                 } else {
2160                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2161                 }
2162         }
2163         nesdev->deepcq_count = 0;
2164 }
2165
2166
2167 /**
2168  * nes_process_ceq
2169  */
2170 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2171 {
2172         u64 u64temp;
2173         struct nes_hw_cq *cq;
2174         u32 head;
2175         u32 ceq_size;
2176
2177         /* nes_debug(NES_DBG_CQ, "\n"); */
2178         head = ceq->ceq_head;
2179         ceq_size = ceq->ceq_size;
2180
2181         do {
2182                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2183                                 NES_CEQE_VALID) {
2184                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2185                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2186                         u64temp <<= 1;
2187                         cq = *((struct nes_hw_cq **)&u64temp);
2188                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2189                         barrier();
2190                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2191
2192                         /* call the event handler */
2193                         cq->ce_handler(nesdev, cq);
2194
2195                         if (++head >= ceq_size)
2196                                 head = 0;
2197                 } else {
2198                         break;
2199                 }
2200
2201         } while (1);
2202
2203         ceq->ceq_head = head;
2204 }
2205
2206
2207 /**
2208  * nes_process_aeq
2209  */
2210 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2211 {
2212         /* u64 u64temp; */
2213         u32 head;
2214         u32 aeq_size;
2215         u32 aeqe_misc;
2216         u32 aeqe_cq_id;
2217         struct nes_hw_aeqe volatile *aeqe;
2218
2219         head = aeq->aeq_head;
2220         aeq_size = aeq->aeq_size;
2221
2222         do {
2223                 aeqe = &aeq->aeq_vbase[head];
2224                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2225                         break;
2226                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2227                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2228                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2229                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2230                                 /* dealing with an accelerated QP related AE */
2231                                 /*
2232                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2233                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2234                                  */
2235                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2236                         } else {
2237                                 /* TODO: dealing with a CQP related AE */
2238                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2239                                                 (u16)(aeqe_misc >> 16));
2240                         }
2241                 }
2242
2243                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2244
2245                 if (++head >= aeq_size)
2246                         head = 0;
2247
2248                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2249         }
2250         while (1);
2251         aeq->aeq_head = head;
2252 }
2253
2254 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2255 {
2256         struct nes_adapter *nesadapter = nesdev->nesadapter;
2257         u32 reset_value;
2258         u32 i=0;
2259         u32 u32temp;
2260
2261         if (nesadapter->hw_rev == NE020_REV) {
2262                 return;
2263         }
2264         mh_detected++;
2265
2266         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2267
2268         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2269                 reset_value |= 0x0000001d;
2270         else
2271                 reset_value |= 0x0000002d;
2272
2273         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2274                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2275                         nesadapter->link_interrupt_count[0] = 0;
2276                         nesadapter->link_interrupt_count[1] = 0;
2277                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2278                         if (0x00000040 & u32temp)
2279                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2280                         else
2281                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2282
2283                         reset_value |= 0x0000003d;
2284                 }
2285                 nesadapter->link_interrupt_count[mac_index] = 0;
2286         }
2287
2288         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2289
2290         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2291                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2292
2293         if (0x0000003d == (reset_value & 0x0000003d)) {
2294                 u32 pcs_control_status0, pcs_control_status1;
2295
2296                 for (i = 0; i < 10; i++) {
2297                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2298                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2299                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2300                              && (pcs_control_status0 & 0x00100000))
2301                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2302                                 && (pcs_control_status1 & 0x00100000)))
2303                                 continue;
2304                         else
2305                                 break;
2306                 }
2307                 if (10 == i) {
2308                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2309                         if (0x00000040 & u32temp)
2310                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2311                         else
2312                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2313
2314                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2315
2316                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2317                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2318                 }
2319         }
2320 }
2321
2322 /**
2323  * nes_process_mac_intr
2324  */
2325 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2326 {
2327         unsigned long flags;
2328         u32 pcs_control_status;
2329         struct nes_adapter *nesadapter = nesdev->nesadapter;
2330         struct nes_vnic *nesvnic;
2331         u32 mac_status;
2332         u32 mac_index = nesdev->mac_index;
2333         u32 u32temp;
2334         u16 phy_data;
2335         u16 temp_phy_data;
2336         u32 pcs_val  = 0x0f0f0000;
2337         u32 pcs_mask = 0x0f1f0000;
2338         u32 cdr_ctrl;
2339
2340         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2341         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2342                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2343                 return;
2344         }
2345         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2346         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2347
2348         /* ack the MAC interrupt */
2349         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2350         /* Clear the interrupt */
2351         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2352
2353         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2354
2355         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2356                 nesdev->link_status_interrupts++;
2357                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2358                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2359                         nes_reset_link(nesdev, mac_index);
2360                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2361                 }
2362                 /* read the PHY interrupt status register */
2363                 if ((nesadapter->OneG_Mode) &&
2364                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2365                         do {
2366                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2367                                                 nesadapter->phy_index[mac_index], &phy_data);
2368                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2369                                                 nesadapter->phy_index[mac_index], phy_data);
2370                         } while (phy_data&0x8000);
2371
2372                         temp_phy_data = 0;
2373                         do {
2374                                 nes_read_1G_phy_reg(nesdev, 0x11,
2375                                                 nesadapter->phy_index[mac_index], &phy_data);
2376                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2377                                                 nesadapter->phy_index[mac_index], phy_data);
2378                                 if (temp_phy_data == phy_data)
2379                                         break;
2380                                 temp_phy_data = phy_data;
2381                         } while (1);
2382
2383                         nes_read_1G_phy_reg(nesdev, 0x1e,
2384                                         nesadapter->phy_index[mac_index], &phy_data);
2385                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2386                                         nesadapter->phy_index[mac_index], phy_data);
2387
2388                         nes_read_1G_phy_reg(nesdev, 1,
2389                                         nesadapter->phy_index[mac_index], &phy_data);
2390                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2391                                         nesadapter->phy_index[mac_index], phy_data);
2392
2393                         if (temp_phy_data & 0x1000) {
2394                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2395                                 phy_data = 4;
2396                         } else {
2397                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2398                         }
2399                 }
2400                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2401                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2402                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2403
2404                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2405                         switch (mac_index) {
2406                         case 1:
2407                         case 3:
2408                                 pcs_control_status = nes_read_indexed(nesdev,
2409                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2410                                 break;
2411                         default:
2412                                 pcs_control_status = nes_read_indexed(nesdev,
2413                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2414                                 break;
2415                         }
2416                 } else {
2417                         pcs_control_status = nes_read_indexed(nesdev,
2418                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2419                         pcs_control_status = nes_read_indexed(nesdev,
2420                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2421                 }
2422
2423                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2424                                 mac_index, pcs_control_status);
2425                 if ((nesadapter->OneG_Mode) &&
2426                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2427                         u32temp = 0x01010000;
2428                         if (nesadapter->port_count > 2) {
2429                                 u32temp |= 0x02020000;
2430                         }
2431                         if ((pcs_control_status & u32temp)!= u32temp) {
2432                                 phy_data = 0;
2433                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2434                         }
2435                 } else {
2436                         switch (nesadapter->phy_type[mac_index]) {
2437                         case NES_PHY_TYPE_IRIS:
2438                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2439                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2440                                 u32temp = 20;
2441                                 do {
2442                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2443                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2444                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2445                                                 break;
2446                                         temp_phy_data = phy_data;
2447                                 } while (1);
2448                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2449                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2450                                 break;
2451
2452                         case NES_PHY_TYPE_ARGUS:
2453                         case NES_PHY_TYPE_SFP_D:
2454                                 /* clear the alarms */
2455                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2456                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2457                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2458                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2459                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2460                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2461                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2462                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2463                                 /* check link status */
2464                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2465                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2466
2467                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2468                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2469                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2470                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2471
2472                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2473
2474                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2475                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2476                                 break;
2477
2478                         case NES_PHY_TYPE_PUMA_1G:
2479                                 if (mac_index < 2)
2480                                         pcs_val = pcs_mask = 0x01010000;
2481                                 else
2482                                         pcs_val = pcs_mask = 0x02020000;
2483                                 /* fall through */
2484                         default:
2485                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2486                                 break;
2487                         }
2488                 }
2489
2490                 if (phy_data & 0x0004) {
2491                         if (wide_ppm_offset &&
2492                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2493                             (nesadapter->hw_rev != NE020_REV)) {
2494                                 cdr_ctrl = nes_read_indexed(nesdev,
2495                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2496                                                             mac_index * 0x200);
2497                                 nes_write_indexed(nesdev,
2498                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2499                                                   mac_index * 0x200,
2500                                                   cdr_ctrl | 0x000F0000);
2501                         }
2502                         nesadapter->mac_link_down[mac_index] = 0;
2503                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2504                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2505                                                 nesvnic->linkup);
2506                                 if (nesvnic->linkup == 0) {
2507                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2508                                                         nesvnic->netdev->name, nesvnic->netdev);
2509                                         if (netif_queue_stopped(nesvnic->netdev))
2510                                                 netif_start_queue(nesvnic->netdev);
2511                                         nesvnic->linkup = 1;
2512                                         netif_carrier_on(nesvnic->netdev);
2513                                 }
2514                         }
2515                 } else {
2516                         if (wide_ppm_offset &&
2517                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2518                             (nesadapter->hw_rev != NE020_REV)) {
2519                                 cdr_ctrl = nes_read_indexed(nesdev,
2520                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2521                                                             mac_index * 0x200);
2522                                 nes_write_indexed(nesdev,
2523                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2524                                                   mac_index * 0x200,
2525                                                   cdr_ctrl & 0xFFF0FFFF);
2526                         }
2527                         nesadapter->mac_link_down[mac_index] = 1;
2528                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2529                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2530                                                 nesvnic->linkup);
2531                                 if (nesvnic->linkup == 1) {
2532                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2533                                                         nesvnic->netdev->name, nesvnic->netdev);
2534                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2535                                                 netif_stop_queue(nesvnic->netdev);
2536                                         nesvnic->linkup = 0;
2537                                         netif_carrier_off(nesvnic->netdev);
2538                                 }
2539                         }
2540                 }
2541         }
2542
2543         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2544 }
2545
2546
2547
2548 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2549 {
2550         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2551
2552         napi_schedule(&nesvnic->napi);
2553 }
2554
2555
2556 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2557 * getting out of nic_ce_handler
2558 */
2559 #define MAX_RQES_TO_PROCESS     384
2560
2561 /**
2562  * nes_nic_ce_handler
2563  */
2564 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2565 {
2566         u64 u64temp;
2567         dma_addr_t bus_address;
2568         struct nes_hw_nic *nesnic;
2569         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2570         struct nes_adapter *nesadapter = nesdev->nesadapter;
2571         struct nes_hw_nic_rq_wqe *nic_rqe;
2572         struct nes_hw_nic_sq_wqe *nic_sqe;
2573         struct sk_buff *skb;
2574         struct sk_buff *rx_skb;
2575         __le16 *wqe_fragment_length;
2576         u32 head;
2577         u32 cq_size;
2578         u32 rx_pkt_size;
2579         u32 cqe_count=0;
2580         u32 cqe_errv;
2581         u32 cqe_misc;
2582         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2583         u16 vlan_tag;
2584         u16 pkt_type;
2585         u16 rqes_processed = 0;
2586         u8 sq_cqes = 0;
2587         u8 nes_use_lro = 0;
2588
2589         head = cq->cq_head;
2590         cq_size = cq->cq_size;
2591         cq->cqes_pending = 1;
2592         if (nesvnic->netdev->features & NETIF_F_LRO)
2593                 nes_use_lro = 1;
2594         do {
2595                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2596                                 NES_NIC_CQE_VALID) {
2597                         nesnic = &nesvnic->nic;
2598                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2599                         if (cqe_misc & NES_NIC_CQE_SQ) {
2600                                 sq_cqes++;
2601                                 wqe_fragment_index = 1;
2602                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2603                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2604                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2605                                 /* bump past the vlan tag */
2606                                 wqe_fragment_length++;
2607                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2608                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2609                                                         wqe_fragment_index * 2]);
2610                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2611                                                         wqe_fragment_index * 2])) << 32;
2612                                         bus_address = (dma_addr_t)u64temp;
2613                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2614                                                 pci_unmap_single(nesdev->pcidev,
2615                                                                 bus_address,
2616                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2617                                                                 PCI_DMA_TODEVICE);
2618                                         }
2619                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2620                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2621                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2622                                                                                 wqe_fragment_index * 2]);
2623                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2624                                                                                 + wqe_fragment_index * 2])) <<32;
2625                                                         bus_address = (dma_addr_t)u64temp;
2626                                                         pci_unmap_page(nesdev->pcidev,
2627                                                                         bus_address,
2628                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2629                                                                         PCI_DMA_TODEVICE);
2630                                                 } else
2631                                                         break;
2632                                         }
2633                                 }
2634                                 if (skb)
2635                                         dev_kfree_skb_any(skb);
2636                                 nesnic->sq_tail++;
2637                                 nesnic->sq_tail &= nesnic->sq_size-1;
2638                                 if (sq_cqes > 128) {
2639                                         barrier();
2640                                 /* restart the queue if it had been stopped */
2641                                 if (netif_queue_stopped(nesvnic->netdev))
2642                                         netif_wake_queue(nesvnic->netdev);
2643                                         sq_cqes = 0;
2644                                 }
2645                         } else {
2646                                 rqes_processed ++;
2647
2648                                 cq->rx_cqes_completed++;
2649                                 cq->rx_pkts_indicated++;
2650                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2651                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2652                                 /* Get the skb */
2653                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2654                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2655                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2656                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2657                                 pci_unmap_single(nesdev->pcidev, bus_address,
2658                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2659                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2660                                 /* rx_skb->len = rx_pkt_size; */
2661                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2662                                 skb_put(rx_skb, rx_pkt_size);
2663                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2664                                 nesnic->rq_tail++;
2665                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2666
2667                                 atomic_inc(&nesvnic->rx_skbs_needed);
2668                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2669                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2670                                                         cq->cq_number | (cqe_count << 16));
2671                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2672                                         nesdev->currcq_count += cqe_count;
2673                                         cqe_count = 0;
2674                                         nes_replenish_nic_rq(nesvnic);
2675                                 }
2676                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2677                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2678                                 rx_skb->ip_summed = CHECKSUM_NONE;
2679
2680                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2681                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2682                                         if ((cqe_errv &
2683                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2684                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2685                                                 if (nesvnic->rx_checksum_disabled == 0) {
2686                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2687                                                 }
2688                                         } else
2689                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2690                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2691                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2692
2693                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2694                                         if ((cqe_errv &
2695                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2696                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2697                                                 if (nesvnic->rx_checksum_disabled == 0) {
2698                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2699                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2700                                                                   nesvnic->netdev->name); */
2701                                                 }
2702                                         } else
2703                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2704                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2705                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2706                                         }
2707                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2708                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2709
2710                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2711                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2712                                                 rx_skb = NULL;
2713                                 }
2714                                 if (rx_skb == NULL)
2715                                         goto skip_rx_indicate0;
2716
2717
2718                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2719                                     (nesvnic->vlan_grp != NULL)) {
2720                                         vlan_tag = (u16)(le32_to_cpu(
2721                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2722                                                         >> 16);
2723                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2724                                                         nesvnic->netdev->name, vlan_tag);
2725                                         if (nes_use_lro)
2726                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2727                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2728                                         else
2729                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2730                                 } else {
2731                                         if (nes_use_lro)
2732                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2733                                         else
2734                                                 nes_netif_rx(rx_skb);
2735                                 }
2736
2737 skip_rx_indicate0:
2738                                 nesvnic->netdev->last_rx = jiffies;
2739                                 /* nesvnic->netstats.rx_packets++; */
2740                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2741                         }
2742
2743                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2744                         /* Accounting... */
2745                         cqe_count++;
2746                         if (++head >= cq_size)
2747                                 head = 0;
2748                         if (cqe_count == 255) {
2749                                 /* Replenish Nic CQ */
2750                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2751                                                 cq->cq_number | (cqe_count << 16));
2752                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2753                                 nesdev->currcq_count += cqe_count;
2754                                 cqe_count = 0;
2755                         }
2756
2757                         if (cq->rx_cqes_completed >= nesvnic->budget)
2758                                 break;
2759                 } else {
2760                         cq->cqes_pending = 0;
2761                         break;
2762                 }
2763
2764         } while (1);
2765
2766         if (nes_use_lro)
2767                 lro_flush_all(&nesvnic->lro_mgr);
2768         if (sq_cqes) {
2769                 barrier();
2770                 /* restart the queue if it had been stopped */
2771                 if (netif_queue_stopped(nesvnic->netdev))
2772                         netif_wake_queue(nesvnic->netdev);
2773         }
2774         cq->cq_head = head;
2775         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2776                         cq->cq_number, cqe_count, cq->cq_head); */
2777         cq->cqe_allocs_pending = cqe_count;
2778         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2779         {
2780                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2781                 nesdev->currcq_count += cqe_count;
2782                 nes_nic_tune_timer(nesdev);
2783         }
2784         if (atomic_read(&nesvnic->rx_skbs_needed))
2785                 nes_replenish_nic_rq(nesvnic);
2786 }
2787
2788
2789 /**
2790  * nes_cqp_ce_handler
2791  */
2792 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2793 {
2794         u64 u64temp;
2795         unsigned long flags;
2796         struct nes_hw_cqp *cqp = NULL;
2797         struct nes_cqp_request *cqp_request;
2798         struct nes_hw_cqp_wqe *cqp_wqe;
2799         u32 head;
2800         u32 cq_size;
2801         u32 cqe_count=0;
2802         u32 error_code;
2803         /* u32 counter; */
2804
2805         head = cq->cq_head;
2806         cq_size = cq->cq_size;
2807
2808         do {
2809                 /* process the CQE */
2810                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2811                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2812
2813                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2814                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2815                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2816                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2817                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2818                         cqp = *((struct nes_hw_cqp **)&u64temp);
2819
2820                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2821                         if (error_code) {
2822                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2823                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2824                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2825                                                 (u16)(error_code >> 16),
2826                                                 (u16)error_code);
2827                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2828                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2829                         }
2830
2831                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2832                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2833                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2834                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2835                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2836                         if (cqp_request) {
2837                                 if (cqp_request->waiting) {
2838                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2839                                         cqp_request->major_code = (u16)(error_code >> 16);
2840                                         cqp_request->minor_code = (u16)error_code;
2841                                         barrier();
2842                                         cqp_request->request_done = 1;
2843                                         wake_up(&cqp_request->waitq);
2844                                         nes_put_cqp_request(nesdev, cqp_request);
2845                                 } else {
2846                                         if (cqp_request->callback)
2847                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2848                                         nes_free_cqp_request(nesdev, cqp_request);
2849                                 }
2850                         } else {
2851                                 wake_up(&nesdev->cqp.waitq);
2852                         }
2853
2854                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2855                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2856                         if (++cqp->sq_tail >= cqp->sq_size)
2857                                 cqp->sq_tail = 0;
2858
2859                         /* Accounting... */
2860                         cqe_count++;
2861                         if (++head >= cq_size)
2862                                 head = 0;
2863                 } else {
2864                         break;
2865                 }
2866         } while (1);
2867         cq->cq_head = head;
2868
2869         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2870         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2871                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2872                         (nesdev->cqp.sq_size - 1)) != 1)) {
2873                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2874                                 struct nes_cqp_request, list);
2875                 list_del_init(&cqp_request->list);
2876                 head = nesdev->cqp.sq_head++;
2877                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2878                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2879                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2880                 barrier();
2881                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2882                         cpu_to_le32((u32)((unsigned long)cqp_request));
2883                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2884                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2885                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2886                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2887                 /* Ring doorbell (1 WQEs) */
2888                 barrier();
2889                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2890         }
2891         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2892
2893         /* Arm the CCQ */
2894         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2895                         cq->cq_number);
2896         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2897 }
2898
2899
2900 /**
2901  * nes_process_iwarp_aeqe
2902  */
2903 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2904                                    struct nes_hw_aeqe *aeqe)
2905 {
2906         u64 context;
2907         u64 aeqe_context = 0;
2908         unsigned long flags;
2909         struct nes_qp *nesqp;
2910         int resource_allocated;
2911         /* struct iw_cm_id *cm_id; */
2912         struct nes_adapter *nesadapter = nesdev->nesadapter;
2913         struct ib_event ibevent;
2914         /* struct iw_cm_event cm_event; */
2915         u32 aeq_info;
2916         u32 next_iwarp_state = 0;
2917         u16 async_event_id;
2918         u8 tcp_state;
2919         u8 iwarp_state;
2920
2921         nes_debug(NES_DBG_AEQ, "\n");
2922         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2923         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2924                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2925                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2926         } else {
2927                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2928                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2929                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2930                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2931                 BUG_ON(!context);
2932         }
2933
2934         async_event_id = (u16)aeq_info;
2935         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2936         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2937         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2938                         " Tcp state = %s, iWARP state = %s\n",
2939                         async_event_id,
2940                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2941                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2942
2943         switch (async_event_id) {
2944                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2945                         nesqp = *((struct nes_qp **)&context);
2946                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2947                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2948                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2949                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2950                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2951                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2952                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2953                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2954                                                 async_event_id, nesqp->last_aeq, tcp_state);
2955                         }
2956                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2957                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2958                                 /* FIN Received but tcp state or IB state moved on,
2959                                                 should expect a close complete */
2960                                 return;
2961                         }
2962                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2963                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2964                 case NES_AEQE_AEID_TERMINATE_SENT:
2965                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2966                 case NES_AEQE_AEID_RESET_SENT:
2967                         nesqp = *((struct nes_qp **)&context);
2968                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2969                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2970                         }
2971                         spin_lock_irqsave(&nesqp->lock, flags);
2972                         nesqp->hw_iwarp_state = iwarp_state;
2973                         nesqp->hw_tcp_state = tcp_state;
2974                         nesqp->last_aeq = async_event_id;
2975
2976                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2977                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2978                                 nesqp->hte_added = 0;
2979                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2980                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2981                                                 nesqp->hwqp.qp_id);
2982                                 nes_hw_modify_qp(nesdev, nesqp,
2983                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2984                                 spin_lock_irqsave(&nesqp->lock, flags);
2985                         }
2986
2987                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2988                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2989                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2990                                 switch (nesqp->hw_iwarp_state) {
2991                                         case NES_AEQE_IWARP_STATE_RTS:
2992                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2993                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2994                                                 break;
2995                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2996                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2997                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2998                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2999                                                         next_iwarp_state |= 0x02000000;
3000                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3001                                                 }
3002                                                 break;
3003                                         default:
3004                                                 next_iwarp_state = 0;
3005                                 }
3006                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3007                                 if (next_iwarp_state) {
3008                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3009                                                         " also added another reference\n",
3010                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3011                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3012                                 }
3013                                 nes_cm_disconn(nesqp);
3014                         } else {
3015                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3016                                         /* FIN Received but ib state not RTS,
3017                                                         close complete will be on its way */
3018                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3019                                         return;
3020                                 }
3021                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3022                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3023                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3024                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3025                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3026                                                         " also added another reference\n",
3027                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3028                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3029                                 }
3030                                 nes_cm_disconn(nesqp);
3031                         }
3032                         break;
3033                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3034                         nesqp = *((struct nes_qp **)&context);
3035                         spin_lock_irqsave(&nesqp->lock, flags);
3036                         nesqp->hw_iwarp_state = iwarp_state;
3037                         nesqp->hw_tcp_state = tcp_state;
3038                         nesqp->last_aeq = async_event_id;
3039                         spin_unlock_irqrestore(&nesqp->lock, flags);
3040                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3041                                         " event on QP%u \n  Q2 Data:\n",
3042                                         nesqp->hwqp.qp_id);
3043                         if (nesqp->ibqp.event_handler) {
3044                                 ibevent.device = nesqp->ibqp.device;
3045                                 ibevent.element.qp = &nesqp->ibqp;
3046                                 ibevent.event = IB_EVENT_QP_FATAL;
3047                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3048                         }
3049                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3050                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3051                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3052                                 nes_cm_disconn(nesqp);
3053                         } else {
3054                                 nesqp->in_disconnect = 0;
3055                                 wake_up(&nesqp->kick_waitq);
3056                         }
3057                         break;
3058                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3059                         nesqp = *((struct nes_qp **)&context);
3060                         spin_lock_irqsave(&nesqp->lock, flags);
3061                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3062                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3063                         nesqp->last_aeq = async_event_id;
3064                         if (nesqp->cm_id) {
3065                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3066                                                 " event on QP%u, remote IP = 0x%08X \n",
3067                                                 nesqp->hwqp.qp_id,
3068                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3069                         } else {
3070                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3071                                                 " event on QP%u \n",
3072                                                 nesqp->hwqp.qp_id);
3073                         }
3074                         spin_unlock_irqrestore(&nesqp->lock, flags);
3075                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3076                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3077                         if (nesqp->ibqp.event_handler) {
3078                                 ibevent.device = nesqp->ibqp.device;
3079                                 ibevent.element.qp = &nesqp->ibqp;
3080                                 ibevent.event = IB_EVENT_QP_FATAL;
3081                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3082                         }
3083                         break;
3084                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3085                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3086                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3087                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3088                         } else {
3089                                 /* TODO: get the actual WQE and mask off wqe index */
3090                                 context &= ~((u64)511);
3091                                 nesqp = *((struct nes_qp **)&context);
3092                         }
3093                         spin_lock_irqsave(&nesqp->lock, flags);
3094                         nesqp->hw_iwarp_state = iwarp_state;
3095                         nesqp->hw_tcp_state = tcp_state;
3096                         nesqp->last_aeq = async_event_id;
3097                         spin_unlock_irqrestore(&nesqp->lock, flags);
3098                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3099                                         nesqp->hwqp.qp_id);
3100                         if (nesqp->ibqp.event_handler) {
3101                                 ibevent.device = nesqp->ibqp.device;
3102                                 ibevent.element.qp = &nesqp->ibqp;
3103                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3104                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3105                         }
3106                         break;
3107                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3108                         nesqp = *((struct nes_qp **)&context);
3109                         spin_lock_irqsave(&nesqp->lock, flags);
3110                         nesqp->hw_iwarp_state = iwarp_state;
3111                         nesqp->hw_tcp_state = tcp_state;
3112                         nesqp->last_aeq = async_event_id;
3113                         spin_unlock_irqrestore(&nesqp->lock, flags);
3114                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3115                                         nesqp->hwqp.qp_id);
3116                         if (nesqp->ibqp.event_handler) {
3117                                 ibevent.device = nesqp->ibqp.device;
3118                                 ibevent.element.qp = &nesqp->ibqp;
3119                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3120                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3121                         }
3122                         break;
3123                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3124                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3125                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3126                         spin_lock_irqsave(&nesqp->lock, flags);
3127                         nesqp->hw_iwarp_state = iwarp_state;
3128                         nesqp->hw_tcp_state = tcp_state;
3129                         nesqp->last_aeq = async_event_id;
3130                         spin_unlock_irqrestore(&nesqp->lock, flags);
3131                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3132                                         " nesqp = %p, AE reported %p\n",
3133                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3134                         if (nesqp->ibqp.event_handler) {
3135                                 ibevent.device = nesqp->ibqp.device;
3136                                 ibevent.element.qp = &nesqp->ibqp;
3137                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3138                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3139                         }
3140                         break;
3141                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3142                         context <<= 1;
3143                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3144                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3145                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3146                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3147                         if (resource_allocated) {
3148                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3149                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3150                         }
3151                         break;
3152                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3153                         nesqp = nesadapter->qp_table[le32_to_cpu(
3154                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3155                         spin_lock_irqsave(&nesqp->lock, flags);
3156                         nesqp->hw_iwarp_state = iwarp_state;
3157                         nesqp->hw_tcp_state = tcp_state;
3158                         nesqp->last_aeq = async_event_id;
3159                         spin_unlock_irqrestore(&nesqp->lock, flags);
3160                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3161                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3162                                         nesqp->hwqp.qp_id);
3163                         if (nesqp->ibqp.event_handler) {
3164                                 ibevent.device = nesqp->ibqp.device;
3165                                 ibevent.element.qp = &nesqp->ibqp;
3166                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3167                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3168                         }
3169                         /* tell cm to disconnect, cm will queue work to thread */
3170                         nes_cm_disconn(nesqp);
3171                         break;
3172                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3173                         nesqp = *((struct nes_qp **)&context);
3174                         spin_lock_irqsave(&nesqp->lock, flags);
3175                         nesqp->hw_iwarp_state = iwarp_state;
3176                         nesqp->hw_tcp_state = tcp_state;
3177                         nesqp->last_aeq = async_event_id;
3178                         spin_unlock_irqrestore(&nesqp->lock, flags);
3179                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3180                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3181                                         nesqp->hwqp.qp_id);
3182                         if (nesqp->ibqp.event_handler) {
3183                                 ibevent.device = nesqp->ibqp.device;
3184                                 ibevent.element.qp = &nesqp->ibqp;
3185                                 ibevent.event = IB_EVENT_QP_FATAL;
3186                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3187                         }
3188                         /* tell cm to disconnect, cm will queue work to thread */
3189                         nes_cm_disconn(nesqp);
3190                         break;
3191                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3192                         nesqp = *((struct nes_qp **)&context);
3193                         spin_lock_irqsave(&nesqp->lock, flags);
3194                         nesqp->hw_iwarp_state = iwarp_state;
3195                         nesqp->hw_tcp_state = tcp_state;
3196                         nesqp->last_aeq = async_event_id;
3197                         spin_unlock_irqrestore(&nesqp->lock, flags);
3198                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3199                                         " event on QP%u \n  Q2 Data:\n",
3200                                         nesqp->hwqp.qp_id);
3201                         if (nesqp->ibqp.event_handler) {
3202                                 ibevent.device = nesqp->ibqp.device;
3203                                 ibevent.element.qp = &nesqp->ibqp;
3204                                 ibevent.event = IB_EVENT_QP_FATAL;
3205                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3206                         }
3207                         /* tell cm to disconnect, cm will queue work to thread */
3208                         nes_cm_disconn(nesqp);
3209                         break;
3210                         /* TODO: additional AEs need to be here */
3211                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3212                         nesqp = *((struct nes_qp **)&context);
3213                         spin_lock_irqsave(&nesqp->lock, flags);
3214                         nesqp->hw_iwarp_state = iwarp_state;
3215                         nesqp->hw_tcp_state = tcp_state;
3216                         nesqp->last_aeq = async_event_id;
3217                         spin_unlock_irqrestore(&nesqp->lock, flags);
3218                         if (nesqp->ibqp.event_handler) {
3219                                 ibevent.device = nesqp->ibqp.device;
3220                                 ibevent.element.qp = &nesqp->ibqp;
3221                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3222                                 nesqp->ibqp.event_handler(&ibevent,
3223                                                 nesqp->ibqp.qp_context);
3224                         }
3225                         nes_cm_disconn(nesqp);
3226                         break;
3227                 default:
3228                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3229                                         async_event_id);
3230                         break;
3231         }
3232
3233 }
3234
3235
3236 /**
3237  * nes_iwarp_ce_handler
3238  */
3239 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3240 {
3241         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3242
3243         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3244                         nescq->hw_cq.cq_number); */
3245         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3246
3247         if (nescq->ibcq.comp_handler)
3248                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3249
3250         return;
3251 }
3252
3253
3254 /**
3255  * nes_manage_apbvt()
3256  */
3257 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3258                 u32 nic_index, u32 add_port)
3259 {
3260         struct nes_device *nesdev = nesvnic->nesdev;
3261         struct nes_hw_cqp_wqe *cqp_wqe;
3262         struct nes_cqp_request *cqp_request;
3263         int ret = 0;
3264         u16 major_code;
3265
3266         /* Send manage APBVT request to CQP */
3267         cqp_request = nes_get_cqp_request(nesdev);
3268         if (cqp_request == NULL) {
3269                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3270                 return -ENOMEM;
3271         }
3272         cqp_request->waiting = 1;
3273         cqp_wqe = &cqp_request->cqp_wqe;
3274
3275         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3276                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3277                         accel_local_port, accel_local_port, nic_index);
3278
3279         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3280         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3281                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3282         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3283                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3284
3285         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3286
3287         atomic_set(&cqp_request->refcount, 2);
3288         nes_post_cqp_request(nesdev, cqp_request);
3289
3290         if (add_port == NES_MANAGE_APBVT_ADD)
3291                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3292                                 NES_EVENT_TIMEOUT);
3293         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3294                         ret, cqp_request->major_code, cqp_request->minor_code);
3295         major_code = cqp_request->major_code;
3296
3297         nes_put_cqp_request(nesdev, cqp_request);
3298
3299         if (!ret)
3300                 return -ETIME;
3301         else if (major_code)
3302                 return -EIO;
3303         else
3304                 return 0;
3305 }
3306
3307
3308 /**
3309  * nes_manage_arp_cache
3310  */
3311 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3312                 u32 ip_addr, u32 action)
3313 {
3314         struct nes_hw_cqp_wqe *cqp_wqe;
3315         struct nes_vnic *nesvnic = netdev_priv(netdev);
3316         struct nes_device *nesdev;
3317         struct nes_cqp_request *cqp_request;
3318         int arp_index;
3319
3320         nesdev = nesvnic->nesdev;
3321         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3322         if (arp_index == -1) {
3323                 return;
3324         }
3325
3326         /* update the ARP entry */
3327         cqp_request = nes_get_cqp_request(nesdev);
3328         if (cqp_request == NULL) {
3329                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3330                 return;
3331         }
3332         cqp_request->waiting = 0;
3333         cqp_wqe = &cqp_request->cqp_wqe;
3334         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3335
3336         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3337                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3338         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3339                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3340         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3341
3342         if (action == NES_ARP_ADD) {
3343                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3344                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3345                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3346                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3347                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3348                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3349         } else {
3350                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3351                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3352         }
3353
3354         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3355                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3356
3357         atomic_set(&cqp_request->refcount, 1);
3358         nes_post_cqp_request(nesdev, cqp_request);
3359 }
3360
3361
3362 /**
3363  * flush_wqes
3364  */
3365 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3366                 u32 which_wq, u32 wait_completion)
3367 {
3368         struct nes_cqp_request *cqp_request;
3369         struct nes_hw_cqp_wqe *cqp_wqe;
3370         int ret;
3371
3372         cqp_request = nes_get_cqp_request(nesdev);
3373         if (cqp_request == NULL) {
3374                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3375                 return;
3376         }
3377         if (wait_completion) {
3378                 cqp_request->waiting = 1;
3379                 atomic_set(&cqp_request->refcount, 2);
3380         } else {
3381                 cqp_request->waiting = 0;
3382         }
3383         cqp_wqe = &cqp_request->cqp_wqe;
3384         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3385
3386         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3387                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3388         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3389
3390         nes_post_cqp_request(nesdev, cqp_request);
3391
3392         if (wait_completion) {
3393                 /* Wait for CQP */
3394                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3395                                 NES_EVENT_TIMEOUT);
3396                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3397                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3398                                 ret, cqp_request->major_code, cqp_request->minor_code);
3399                 nes_put_cqp_request(nesdev, cqp_request);
3400         }
3401 }