git.oblomov.eu Git - linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77
  78 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  79 static unsigned char *nes_iwarp_state_str[] = {
  80         "Non-Existant",
  81         "Idle",
  82         "RTS",
  83         "Closing",
  84         "RSVD1",
  85         "Terminate",
  86         "Error",
  87         "RSVD2",
  88 };
  89
  90 static unsigned char *nes_tcp_state_str[] = {
  91         "Non-Existant",
  92         "Closed",
  93         "Listen",
  94         "SYN Sent",
  95         "SYN Rcvd",
  96         "Established",
  97         "Close Wait",
  98         "FIN Wait 1",
  99         "Closing",
 100         "Last Ack",
 101         "FIN Wait 2",
 102         "Time Wait",
 103         "RSVD1",
 104         "RSVD2",
 105         "RSVD3",
 106         "RSVD4",
 107 };
 108 #endif
 109
 110
 111 /**
 112  * nes_nic_init_timer_defaults
 113  */
 114 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 115 {
 116         unsigned long flags;
 117         struct nes_adapter *nesadapter = nesdev->nesadapter;
 118         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 119
 120         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 121
 122         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 123         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 124         if (jumbomode) {
 125                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 128         } else {
 129                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 130                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 131                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 132         }
 133
 134         /* todo use netdev->mtu to set thresholds */
 135         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 136 }
 137
 138
 139 /**
 140  * nes_nic_init_timer
 141  */
 142 static void  nes_nic_init_timer(struct nes_device *nesdev)
 143 {
 144         unsigned long flags;
 145         struct nes_adapter *nesadapter = nesdev->nesadapter;
 146         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 147
 148         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 149
 150         if (shared_timer->timer_in_use_old == 0) {
 151                 nesdev->deepcq_count = 0;
 152                 shared_timer->timer_direction_upward = 0;
 153                 shared_timer->timer_direction_downward = 0;
 154                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 155                 shared_timer->timer_in_use_old = 0;
 156
 157         }
 158         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 159                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 160                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 161                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 162         }
 163         /* todo use netdev->mtu to set thresholds */
 164         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 165 }
 166
 167
 168 /**
 169  * nes_nic_tune_timer
 170  */
 171 static void nes_nic_tune_timer(struct nes_device *nesdev)
 172 {
 173         unsigned long flags;
 174         struct nes_adapter *nesadapter = nesdev->nesadapter;
 175         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 176         u16 cq_count = nesdev->currcq_count;
 177
 178         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 179
 180         if (shared_timer->cq_count_old <= cq_count)
 181                 shared_timer->cq_direction_downward = 0;
 182         else
 183                 shared_timer->cq_direction_downward++;
 184         shared_timer->cq_count_old = cq_count;
 185         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 186                 if (cq_count <= shared_timer->threshold_low &&
 187                     shared_timer->threshold_low > 4) {
 188                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 189                         shared_timer->cq_direction_downward=0;
 190                         nesdev->currcq_count = 0;
 191                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 192                         return;
 193                 }
 194         }
 195
 196         if (cq_count > 1) {
 197                 nesdev->deepcq_count += cq_count;
 198                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 199                         shared_timer->timer_direction_upward++;
 200                         shared_timer->timer_direction_downward = 0;
 201                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 202                         shared_timer->timer_direction_upward = 0;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 205                         shared_timer->timer_direction_downward++;
 206                         shared_timer->timer_direction_upward = 0;
 207                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 208                         shared_timer->timer_in_use -= 2;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 } else {
 212                         shared_timer->timer_in_use -= 4;
 213                         shared_timer->timer_direction_upward = 0;
 214                         shared_timer->timer_direction_downward++;
 215                 }
 216
 217                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 218                         shared_timer->timer_in_use += 3;
 219                         shared_timer->timer_direction_upward = 0;
 220                         shared_timer->timer_direction_downward = 0;
 221                 }
 222                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 223                         shared_timer->timer_in_use -= 4 ;
 224                         shared_timer->timer_direction_downward = 0;
 225                         shared_timer->timer_direction_upward = 0;
 226                 }
 227         }
 228
 229         /* boundary checking */
 230         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 231                 shared_timer->timer_in_use = shared_timer->threshold_high;
 232         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 233                 shared_timer->timer_in_use = shared_timer->threshold_low;
 234
 235         nesdev->currcq_count = 0;
 236
 237         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 238 }
 239
 240
 241 /**
 242  * nes_init_adapter - initialize adapter
 243  */
 244 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 245         struct nes_adapter *nesadapter = NULL;
 246         unsigned long num_pds;
 247         u32 u32temp;
 248         u32 port_count;
 249         u16 max_rq_wrs;
 250         u16 max_sq_wrs;
 251         u32 max_mr;
 252         u32 max_256pbl;
 253         u32 max_4kpbl;
 254         u32 max_qp;
 255         u32 max_irrq;
 256         u32 max_cq;
 257         u32 hte_index_mask;
 258         u32 adapter_size;
 259         u32 arp_table_size;
 260         u16 vendor_id;
 261         u16 device_id;
 262         u8  OneG_Mode;
 263         u8  func_index;
 264
 265         /* search the list of existing adapters */
 266         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 267                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 268                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 269                                 nesdev->pcidev->devfn,
 270                                 PCI_SLOT(nesadapter->devfn),
 271                                 nesadapter->bus_number,
 272                                 PCI_SLOT(nesdev->pcidev->devfn),
 273                                 nesdev->pcidev->bus->number );
 274                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 275                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 276                         nesadapter->ref_count++;
 277                         return nesadapter;
 278                 }
 279         }
 280
 281         /* no adapter found */
 282         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 283         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 284                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 285                                 hw_rev);
 286                 return NULL;
 287         }
 288
 289         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 290                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 291                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 292                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 294
 295         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 296
 297
 298         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 299                 return NULL;
 300
 301         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 302         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 303
 304         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 305         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 306                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 307                                 max_qp, u32temp);
 308                 max_qp = (u32)1 << (u32temp & 0x001f);
 309         }
 310
 311         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 312         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 313                         max_qp, hte_index_mask);
 314
 315         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 316
 317         max_irrq = 1 << (u32temp & 0x001f);
 318
 319         if (max_qp > max_irrq) {
 320                 max_qp = max_irrq;
 321                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 322                                 max_qp);
 323         }
 324
 325         /* there should be no reason to allocate more pds than qps */
 326         if (num_pds > max_qp)
 327                 num_pds = max_qp;
 328
 329         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 330         max_mr = (u32)8192 << (u32temp & 0x7);
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 333         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 334         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 335         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 336
 337         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 338         arp_table_size = 1 << u32temp;
 339
 340         adapter_size = (sizeof(struct nes_adapter) +
 341                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 342         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 343         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 347         adapter_size += sizeof(struct nes_qp **) * max_qp;
 348
 349         /* allocate a new adapter struct */
 350         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 351         if (nesadapter == NULL) {
 352                 return NULL;
 353         }
 354
 355         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 356                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 357
 358         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 359                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 360                 kfree(nesadapter);
 361                 return NULL;
 362         }
 363
 364         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 365                                 (nesadapter->mac_addr_low >> 24);
 366
 367         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 368                                  PCI_DEVICE_ID, &device_id);
 369         nesadapter->vendor_part_id = device_id;
 370
 371         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 372                                                         OneG_Mode)) {
 373                 kfree(nesadapter);
 374                 return NULL;
 375         }
 376         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 377
 378         memset(nesadapter->pft_mcast_map, 255,
 379                sizeof nesadapter->pft_mcast_map);
 380
 381         /* populate the new nesadapter */
 382         nesadapter->devfn = nesdev->pcidev->devfn;
 383         nesadapter->bus_number = nesdev->pcidev->bus->number;
 384         nesadapter->ref_count = 1;
 385         nesadapter->timer_int_req = 0xffff0000;
 386         nesadapter->OneG_Mode = OneG_Mode;
 387         nesadapter->doorbell_start = nesdev->doorbell_region;
 388
 389         /* nesadapter->tick_delta = clk_divisor; */
 390         nesadapter->hw_rev = hw_rev;
 391         nesadapter->port_count = port_count;
 392
 393         nesadapter->max_qp = max_qp;
 394         nesadapter->hte_index_mask = hte_index_mask;
 395         nesadapter->max_irrq = max_irrq;
 396         nesadapter->max_mr = max_mr;
 397         nesadapter->max_256pbl = max_256pbl - 1;
 398         nesadapter->max_4kpbl = max_4kpbl - 1;
 399         nesadapter->max_cq = max_cq;
 400         nesadapter->free_256pbl = max_256pbl - 1;
 401         nesadapter->free_4kpbl = max_4kpbl - 1;
 402         nesadapter->max_pd = num_pds;
 403         nesadapter->arp_table_size = arp_table_size;
 404
 405         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 406         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 407                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 408                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 409                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 410         } else {
 411                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 412                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 413                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 414                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 415         }
 416         /* Setup and enable the periodic timer */
 417         if (nesadapter->et_rx_coalesce_usecs_irq)
 418                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 419                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 420         else
 421                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 422
 423         nesadapter->base_pd = 1;
 424
 425         nesadapter->device_cap_flags =
 426                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 427
 428         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 429                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 430         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 431         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 432         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 433         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 434         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 435
 436
 437         /* mark the usual suspect QPs and CQs as in use */
 438         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 439                 set_bit(u32temp, nesadapter->allocated_qps);
 440                 set_bit(u32temp, nesadapter->allocated_cqs);
 441         }
 442
 443         for (u32temp = 0; u32temp < 20; u32temp++)
 444                 set_bit(u32temp, nesadapter->allocated_pds);
 445         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 446
 447         max_rq_wrs = ((u32temp >> 8) & 3);
 448         switch (max_rq_wrs) {
 449                 case 0:
 450                         max_rq_wrs = 4;
 451                         break;
 452                 case 1:
 453                         max_rq_wrs = 16;
 454                         break;
 455                 case 2:
 456                         max_rq_wrs = 32;
 457                         break;
 458                 case 3:
 459                         max_rq_wrs = 512;
 460                         break;
 461         }
 462
 463         max_sq_wrs = (u32temp & 3);
 464         switch (max_sq_wrs) {
 465                 case 0:
 466                         max_sq_wrs = 4;
 467                         break;
 468                 case 1:
 469                         max_sq_wrs = 16;
 470                         break;
 471                 case 2:
 472                         max_sq_wrs = 32;
 473                         break;
 474                 case 3:
 475                         max_sq_wrs = 512;
 476                         break;
 477         }
 478         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 479         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 480
 481         nesadapter->max_sge = 4;
 482         nesadapter->max_cqe = 32767;
 483
 484         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 485                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 486                 kfree(nesadapter);
 487                 return NULL;
 488         }
 489
 490         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 491         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 492                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 493
 494         /* setup port configuration */
 495         if (nesadapter->port_count == 1) {
 496                 nesadapter->log_port = 0x00000000;
 497                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 498                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 499                 else
 500                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 501         } else {
 502                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 503                         nesadapter->log_port = 0x000000D8;
 504                 } else {
 505                         if (nesadapter->port_count == 2)
 506                                 nesadapter->log_port = 0x00000044;
 507                         else
 508                                 nesadapter->log_port = 0x000000e4;
 509                 }
 510                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 511         }
 512
 513         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 514                                                 nesadapter->log_port);
 515         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 516                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 517
 518         spin_lock_init(&nesadapter->resource_lock);
 519         spin_lock_init(&nesadapter->phy_lock);
 520         spin_lock_init(&nesadapter->pbl_lock);
 521         spin_lock_init(&nesadapter->periodic_timer_lock);
 522
 523         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 524         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 525         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 527
 528         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 529                 u32 pcs_control_status0, pcs_control_status1;
 530                 u32 reset_value;
 531                 u32 i = 0;
 532                 u32 int_cnt = 0;
 533                 u32 ext_cnt = 0;
 534                 unsigned long flags;
 535                 u32 j = 0;
 536
 537                 pcs_control_status0 = nes_read_indexed(nesdev,
 538                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 539                 pcs_control_status1 = nes_read_indexed(nesdev,
 540                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 541
 542                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 543                         pcs_control_status0 = nes_read_indexed(nesdev,
 544                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 545                         pcs_control_status1 = nes_read_indexed(nesdev,
 546                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 547                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 548                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 549                                 int_cnt++;
 550                         msleep(1);
 551                 }
 552                 if (int_cnt > 1) {
 553                         u32 sds;
 554                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 555                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 556                         sds |= 0x00000040;
 557                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 558                         mh_detected++;
 559                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 560                         reset_value |= 0x0000003d;
 561                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 562
 563                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 564                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 565                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 566
 567                         pcs_control_status0 = nes_read_indexed(nesdev,
 568                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 569                         pcs_control_status1 = nes_read_indexed(nesdev,
 570                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 571
 572                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 573                                 pcs_control_status0 = nes_read_indexed(nesdev,
 574                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 575                                 pcs_control_status1 = nes_read_indexed(nesdev,
 576                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 577                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 578                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 579                                         if (++ext_cnt > int_cnt) {
 580                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 581                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 582                                                                 0x0000F0C8);
 583                                                 mh_detected++;
 584                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 585                                                 reset_value |= 0x0000003d;
 586                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 587
 588                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 589                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 590                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 591                                                 break;
 592                                         }
 593                                 }
 594                                 msleep(1);
 595                         }
 596                 }
 597         }
 598
 599         if (nesadapter->hw_rev == NE020_REV) {
 600                 init_timer(&nesadapter->mh_timer);
 601                 nesadapter->mh_timer.function = nes_mh_fix;
 602                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 603                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 604                 add_timer(&nesadapter->mh_timer);
 605         } else {
 606                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 607         }
 608
 609         init_timer(&nesadapter->lc_timer);
 610         nesadapter->lc_timer.function = nes_clc;
 611         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 612         nesadapter->lc_timer.data = (unsigned long)nesdev;
 613         add_timer(&nesadapter->lc_timer);
 614
 615         list_add_tail(&nesadapter->list, &nes_adapter_list);
 616
 617         for (func_index = 0; func_index < 8; func_index++) {
 618                 pci_bus_read_config_word(nesdev->pcidev->bus,
 619                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 620                                         func_index), 0, &vendor_id);
 621                 if (vendor_id == 0xffff)
 622                         break;
 623         }
 624         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 625                 func_index, pci_name(nesdev->pcidev));
 626         nesadapter->adapter_fcn_count = func_index;
 627
 628         return nesadapter;
 629 }
 630
 631
 632 /**
 633  * nes_reset_adapter_ne020
 634  */
 635 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 636 {
 637         u32 port_count;
 638         u32 u32temp;
 639         u32 i;
 640
 641         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 642         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 643         /* TODO: assuming that both SERDES are set the same for now */
 644         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 645         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 646                         u32temp, port_count);
 647         if (*OneG_Mode)
 648                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 649         u32temp &= 0xff00ffc0;
 650         switch (port_count) {
 651                 case 1:
 652                         u32temp |= 0x00ee0000;
 653                         break;
 654                 case 2:
 655                         u32temp |= 0x00cc0000;
 656                         break;
 657                 case 4:
 658                         u32temp |= 0x00000000;
 659                         break;
 660                 default:
 661                         return 0;
 662                         break;
 663         }
 664
 665         /* check and do full reset if needed */
 666         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 667                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 668                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 669
 670                 i = 0;
 671                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 672                         mdelay(1);
 673                 if (i >= 10000) {
 674                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 675                         return 0;
 676                 }
 677
 678                 i = 0;
 679                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 680                         mdelay(1);
 681                 if (i >= 10000) {
 682                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 683                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 684                         return 0;
 685                 }
 686         }
 687
 688         /* port reset */
 689         switch (port_count) {
 690                 case 1:
 691                         u32temp |= 0x00ee0010;
 692                         break;
 693                 case 2:
 694                         u32temp |= 0x00cc0030;
 695                         break;
 696                 case 4:
 697                         u32temp |= 0x00000030;
 698                         break;
 699         }
 700
 701         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 702         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 703
 704         i = 0;
 705         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 706                 mdelay(1);
 707         if (i >= 10000) {
 708                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 709                 return 0;
 710         }
 711
 712         /* serdes 0 */
 713         i = 0;
 714         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 715                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 716                 mdelay(1);
 717         if (i >= 5000) {
 718                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 719                 return 0;
 720         }
 721
 722         /* serdes 1 */
 723         if (port_count > 1) {
 724                 i = 0;
 725                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 726                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 727                         mdelay(1);
 728                 if (i >= 5000) {
 729                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 730                         return 0;
 731                 }
 732         }
 733
 734         return port_count;
 735 }
 736
 737
 738 /**
 739  * nes_init_serdes
 740  */
 741 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 742                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 743 {
 744         int i;
 745         u32 u32temp;
 746         u32 sds;
 747
 748         if (hw_rev != NE020_REV) {
 749                 /* init serdes 0 */
 750                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
 751                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 752                 else
 753                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 754
 755                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 756                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 757                         sds |= 0x00000100;
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 759                 }
 760                 if (!OneG_Mode)
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 762
 763                 if (port_count < 2)
 764                         return 0;
 765
 766                 /* init serdes 1 */
 767                 switch (nesadapter->phy_type[1]) {
 768                 case  NES_PHY_TYPE_ARGUS:
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 770                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 771                         break;
 772                 case NES_PHY_TYPE_CX4:
 773                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 774                         sds &= 0xFFFFFFBF;
 775                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 776                         if (wide_ppm_offset)
 777                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 778                         else
 779                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 780                         break;
 781                 case NES_PHY_TYPE_PUMA_1G:
 782                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 783                         sds |= 0x000000100;
 784                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 785                 }
 786                 if (!OneG_Mode)
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 788         } else {
 789                 /* init serdes 0 */
 790                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 791                 i = 0;
 792                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 793                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 794                         mdelay(1);
 795                 if (i >= 5000) {
 796                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 797                         return 1;
 798                 }
 799                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 800                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 801                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 802                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 805                 if (OneG_Mode)
 806                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 807                 else
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 809
 810                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 811                 if (port_count > 1) {
 812                         /* init serdes 1 */
 813                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 814                         i = 0;
 815                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 816                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 817                                 mdelay(1);
 818                         if (i >= 5000) {
 819                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 820                                 /* return 1; */
 821                         }
 822                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 823                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 824                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 830                 }
 831         }
 832         return 0;
 833 }
 834
 835
 836 /**
 837  * nes_init_csr_ne020
 838  * Initialize registers for ne020 hardware
 839  */
 840 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 841 {
 842         u32 u32temp;
 843
 844         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 845
 846         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 847         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 848         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 849         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 850         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 851         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 852         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 853         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 854
 855         /* TODO: move these MAC register settings to NIC bringup */
 856         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 857         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 858         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 859         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 860         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 861         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 862         if (port_count > 1) {
 863                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 864                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 865                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 866                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 867                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 868                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 869                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 870         }
 871         if (port_count > 2) {
 872                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 873                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 874                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 875                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 876                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 877                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 878                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 879
 880                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 881                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 882                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 883                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 885                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 886                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 887         }
 888
 889         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 890         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 891         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 892                                                          0x00000001);
 893         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 894         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 895         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 896         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 897         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 898
 899         /* TODO: move this to code, get from EEPROM */
 900         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 901         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 902         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 903
 904         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 905         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 906
 907         if (hw_rev != NE020_REV) {
 908                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 909                 u32temp |= 0x80000000;
 910                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 911                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 912                 u32temp &= 0x7fffffff;
 913                 u32temp |= 0x7fff0010;
 914                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 915         }
 916 }
 917
 918
 919 /**
 920  * nes_destroy_adapter - destroy the adapter structure
 921  */
 922 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 923 {
 924         struct nes_adapter *tmp_adapter;
 925
 926         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 927                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 928                                 tmp_adapter);
 929         }
 930
 931         nesadapter->ref_count--;
 932         if (!nesadapter->ref_count) {
 933                 if (nesadapter->hw_rev == NE020_REV) {
 934                         del_timer(&nesadapter->mh_timer);
 935                 }
 936                 del_timer(&nesadapter->lc_timer);
 937
 938                 list_del(&nesadapter->list);
 939                 kfree(nesadapter);
 940         }
 941 }
 942
 943
 944 /**
 945  * nes_init_cqp
 946  */
 947 int nes_init_cqp(struct nes_device *nesdev)
 948 {
 949         struct nes_adapter *nesadapter = nesdev->nesadapter;
 950         struct nes_hw_cqp_qp_context *cqp_qp_context;
 951         struct nes_hw_cqp_wqe *cqp_wqe;
 952         struct nes_hw_ceq *ceq;
 953         struct nes_hw_ceq *nic_ceq;
 954         struct nes_hw_aeq *aeq;
 955         void *vmem;
 956         dma_addr_t pmem;
 957         u32 count=0;
 958         u32 cqp_head;
 959         u64 u64temp;
 960         u32 u32temp;
 961
 962         /* allocate CQP memory */
 963         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 964         /* SQ is 512 byte aligned, others are 256 byte aligned */
 965         nesdev->cqp_mem_size = 512 +
 966                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 967                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 968                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 969                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 970                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 971                         sizeof(struct nes_hw_cqp_qp_context);
 972
 973         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 974                         &nesdev->cqp_pbase);
 975         if (!nesdev->cqp_vbase) {
 976                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 977                 return -ENOMEM;
 978         }
 979         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 980
 981         /* Allocate a twice the number of CQP requests as the SQ size */
 982         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 983                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 984         if (nesdev->nes_cqp_requests == NULL) {
 985                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 986                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 987                                 nesdev->cqp.sq_pbase);
 988                 return -ENOMEM;
 989         }
 990
 991         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 992                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 993
 994         spin_lock_init(&nesdev->cqp.lock);
 995         init_waitqueue_head(&nesdev->cqp.waitq);
 996
 997         /* Setup Various Structures */
 998         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 999                         ~(unsigned long)(512 - 1));
1000         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1001                         ~(unsigned long long)(512 - 1));
1002
1003         nesdev->cqp.sq_vbase = vmem;
1004         nesdev->cqp.sq_pbase = pmem;
1005         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1006         nesdev->cqp.sq_head = 0;
1007         nesdev->cqp.sq_tail = 0;
1008         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1009
1010         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1011         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1012
1013         nesdev->ccq.cq_vbase = vmem;
1014         nesdev->ccq.cq_pbase = pmem;
1015         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1016         nesdev->ccq.cq_head = 0;
1017         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1018         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1019
1020         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1021         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1022
1023         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1024         ceq = &nesadapter->ceq[nesdev->ceq_index];
1025         ceq->ceq_vbase = vmem;
1026         ceq->ceq_pbase = pmem;
1027         ceq->ceq_size = NES_CCEQ_SIZE;
1028         ceq->ceq_head = 0;
1029
1030         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1031         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1032
1033         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1034         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1035         nic_ceq->ceq_vbase = vmem;
1036         nic_ceq->ceq_pbase = pmem;
1037         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1038         nic_ceq->ceq_head = 0;
1039
1040         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1041         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1042
1043         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1044         aeq->aeq_vbase = vmem;
1045         aeq->aeq_pbase = pmem;
1046         aeq->aeq_size = nesadapter->max_qp;
1047         aeq->aeq_head = 0;
1048
1049         /* Setup QP Context */
1050         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1051         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1052
1053         cqp_qp_context = vmem;
1054         cqp_qp_context->context_words[0] =
1055                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1056         cqp_qp_context->context_words[1] = 0;
1057         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1058         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1059
1060
1061         /* Write the address to Create CQP */
1062         if ((sizeof(dma_addr_t) > 4)) {
1063                 nes_write_indexed(nesdev,
1064                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1065                                 ((u64)pmem) >> 32);
1066         } else {
1067                 nes_write_indexed(nesdev,
1068                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1069         }
1070         nes_write_indexed(nesdev,
1071                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1072                         (u32)pmem);
1073
1074         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1075         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1076
1077         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1078                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1079                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1080         }
1081
1082         /* Write Create CCQ WQE */
1083         cqp_head = nesdev->cqp.sq_head++;
1084         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1085         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1086         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1087                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1088                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1089         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1090                             (nesdev->ccq.cq_number |
1091                              ((u32)nesdev->ceq_index << 16)));
1092         u64temp = (u64)nesdev->ccq.cq_pbase;
1093         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1094         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1095         u64temp = (unsigned long)&nesdev->ccq;
1096         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1097                         cpu_to_le32((u32)(u64temp >> 1));
1098         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1099                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1100         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1101
1102         /* Write Create CEQ WQE */
1103         cqp_head = nesdev->cqp.sq_head++;
1104         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1105         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1106         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1107                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1108         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1109         u64temp = (u64)ceq->ceq_pbase;
1110         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1111
1112         /* Write Create AEQ WQE */
1113         cqp_head = nesdev->cqp.sq_head++;
1114         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1115         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1116         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1117                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1118         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1119         u64temp = (u64)aeq->aeq_pbase;
1120         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1121
1122         /* Write Create NIC CEQ WQE */
1123         cqp_head = nesdev->cqp.sq_head++;
1124         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1125         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1126         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1127                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1128         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1129         u64temp = (u64)nic_ceq->ceq_pbase;
1130         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1131
1132         /* Poll until CCQP done */
1133         count = 0;
1134         do {
1135                 if (count++ > 1000) {
1136                         printk(KERN_ERR PFX "Error creating CQP\n");
1137                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1138                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1139                         return -1;
1140                 }
1141                 udelay(10);
1142         } while (!(nes_read_indexed(nesdev,
1143                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1144
1145         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1146                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1147
1148         u32temp = 0x04800000;
1149         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1150
1151         /* wait for the CCQ, CEQ, and AEQ to get created */
1152         count = 0;
1153         do {
1154                 if (count++ > 1000) {
1155                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1156                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1157                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1158                         return -1;
1159                 }
1160                 udelay(10);
1161         } while (((nes_read_indexed(nesdev,
1162                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1163
1164         /* dump the QP status value */
1165         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1166                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1167
1168         nesdev->cqp.sq_tail++;
1169
1170         return 0;
1171 }
1172
1173
1174 /**
1175  * nes_destroy_cqp
1176  */
1177 int nes_destroy_cqp(struct nes_device *nesdev)
1178 {
1179         struct nes_hw_cqp_wqe *cqp_wqe;
1180         u32 count = 0;
1181         u32 cqp_head;
1182         unsigned long flags;
1183
1184         do {
1185                 if (count++ > 1000)
1186                         break;
1187                 udelay(10);
1188         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1189
1190         /* Reset CCQ */
1191         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1192                         nesdev->ccq.cq_number);
1193
1194         /* Disable device interrupts */
1195         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1196
1197         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1198
1199         /* Destroy the AEQ */
1200         cqp_head = nesdev->cqp.sq_head++;
1201         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1202         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1203         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1204                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1205         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1206
1207         /* Destroy the NIC CEQ */
1208         cqp_head = nesdev->cqp.sq_head++;
1209         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1210         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1211         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1212                         ((u32)nesdev->nic_ceq_index << 8));
1213
1214         /* Destroy the CEQ */
1215         cqp_head = nesdev->cqp.sq_head++;
1216         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1217         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1218         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1219                         (nesdev->ceq_index << 8));
1220
1221         /* Destroy the CCQ */
1222         cqp_head = nesdev->cqp.sq_head++;
1223         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1224         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1225         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1226         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1227                         ((u32)nesdev->ceq_index << 16));
1228
1229         /* Destroy CQP */
1230         cqp_head = nesdev->cqp.sq_head++;
1231         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1232         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1233         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1234                         NES_CQP_QP_TYPE_CQP);
1235         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1236
1237         barrier();
1238         /* Ring doorbell (5 WQEs) */
1239         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1240
1241         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1242
1243         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1244         count = 0;
1245         do {
1246                 if (count++ > 1000) {
1247                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1248                                         PCI_FUNC(nesdev->pcidev->devfn));
1249                         break;
1250                 }
1251                 udelay(10);
1252         } while (((nes_read_indexed(nesdev,
1253                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1254
1255         /* dump the QP status value */
1256         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1257                         PCI_FUNC(nesdev->pcidev->devfn),
1258                         nes_read_indexed(nesdev,
1259                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1260
1261         kfree(nesdev->nes_cqp_requests);
1262
1263         /* Free the control structures */
1264         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1265                         nesdev->cqp.sq_pbase);
1266
1267         return 0;
1268 }
1269
1270
1271 /**
1272  * nes_init_phy
1273  */
1274 int nes_init_phy(struct nes_device *nesdev)
1275 {
1276         struct nes_adapter *nesadapter = nesdev->nesadapter;
1277         u32 counter = 0;
1278         u32 sds;
1279         u32 mac_index = nesdev->mac_index;
1280         u32 tx_config = 0;
1281         u16 phy_data;
1282         u32 temp_phy_data = 0;
1283         u32 temp_phy_data2 = 0;
1284         u8  phy_type = nesadapter->phy_type[mac_index];
1285         u8  phy_index = nesadapter->phy_index[mac_index];
1286
1287         if ((nesadapter->OneG_Mode) &&
1288             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1289                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1290                 if (phy_type == NES_PHY_TYPE_1G) {
1291                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1292                         tx_config &= 0xFFFFFFE3;
1293                         tx_config |= 0x04;
1294                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1295                 }
1296
1297                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1298                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1299
1300                 /* Reset the PHY */
1301                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1302                 udelay(100);
1303                 counter = 0;
1304                 do {
1305                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1306                         if (counter++ > 100)
1307                                 break;
1308                 } while (phy_data & 0x8000);
1309
1310                 /* Setting no phy loopback */
1311                 phy_data &= 0xbfff;
1312                 phy_data |= 0x1140;
1313                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1314                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1315                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1316                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1317
1318                 /* Setting the interrupt mask */
1319                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1320                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1321                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1322
1323                 /* turning on flow control */
1324                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1325                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1326                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1327
1328                 /* Clear Half duplex */
1329                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1330                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1331                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1332
1333                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1334                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1335
1336                 return 0;
1337         }
1338
1339         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1340             (phy_type == NES_PHY_TYPE_ARGUS)) {
1341                 /* setup 10G MDIO operation */
1342                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1343                 tx_config &= 0xFFFFFFE3;
1344                 tx_config |= 0x15;
1345                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1346         }
1347         if ((phy_type == NES_PHY_TYPE_ARGUS)) {
1348                 /* Check firmware heartbeat */
1349                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1350                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1351                 udelay(1500);
1352                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1353                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1354
1355                 if (temp_phy_data != temp_phy_data2)
1356                         return 0;
1357
1358                 /* no heartbeat, configure the PHY */
1359                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1360                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1361                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1362                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1363                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1364                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1365                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1366                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1367                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1368
1369                 /* setup LEDs */
1370                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1371                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1372                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1373
1374                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1375
1376                 /* Bring PHY out of reset */
1377                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1378
1379                 /* Check for heartbeat */
1380                 counter = 0;
1381                 mdelay(690);
1382                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1383                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1384                 do {
1385                         if (counter++ > 150) {
1386                                 nes_debug(NES_DBG_PHY, "No PHY heartbeat\n");
1387                                 break;
1388                         }
1389                         mdelay(1);
1390                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1391                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1392                 } while ((temp_phy_data2 == temp_phy_data));
1393
1394                 /* wait for tracking */
1395                 counter = 0;
1396                 do {
1397                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1398                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1399                         if (counter++ > 300) {
1400                                 nes_debug(NES_DBG_PHY, "PHY did not track\n");
1401                                 break;
1402                         }
1403                         mdelay(10);
1404                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1405
1406                 /* setup signal integrity */
1407                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1408                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1409                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1412
1413                 /* reset serdes */
1414                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1415                                        mac_index * 0x200);
1416                 sds |= 0x1;
1417                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1418                                   mac_index * 0x200, sds);
1419                 sds &= 0xfffffffe;
1420                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1421                                   mac_index * 0x200, sds);
1422
1423                 counter = 0;
1424                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1425                                 && (counter++ < 5000))
1426                         ;
1427         }
1428         return 0;
1429 }
1430
1431
1432 /**
1433  * nes_replenish_nic_rq
1434  */
1435 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1436 {
1437         unsigned long flags;
1438         dma_addr_t bus_address;
1439         struct sk_buff *skb;
1440         struct nes_hw_nic_rq_wqe *nic_rqe;
1441         struct nes_hw_nic *nesnic;
1442         struct nes_device *nesdev;
1443         u32 rx_wqes_posted = 0;
1444
1445         nesnic = &nesvnic->nic;
1446         nesdev = nesvnic->nesdev;
1447         spin_lock_irqsave(&nesnic->rq_lock, flags);
1448         if (nesnic->replenishing_rq !=0) {
1449                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1450                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1451                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1452                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1453                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1454                         add_timer(&nesvnic->rq_wqes_timer);
1455                 } else
1456                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1457                 return;
1458         }
1459         nesnic->replenishing_rq = 1;
1460         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1461         do {
1462                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1463                 if (skb) {
1464                         skb->dev = nesvnic->netdev;
1465
1466                         bus_address = pci_map_single(nesdev->pcidev,
1467                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1468
1469                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1470                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1471                                         cpu_to_le32(nesvnic->max_frame_size);
1472                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1473                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1474                                         cpu_to_le32((u32)bus_address);
1475                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1476                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1477                         nesnic->rx_skb[nesnic->rq_head] = skb;
1478                         nesnic->rq_head++;
1479                         nesnic->rq_head &= nesnic->rq_size - 1;
1480                         atomic_dec(&nesvnic->rx_skbs_needed);
1481                         barrier();
1482                         if (++rx_wqes_posted == 255) {
1483                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1484                                 rx_wqes_posted = 0;
1485                         }
1486                 } else {
1487                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1488                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1489                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1490                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1491                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1492                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1493                                 add_timer(&nesvnic->rq_wqes_timer);
1494                         } else
1495                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1496                         break;
1497                 }
1498         } while (atomic_read(&nesvnic->rx_skbs_needed));
1499         barrier();
1500         if (rx_wqes_posted)
1501                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1502         nesnic->replenishing_rq = 0;
1503 }
1504
1505
1506 /**
1507  * nes_rq_wqes_timeout
1508  */
1509 static void nes_rq_wqes_timeout(unsigned long parm)
1510 {
1511         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1512         printk("%s: Timer fired.\n", __func__);
1513         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1514         if (atomic_read(&nesvnic->rx_skbs_needed))
1515                 nes_replenish_nic_rq(nesvnic);
1516 }
1517
1518
1519 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1520                                void **tcph, u64 *hdr_flags, void *priv)
1521 {
1522         unsigned int ip_len;
1523         struct iphdr *iph;
1524         skb_reset_network_header(skb);
1525         iph = ip_hdr(skb);
1526         if (iph->protocol != IPPROTO_TCP)
1527                 return -1;
1528         ip_len = ip_hdrlen(skb);
1529         skb_set_transport_header(skb, ip_len);
1530         *tcph = tcp_hdr(skb);
1531
1532         *hdr_flags = LRO_IPV4 | LRO_TCP;
1533         *iphdr = iph;
1534         return 0;
1535 }
1536
1537
1538 /**
1539  * nes_init_nic_qp
1540  */
1541 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1542 {
1543         struct nes_hw_cqp_wqe *cqp_wqe;
1544         struct nes_hw_nic_sq_wqe *nic_sqe;
1545         struct nes_hw_nic_qp_context *nic_context;
1546         struct sk_buff *skb;
1547         struct nes_hw_nic_rq_wqe *nic_rqe;
1548         struct nes_vnic *nesvnic = netdev_priv(netdev);
1549         unsigned long flags;
1550         void *vmem;
1551         dma_addr_t pmem;
1552         u64 u64temp;
1553         int ret;
1554         u32 cqp_head;
1555         u32 counter;
1556         u32 wqe_count;
1557         u8 jumbomode=0;
1558
1559         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1560         nesvnic->nic_mem_size = 256 +
1561                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1562                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1563                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1564                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1565                         sizeof(struct nes_hw_nic_qp_context);
1566
1567         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1568                         &nesvnic->nic_pbase);
1569         if (!nesvnic->nic_vbase) {
1570                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1571                 return -ENOMEM;
1572         }
1573         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1574         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1575                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1576
1577         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1578                         ~(unsigned long)(256 - 1));
1579         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1580                         ~(unsigned long long)(256 - 1));
1581
1582         /* Setup the first Fragment buffers */
1583         nesvnic->nic.first_frag_vbase = vmem;
1584
1585         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1586                 nesvnic->nic.frag_paddr[counter] = pmem;
1587                 pmem += sizeof(struct nes_first_frag);
1588         }
1589
1590         /* setup the SQ */
1591         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1592
1593         nesvnic->nic.sq_vbase = (void *)vmem;
1594         nesvnic->nic.sq_pbase = pmem;
1595         nesvnic->nic.sq_head = 0;
1596         nesvnic->nic.sq_tail = 0;
1597         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1598         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1599                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1600                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1601                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1602                                 NES_NIC_SQ_WQE_COMPLETION);
1603                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1604                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1605                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1606                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1607                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1608                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1609         }
1610
1611         nesvnic->get_cqp_request = nes_get_cqp_request;
1612         nesvnic->post_cqp_request = nes_post_cqp_request;
1613         nesvnic->mcrq_mcast_filter = NULL;
1614
1615         spin_lock_init(&nesvnic->nic.rq_lock);
1616
1617         /* setup the RQ */
1618         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1619         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1620
1621
1622         nesvnic->nic.rq_vbase = vmem;
1623         nesvnic->nic.rq_pbase = pmem;
1624         nesvnic->nic.rq_head = 0;
1625         nesvnic->nic.rq_tail = 0;
1626         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1627
1628         /* setup the CQ */
1629         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1630         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1631
1632         if (nesdev->nesadapter->netdev_count > 2)
1633                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1634         else
1635                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1636
1637         nesvnic->nic_cq.cq_vbase = vmem;
1638         nesvnic->nic_cq.cq_pbase = pmem;
1639         nesvnic->nic_cq.cq_head = 0;
1640         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1641
1642         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1643
1644         /* Send CreateCQ request to CQP */
1645         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1646         cqp_head = nesdev->cqp.sq_head;
1647
1648         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1649         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1650
1651         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1652                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1653                         ((u32)nesvnic->nic_cq.cq_size << 16));
1654         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1655                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1656         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1657         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1658         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1659         u64temp = (unsigned long)&nesvnic->nic_cq;
1660         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1661         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1662                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1663         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1664         if (++cqp_head >= nesdev->cqp.sq_size)
1665                 cqp_head = 0;
1666         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1667         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1668
1669         /* Send CreateQP request to CQP */
1670         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1671         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1672                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1673                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1674         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1675                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1676                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1677         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1678                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1679         }
1680
1681         u64temp = (u64)nesvnic->nic.sq_pbase;
1682         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1683         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1684         u64temp = (u64)nesvnic->nic.rq_pbase;
1685         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1686         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1687
1688         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1689                         NES_CQP_QP_TYPE_NIC);
1690         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1691         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1692                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1693         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1694
1695         if (++cqp_head >= nesdev->cqp.sq_size)
1696                 cqp_head = 0;
1697         nesdev->cqp.sq_head = cqp_head;
1698
1699         barrier();
1700
1701         /* Ring doorbell (2 WQEs) */
1702         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1703
1704         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1705         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1706                         nesvnic->nic.qp_id);
1707
1708         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1709                         NES_EVENT_TIMEOUT);
1710         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1711                         nesvnic->nic.qp_id, ret);
1712         if (!ret) {
1713                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1714                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1715                                 nesvnic->nic_pbase);
1716                 return -EIO;
1717         }
1718
1719         /* Populate the RQ */
1720         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1721                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1722                 if (!skb) {
1723                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1724
1725                         nes_destroy_nic_qp(nesvnic);
1726                         return -ENOMEM;
1727                 }
1728
1729                 skb->dev = netdev;
1730
1731                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1732                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1733
1734                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1735                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1736                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1737                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1738                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1739                 nesvnic->nic.rx_skb[counter] = skb;
1740         }
1741
1742         wqe_count = NES_NIC_WQ_SIZE - 1;
1743         nesvnic->nic.rq_head = wqe_count;
1744         barrier();
1745         do {
1746                 counter = min(wqe_count, ((u32)255));
1747                 wqe_count -= counter;
1748                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1749         } while (wqe_count);
1750         init_timer(&nesvnic->rq_wqes_timer);
1751         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1752         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1753         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1754         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1755         {
1756                 nes_nic_init_timer(nesdev);
1757                 if (netdev->mtu > 1500)
1758                         jumbomode = 1;
1759                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1760         }
1761         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1762         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1763         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1764         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1765         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1766         nesvnic->lro_mgr.dev            = netdev;
1767         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1768         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1769         return 0;
1770 }
1771
1772
1773 /**
1774  * nes_destroy_nic_qp
1775  */
1776 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1777 {
1778         u64 u64temp;
1779         dma_addr_t bus_address;
1780         struct nes_device *nesdev = nesvnic->nesdev;
1781         struct nes_hw_cqp_wqe *cqp_wqe;
1782         struct nes_hw_nic_sq_wqe *nic_sqe;
1783         struct nes_hw_nic_rq_wqe *nic_rqe;
1784         __le16 *wqe_fragment_length;
1785         u16  wqe_fragment_index;
1786         u64 wqe_frag;
1787         u32 cqp_head;
1788         unsigned long flags;
1789         int ret;
1790
1791         /* Free remaining NIC receive buffers */
1792         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1793                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1794                 wqe_frag  = (u64)le32_to_cpu(
1795                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1796                 wqe_frag |= ((u64)le32_to_cpu(
1797                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1798                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1799                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1800                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1801                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1802         }
1803
1804         /* Free remaining NIC transmit buffers */
1805         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1806                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1807                 wqe_fragment_index = 1;
1808                 wqe_fragment_length = (__le16 *)
1809                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1810                 /* bump past the vlan tag */
1811                 wqe_fragment_length++;
1812                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1813                         u64temp = (u64)le32_to_cpu(
1814                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1815                                 wqe_fragment_index*2]);
1816                         u64temp += ((u64)le32_to_cpu(
1817                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1818                                 + wqe_fragment_index*2]))<<32;
1819                         bus_address = (dma_addr_t)u64temp;
1820                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1821                                         nesvnic->nic.first_frag_overflow)) {
1822                                 pci_unmap_single(nesdev->pcidev,
1823                                                 bus_address,
1824                                                 le16_to_cpu(wqe_fragment_length[
1825                                                         wqe_fragment_index++]),
1826                                                 PCI_DMA_TODEVICE);
1827                         }
1828                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1829                                 if (wqe_fragment_length[wqe_fragment_index]) {
1830                                         u64temp = le32_to_cpu(
1831                                                 nic_sqe->wqe_words[
1832                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1833                                                 wqe_fragment_index*2]);
1834                                         u64temp += ((u64)le32_to_cpu(
1835                                                 nic_sqe->wqe_words[
1836                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1837                                                 wqe_fragment_index*2]))<<32;
1838                                         bus_address = (dma_addr_t)u64temp;
1839                                         pci_unmap_page(nesdev->pcidev,
1840                                                         bus_address,
1841                                                         le16_to_cpu(
1842                                                         wqe_fragment_length[
1843                                                         wqe_fragment_index]),
1844                                                         PCI_DMA_TODEVICE);
1845                                 } else
1846                                         break;
1847                         }
1848                 }
1849                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1850                         dev_kfree_skb(
1851                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1852
1853                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1854                                         & (nesvnic->nic.sq_size - 1);
1855         }
1856
1857         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1858
1859         /* Destroy NIC QP */
1860         cqp_head = nesdev->cqp.sq_head;
1861         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1862         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1863
1864         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1865                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1866         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1867                 nesvnic->nic.qp_id);
1868
1869         if (++cqp_head >= nesdev->cqp.sq_size)
1870                 cqp_head = 0;
1871
1872         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1873
1874         /* Destroy NIC CQ */
1875         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1876         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1877                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1878         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1879                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1880
1881         if (++cqp_head >= nesdev->cqp.sq_size)
1882                 cqp_head = 0;
1883
1884         nesdev->cqp.sq_head = cqp_head;
1885         barrier();
1886
1887         /* Ring doorbell (2 WQEs) */
1888         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1889
1890         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1891         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1892                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1893                         cqp_head, nesdev->cqp.sq_head,
1894                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1895
1896         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1897                         NES_EVENT_TIMEOUT);
1898
1899         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1900                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1901                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1902         if (!ret) {
1903                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1904                                 nesvnic->nic.qp_id);
1905         }
1906
1907         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1908                         nesvnic->nic_pbase);
1909 }
1910
1911 /**
1912  * nes_napi_isr
1913  */
1914 int nes_napi_isr(struct nes_device *nesdev)
1915 {
1916         struct nes_adapter *nesadapter = nesdev->nesadapter;
1917         u32 int_stat;
1918
1919         if (nesdev->napi_isr_ran) {
1920                 /* interrupt status has already been read in ISR */
1921                 int_stat = nesdev->int_stat;
1922         } else {
1923                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1924                 nesdev->int_stat = int_stat;
1925                 nesdev->napi_isr_ran = 1;
1926         }
1927
1928         int_stat &= nesdev->int_req;
1929         /* iff NIC, process here, else wait for DPC */
1930         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1931                 nesdev->napi_isr_ran = 0;
1932                 nes_write32(nesdev->regs + NES_INT_STAT,
1933                         (int_stat &
1934                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1935
1936                 /* Process the CEQs */
1937                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1938
1939                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1940                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1941                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1942                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1943                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1944                                 /* Enable Periodic timer interrupts */
1945                                 nesdev->int_req |= NES_INT_TIMER;
1946                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1947                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1948                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1949                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1950                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1951                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1952                         }
1953
1954                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1955                         {
1956                                 nes_nic_init_timer(nesdev);
1957                         }
1958                         /* Enable interrupts, except CEQs */
1959                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1960                 } else {
1961                         /* Enable interrupts, make sure timer is off */
1962                         nesdev->int_req &= ~NES_INT_TIMER;
1963                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1964                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1965                 }
1966                 nesdev->deepcq_count = 0;
1967                 return 1;
1968         } else {
1969                 return 0;
1970         }
1971 }
1972
1973 static void process_critical_error(struct nes_device *nesdev)
1974 {
1975         u32 debug_error;
1976         u32 nes_idx_debug_error_masks0 = 0;
1977         u16 error_module = 0;
1978
1979         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1980         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1981                         (u16)debug_error);
1982         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1983                         0x01010000 | (debug_error & 0x0000ffff));
1984         if (crit_err_count++ > 10)
1985                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1986         error_module = (u16) (debug_error & 0x1F00) >> 8;
1987         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
1988                         nes_max_critical_error_count) {
1989                 printk(KERN_ERR PFX "Masking off critical error for module "
1990                         "0x%02X\n", (u16)error_module);
1991                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
1992                         NES_IDX_DEBUG_ERROR_MASKS0);
1993                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
1994                         nes_idx_debug_error_masks0 | (1 << error_module));
1995         }
1996 }
1997 /**
1998  * nes_dpc
1999  */
2000 void nes_dpc(unsigned long param)
2001 {
2002         struct nes_device *nesdev = (struct nes_device *)param;
2003         struct nes_adapter *nesadapter = nesdev->nesadapter;
2004         u32 counter;
2005         u32 loop_counter = 0;
2006         u32 int_status_bit;
2007         u32 int_stat;
2008         u32 timer_stat;
2009         u32 temp_int_stat;
2010         u32 intf_int_stat;
2011         u32 processed_intf_int = 0;
2012         u16 processed_timer_int = 0;
2013         u16 completion_ints = 0;
2014         u16 timer_ints = 0;
2015
2016         /* nes_debug(NES_DBG_ISR, "\n"); */
2017
2018         do {
2019                 timer_stat = 0;
2020                 if (nesdev->napi_isr_ran) {
2021                         nesdev->napi_isr_ran = 0;
2022                         int_stat = nesdev->int_stat;
2023                 } else
2024                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2025                 if (processed_intf_int != 0)
2026                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2027                 else
2028                         int_stat &= nesdev->int_req;
2029                 if (processed_timer_int == 0) {
2030                         processed_timer_int = 1;
2031                         if (int_stat & NES_INT_TIMER) {
2032                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2033                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2034                                         int_stat &= ~NES_INT_TIMER;
2035                                 }
2036                         }
2037                 } else {
2038                         int_stat &= ~NES_INT_TIMER;
2039                 }
2040
2041                 if (int_stat) {
2042                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2043                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2044                                 /* Ack the interrupts */
2045                                 nes_write32(nesdev->regs+NES_INT_STAT,
2046                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2047                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2048                         }
2049
2050                         temp_int_stat = int_stat;
2051                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2052                                 if (int_stat & int_status_bit) {
2053                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2054                                         temp_int_stat &= ~int_status_bit;
2055                                         completion_ints = 1;
2056                                 }
2057                                 if (!(temp_int_stat & 0x0000ffff))
2058                                         break;
2059                                 int_status_bit <<= 1;
2060                         }
2061
2062                         /* Process the AEQ for this pci function */
2063                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2064                         if (int_stat & int_status_bit) {
2065                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2066                         }
2067
2068                         /* Process the MAC interrupt for this pci function */
2069                         int_status_bit = 1 << (24 + nesdev->mac_index);
2070                         if (int_stat & int_status_bit) {
2071                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2072                         }
2073
2074                         if (int_stat & NES_INT_TIMER) {
2075                                 if (timer_stat & nesdev->timer_int_req) {
2076                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2077                                                         (timer_stat & nesdev->timer_int_req) |
2078                                                         ~(nesdev->nesadapter->timer_int_req));
2079                                         timer_ints = 1;
2080                                 }
2081                         }
2082
2083                         if (int_stat & NES_INT_INTF) {
2084                                 processed_intf_int = 1;
2085                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2086                                 intf_int_stat &= nesdev->intf_int_req;
2087                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2088                                         process_critical_error(nesdev);
2089                                 }
2090                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2091                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2092                                         BUG();
2093                                 }
2094                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2095                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2096                                         BUG();
2097                                 }
2098                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2099                         }
2100
2101                         if (int_stat & NES_INT_TSW) {
2102                         }
2103                 }
2104                 /* Don't use the interface interrupt bit stay in loop */
2105                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2106                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2107         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2108
2109         if (timer_ints == 1) {
2110                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2111                         if (completion_ints == 0) {
2112                                 nesdev->timer_only_int_count++;
2113                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2114                                         nesdev->timer_only_int_count = 0;
2115                                         nesdev->int_req &= ~NES_INT_TIMER;
2116                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2117                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2118                                 } else {
2119                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2120                                 }
2121                         } else {
2122                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2123                                 {
2124                                         nes_nic_init_timer(nesdev);
2125                                 }
2126                                 nesdev->timer_only_int_count = 0;
2127                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2128                         }
2129                 } else {
2130                         nesdev->timer_only_int_count = 0;
2131                         nesdev->int_req &= ~NES_INT_TIMER;
2132                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2133                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2134                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2135                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2136                 }
2137         } else {
2138                 if ( (completion_ints == 1) &&
2139                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2140                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2141                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2142                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2143                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2144                         nesdev->timer_only_int_count = 0;
2145                         nesdev->int_req |= NES_INT_TIMER;
2146                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2147                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2148                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2149                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2150                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2151                 } else {
2152                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2153                 }
2154         }
2155         nesdev->deepcq_count = 0;
2156 }
2157
2158
2159 /**
2160  * nes_process_ceq
2161  */
2162 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2163 {
2164         u64 u64temp;
2165         struct nes_hw_cq *cq;
2166         u32 head;
2167         u32 ceq_size;
2168
2169         /* nes_debug(NES_DBG_CQ, "\n"); */
2170         head = ceq->ceq_head;
2171         ceq_size = ceq->ceq_size;
2172
2173         do {
2174                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2175                                 NES_CEQE_VALID) {
2176                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2177                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2178                         u64temp <<= 1;
2179                         cq = *((struct nes_hw_cq **)&u64temp);
2180                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2181                         barrier();
2182                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2183
2184                         /* call the event handler */
2185                         cq->ce_handler(nesdev, cq);
2186
2187                         if (++head >= ceq_size)
2188                                 head = 0;
2189                 } else {
2190                         break;
2191                 }
2192
2193         } while (1);
2194
2195         ceq->ceq_head = head;
2196 }
2197
2198
2199 /**
2200  * nes_process_aeq
2201  */
2202 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2203 {
2204         /* u64 u64temp; */
2205         u32 head;
2206         u32 aeq_size;
2207         u32 aeqe_misc;
2208         u32 aeqe_cq_id;
2209         struct nes_hw_aeqe volatile *aeqe;
2210
2211         head = aeq->aeq_head;
2212         aeq_size = aeq->aeq_size;
2213
2214         do {
2215                 aeqe = &aeq->aeq_vbase[head];
2216                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2217                         break;
2218                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2219                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2220                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2221                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2222                                 /* dealing with an accelerated QP related AE */
2223                                 /*
2224                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2225                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2226                                  */
2227                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2228                         } else {
2229                                 /* TODO: dealing with a CQP related AE */
2230                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2231                                                 (u16)(aeqe_misc >> 16));
2232                         }
2233                 }
2234
2235                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2236
2237                 if (++head >= aeq_size)
2238                         head = 0;
2239
2240                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2241         }
2242         while (1);
2243         aeq->aeq_head = head;
2244 }
2245
2246 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2247 {
2248         struct nes_adapter *nesadapter = nesdev->nesadapter;
2249         u32 reset_value;
2250         u32 i=0;
2251         u32 u32temp;
2252
2253         if (nesadapter->hw_rev == NE020_REV) {
2254                 return;
2255         }
2256         mh_detected++;
2257
2258         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2259
2260         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2261                 reset_value |= 0x0000001d;
2262         else
2263                 reset_value |= 0x0000002d;
2264
2265         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2266                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2267                         nesadapter->link_interrupt_count[0] = 0;
2268                         nesadapter->link_interrupt_count[1] = 0;
2269                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2270                         if (0x00000040 & u32temp)
2271                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2272                         else
2273                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2274
2275                         reset_value |= 0x0000003d;
2276                 }
2277                 nesadapter->link_interrupt_count[mac_index] = 0;
2278         }
2279
2280         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2281
2282         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2283                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2284
2285         if (0x0000003d == (reset_value & 0x0000003d)) {
2286                 u32 pcs_control_status0, pcs_control_status1;
2287
2288                 for (i = 0; i < 10; i++) {
2289                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2290                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2291                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2292                              && (pcs_control_status0 & 0x00100000))
2293                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2294                                 && (pcs_control_status1 & 0x00100000)))
2295                                 continue;
2296                         else
2297                                 break;
2298                 }
2299                 if (10 == i) {
2300                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2301                         if (0x00000040 & u32temp)
2302                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2303                         else
2304                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2305
2306                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2307
2308                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2309                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2310                 }
2311         }
2312 }
2313
2314 /**
2315  * nes_process_mac_intr
2316  */
2317 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2318 {
2319         unsigned long flags;
2320         u32 pcs_control_status;
2321         struct nes_adapter *nesadapter = nesdev->nesadapter;
2322         struct nes_vnic *nesvnic;
2323         u32 mac_status;
2324         u32 mac_index = nesdev->mac_index;
2325         u32 u32temp;
2326         u16 phy_data;
2327         u16 temp_phy_data;
2328         u32 pcs_val  = 0x0f0f0000;
2329         u32 pcs_mask = 0x0f1f0000;
2330         u32 cdr_ctrl;
2331
2332         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2333         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2334                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2335                 return;
2336         }
2337         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2338         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2339
2340         /* ack the MAC interrupt */
2341         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2342         /* Clear the interrupt */
2343         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2344
2345         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2346
2347         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2348                 nesdev->link_status_interrupts++;
2349                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2350                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2351                         nes_reset_link(nesdev, mac_index);
2352                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2353                 }
2354                 /* read the PHY interrupt status register */
2355                 if ((nesadapter->OneG_Mode) &&
2356                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2357                         do {
2358                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2359                                                 nesadapter->phy_index[mac_index], &phy_data);
2360                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2361                                                 nesadapter->phy_index[mac_index], phy_data);
2362                         } while (phy_data&0x8000);
2363
2364                         temp_phy_data = 0;
2365                         do {
2366                                 nes_read_1G_phy_reg(nesdev, 0x11,
2367                                                 nesadapter->phy_index[mac_index], &phy_data);
2368                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2369                                                 nesadapter->phy_index[mac_index], phy_data);
2370                                 if (temp_phy_data == phy_data)
2371                                         break;
2372                                 temp_phy_data = phy_data;
2373                         } while (1);
2374
2375                         nes_read_1G_phy_reg(nesdev, 0x1e,
2376                                         nesadapter->phy_index[mac_index], &phy_data);
2377                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2378                                         nesadapter->phy_index[mac_index], phy_data);
2379
2380                         nes_read_1G_phy_reg(nesdev, 1,
2381                                         nesadapter->phy_index[mac_index], &phy_data);
2382                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2383                                         nesadapter->phy_index[mac_index], phy_data);
2384
2385                         if (temp_phy_data & 0x1000) {
2386                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2387                                 phy_data = 4;
2388                         } else {
2389                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2390                         }
2391                 }
2392                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2393                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2394                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2395
2396                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2397                         switch (mac_index) {
2398                         case 1:
2399                         case 3:
2400                                 pcs_control_status = nes_read_indexed(nesdev,
2401                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2402                                 break;
2403                         default:
2404                                 pcs_control_status = nes_read_indexed(nesdev,
2405                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2406                                 break;
2407                         }
2408                 } else {
2409                         pcs_control_status = nes_read_indexed(nesdev,
2410                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2411                         pcs_control_status = nes_read_indexed(nesdev,
2412                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2413                 }
2414
2415                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2416                                 mac_index, pcs_control_status);
2417                 if ((nesadapter->OneG_Mode) &&
2418                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2419                         u32temp = 0x01010000;
2420                         if (nesadapter->port_count > 2) {
2421                                 u32temp |= 0x02020000;
2422                         }
2423                         if ((pcs_control_status & u32temp)!= u32temp) {
2424                                 phy_data = 0;
2425                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2426                         }
2427                 } else {
2428                         switch (nesadapter->phy_type[mac_index]) {
2429                         case NES_PHY_TYPE_IRIS:
2430                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2431                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2432                                 u32temp = 20;
2433                                 do {
2434                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2435                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2436                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2437                                                 break;
2438                                         temp_phy_data = phy_data;
2439                                 } while (1);
2440                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2441                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2442                                 break;
2443
2444                         case NES_PHY_TYPE_ARGUS:
2445                                 /* clear the alarms */
2446                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2447                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2448                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2449                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2450                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2451                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2452                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2453                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2454                                 /* check link status */
2455                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2456                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2457
2458                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2459                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2460                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2461                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2462
2463                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2464
2465                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2466                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2467                                 break;
2468
2469                         case NES_PHY_TYPE_PUMA_1G:
2470                                 if (mac_index < 2)
2471                                         pcs_val = pcs_mask = 0x01010000;
2472                                 else
2473                                         pcs_val = pcs_mask = 0x02020000;
2474                                 /* fall through */
2475                         default:
2476                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2477                                 break;
2478                         }
2479                 }
2480
2481                 if (phy_data & 0x0004) {
2482                         if (wide_ppm_offset &&
2483                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2484                             (nesadapter->hw_rev != NE020_REV)) {
2485                                 cdr_ctrl = nes_read_indexed(nesdev,
2486                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2487                                                             mac_index * 0x200);
2488                                 nes_write_indexed(nesdev,
2489                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2490                                                   mac_index * 0x200,
2491                                                   cdr_ctrl | 0x000F0000);
2492                         }
2493                         nesadapter->mac_link_down[mac_index] = 0;
2494                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2495                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2496                                                 nesvnic->linkup);
2497                                 if (nesvnic->linkup == 0) {
2498                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2499                                                         nesvnic->netdev->name, nesvnic->netdev);
2500                                         if (netif_queue_stopped(nesvnic->netdev))
2501                                                 netif_start_queue(nesvnic->netdev);
2502                                         nesvnic->linkup = 1;
2503                                         netif_carrier_on(nesvnic->netdev);
2504                                 }
2505                         }
2506                 } else {
2507                         if (wide_ppm_offset &&
2508                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2509                             (nesadapter->hw_rev != NE020_REV)) {
2510                                 cdr_ctrl = nes_read_indexed(nesdev,
2511                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2512                                                             mac_index * 0x200);
2513                                 nes_write_indexed(nesdev,
2514                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2515                                                   mac_index * 0x200,
2516                                                   cdr_ctrl & 0xFFF0FFFF);
2517                         }
2518                         nesadapter->mac_link_down[mac_index] = 1;
2519                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2520                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2521                                                 nesvnic->linkup);
2522                                 if (nesvnic->linkup == 1) {
2523                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2524                                                         nesvnic->netdev->name, nesvnic->netdev);
2525                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2526                                                 netif_stop_queue(nesvnic->netdev);
2527                                         nesvnic->linkup = 0;
2528                                         netif_carrier_off(nesvnic->netdev);
2529                                 }
2530                         }
2531                 }
2532         }
2533
2534         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2535 }
2536
2537
2538
2539 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2540 {
2541         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2542
2543         napi_schedule(&nesvnic->napi);
2544 }
2545
2546
2547 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2548 * getting out of nic_ce_handler
2549 */
2550 #define MAX_RQES_TO_PROCESS     384
2551
2552 /**
2553  * nes_nic_ce_handler
2554  */
2555 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2556 {
2557         u64 u64temp;
2558         dma_addr_t bus_address;
2559         struct nes_hw_nic *nesnic;
2560         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2561         struct nes_adapter *nesadapter = nesdev->nesadapter;
2562         struct nes_hw_nic_rq_wqe *nic_rqe;
2563         struct nes_hw_nic_sq_wqe *nic_sqe;
2564         struct sk_buff *skb;
2565         struct sk_buff *rx_skb;
2566         __le16 *wqe_fragment_length;
2567         u32 head;
2568         u32 cq_size;
2569         u32 rx_pkt_size;
2570         u32 cqe_count=0;
2571         u32 cqe_errv;
2572         u32 cqe_misc;
2573         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2574         u16 vlan_tag;
2575         u16 pkt_type;
2576         u16 rqes_processed = 0;
2577         u8 sq_cqes = 0;
2578         u8 nes_use_lro = 0;
2579
2580         head = cq->cq_head;
2581         cq_size = cq->cq_size;
2582         cq->cqes_pending = 1;
2583         if (nesvnic->netdev->features & NETIF_F_LRO)
2584                 nes_use_lro = 1;
2585         do {
2586                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2587                                 NES_NIC_CQE_VALID) {
2588                         nesnic = &nesvnic->nic;
2589                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2590                         if (cqe_misc & NES_NIC_CQE_SQ) {
2591                                 sq_cqes++;
2592                                 wqe_fragment_index = 1;
2593                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2594                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2595                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2596                                 /* bump past the vlan tag */
2597                                 wqe_fragment_length++;
2598                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2599                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2600                                                         wqe_fragment_index * 2]);
2601                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2602                                                         wqe_fragment_index * 2])) << 32;
2603                                         bus_address = (dma_addr_t)u64temp;
2604                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2605                                                 pci_unmap_single(nesdev->pcidev,
2606                                                                 bus_address,
2607                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2608                                                                 PCI_DMA_TODEVICE);
2609                                         }
2610                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2611                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2612                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2613                                                                                 wqe_fragment_index * 2]);
2614                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2615                                                                                 + wqe_fragment_index * 2])) <<32;
2616                                                         bus_address = (dma_addr_t)u64temp;
2617                                                         pci_unmap_page(nesdev->pcidev,
2618                                                                         bus_address,
2619                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2620                                                                         PCI_DMA_TODEVICE);
2621                                                 } else
2622                                                         break;
2623                                         }
2624                                 }
2625                                 if (skb)
2626                                         dev_kfree_skb_any(skb);
2627                                 nesnic->sq_tail++;
2628                                 nesnic->sq_tail &= nesnic->sq_size-1;
2629                                 if (sq_cqes > 128) {
2630                                         barrier();
2631                                 /* restart the queue if it had been stopped */
2632                                 if (netif_queue_stopped(nesvnic->netdev))
2633                                         netif_wake_queue(nesvnic->netdev);
2634                                         sq_cqes = 0;
2635                                 }
2636                         } else {
2637                                 rqes_processed ++;
2638
2639                                 cq->rx_cqes_completed++;
2640                                 cq->rx_pkts_indicated++;
2641                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2642                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2643                                 /* Get the skb */
2644                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2645                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2646                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2647                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2648                                 pci_unmap_single(nesdev->pcidev, bus_address,
2649                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2650                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2651                                 /* rx_skb->len = rx_pkt_size; */
2652                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2653                                 skb_put(rx_skb, rx_pkt_size);
2654                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2655                                 nesnic->rq_tail++;
2656                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2657
2658                                 atomic_inc(&nesvnic->rx_skbs_needed);
2659                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2660                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2661                                                         cq->cq_number | (cqe_count << 16));
2662                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2663                                         nesdev->currcq_count += cqe_count;
2664                                         cqe_count = 0;
2665                                         nes_replenish_nic_rq(nesvnic);
2666                                 }
2667                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2668                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2669                                 rx_skb->ip_summed = CHECKSUM_NONE;
2670
2671                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2672                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2673                                         if ((cqe_errv &
2674                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2675                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2676                                                 if (nesvnic->rx_checksum_disabled == 0) {
2677                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2678                                                 }
2679                                         } else
2680                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2681                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2682                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2683
2684                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2685                                         if ((cqe_errv &
2686                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2687                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2688                                                 if (nesvnic->rx_checksum_disabled == 0) {
2689                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2690                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2691                                                                   nesvnic->netdev->name); */
2692                                                 }
2693                                         } else
2694                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2695                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2696                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2697                                         }
2698                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2699                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2700
2701                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2702                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2703                                                 rx_skb = NULL;
2704                                 }
2705                                 if (rx_skb == NULL)
2706                                         goto skip_rx_indicate0;
2707
2708
2709                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2710                                     (nesvnic->vlan_grp != NULL)) {
2711                                         vlan_tag = (u16)(le32_to_cpu(
2712                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2713                                                         >> 16);
2714                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2715                                                         nesvnic->netdev->name, vlan_tag);
2716                                         if (nes_use_lro)
2717                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2718                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2719                                         else
2720                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2721                                 } else {
2722                                         if (nes_use_lro)
2723                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2724                                         else
2725                                                 nes_netif_rx(rx_skb);
2726                                 }
2727
2728 skip_rx_indicate0:
2729                                 nesvnic->netdev->last_rx = jiffies;
2730                                 /* nesvnic->netstats.rx_packets++; */
2731                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2732                         }
2733
2734                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2735                         /* Accounting... */
2736                         cqe_count++;
2737                         if (++head >= cq_size)
2738                                 head = 0;
2739                         if (cqe_count == 255) {
2740                                 /* Replenish Nic CQ */
2741                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2742                                                 cq->cq_number | (cqe_count << 16));
2743                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2744                                 nesdev->currcq_count += cqe_count;
2745                                 cqe_count = 0;
2746                         }
2747
2748                         if (cq->rx_cqes_completed >= nesvnic->budget)
2749                                 break;
2750                 } else {
2751                         cq->cqes_pending = 0;
2752                         break;
2753                 }
2754
2755         } while (1);
2756
2757         if (nes_use_lro)
2758                 lro_flush_all(&nesvnic->lro_mgr);
2759         if (sq_cqes) {
2760                 barrier();
2761                 /* restart the queue if it had been stopped */
2762                 if (netif_queue_stopped(nesvnic->netdev))
2763                         netif_wake_queue(nesvnic->netdev);
2764         }
2765         cq->cq_head = head;
2766         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2767                         cq->cq_number, cqe_count, cq->cq_head); */
2768         cq->cqe_allocs_pending = cqe_count;
2769         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2770         {
2771                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2772                 nesdev->currcq_count += cqe_count;
2773                 nes_nic_tune_timer(nesdev);
2774         }
2775         if (atomic_read(&nesvnic->rx_skbs_needed))
2776                 nes_replenish_nic_rq(nesvnic);
2777 }
2778
2779
2780 /**
2781  * nes_cqp_ce_handler
2782  */
2783 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2784 {
2785         u64 u64temp;
2786         unsigned long flags;
2787         struct nes_hw_cqp *cqp = NULL;
2788         struct nes_cqp_request *cqp_request;
2789         struct nes_hw_cqp_wqe *cqp_wqe;
2790         u32 head;
2791         u32 cq_size;
2792         u32 cqe_count=0;
2793         u32 error_code;
2794         /* u32 counter; */
2795
2796         head = cq->cq_head;
2797         cq_size = cq->cq_size;
2798
2799         do {
2800                 /* process the CQE */
2801                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2802                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2803
2804                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2805                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2806                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2807                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2808                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2809                         cqp = *((struct nes_hw_cqp **)&u64temp);
2810
2811                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2812                         if (error_code) {
2813                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2814                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2815                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2816                                                 (u16)(error_code >> 16),
2817                                                 (u16)error_code);
2818                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2819                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2820                         }
2821
2822                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2823                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2824                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2825                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2826                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2827                         if (cqp_request) {
2828                                 if (cqp_request->waiting) {
2829                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2830                                         cqp_request->major_code = (u16)(error_code >> 16);
2831                                         cqp_request->minor_code = (u16)error_code;
2832                                         barrier();
2833                                         cqp_request->request_done = 1;
2834                                         wake_up(&cqp_request->waitq);
2835                                         nes_put_cqp_request(nesdev, cqp_request);
2836                                 } else {
2837                                         if (cqp_request->callback)
2838                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2839                                         nes_free_cqp_request(nesdev, cqp_request);
2840                                 }
2841                         } else {
2842                                 wake_up(&nesdev->cqp.waitq);
2843                         }
2844
2845                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2846                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2847                         if (++cqp->sq_tail >= cqp->sq_size)
2848                                 cqp->sq_tail = 0;
2849
2850                         /* Accounting... */
2851                         cqe_count++;
2852                         if (++head >= cq_size)
2853                                 head = 0;
2854                 } else {
2855                         break;
2856                 }
2857         } while (1);
2858         cq->cq_head = head;
2859
2860         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2861         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2862                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2863                         (nesdev->cqp.sq_size - 1)) != 1)) {
2864                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2865                                 struct nes_cqp_request, list);
2866                 list_del_init(&cqp_request->list);
2867                 head = nesdev->cqp.sq_head++;
2868                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2869                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2870                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2871                 barrier();
2872                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2873                         cpu_to_le32((u32)((unsigned long)cqp_request));
2874                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2875                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2876                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2877                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2878                 /* Ring doorbell (1 WQEs) */
2879                 barrier();
2880                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2881         }
2882         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2883
2884         /* Arm the CCQ */
2885         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2886                         cq->cq_number);
2887         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2888 }
2889
2890
2891 /**
2892  * nes_process_iwarp_aeqe
2893  */
2894 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2895                                    struct nes_hw_aeqe *aeqe)
2896 {
2897         u64 context;
2898         u64 aeqe_context = 0;
2899         unsigned long flags;
2900         struct nes_qp *nesqp;
2901         int resource_allocated;
2902         /* struct iw_cm_id *cm_id; */
2903         struct nes_adapter *nesadapter = nesdev->nesadapter;
2904         struct ib_event ibevent;
2905         /* struct iw_cm_event cm_event; */
2906         u32 aeq_info;
2907         u32 next_iwarp_state = 0;
2908         u16 async_event_id;
2909         u8 tcp_state;
2910         u8 iwarp_state;
2911
2912         nes_debug(NES_DBG_AEQ, "\n");
2913         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2914         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2915                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2916                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2917         } else {
2918                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2919                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2920                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2921                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2922                 BUG_ON(!context);
2923         }
2924
2925         async_event_id = (u16)aeq_info;
2926         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2927         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2928         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2929                         " Tcp state = %s, iWARP state = %s\n",
2930                         async_event_id,
2931                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2932                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2933
2934         switch (async_event_id) {
2935                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2936                         nesqp = *((struct nes_qp **)&context);
2937                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2938                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2939                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2940                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2941                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2942                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2943                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2944                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2945                                                 async_event_id, nesqp->last_aeq, tcp_state);
2946                         }
2947                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2948                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2949                                 /* FIN Received but tcp state or IB state moved on,
2950                                                 should expect a close complete */
2951                                 return;
2952                         }
2953                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2954                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2955                 case NES_AEQE_AEID_TERMINATE_SENT:
2956                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2957                 case NES_AEQE_AEID_RESET_SENT:
2958                         nesqp = *((struct nes_qp **)&context);
2959                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2960                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2961                         }
2962                         spin_lock_irqsave(&nesqp->lock, flags);
2963                         nesqp->hw_iwarp_state = iwarp_state;
2964                         nesqp->hw_tcp_state = tcp_state;
2965                         nesqp->last_aeq = async_event_id;
2966
2967                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2968                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2969                                 nesqp->hte_added = 0;
2970                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2971                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2972                                                 nesqp->hwqp.qp_id);
2973                                 nes_hw_modify_qp(nesdev, nesqp,
2974                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2975                                 spin_lock_irqsave(&nesqp->lock, flags);
2976                         }
2977
2978                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2979                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2980                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2981                                 switch (nesqp->hw_iwarp_state) {
2982                                         case NES_AEQE_IWARP_STATE_RTS:
2983                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2984                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2985                                                 break;
2986                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2987                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2988                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2989                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2990                                                         next_iwarp_state |= 0x02000000;
2991                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2992                                                 }
2993                                                 break;
2994                                         default:
2995                                                 next_iwarp_state = 0;
2996                                 }
2997                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2998                                 if (next_iwarp_state) {
2999                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3000                                                         " also added another reference\n",
3001                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3002                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3003                                 }
3004                                 nes_cm_disconn(nesqp);
3005                         } else {
3006                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3007                                         /* FIN Received but ib state not RTS,
3008                                                         close complete will be on its way */
3009                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3010                                         return;
3011                                 }
3012                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3013                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3014                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3015                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3016                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3017                                                         " also added another reference\n",
3018                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3019                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3020                                 }
3021                                 nes_cm_disconn(nesqp);
3022                         }
3023                         break;
3024                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3025                         nesqp = *((struct nes_qp **)&context);
3026                         spin_lock_irqsave(&nesqp->lock, flags);
3027                         nesqp->hw_iwarp_state = iwarp_state;
3028                         nesqp->hw_tcp_state = tcp_state;
3029                         nesqp->last_aeq = async_event_id;
3030                         spin_unlock_irqrestore(&nesqp->lock, flags);
3031                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3032                                         " event on QP%u \n  Q2 Data:\n",
3033                                         nesqp->hwqp.qp_id);
3034                         if (nesqp->ibqp.event_handler) {
3035                                 ibevent.device = nesqp->ibqp.device;
3036                                 ibevent.element.qp = &nesqp->ibqp;
3037                                 ibevent.event = IB_EVENT_QP_FATAL;
3038                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3039                         }
3040                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3041                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3042                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3043                                 nes_cm_disconn(nesqp);
3044                         } else {
3045                                 nesqp->in_disconnect = 0;
3046                                 wake_up(&nesqp->kick_waitq);
3047                         }
3048                         break;
3049                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3050                         nesqp = *((struct nes_qp **)&context);
3051                         spin_lock_irqsave(&nesqp->lock, flags);
3052                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3053                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3054                         nesqp->last_aeq = async_event_id;
3055                         if (nesqp->cm_id) {
3056                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3057                                                 " event on QP%u, remote IP = 0x%08X \n",
3058                                                 nesqp->hwqp.qp_id,
3059                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3060                         } else {
3061                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3062                                                 " event on QP%u \n",
3063                                                 nesqp->hwqp.qp_id);
3064                         }
3065                         spin_unlock_irqrestore(&nesqp->lock, flags);
3066                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3067                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3068                         if (nesqp->ibqp.event_handler) {
3069                                 ibevent.device = nesqp->ibqp.device;
3070                                 ibevent.element.qp = &nesqp->ibqp;
3071                                 ibevent.event = IB_EVENT_QP_FATAL;
3072                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3073                         }
3074                         break;
3075                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3076                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3077                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3078                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3079                         } else {
3080                                 /* TODO: get the actual WQE and mask off wqe index */
3081                                 context &= ~((u64)511);
3082                                 nesqp = *((struct nes_qp **)&context);
3083                         }
3084                         spin_lock_irqsave(&nesqp->lock, flags);
3085                         nesqp->hw_iwarp_state = iwarp_state;
3086                         nesqp->hw_tcp_state = tcp_state;
3087                         nesqp->last_aeq = async_event_id;
3088                         spin_unlock_irqrestore(&nesqp->lock, flags);
3089                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3090                                         nesqp->hwqp.qp_id);
3091                         if (nesqp->ibqp.event_handler) {
3092                                 ibevent.device = nesqp->ibqp.device;
3093                                 ibevent.element.qp = &nesqp->ibqp;
3094                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3095                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3096                         }
3097                         break;
3098                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3099                         nesqp = *((struct nes_qp **)&context);
3100                         spin_lock_irqsave(&nesqp->lock, flags);
3101                         nesqp->hw_iwarp_state = iwarp_state;
3102                         nesqp->hw_tcp_state = tcp_state;
3103                         nesqp->last_aeq = async_event_id;
3104                         spin_unlock_irqrestore(&nesqp->lock, flags);
3105                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3106                                         nesqp->hwqp.qp_id);
3107                         if (nesqp->ibqp.event_handler) {
3108                                 ibevent.device = nesqp->ibqp.device;
3109                                 ibevent.element.qp = &nesqp->ibqp;
3110                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3111                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3112                         }
3113                         break;
3114                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3115                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3116                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3117                         spin_lock_irqsave(&nesqp->lock, flags);
3118                         nesqp->hw_iwarp_state = iwarp_state;
3119                         nesqp->hw_tcp_state = tcp_state;
3120                         nesqp->last_aeq = async_event_id;
3121                         spin_unlock_irqrestore(&nesqp->lock, flags);
3122                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3123                                         " nesqp = %p, AE reported %p\n",
3124                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3125                         if (nesqp->ibqp.event_handler) {
3126                                 ibevent.device = nesqp->ibqp.device;
3127                                 ibevent.element.qp = &nesqp->ibqp;
3128                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3129                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3130                         }
3131                         break;
3132                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3133                         context <<= 1;
3134                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3135                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3136                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3137                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3138                         if (resource_allocated) {
3139                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3140                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3141                         }
3142                         break;
3143                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3144                         nesqp = nesadapter->qp_table[le32_to_cpu(
3145                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3146                         spin_lock_irqsave(&nesqp->lock, flags);
3147                         nesqp->hw_iwarp_state = iwarp_state;
3148                         nesqp->hw_tcp_state = tcp_state;
3149                         nesqp->last_aeq = async_event_id;
3150                         spin_unlock_irqrestore(&nesqp->lock, flags);
3151                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3152                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3153                                         nesqp->hwqp.qp_id);
3154                         if (nesqp->ibqp.event_handler) {
3155                                 ibevent.device = nesqp->ibqp.device;
3156                                 ibevent.element.qp = &nesqp->ibqp;
3157                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3158                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3159                         }
3160                         /* tell cm to disconnect, cm will queue work to thread */
3161                         nes_cm_disconn(nesqp);
3162                         break;
3163                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3164                         nesqp = *((struct nes_qp **)&context);
3165                         spin_lock_irqsave(&nesqp->lock, flags);
3166                         nesqp->hw_iwarp_state = iwarp_state;
3167                         nesqp->hw_tcp_state = tcp_state;
3168                         nesqp->last_aeq = async_event_id;
3169                         spin_unlock_irqrestore(&nesqp->lock, flags);
3170                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3171                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3172                                         nesqp->hwqp.qp_id);
3173                         if (nesqp->ibqp.event_handler) {
3174                                 ibevent.device = nesqp->ibqp.device;
3175                                 ibevent.element.qp = &nesqp->ibqp;
3176                                 ibevent.event = IB_EVENT_QP_FATAL;
3177                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3178                         }
3179                         /* tell cm to disconnect, cm will queue work to thread */
3180                         nes_cm_disconn(nesqp);
3181                         break;
3182                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3183                         nesqp = *((struct nes_qp **)&context);
3184                         spin_lock_irqsave(&nesqp->lock, flags);
3185                         nesqp->hw_iwarp_state = iwarp_state;
3186                         nesqp->hw_tcp_state = tcp_state;
3187                         nesqp->last_aeq = async_event_id;
3188                         spin_unlock_irqrestore(&nesqp->lock, flags);
3189                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3190                                         " event on QP%u \n  Q2 Data:\n",
3191                                         nesqp->hwqp.qp_id);
3192                         if (nesqp->ibqp.event_handler) {
3193                                 ibevent.device = nesqp->ibqp.device;
3194                                 ibevent.element.qp = &nesqp->ibqp;
3195                                 ibevent.event = IB_EVENT_QP_FATAL;
3196                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3197                         }
3198                         /* tell cm to disconnect, cm will queue work to thread */
3199                         nes_cm_disconn(nesqp);
3200                         break;
3201                         /* TODO: additional AEs need to be here */
3202                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3203                         nesqp = *((struct nes_qp **)&context);
3204                         spin_lock_irqsave(&nesqp->lock, flags);
3205                         nesqp->hw_iwarp_state = iwarp_state;
3206                         nesqp->hw_tcp_state = tcp_state;
3207                         nesqp->last_aeq = async_event_id;
3208                         spin_unlock_irqrestore(&nesqp->lock, flags);
3209                         if (nesqp->ibqp.event_handler) {
3210                                 ibevent.device = nesqp->ibqp.device;
3211                                 ibevent.element.qp = &nesqp->ibqp;
3212                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3213                                 nesqp->ibqp.event_handler(&ibevent,
3214                                                 nesqp->ibqp.qp_context);
3215                         }
3216                         nes_cm_disconn(nesqp);
3217                         break;
3218                 default:
3219                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3220                                         async_event_id);
3221                         break;
3222         }
3223
3224 }
3225
3226
3227 /**
3228  * nes_iwarp_ce_handler
3229  */
3230 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3231 {
3232         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3233
3234         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3235                         nescq->hw_cq.cq_number); */
3236         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3237
3238         if (nescq->ibcq.comp_handler)
3239                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3240
3241         return;
3242 }
3243
3244
3245 /**
3246  * nes_manage_apbvt()
3247  */
3248 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3249                 u32 nic_index, u32 add_port)
3250 {
3251         struct nes_device *nesdev = nesvnic->nesdev;
3252         struct nes_hw_cqp_wqe *cqp_wqe;
3253         struct nes_cqp_request *cqp_request;
3254         int ret = 0;
3255         u16 major_code;
3256
3257         /* Send manage APBVT request to CQP */
3258         cqp_request = nes_get_cqp_request(nesdev);
3259         if (cqp_request == NULL) {
3260                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3261                 return -ENOMEM;
3262         }
3263         cqp_request->waiting = 1;
3264         cqp_wqe = &cqp_request->cqp_wqe;
3265
3266         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3267                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3268                         accel_local_port, accel_local_port, nic_index);
3269
3270         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3271         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3272                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3273         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3274                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3275
3276         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3277
3278         atomic_set(&cqp_request->refcount, 2);
3279         nes_post_cqp_request(nesdev, cqp_request);
3280
3281         if (add_port == NES_MANAGE_APBVT_ADD)
3282                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3283                                 NES_EVENT_TIMEOUT);
3284         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3285                         ret, cqp_request->major_code, cqp_request->minor_code);
3286         major_code = cqp_request->major_code;
3287
3288         nes_put_cqp_request(nesdev, cqp_request);
3289
3290         if (!ret)
3291                 return -ETIME;
3292         else if (major_code)
3293                 return -EIO;
3294         else
3295                 return 0;
3296 }
3297
3298
3299 /**
3300  * nes_manage_arp_cache
3301  */
3302 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3303                 u32 ip_addr, u32 action)
3304 {
3305         struct nes_hw_cqp_wqe *cqp_wqe;
3306         struct nes_vnic *nesvnic = netdev_priv(netdev);
3307         struct nes_device *nesdev;
3308         struct nes_cqp_request *cqp_request;
3309         int arp_index;
3310
3311         nesdev = nesvnic->nesdev;
3312         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3313         if (arp_index == -1) {
3314                 return;
3315         }
3316
3317         /* update the ARP entry */
3318         cqp_request = nes_get_cqp_request(nesdev);
3319         if (cqp_request == NULL) {
3320                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3321                 return;
3322         }
3323         cqp_request->waiting = 0;
3324         cqp_wqe = &cqp_request->cqp_wqe;
3325         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3326
3327         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3328                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3329         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3330                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3331         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3332
3333         if (action == NES_ARP_ADD) {
3334                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3335                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3336                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3337                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3338                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3339                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3340         } else {
3341                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3342                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3343         }
3344
3345         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3346                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3347
3348         atomic_set(&cqp_request->refcount, 1);
3349         nes_post_cqp_request(nesdev, cqp_request);
3350 }
3351
3352
3353 /**
3354  * flush_wqes
3355  */
3356 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3357                 u32 which_wq, u32 wait_completion)
3358 {
3359         struct nes_cqp_request *cqp_request;
3360         struct nes_hw_cqp_wqe *cqp_wqe;
3361         int ret;
3362
3363         cqp_request = nes_get_cqp_request(nesdev);
3364         if (cqp_request == NULL) {
3365                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3366                 return;
3367         }
3368         if (wait_completion) {
3369                 cqp_request->waiting = 1;
3370                 atomic_set(&cqp_request->refcount, 2);
3371         } else {
3372                 cqp_request->waiting = 0;
3373         }
3374         cqp_wqe = &cqp_request->cqp_wqe;
3375         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3376
3377         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3378                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3379         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3380
3381         nes_post_cqp_request(nesdev, cqp_request);
3382
3383         if (wait_completion) {
3384                 /* Wait for CQP */
3385                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3386                                 NES_EVENT_TIMEOUT);
3387                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3388                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3389                                 ret, cqp_request->major_code, cqp_request->minor_code);
3390                 nes_put_cqp_request(nesdev, cqp_request);
3391         }
3392 }