git.oblomov.eu Git - linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static u32 crit_err_count;
  50 u32 int_mod_timer_init;
  51 u32 int_mod_cq_depth_256;
  52 u32 int_mod_cq_depth_128;
  53 u32 int_mod_cq_depth_32;
  54 u32 int_mod_cq_depth_24;
  55 u32 int_mod_cq_depth_16;
  56 u32 int_mod_cq_depth_4;
  57 u32 int_mod_cq_depth_1;
  58 static const u8 nes_max_critical_error_count = 100;
  59 #include "nes_cm.h"
  60
  61 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  62 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  63 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  64                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  65 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  66 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  67 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  68 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  69                                    struct nes_hw_aeqe *aeqe);
  70 static void process_critical_error(struct nes_device *nesdev);
  71 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  72 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  73
  74 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  75 static unsigned char *nes_iwarp_state_str[] = {
  76         "Non-Existant",
  77         "Idle",
  78         "RTS",
  79         "Closing",
  80         "RSVD1",
  81         "Terminate",
  82         "Error",
  83         "RSVD2",
  84 };
  85
  86 static unsigned char *nes_tcp_state_str[] = {
  87         "Non-Existant",
  88         "Closed",
  89         "Listen",
  90         "SYN Sent",
  91         "SYN Rcvd",
  92         "Established",
  93         "Close Wait",
  94         "FIN Wait 1",
  95         "Closing",
  96         "Last Ack",
  97         "FIN Wait 2",
  98         "Time Wait",
  99         "RSVD1",
 100         "RSVD2",
 101         "RSVD3",
 102         "RSVD4",
 103 };
 104 #endif
 105
 106
 107 /**
 108  * nes_nic_init_timer_defaults
 109  */
 110 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 111 {
 112         unsigned long flags;
 113         struct nes_adapter *nesadapter = nesdev->nesadapter;
 114         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 115
 116         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 117
 118         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 119         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 120         if (jumbomode) {
 121                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 122                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 123                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 124         } else {
 125                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 128         }
 129
 130         /* todo use netdev->mtu to set thresholds */
 131         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 132 }
 133
 134
 135 /**
 136  * nes_nic_init_timer
 137  */
 138 static void  nes_nic_init_timer(struct nes_device *nesdev)
 139 {
 140         unsigned long flags;
 141         struct nes_adapter *nesadapter = nesdev->nesadapter;
 142         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 143
 144         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 145
 146         if (shared_timer->timer_in_use_old == 0) {
 147                 nesdev->deepcq_count = 0;
 148                 shared_timer->timer_direction_upward = 0;
 149                 shared_timer->timer_direction_downward = 0;
 150                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 151                 shared_timer->timer_in_use_old = 0;
 152
 153         }
 154         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 155                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 156                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 157                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 158         }
 159         /* todo use netdev->mtu to set thresholds */
 160         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 161 }
 162
 163
 164 /**
 165  * nes_nic_tune_timer
 166  */
 167 static void nes_nic_tune_timer(struct nes_device *nesdev)
 168 {
 169         unsigned long flags;
 170         struct nes_adapter *nesadapter = nesdev->nesadapter;
 171         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 172         u16 cq_count = nesdev->currcq_count;
 173
 174         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 175
 176         if (shared_timer->cq_count_old <= cq_count)
 177                 shared_timer->cq_direction_downward = 0;
 178         else
 179                 shared_timer->cq_direction_downward++;
 180         shared_timer->cq_count_old = cq_count;
 181         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 182                 if (cq_count <= shared_timer->threshold_low &&
 183                     shared_timer->threshold_low > 4) {
 184                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 185                         shared_timer->cq_direction_downward=0;
 186                         nesdev->currcq_count = 0;
 187                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 188                         return;
 189                 }
 190         }
 191
 192         if (cq_count > 1) {
 193                 nesdev->deepcq_count += cq_count;
 194                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 195                         shared_timer->timer_direction_upward++;
 196                         shared_timer->timer_direction_downward = 0;
 197                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 198                         shared_timer->timer_direction_upward = 0;
 199                         shared_timer->timer_direction_downward = 0;
 200                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 201                         shared_timer->timer_direction_downward++;
 202                         shared_timer->timer_direction_upward = 0;
 203                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 204                         shared_timer->timer_in_use -= 2;
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward++;
 207                 } else {
 208                         shared_timer->timer_in_use -= 4;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 }
 212
 213                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 214                         shared_timer->timer_in_use += 3;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward = 0;
 217                 }
 218                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 219                         shared_timer->timer_in_use -= 4 ;
 220                         shared_timer->timer_direction_downward = 0;
 221                         shared_timer->timer_direction_upward = 0;
 222                 }
 223         }
 224
 225         /* boundary checking */
 226         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 227                 shared_timer->timer_in_use = shared_timer->threshold_high;
 228         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 229                 shared_timer->timer_in_use = shared_timer->threshold_low;
 230
 231         nesdev->currcq_count = 0;
 232
 233         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 234 }
 235
 236
 237 /**
 238  * nes_init_adapter - initialize adapter
 239  */
 240 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 241         struct nes_adapter *nesadapter = NULL;
 242         unsigned long num_pds;
 243         u32 u32temp;
 244         u32 port_count;
 245         u16 max_rq_wrs;
 246         u16 max_sq_wrs;
 247         u32 max_mr;
 248         u32 max_256pbl;
 249         u32 max_4kpbl;
 250         u32 max_qp;
 251         u32 max_irrq;
 252         u32 max_cq;
 253         u32 hte_index_mask;
 254         u32 adapter_size;
 255         u32 arp_table_size;
 256         u16 vendor_id;
 257         u8  OneG_Mode;
 258         u8  func_index;
 259
 260         /* search the list of existing adapters */
 261         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 262                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 263                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 264                                 nesdev->pcidev->devfn,
 265                                 PCI_SLOT(nesadapter->devfn),
 266                                 nesadapter->bus_number,
 267                                 PCI_SLOT(nesdev->pcidev->devfn),
 268                                 nesdev->pcidev->bus->number );
 269                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 270                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 271                         nesadapter->ref_count++;
 272                         return nesadapter;
 273                 }
 274         }
 275
 276         /* no adapter found */
 277         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 278         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 279                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 280                                 hw_rev);
 281                 return NULL;
 282         }
 283
 284         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 285                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 286                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 287                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 288                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 289
 290         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 291
 292
 293         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 294                 return NULL;
 295
 296         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 297         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 298
 299         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 300         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 301                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 302                                 max_qp, u32temp);
 303                 max_qp = (u32)1 << (u32temp & 0x001f);
 304         }
 305
 306         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 307         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 308                         max_qp, hte_index_mask);
 309
 310         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 311
 312         max_irrq = 1 << (u32temp & 0x001f);
 313
 314         if (max_qp > max_irrq) {
 315                 max_qp = max_irrq;
 316                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 317                                 max_qp);
 318         }
 319
 320         /* there should be no reason to allocate more pds than qps */
 321         if (num_pds > max_qp)
 322                 num_pds = max_qp;
 323
 324         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 325         max_mr = (u32)8192 << (u32temp & 0x7);
 326
 327         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 328         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 329         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 330         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 333         arp_table_size = 1 << u32temp;
 334
 335         adapter_size = (sizeof(struct nes_adapter) +
 336                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 337         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 341         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 342         adapter_size += sizeof(struct nes_qp **) * max_qp;
 343
 344         /* allocate a new adapter struct */
 345         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 346         if (nesadapter == NULL) {
 347                 return NULL;
 348         }
 349
 350         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 351                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 352
 353         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 354                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 355                 kfree(nesadapter);
 356                 return NULL;
 357         }
 358
 359         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 360                                                         OneG_Mode)) {
 361                 kfree(nesadapter);
 362                 return NULL;
 363         }
 364         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 365
 366         memset(nesadapter->pft_mcast_map, 255,
 367                sizeof nesadapter->pft_mcast_map);
 368
 369         /* populate the new nesadapter */
 370         nesadapter->devfn = nesdev->pcidev->devfn;
 371         nesadapter->bus_number = nesdev->pcidev->bus->number;
 372         nesadapter->ref_count = 1;
 373         nesadapter->timer_int_req = 0xffff0000;
 374         nesadapter->OneG_Mode = OneG_Mode;
 375         nesadapter->doorbell_start = nesdev->doorbell_region;
 376
 377         /* nesadapter->tick_delta = clk_divisor; */
 378         nesadapter->hw_rev = hw_rev;
 379         nesadapter->port_count = port_count;
 380
 381         nesadapter->max_qp = max_qp;
 382         nesadapter->hte_index_mask = hte_index_mask;
 383         nesadapter->max_irrq = max_irrq;
 384         nesadapter->max_mr = max_mr;
 385         nesadapter->max_256pbl = max_256pbl - 1;
 386         nesadapter->max_4kpbl = max_4kpbl - 1;
 387         nesadapter->max_cq = max_cq;
 388         nesadapter->free_256pbl = max_256pbl - 1;
 389         nesadapter->free_4kpbl = max_4kpbl - 1;
 390         nesadapter->max_pd = num_pds;
 391         nesadapter->arp_table_size = arp_table_size;
 392
 393         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 394         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 395                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 396                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 397                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 398         } else {
 399                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 400                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 401                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 402                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 403         }
 404         /* Setup and enable the periodic timer */
 405         if (nesadapter->et_rx_coalesce_usecs_irq)
 406                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 407                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 408         else
 409                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 410
 411         nesadapter->base_pd = 1;
 412
 413         nesadapter->device_cap_flags =
 414                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 415
 416         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 417                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 418         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 419         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 420         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 421         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 422         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 423
 424
 425         /* mark the usual suspect QPs and CQs as in use */
 426         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 427                 set_bit(u32temp, nesadapter->allocated_qps);
 428                 set_bit(u32temp, nesadapter->allocated_cqs);
 429         }
 430
 431         for (u32temp = 0; u32temp < 20; u32temp++)
 432                 set_bit(u32temp, nesadapter->allocated_pds);
 433         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 434
 435         max_rq_wrs = ((u32temp >> 8) & 3);
 436         switch (max_rq_wrs) {
 437                 case 0:
 438                         max_rq_wrs = 4;
 439                         break;
 440                 case 1:
 441                         max_rq_wrs = 16;
 442                         break;
 443                 case 2:
 444                         max_rq_wrs = 32;
 445                         break;
 446                 case 3:
 447                         max_rq_wrs = 512;
 448                         break;
 449         }
 450
 451         max_sq_wrs = (u32temp & 3);
 452         switch (max_sq_wrs) {
 453                 case 0:
 454                         max_sq_wrs = 4;
 455                         break;
 456                 case 1:
 457                         max_sq_wrs = 16;
 458                         break;
 459                 case 2:
 460                         max_sq_wrs = 32;
 461                         break;
 462                 case 3:
 463                         max_sq_wrs = 512;
 464                         break;
 465         }
 466         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 467         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 468
 469         nesadapter->max_sge = 4;
 470         nesadapter->max_cqe = 32767;
 471
 472         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 473                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 474                 kfree(nesadapter);
 475                 return NULL;
 476         }
 477
 478         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 479         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 480                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 481
 482         /* setup port configuration */
 483         if (nesadapter->port_count == 1) {
 484                 nesadapter->log_port = 0x00000000;
 485                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 486                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 487                 else
 488                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 489         } else {
 490                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 491                         nesadapter->log_port = 0x000000D8;
 492                 } else {
 493                         if (nesadapter->port_count == 2)
 494                                 nesadapter->log_port = 0x00000044;
 495                         else
 496                                 nesadapter->log_port = 0x000000e4;
 497                 }
 498                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 499         }
 500
 501         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 502                                                 nesadapter->log_port);
 503         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 504                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 505
 506         spin_lock_init(&nesadapter->resource_lock);
 507         spin_lock_init(&nesadapter->phy_lock);
 508         spin_lock_init(&nesadapter->pbl_lock);
 509         spin_lock_init(&nesadapter->periodic_timer_lock);
 510
 511         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 512         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 513         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 514         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 515
 516         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 517                 u32 pcs_control_status0, pcs_control_status1;
 518                 u32 reset_value;
 519                 u32 i = 0;
 520                 u32 int_cnt = 0;
 521                 u32 ext_cnt = 0;
 522                 unsigned long flags;
 523                 u32 j = 0;
 524
 525                 pcs_control_status0 = nes_read_indexed(nesdev,
 526                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 527                 pcs_control_status1 = nes_read_indexed(nesdev,
 528                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 529
 530                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 531                         pcs_control_status0 = nes_read_indexed(nesdev,
 532                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 533                         pcs_control_status1 = nes_read_indexed(nesdev,
 534                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 535                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 536                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 537                                 int_cnt++;
 538                         msleep(1);
 539                 }
 540                 if (int_cnt > 1) {
 541                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 542                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 543                         mh_detected++;
 544                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 545                         reset_value |= 0x0000003d;
 546                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 547
 548                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 549                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 550                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 551
 552                         pcs_control_status0 = nes_read_indexed(nesdev,
 553                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 554                         pcs_control_status1 = nes_read_indexed(nesdev,
 555                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 556
 557                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 558                                 pcs_control_status0 = nes_read_indexed(nesdev,
 559                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 560                                 pcs_control_status1 = nes_read_indexed(nesdev,
 561                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 562                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 563                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 564                                         if (++ext_cnt > int_cnt) {
 565                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 566                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 567                                                                 0x0000F0C8);
 568                                                 mh_detected++;
 569                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 570                                                 reset_value |= 0x0000003d;
 571                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 572
 573                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 574                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 575                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 576                                                 break;
 577                                         }
 578                                 }
 579                                 msleep(1);
 580                         }
 581                 }
 582         }
 583
 584         if (nesadapter->hw_rev == NE020_REV) {
 585                 init_timer(&nesadapter->mh_timer);
 586                 nesadapter->mh_timer.function = nes_mh_fix;
 587                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 588                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 589                 add_timer(&nesadapter->mh_timer);
 590         } else {
 591                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 592         }
 593
 594         init_timer(&nesadapter->lc_timer);
 595         nesadapter->lc_timer.function = nes_clc;
 596         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 597         nesadapter->lc_timer.data = (unsigned long)nesdev;
 598         add_timer(&nesadapter->lc_timer);
 599
 600         list_add_tail(&nesadapter->list, &nes_adapter_list);
 601
 602         for (func_index = 0; func_index < 8; func_index++) {
 603                 pci_bus_read_config_word(nesdev->pcidev->bus,
 604                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 605                                         func_index), 0, &vendor_id);
 606                 if (vendor_id == 0xffff)
 607                         break;
 608         }
 609         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 610                 func_index, pci_name(nesdev->pcidev));
 611         nesadapter->adapter_fcn_count = func_index;
 612
 613         return nesadapter;
 614 }
 615
 616
 617 /**
 618  * nes_reset_adapter_ne020
 619  */
 620 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 621 {
 622         u32 port_count;
 623         u32 u32temp;
 624         u32 i;
 625
 626         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 627         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 628         /* TODO: assuming that both SERDES are set the same for now */
 629         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 630         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 631                         u32temp, port_count);
 632         if (*OneG_Mode)
 633                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 634         u32temp &= 0xff00ffc0;
 635         switch (port_count) {
 636                 case 1:
 637                         u32temp |= 0x00ee0000;
 638                         break;
 639                 case 2:
 640                         u32temp |= 0x00cc0000;
 641                         break;
 642                 case 4:
 643                         u32temp |= 0x00000000;
 644                         break;
 645                 default:
 646                         return 0;
 647                         break;
 648         }
 649
 650         /* check and do full reset if needed */
 651         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 652                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 653                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 654
 655                 i = 0;
 656                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 657                         mdelay(1);
 658                 if (i >= 10000) {
 659                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 660                         return 0;
 661                 }
 662
 663                 i = 0;
 664                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 665                         mdelay(1);
 666                 if (i >= 10000) {
 667                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 668                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 669                         return 0;
 670                 }
 671         }
 672
 673         /* port reset */
 674         switch (port_count) {
 675                 case 1:
 676                         u32temp |= 0x00ee0010;
 677                         break;
 678                 case 2:
 679                         u32temp |= 0x00cc0030;
 680                         break;
 681                 case 4:
 682                         u32temp |= 0x00000030;
 683                         break;
 684         }
 685
 686         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 687         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 688
 689         i = 0;
 690         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 691                 mdelay(1);
 692         if (i >= 10000) {
 693                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 694                 return 0;
 695         }
 696
 697         /* serdes 0 */
 698         i = 0;
 699         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 700                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 701                 mdelay(1);
 702         if (i >= 5000) {
 703                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 704                 return 0;
 705         }
 706
 707         /* serdes 1 */
 708         if (port_count > 1) {
 709                 i = 0;
 710                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 711                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 712                         mdelay(1);
 713                 if (i >= 5000) {
 714                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 715                         return 0;
 716                 }
 717         }
 718
 719         return port_count;
 720 }
 721
 722
 723 /**
 724  * nes_init_serdes
 725  */
 726 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 727                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 728 {
 729         int i;
 730         u32 u32temp;
 731         u32 serdes_common_control;
 732
 733         if (hw_rev != NE020_REV) {
 734                 /* init serdes 0 */
 735
 736                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 737                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 738                         serdes_common_control = nes_read_indexed(nesdev,
 739                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 740                         serdes_common_control |= 0x000000100;
 741                         nes_write_indexed(nesdev,
 742                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0,
 743                                         serdes_common_control);
 744                 } else if (!OneG_Mode) {
 745                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 746                 }
 747                 if (((port_count > 1) &&
 748                         (nesadapter->phy_type[0] != NES_PHY_TYPE_PUMA_1G)) ||
 749                         ((port_count > 2) &&
 750                         (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G))) {
 751                         /* init serdes 1 */
 752                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 753                         if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 754                                 serdes_common_control = nes_read_indexed(nesdev,
 755                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 756                                 serdes_common_control |= 0x000000100;
 757                                 nes_write_indexed(nesdev,
 758                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 759                                         serdes_common_control);
 760                         } else if (!OneG_Mode) {
 761                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 762                         }
 763                 }
 764         } else {
 765                 /* init serdes 0 */
 766                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 767                 i = 0;
 768                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 769                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 770                         mdelay(1);
 771                 if (i >= 5000) {
 772                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 773                         return 1;
 774                 }
 775                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 776                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 777                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 778                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 779                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 780                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 781                 if (OneG_Mode)
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 783                 else
 784                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 785
 786                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 787                 if (port_count > 1) {
 788                         /* init serdes 1 */
 789                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 790                         i = 0;
 791                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 792                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 793                                 mdelay(1);
 794                         if (i >= 5000) {
 795                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 796                                 /* return 1; */
 797                         }
 798                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 799                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 800                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 801                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 803                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 804                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 805                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 806                 }
 807         }
 808         return 0;
 809 }
 810
 811
 812 /**
 813  * nes_init_csr_ne020
 814  * Initialize registers for ne020 hardware
 815  */
 816 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 817 {
 818         u32 u32temp;
 819
 820         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 821
 822         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 823         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 824         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 825         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 826         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 827         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 828         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 829         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 830
 831         /* TODO: move these MAC register settings to NIC bringup */
 832         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 833         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 834         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 835         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 836         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 837         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 838         if (port_count > 1) {
 839                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 840                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 841                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 842                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 843                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 844                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 845                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 846         }
 847         if (port_count > 2) {
 848                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 849                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 850                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 851                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 852                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 853                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 854                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 855
 856                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 857                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 858                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 859                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 860                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 861                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 862                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 863         }
 864
 865         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 866         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 867         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 868                                                          0x00000001);
 869         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 870         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 871         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 872         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 873         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 874
 875         /* TODO: move this to code, get from EEPROM */
 876         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 877         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 878         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 879
 880         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 881         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 882
 883         if (hw_rev != NE020_REV) {
 884                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 885                 u32temp |= 0x80000000;
 886                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 887                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 888                 u32temp &= 0x7fffffff;
 889                 u32temp |= 0x7fff0010;
 890                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 891         }
 892 }
 893
 894
 895 /**
 896  * nes_destroy_adapter - destroy the adapter structure
 897  */
 898 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 899 {
 900         struct nes_adapter *tmp_adapter;
 901
 902         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 903                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 904                                 tmp_adapter);
 905         }
 906
 907         nesadapter->ref_count--;
 908         if (!nesadapter->ref_count) {
 909                 if (nesadapter->hw_rev == NE020_REV) {
 910                         del_timer(&nesadapter->mh_timer);
 911                 }
 912                 del_timer(&nesadapter->lc_timer);
 913
 914                 list_del(&nesadapter->list);
 915                 kfree(nesadapter);
 916         }
 917 }
 918
 919
 920 /**
 921  * nes_init_cqp
 922  */
 923 int nes_init_cqp(struct nes_device *nesdev)
 924 {
 925         struct nes_adapter *nesadapter = nesdev->nesadapter;
 926         struct nes_hw_cqp_qp_context *cqp_qp_context;
 927         struct nes_hw_cqp_wqe *cqp_wqe;
 928         struct nes_hw_ceq *ceq;
 929         struct nes_hw_ceq *nic_ceq;
 930         struct nes_hw_aeq *aeq;
 931         void *vmem;
 932         dma_addr_t pmem;
 933         u32 count=0;
 934         u32 cqp_head;
 935         u64 u64temp;
 936         u32 u32temp;
 937
 938         /* allocate CQP memory */
 939         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 940         /* SQ is 512 byte aligned, others are 256 byte aligned */
 941         nesdev->cqp_mem_size = 512 +
 942                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 943                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 944                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 945                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 946                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 947                         sizeof(struct nes_hw_cqp_qp_context);
 948
 949         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 950                         &nesdev->cqp_pbase);
 951         if (!nesdev->cqp_vbase) {
 952                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 953                 return -ENOMEM;
 954         }
 955         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 956
 957         /* Allocate a twice the number of CQP requests as the SQ size */
 958         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 959                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 960         if (nesdev->nes_cqp_requests == NULL) {
 961                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 962                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 963                                 nesdev->cqp.sq_pbase);
 964                 return -ENOMEM;
 965         }
 966
 967         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 968                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 969
 970         spin_lock_init(&nesdev->cqp.lock);
 971         init_waitqueue_head(&nesdev->cqp.waitq);
 972
 973         /* Setup Various Structures */
 974         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 975                         ~(unsigned long)(512 - 1));
 976         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 977                         ~(unsigned long long)(512 - 1));
 978
 979         nesdev->cqp.sq_vbase = vmem;
 980         nesdev->cqp.sq_pbase = pmem;
 981         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 982         nesdev->cqp.sq_head = 0;
 983         nesdev->cqp.sq_tail = 0;
 984         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 985
 986         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 987         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 988
 989         nesdev->ccq.cq_vbase = vmem;
 990         nesdev->ccq.cq_pbase = pmem;
 991         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 992         nesdev->ccq.cq_head = 0;
 993         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 994         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 995
 996         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 997         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 998
 999         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1000         ceq = &nesadapter->ceq[nesdev->ceq_index];
1001         ceq->ceq_vbase = vmem;
1002         ceq->ceq_pbase = pmem;
1003         ceq->ceq_size = NES_CCEQ_SIZE;
1004         ceq->ceq_head = 0;
1005
1006         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1007         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1008
1009         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1010         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1011         nic_ceq->ceq_vbase = vmem;
1012         nic_ceq->ceq_pbase = pmem;
1013         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1014         nic_ceq->ceq_head = 0;
1015
1016         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1017         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1018
1019         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1020         aeq->aeq_vbase = vmem;
1021         aeq->aeq_pbase = pmem;
1022         aeq->aeq_size = nesadapter->max_qp;
1023         aeq->aeq_head = 0;
1024
1025         /* Setup QP Context */
1026         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1027         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1028
1029         cqp_qp_context = vmem;
1030         cqp_qp_context->context_words[0] =
1031                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1032         cqp_qp_context->context_words[1] = 0;
1033         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1034         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1035
1036
1037         /* Write the address to Create CQP */
1038         if ((sizeof(dma_addr_t) > 4)) {
1039                 nes_write_indexed(nesdev,
1040                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1041                                 ((u64)pmem) >> 32);
1042         } else {
1043                 nes_write_indexed(nesdev,
1044                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1045         }
1046         nes_write_indexed(nesdev,
1047                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1048                         (u32)pmem);
1049
1050         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1051         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1052
1053         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1054                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1055                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1056         }
1057
1058         /* Write Create CCQ WQE */
1059         cqp_head = nesdev->cqp.sq_head++;
1060         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1061         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1062         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1063                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1064                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1065         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1066                             (nesdev->ccq.cq_number |
1067                              ((u32)nesdev->ceq_index << 16)));
1068         u64temp = (u64)nesdev->ccq.cq_pbase;
1069         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1070         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1071         u64temp = (unsigned long)&nesdev->ccq;
1072         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1073                         cpu_to_le32((u32)(u64temp >> 1));
1074         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1075                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1076         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1077
1078         /* Write Create CEQ WQE */
1079         cqp_head = nesdev->cqp.sq_head++;
1080         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1081         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1082         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1083                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1084         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1085         u64temp = (u64)ceq->ceq_pbase;
1086         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1087
1088         /* Write Create AEQ WQE */
1089         cqp_head = nesdev->cqp.sq_head++;
1090         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1091         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1092         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1093                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1094         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1095         u64temp = (u64)aeq->aeq_pbase;
1096         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1097
1098         /* Write Create NIC CEQ WQE */
1099         cqp_head = nesdev->cqp.sq_head++;
1100         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1101         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1102         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1103                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1104         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1105         u64temp = (u64)nic_ceq->ceq_pbase;
1106         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1107
1108         /* Poll until CCQP done */
1109         count = 0;
1110         do {
1111                 if (count++ > 1000) {
1112                         printk(KERN_ERR PFX "Error creating CQP\n");
1113                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1114                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1115                         return -1;
1116                 }
1117                 udelay(10);
1118         } while (!(nes_read_indexed(nesdev,
1119                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1120
1121         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1122                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1123
1124         u32temp = 0x04800000;
1125         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1126
1127         /* wait for the CCQ, CEQ, and AEQ to get created */
1128         count = 0;
1129         do {
1130                 if (count++ > 1000) {
1131                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1132                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1133                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1134                         return -1;
1135                 }
1136                 udelay(10);
1137         } while (((nes_read_indexed(nesdev,
1138                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1139
1140         /* dump the QP status value */
1141         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1142                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1143
1144         nesdev->cqp.sq_tail++;
1145
1146         return 0;
1147 }
1148
1149
1150 /**
1151  * nes_destroy_cqp
1152  */
1153 int nes_destroy_cqp(struct nes_device *nesdev)
1154 {
1155         struct nes_hw_cqp_wqe *cqp_wqe;
1156         u32 count = 0;
1157         u32 cqp_head;
1158         unsigned long flags;
1159
1160         do {
1161                 if (count++ > 1000)
1162                         break;
1163                 udelay(10);
1164         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1165
1166         /* Reset CCQ */
1167         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1168                         nesdev->ccq.cq_number);
1169
1170         /* Disable device interrupts */
1171         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1172
1173         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1174
1175         /* Destroy the AEQ */
1176         cqp_head = nesdev->cqp.sq_head++;
1177         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1178         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1179         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1180                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1181         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1182
1183         /* Destroy the NIC CEQ */
1184         cqp_head = nesdev->cqp.sq_head++;
1185         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1186         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1187         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1188                         ((u32)nesdev->nic_ceq_index << 8));
1189
1190         /* Destroy the CEQ */
1191         cqp_head = nesdev->cqp.sq_head++;
1192         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1193         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1194         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1195                         (nesdev->ceq_index << 8));
1196
1197         /* Destroy the CCQ */
1198         cqp_head = nesdev->cqp.sq_head++;
1199         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1200         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1201         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1202         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1203                         ((u32)nesdev->ceq_index << 16));
1204
1205         /* Destroy CQP */
1206         cqp_head = nesdev->cqp.sq_head++;
1207         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1208         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1209         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1210                         NES_CQP_QP_TYPE_CQP);
1211         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1212
1213         barrier();
1214         /* Ring doorbell (5 WQEs) */
1215         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1216
1217         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1218
1219         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1220         count = 0;
1221         do {
1222                 if (count++ > 1000) {
1223                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1224                                         PCI_FUNC(nesdev->pcidev->devfn));
1225                         break;
1226                 }
1227                 udelay(10);
1228         } while (((nes_read_indexed(nesdev,
1229                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1230
1231         /* dump the QP status value */
1232         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1233                         PCI_FUNC(nesdev->pcidev->devfn),
1234                         nes_read_indexed(nesdev,
1235                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1236
1237         kfree(nesdev->nes_cqp_requests);
1238
1239         /* Free the control structures */
1240         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1241                         nesdev->cqp.sq_pbase);
1242
1243         return 0;
1244 }
1245
1246
1247 /**
1248  * nes_init_phy
1249  */
1250 int nes_init_phy(struct nes_device *nesdev)
1251 {
1252         struct nes_adapter *nesadapter = nesdev->nesadapter;
1253         u32 counter = 0;
1254         u32 sds_common_control0;
1255         u32 mac_index = nesdev->mac_index;
1256         u32 tx_config = 0;
1257         u16 phy_data;
1258         u32 temp_phy_data = 0;
1259         u32 temp_phy_data2 = 0;
1260         u32 i = 0;
1261
1262         if ((nesadapter->OneG_Mode) &&
1263             (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
1264                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1265                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1266                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1267                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1268                         tx_config &= 0xFFFFFFE3;
1269                         tx_config |= 0x04;
1270                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1271                 }
1272
1273                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1274                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1275                                 nesadapter->phy_index[mac_index], phy_data);
1276                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index], 0xb000);
1277
1278                 /* Reset the PHY */
1279                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1280                 udelay(100);
1281                 counter = 0;
1282                 do {
1283                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1284                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1285                         if (counter++ > 100) break;
1286                 } while (phy_data & 0x8000);
1287
1288                 /* Setting no phy loopback */
1289                 phy_data &= 0xbfff;
1290                 phy_data |= 0x1140;
1291                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1292                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1293                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1294
1295                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1296                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1297
1298                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1299                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1300
1301                 /* Setting the interrupt mask */
1302                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1303                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1304                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1305
1306                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1307                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1308
1309                 /* turning on flow control */
1310                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1311                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1312                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1313                                 (phy_data & ~(0x03E0)) | 0xc00);
1314                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1315                                 phy_data | 0xc00); */
1316                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1317                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1318
1319                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1320                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1321                 /* Clear Half duplex */
1322                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1323                                 phy_data & ~(0x0100));
1324                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1325                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1326
1327                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1328                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1329         } else {
1330                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
1331                     (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1332                         /* setup 10G MDIO operation */
1333                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1334                         tx_config &= 0xFFFFFFE3;
1335                         tx_config |= 0x15;
1336                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1337                 }
1338                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1339                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1340
1341                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1342                         mdelay(10);
1343                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1344                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1345
1346                         /*
1347                          * if firmware is already running (like from a
1348                          * driver un-load/load, don't do anything.
1349                          */
1350                         if (temp_phy_data == temp_phy_data2) {
1351                                 /* configure QT2505 AMCC PHY */
1352                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
1353                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
1354                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
1355                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
1356                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
1357                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
1358                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
1359                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0001);
1360                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
1361
1362                                 /*
1363                                  * remove micro from reset; chip boots from ROM,
1364                                  * uploads EEPROM f/w image, uC executes f/w
1365                                  */
1366                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
1367
1368                                 /*
1369                                  * wait for heart beat to start to
1370                                  * know loading is done
1371                                  */
1372                                 counter = 0;
1373                                 do {
1374                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1375                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1376                                         if (counter++ > 1000) {
1377                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
1378                                                 break;
1379                                         }
1380                                         mdelay(100);
1381                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1382                                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1383                                 } while ((temp_phy_data2 == temp_phy_data));
1384
1385                                 /*
1386                                  * wait for tracking to start to know
1387                                  * f/w is good to go
1388                                  */
1389                                 counter = 0;
1390                                 do {
1391                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
1392                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1393                                         if (counter++ > 1000) {
1394                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
1395                                                 break;
1396                                         }
1397                                         mdelay(1000);
1398                                         /*
1399                                          * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
1400                                          *                      temp_phy_data);
1401                                          */
1402                                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1403
1404                                 /* set LOS Control invert RXLOSB_I_PADINV */
1405                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
1406                                 /* set LOS Control to mask of RXLOSB_I */
1407                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
1408                                 /* set LED1 to input mode (LED1 and LED2 share same LED) */
1409                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
1410                                 /* set LED2 to RX link_status and activity */
1411                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
1412                                 /* set LED3 to RX link_status */
1413                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
1414
1415                                 /*
1416                                  * reset the res-calibration on t2
1417                                  * serdes; ensures it is stable after
1418                                  * the amcc phy is stable
1419                                  */
1420
1421                                 sds_common_control0  = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
1422                                 sds_common_control0 |= 0x1;
1423                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1424
1425                                 /* release the res-calibration reset */
1426                                 sds_common_control0 &= 0xfffffffe;
1427                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1428
1429                                 i = 0;
1430                                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1431                                                 && (i++ < 5000)) {
1432                                         /* mdelay(1); */
1433                                 }
1434
1435                                 /*
1436                                  * wait for link train done before moving on,
1437                                  * or will get an interupt storm
1438                                  */
1439                                 counter = 0;
1440                                 do {
1441                                         temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
1442                                                                 (0x200 * (nesdev->mac_index & 1)));
1443                                         if (counter++ > 1000) {
1444                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
1445                                                 break;
1446                                         }
1447                                         mdelay(1);
1448                                 } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
1449                         }
1450                 }
1451         }
1452         return 0;
1453 }
1454
1455
1456 /**
1457  * nes_replenish_nic_rq
1458  */
1459 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1460 {
1461         unsigned long flags;
1462         dma_addr_t bus_address;
1463         struct sk_buff *skb;
1464         struct nes_hw_nic_rq_wqe *nic_rqe;
1465         struct nes_hw_nic *nesnic;
1466         struct nes_device *nesdev;
1467         u32 rx_wqes_posted = 0;
1468
1469         nesnic = &nesvnic->nic;
1470         nesdev = nesvnic->nesdev;
1471         spin_lock_irqsave(&nesnic->rq_lock, flags);
1472         if (nesnic->replenishing_rq !=0) {
1473                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1474                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1475                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1476                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1477                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1478                         add_timer(&nesvnic->rq_wqes_timer);
1479                 } else
1480                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1481                 return;
1482         }
1483         nesnic->replenishing_rq = 1;
1484         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1485         do {
1486                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1487                 if (skb) {
1488                         skb->dev = nesvnic->netdev;
1489
1490                         bus_address = pci_map_single(nesdev->pcidev,
1491                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1492
1493                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1494                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1495                                         cpu_to_le32(nesvnic->max_frame_size);
1496                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1497                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1498                                         cpu_to_le32((u32)bus_address);
1499                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1500                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1501                         nesnic->rx_skb[nesnic->rq_head] = skb;
1502                         nesnic->rq_head++;
1503                         nesnic->rq_head &= nesnic->rq_size - 1;
1504                         atomic_dec(&nesvnic->rx_skbs_needed);
1505                         barrier();
1506                         if (++rx_wqes_posted == 255) {
1507                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1508                                 rx_wqes_posted = 0;
1509                         }
1510                 } else {
1511                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1512                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1513                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1514                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1515                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1516                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1517                                 add_timer(&nesvnic->rq_wqes_timer);
1518                         } else
1519                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1520                         break;
1521                 }
1522         } while (atomic_read(&nesvnic->rx_skbs_needed));
1523         barrier();
1524         if (rx_wqes_posted)
1525                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1526         nesnic->replenishing_rq = 0;
1527 }
1528
1529
1530 /**
1531  * nes_rq_wqes_timeout
1532  */
1533 static void nes_rq_wqes_timeout(unsigned long parm)
1534 {
1535         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1536         printk("%s: Timer fired.\n", __func__);
1537         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1538         if (atomic_read(&nesvnic->rx_skbs_needed))
1539                 nes_replenish_nic_rq(nesvnic);
1540 }
1541
1542
1543 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1544                                void **tcph, u64 *hdr_flags, void *priv)
1545 {
1546         unsigned int ip_len;
1547         struct iphdr *iph;
1548         skb_reset_network_header(skb);
1549         iph = ip_hdr(skb);
1550         if (iph->protocol != IPPROTO_TCP)
1551                 return -1;
1552         ip_len = ip_hdrlen(skb);
1553         skb_set_transport_header(skb, ip_len);
1554         *tcph = tcp_hdr(skb);
1555
1556         *hdr_flags = LRO_IPV4 | LRO_TCP;
1557         *iphdr = iph;
1558         return 0;
1559 }
1560
1561
1562 /**
1563  * nes_init_nic_qp
1564  */
1565 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1566 {
1567         struct nes_hw_cqp_wqe *cqp_wqe;
1568         struct nes_hw_nic_sq_wqe *nic_sqe;
1569         struct nes_hw_nic_qp_context *nic_context;
1570         struct sk_buff *skb;
1571         struct nes_hw_nic_rq_wqe *nic_rqe;
1572         struct nes_vnic *nesvnic = netdev_priv(netdev);
1573         unsigned long flags;
1574         void *vmem;
1575         dma_addr_t pmem;
1576         u64 u64temp;
1577         int ret;
1578         u32 cqp_head;
1579         u32 counter;
1580         u32 wqe_count;
1581         u8 jumbomode=0;
1582
1583         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1584         nesvnic->nic_mem_size = 256 +
1585                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1586                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1587                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1588                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1589                         sizeof(struct nes_hw_nic_qp_context);
1590
1591         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1592                         &nesvnic->nic_pbase);
1593         if (!nesvnic->nic_vbase) {
1594                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1595                 return -ENOMEM;
1596         }
1597         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1598         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1599                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1600
1601         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1602                         ~(unsigned long)(256 - 1));
1603         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1604                         ~(unsigned long long)(256 - 1));
1605
1606         /* Setup the first Fragment buffers */
1607         nesvnic->nic.first_frag_vbase = vmem;
1608
1609         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1610                 nesvnic->nic.frag_paddr[counter] = pmem;
1611                 pmem += sizeof(struct nes_first_frag);
1612         }
1613
1614         /* setup the SQ */
1615         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1616
1617         nesvnic->nic.sq_vbase = (void *)vmem;
1618         nesvnic->nic.sq_pbase = pmem;
1619         nesvnic->nic.sq_head = 0;
1620         nesvnic->nic.sq_tail = 0;
1621         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1622         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1623                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1624                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1625                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1626                                 NES_NIC_SQ_WQE_COMPLETION);
1627                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1628                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1629                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1630                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1631                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1632                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1633         }
1634
1635         nesvnic->get_cqp_request = nes_get_cqp_request;
1636         nesvnic->post_cqp_request = nes_post_cqp_request;
1637         nesvnic->mcrq_mcast_filter = NULL;
1638
1639         spin_lock_init(&nesvnic->nic.sq_lock);
1640         spin_lock_init(&nesvnic->nic.rq_lock);
1641
1642         /* setup the RQ */
1643         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1644         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1645
1646
1647         nesvnic->nic.rq_vbase = vmem;
1648         nesvnic->nic.rq_pbase = pmem;
1649         nesvnic->nic.rq_head = 0;
1650         nesvnic->nic.rq_tail = 0;
1651         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1652
1653         /* setup the CQ */
1654         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1655         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1656
1657         if (nesdev->nesadapter->netdev_count > 2)
1658                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1659         else
1660                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1661
1662         nesvnic->nic_cq.cq_vbase = vmem;
1663         nesvnic->nic_cq.cq_pbase = pmem;
1664         nesvnic->nic_cq.cq_head = 0;
1665         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1666
1667         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1668
1669         /* Send CreateCQ request to CQP */
1670         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1671         cqp_head = nesdev->cqp.sq_head;
1672
1673         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1674         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1675
1676         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1677                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1678                         ((u32)nesvnic->nic_cq.cq_size << 16));
1679         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1680                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1681         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1682         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1683         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1684         u64temp = (unsigned long)&nesvnic->nic_cq;
1685         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1686         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1687                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1688         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1689         if (++cqp_head >= nesdev->cqp.sq_size)
1690                 cqp_head = 0;
1691         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1692         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1693
1694         /* Send CreateQP request to CQP */
1695         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1696         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1697                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1698                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1699         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1700                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1701                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1702         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1703                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1704         }
1705
1706         u64temp = (u64)nesvnic->nic.sq_pbase;
1707         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1708         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1709         u64temp = (u64)nesvnic->nic.rq_pbase;
1710         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1711         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1712
1713         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1714                         NES_CQP_QP_TYPE_NIC);
1715         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1716         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1717                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1718         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1719
1720         if (++cqp_head >= nesdev->cqp.sq_size)
1721                 cqp_head = 0;
1722         nesdev->cqp.sq_head = cqp_head;
1723
1724         barrier();
1725
1726         /* Ring doorbell (2 WQEs) */
1727         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1728
1729         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1730         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1731                         nesvnic->nic.qp_id);
1732
1733         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1734                         NES_EVENT_TIMEOUT);
1735         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1736                         nesvnic->nic.qp_id, ret);
1737         if (!ret) {
1738                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1739                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1740                                 nesvnic->nic_pbase);
1741                 return -EIO;
1742         }
1743
1744         /* Populate the RQ */
1745         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1746                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1747                 if (!skb) {
1748                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1749
1750                         nes_destroy_nic_qp(nesvnic);
1751                         return -ENOMEM;
1752                 }
1753
1754                 skb->dev = netdev;
1755
1756                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1757                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1758
1759                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1760                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1761                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1762                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1763                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1764                 nesvnic->nic.rx_skb[counter] = skb;
1765         }
1766
1767         wqe_count = NES_NIC_WQ_SIZE - 1;
1768         nesvnic->nic.rq_head = wqe_count;
1769         barrier();
1770         do {
1771                 counter = min(wqe_count, ((u32)255));
1772                 wqe_count -= counter;
1773                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1774         } while (wqe_count);
1775         init_timer(&nesvnic->rq_wqes_timer);
1776         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1777         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1778         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1779         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1780         {
1781                 nes_nic_init_timer(nesdev);
1782                 if (netdev->mtu > 1500)
1783                         jumbomode = 1;
1784                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1785         }
1786         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1787         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1788         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1789         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1790         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1791         nesvnic->lro_mgr.dev            = netdev;
1792         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1793         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1794         return 0;
1795 }
1796
1797
1798 /**
1799  * nes_destroy_nic_qp
1800  */
1801 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1802 {
1803         u64 u64temp;
1804         dma_addr_t bus_address;
1805         struct nes_device *nesdev = nesvnic->nesdev;
1806         struct nes_hw_cqp_wqe *cqp_wqe;
1807         struct nes_hw_nic_sq_wqe *nic_sqe;
1808         struct nes_hw_nic_rq_wqe *nic_rqe;
1809         __le16 *wqe_fragment_length;
1810         u16  wqe_fragment_index;
1811         u64 wqe_frag;
1812         u32 cqp_head;
1813         unsigned long flags;
1814         int ret;
1815
1816         /* Free remaining NIC receive buffers */
1817         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1818                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1819                 wqe_frag  = (u64)le32_to_cpu(
1820                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1821                 wqe_frag |= ((u64)le32_to_cpu(
1822                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1823                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1824                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1825                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1826                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1827         }
1828
1829         /* Free remaining NIC transmit buffers */
1830         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1831                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1832                 wqe_fragment_index = 1;
1833                 wqe_fragment_length = (__le16 *)
1834                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1835                 /* bump past the vlan tag */
1836                 wqe_fragment_length++;
1837                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1838                         u64temp = (u64)le32_to_cpu(
1839                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1840                                 wqe_fragment_index*2]);
1841                         u64temp += ((u64)le32_to_cpu(
1842                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1843                                 + wqe_fragment_index*2]))<<32;
1844                         bus_address = (dma_addr_t)u64temp;
1845                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1846                                         nesvnic->nic.first_frag_overflow)) {
1847                                 pci_unmap_single(nesdev->pcidev,
1848                                                 bus_address,
1849                                                 le16_to_cpu(wqe_fragment_length[
1850                                                         wqe_fragment_index++]),
1851                                                 PCI_DMA_TODEVICE);
1852                         }
1853                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1854                                 if (wqe_fragment_length[wqe_fragment_index]) {
1855                                         u64temp = le32_to_cpu(
1856                                                 nic_sqe->wqe_words[
1857                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1858                                                 wqe_fragment_index*2]);
1859                                         u64temp += ((u64)le32_to_cpu(
1860                                                 nic_sqe->wqe_words[
1861                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1862                                                 wqe_fragment_index*2]))<<32;
1863                                         bus_address = (dma_addr_t)u64temp;
1864                                         pci_unmap_page(nesdev->pcidev,
1865                                                         bus_address,
1866                                                         le16_to_cpu(
1867                                                         wqe_fragment_length[
1868                                                         wqe_fragment_index]),
1869                                                         PCI_DMA_TODEVICE);
1870                                 } else
1871                                         break;
1872                         }
1873                 }
1874                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1875                         dev_kfree_skb(
1876                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1877
1878                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1879                                         & (nesvnic->nic.sq_size - 1);
1880         }
1881
1882         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1883
1884         /* Destroy NIC QP */
1885         cqp_head = nesdev->cqp.sq_head;
1886         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1887         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1888
1889         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1890                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1891         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1892                 nesvnic->nic.qp_id);
1893
1894         if (++cqp_head >= nesdev->cqp.sq_size)
1895                 cqp_head = 0;
1896
1897         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1898
1899         /* Destroy NIC CQ */
1900         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1901         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1902                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1903         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1904                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1905
1906         if (++cqp_head >= nesdev->cqp.sq_size)
1907                 cqp_head = 0;
1908
1909         nesdev->cqp.sq_head = cqp_head;
1910         barrier();
1911
1912         /* Ring doorbell (2 WQEs) */
1913         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1914
1915         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1916         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1917                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1918                         cqp_head, nesdev->cqp.sq_head,
1919                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1920
1921         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1922                         NES_EVENT_TIMEOUT);
1923
1924         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1925                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1926                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1927         if (!ret) {
1928                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1929                                 nesvnic->nic.qp_id);
1930         }
1931
1932         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1933                         nesvnic->nic_pbase);
1934 }
1935
1936 /**
1937  * nes_napi_isr
1938  */
1939 int nes_napi_isr(struct nes_device *nesdev)
1940 {
1941         struct nes_adapter *nesadapter = nesdev->nesadapter;
1942         u32 int_stat;
1943
1944         if (nesdev->napi_isr_ran) {
1945                 /* interrupt status has already been read in ISR */
1946                 int_stat = nesdev->int_stat;
1947         } else {
1948                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1949                 nesdev->int_stat = int_stat;
1950                 nesdev->napi_isr_ran = 1;
1951         }
1952
1953         int_stat &= nesdev->int_req;
1954         /* iff NIC, process here, else wait for DPC */
1955         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1956                 nesdev->napi_isr_ran = 0;
1957                 nes_write32(nesdev->regs + NES_INT_STAT,
1958                         (int_stat &
1959                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1960
1961                 /* Process the CEQs */
1962                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1963
1964                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1965                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1966                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1967                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1968                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1969                                 /* Enable Periodic timer interrupts */
1970                                 nesdev->int_req |= NES_INT_TIMER;
1971                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1972                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1973                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1974                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1975                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1976                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1977                         }
1978
1979                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1980                         {
1981                                 nes_nic_init_timer(nesdev);
1982                         }
1983                         /* Enable interrupts, except CEQs */
1984                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1985                 } else {
1986                         /* Enable interrupts, make sure timer is off */
1987                         nesdev->int_req &= ~NES_INT_TIMER;
1988                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1989                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1990                 }
1991                 nesdev->deepcq_count = 0;
1992                 return 1;
1993         } else {
1994                 return 0;
1995         }
1996 }
1997
1998 static void process_critical_error(struct nes_device *nesdev)
1999 {
2000         u32 debug_error;
2001         u32 nes_idx_debug_error_masks0 = 0;
2002         u16 error_module = 0;
2003
2004         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2005         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2006                         (u16)debug_error);
2007         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2008                         0x01010000 | (debug_error & 0x0000ffff));
2009         if (crit_err_count++ > 10)
2010                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2011         error_module = (u16) (debug_error & 0x0F00) >> 8;
2012         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2013                         nes_max_critical_error_count) {
2014                 printk(KERN_ERR PFX "Masking off critical error for module "
2015                         "0x%02X\n", (u16)error_module);
2016                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2017                         NES_IDX_DEBUG_ERROR_MASKS0);
2018                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2019                         nes_idx_debug_error_masks0 | (1 << error_module));
2020         }
2021 }
2022 /**
2023  * nes_dpc
2024  */
2025 void nes_dpc(unsigned long param)
2026 {
2027         struct nes_device *nesdev = (struct nes_device *)param;
2028         struct nes_adapter *nesadapter = nesdev->nesadapter;
2029         u32 counter;
2030         u32 loop_counter = 0;
2031         u32 int_status_bit;
2032         u32 int_stat;
2033         u32 timer_stat;
2034         u32 temp_int_stat;
2035         u32 intf_int_stat;
2036         u32 processed_intf_int = 0;
2037         u16 processed_timer_int = 0;
2038         u16 completion_ints = 0;
2039         u16 timer_ints = 0;
2040
2041         /* nes_debug(NES_DBG_ISR, "\n"); */
2042
2043         do {
2044                 timer_stat = 0;
2045                 if (nesdev->napi_isr_ran) {
2046                         nesdev->napi_isr_ran = 0;
2047                         int_stat = nesdev->int_stat;
2048                 } else
2049                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2050                 if (processed_intf_int != 0)
2051                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2052                 else
2053                         int_stat &= nesdev->int_req;
2054                 if (processed_timer_int == 0) {
2055                         processed_timer_int = 1;
2056                         if (int_stat & NES_INT_TIMER) {
2057                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2058                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2059                                         int_stat &= ~NES_INT_TIMER;
2060                                 }
2061                         }
2062                 } else {
2063                         int_stat &= ~NES_INT_TIMER;
2064                 }
2065
2066                 if (int_stat) {
2067                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2068                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2069                                 /* Ack the interrupts */
2070                                 nes_write32(nesdev->regs+NES_INT_STAT,
2071                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2072                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2073                         }
2074
2075                         temp_int_stat = int_stat;
2076                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2077                                 if (int_stat & int_status_bit) {
2078                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2079                                         temp_int_stat &= ~int_status_bit;
2080                                         completion_ints = 1;
2081                                 }
2082                                 if (!(temp_int_stat & 0x0000ffff))
2083                                         break;
2084                                 int_status_bit <<= 1;
2085                         }
2086
2087                         /* Process the AEQ for this pci function */
2088                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2089                         if (int_stat & int_status_bit) {
2090                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2091                         }
2092
2093                         /* Process the MAC interrupt for this pci function */
2094                         int_status_bit = 1 << (24 + nesdev->mac_index);
2095                         if (int_stat & int_status_bit) {
2096                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2097                         }
2098
2099                         if (int_stat & NES_INT_TIMER) {
2100                                 if (timer_stat & nesdev->timer_int_req) {
2101                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2102                                                         (timer_stat & nesdev->timer_int_req) |
2103                                                         ~(nesdev->nesadapter->timer_int_req));
2104                                         timer_ints = 1;
2105                                 }
2106                         }
2107
2108                         if (int_stat & NES_INT_INTF) {
2109                                 processed_intf_int = 1;
2110                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2111                                 intf_int_stat &= nesdev->intf_int_req;
2112                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2113                                         process_critical_error(nesdev);
2114                                 }
2115                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2116                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2117                                         BUG();
2118                                 }
2119                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2120                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2121                                         BUG();
2122                                 }
2123                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2124                         }
2125
2126                         if (int_stat & NES_INT_TSW) {
2127                         }
2128                 }
2129                 /* Don't use the interface interrupt bit stay in loop */
2130                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2131                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2132         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2133
2134         if (timer_ints == 1) {
2135                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2136                         if (completion_ints == 0) {
2137                                 nesdev->timer_only_int_count++;
2138                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2139                                         nesdev->timer_only_int_count = 0;
2140                                         nesdev->int_req &= ~NES_INT_TIMER;
2141                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2142                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2143                                 } else {
2144                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2145                                 }
2146                         } else {
2147                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2148                                 {
2149                                         nes_nic_init_timer(nesdev);
2150                                 }
2151                                 nesdev->timer_only_int_count = 0;
2152                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2153                         }
2154                 } else {
2155                         nesdev->timer_only_int_count = 0;
2156                         nesdev->int_req &= ~NES_INT_TIMER;
2157                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2158                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2159                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2160                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2161                 }
2162         } else {
2163                 if ( (completion_ints == 1) &&
2164                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2165                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2166                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2167                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2168                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2169                         nesdev->timer_only_int_count = 0;
2170                         nesdev->int_req |= NES_INT_TIMER;
2171                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2172                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2173                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2174                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2175                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2176                 } else {
2177                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2178                 }
2179         }
2180         nesdev->deepcq_count = 0;
2181 }
2182
2183
2184 /**
2185  * nes_process_ceq
2186  */
2187 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2188 {
2189         u64 u64temp;
2190         struct nes_hw_cq *cq;
2191         u32 head;
2192         u32 ceq_size;
2193
2194         /* nes_debug(NES_DBG_CQ, "\n"); */
2195         head = ceq->ceq_head;
2196         ceq_size = ceq->ceq_size;
2197
2198         do {
2199                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2200                                 NES_CEQE_VALID) {
2201                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2202                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2203                         u64temp <<= 1;
2204                         cq = *((struct nes_hw_cq **)&u64temp);
2205                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2206                         barrier();
2207                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2208
2209                         /* call the event handler */
2210                         cq->ce_handler(nesdev, cq);
2211
2212                         if (++head >= ceq_size)
2213                                 head = 0;
2214                 } else {
2215                         break;
2216                 }
2217
2218         } while (1);
2219
2220         ceq->ceq_head = head;
2221 }
2222
2223
2224 /**
2225  * nes_process_aeq
2226  */
2227 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2228 {
2229         /* u64 u64temp; */
2230         u32 head;
2231         u32 aeq_size;
2232         u32 aeqe_misc;
2233         u32 aeqe_cq_id;
2234         struct nes_hw_aeqe volatile *aeqe;
2235
2236         head = aeq->aeq_head;
2237         aeq_size = aeq->aeq_size;
2238
2239         do {
2240                 aeqe = &aeq->aeq_vbase[head];
2241                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2242                         break;
2243                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2244                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2245                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2246                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2247                                 /* dealing with an accelerated QP related AE */
2248                                 /*
2249                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2250                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2251                                  */
2252                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2253                         } else {
2254                                 /* TODO: dealing with a CQP related AE */
2255                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2256                                                 (u16)(aeqe_misc >> 16));
2257                         }
2258                 }
2259
2260                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2261
2262                 if (++head >= aeq_size)
2263                         head = 0;
2264         }
2265         while (1);
2266         aeq->aeq_head = head;
2267 }
2268
2269 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2270 {
2271         struct nes_adapter *nesadapter = nesdev->nesadapter;
2272         u32 reset_value;
2273         u32 i=0;
2274         u32 u32temp;
2275
2276         if (nesadapter->hw_rev == NE020_REV) {
2277                 return;
2278         }
2279         mh_detected++;
2280
2281         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2282
2283         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2284                 reset_value |= 0x0000001d;
2285         else
2286                 reset_value |= 0x0000002d;
2287
2288         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2289                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2290                         nesadapter->link_interrupt_count[0] = 0;
2291                         nesadapter->link_interrupt_count[1] = 0;
2292                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2293                         if (0x00000040 & u32temp)
2294                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2295                         else
2296                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2297
2298                         reset_value |= 0x0000003d;
2299                 }
2300                 nesadapter->link_interrupt_count[mac_index] = 0;
2301         }
2302
2303         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2304
2305         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2306                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2307
2308         if (0x0000003d == (reset_value & 0x0000003d)) {
2309                 u32 pcs_control_status0, pcs_control_status1;
2310
2311                 for (i = 0; i < 10; i++) {
2312                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2313                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2314                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2315                              && (pcs_control_status0 & 0x00100000))
2316                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2317                                 && (pcs_control_status1 & 0x00100000)))
2318                                 continue;
2319                         else
2320                                 break;
2321                 }
2322                 if (10 == i) {
2323                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2324                         if (0x00000040 & u32temp)
2325                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2326                         else
2327                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2328
2329                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2330
2331                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2332                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2333                 }
2334         }
2335 }
2336
2337 /**
2338  * nes_process_mac_intr
2339  */
2340 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2341 {
2342         unsigned long flags;
2343         u32 pcs_control_status;
2344         struct nes_adapter *nesadapter = nesdev->nesadapter;
2345         struct nes_vnic *nesvnic;
2346         u32 mac_status;
2347         u32 mac_index = nesdev->mac_index;
2348         u32 u32temp;
2349         u16 phy_data;
2350         u16 temp_phy_data;
2351         u32 pcs_val  = 0x0f0f0000;
2352         u32 pcs_mask = 0x0f1f0000;
2353
2354         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2355         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2356                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2357                 return;
2358         }
2359         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2360         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2361
2362         /* ack the MAC interrupt */
2363         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2364         /* Clear the interrupt */
2365         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2366
2367         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2368
2369         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2370                 nesdev->link_status_interrupts++;
2371                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2372                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2373                         nes_reset_link(nesdev, mac_index);
2374                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2375                 }
2376                 /* read the PHY interrupt status register */
2377                 if ((nesadapter->OneG_Mode) &&
2378                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2379                         do {
2380                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2381                                                 nesadapter->phy_index[mac_index], &phy_data);
2382                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2383                                                 nesadapter->phy_index[mac_index], phy_data);
2384                         } while (phy_data&0x8000);
2385
2386                         temp_phy_data = 0;
2387                         do {
2388                                 nes_read_1G_phy_reg(nesdev, 0x11,
2389                                                 nesadapter->phy_index[mac_index], &phy_data);
2390                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2391                                                 nesadapter->phy_index[mac_index], phy_data);
2392                                 if (temp_phy_data == phy_data)
2393                                         break;
2394                                 temp_phy_data = phy_data;
2395                         } while (1);
2396
2397                         nes_read_1G_phy_reg(nesdev, 0x1e,
2398                                         nesadapter->phy_index[mac_index], &phy_data);
2399                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2400                                         nesadapter->phy_index[mac_index], phy_data);
2401
2402                         nes_read_1G_phy_reg(nesdev, 1,
2403                                         nesadapter->phy_index[mac_index], &phy_data);
2404                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2405                                         nesadapter->phy_index[mac_index], phy_data);
2406
2407                         if (temp_phy_data & 0x1000) {
2408                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2409                                 phy_data = 4;
2410                         } else {
2411                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2412                         }
2413                 }
2414                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2415                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2416                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2417
2418                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2419                         switch (mac_index) {
2420                         case 1:
2421                         case 3:
2422                                 pcs_control_status = nes_read_indexed(nesdev,
2423                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2424                                 break;
2425                         default:
2426                                 pcs_control_status = nes_read_indexed(nesdev,
2427                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2428                                 break;
2429                         }
2430                 } else {
2431                         pcs_control_status = nes_read_indexed(nesdev,
2432                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2433                         pcs_control_status = nes_read_indexed(nesdev,
2434                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2435                 }
2436
2437                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2438                                 mac_index, pcs_control_status);
2439                 if ((nesadapter->OneG_Mode) &&
2440                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2441                         u32temp = 0x01010000;
2442                         if (nesadapter->port_count > 2) {
2443                                 u32temp |= 0x02020000;
2444                         }
2445                         if ((pcs_control_status & u32temp)!= u32temp) {
2446                                 phy_data = 0;
2447                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2448                         }
2449                 } else {
2450                         switch (nesadapter->phy_type[mac_index]) {
2451                         case NES_PHY_TYPE_IRIS:
2452                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2453                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2454                                 u32temp = 20;
2455                                 do {
2456                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2457                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2458                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2459                                                 break;
2460                                         temp_phy_data = phy_data;
2461                                 } while (1);
2462                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2463                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2464                                 break;
2465
2466                         case NES_PHY_TYPE_ARGUS:
2467                                 /* clear the alarms */
2468                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2469                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2470                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2471                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2472                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2473                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2474                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2475                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2476                                 /* check link status */
2477                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2478                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2479                                 u32temp = 100;
2480                                 do {
2481                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2482
2483                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2484                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2485                                                 break;
2486                                         temp_phy_data = phy_data;
2487                                 } while (1);
2488                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2489                                         __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2490                                 break;
2491
2492                         case NES_PHY_TYPE_PUMA_1G:
2493                                 if (mac_index < 2)
2494                                         pcs_val = pcs_mask = 0x01010000;
2495                                 else
2496                                         pcs_val = pcs_mask = 0x02020000;
2497                                 /* fall through */
2498                         default:
2499                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2500                                 break;
2501                         }
2502                 }
2503
2504                 if (phy_data & 0x0004) {
2505                         nesadapter->mac_link_down[mac_index] = 0;
2506                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2507                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2508                                                 nesvnic->linkup);
2509                                 if (nesvnic->linkup == 0) {
2510                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2511                                                         nesvnic->netdev->name, nesvnic->netdev);
2512                                         if (netif_queue_stopped(nesvnic->netdev))
2513                                                 netif_start_queue(nesvnic->netdev);
2514                                         nesvnic->linkup = 1;
2515                                         netif_carrier_on(nesvnic->netdev);
2516                                 }
2517                         }
2518                 } else {
2519                         nesadapter->mac_link_down[mac_index] = 1;
2520                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2521                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2522                                                 nesvnic->linkup);
2523                                 if (nesvnic->linkup == 1) {
2524                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2525                                                         nesvnic->netdev->name, nesvnic->netdev);
2526                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2527                                                 netif_stop_queue(nesvnic->netdev);
2528                                         nesvnic->linkup = 0;
2529                                         netif_carrier_off(nesvnic->netdev);
2530                                 }
2531                         }
2532                 }
2533         }
2534
2535         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2536 }
2537
2538
2539
2540 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2541 {
2542         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2543
2544         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2545 }
2546
2547
2548 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2549 * getting out of nic_ce_handler
2550 */
2551 #define MAX_RQES_TO_PROCESS     384
2552
2553 /**
2554  * nes_nic_ce_handler
2555  */
2556 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2557 {
2558         u64 u64temp;
2559         dma_addr_t bus_address;
2560         struct nes_hw_nic *nesnic;
2561         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2562         struct nes_adapter *nesadapter = nesdev->nesadapter;
2563         struct nes_hw_nic_rq_wqe *nic_rqe;
2564         struct nes_hw_nic_sq_wqe *nic_sqe;
2565         struct sk_buff *skb;
2566         struct sk_buff *rx_skb;
2567         __le16 *wqe_fragment_length;
2568         u32 head;
2569         u32 cq_size;
2570         u32 rx_pkt_size;
2571         u32 cqe_count=0;
2572         u32 cqe_errv;
2573         u32 cqe_misc;
2574         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2575         u16 vlan_tag;
2576         u16 pkt_type;
2577         u16 rqes_processed = 0;
2578         u8 sq_cqes = 0;
2579         u8 nes_use_lro = 0;
2580
2581         head = cq->cq_head;
2582         cq_size = cq->cq_size;
2583         cq->cqes_pending = 1;
2584         if (nesvnic->netdev->features & NETIF_F_LRO)
2585                 nes_use_lro = 1;
2586         do {
2587                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2588                                 NES_NIC_CQE_VALID) {
2589                         nesnic = &nesvnic->nic;
2590                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2591                         if (cqe_misc & NES_NIC_CQE_SQ) {
2592                                 sq_cqes++;
2593                                 wqe_fragment_index = 1;
2594                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2595                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2596                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2597                                 /* bump past the vlan tag */
2598                                 wqe_fragment_length++;
2599                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2600                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2601                                                         wqe_fragment_index * 2]);
2602                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2603                                                         wqe_fragment_index * 2])) << 32;
2604                                         bus_address = (dma_addr_t)u64temp;
2605                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2606                                                 pci_unmap_single(nesdev->pcidev,
2607                                                                 bus_address,
2608                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2609                                                                 PCI_DMA_TODEVICE);
2610                                         }
2611                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2612                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2613                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2614                                                                                 wqe_fragment_index * 2]);
2615                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2616                                                                                 + wqe_fragment_index * 2])) <<32;
2617                                                         bus_address = (dma_addr_t)u64temp;
2618                                                         pci_unmap_page(nesdev->pcidev,
2619                                                                         bus_address,
2620                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2621                                                                         PCI_DMA_TODEVICE);
2622                                                 } else
2623                                                         break;
2624                                         }
2625                                         if (skb)
2626                                                 dev_kfree_skb_any(skb);
2627                                 }
2628                                 nesnic->sq_tail++;
2629                                 nesnic->sq_tail &= nesnic->sq_size-1;
2630                                 if (sq_cqes > 128) {
2631                                         barrier();
2632                                 /* restart the queue if it had been stopped */
2633                                 if (netif_queue_stopped(nesvnic->netdev))
2634                                         netif_wake_queue(nesvnic->netdev);
2635                                         sq_cqes = 0;
2636                                 }
2637                         } else {
2638                                 rqes_processed ++;
2639
2640                                 cq->rx_cqes_completed++;
2641                                 cq->rx_pkts_indicated++;
2642                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2643                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2644                                 /* Get the skb */
2645                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2646                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2647                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2648                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2649                                 pci_unmap_single(nesdev->pcidev, bus_address,
2650                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2651                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2652                                 /* rx_skb->len = rx_pkt_size; */
2653                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2654                                 skb_put(rx_skb, rx_pkt_size);
2655                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2656                                 nesnic->rq_tail++;
2657                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2658
2659                                 atomic_inc(&nesvnic->rx_skbs_needed);
2660                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2661                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2662                                                         cq->cq_number | (cqe_count << 16));
2663                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2664                                         nesdev->currcq_count += cqe_count;
2665                                         cqe_count = 0;
2666                                         nes_replenish_nic_rq(nesvnic);
2667                                 }
2668                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2669                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2670                                 rx_skb->ip_summed = CHECKSUM_NONE;
2671
2672                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2673                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2674                                         if ((cqe_errv &
2675                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2676                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2677                                                 if (nesvnic->rx_checksum_disabled == 0) {
2678                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2679                                                 }
2680                                         } else
2681                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2682                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2683                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2684
2685                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2686                                         if ((cqe_errv &
2687                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2688                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2689                                                 if (nesvnic->rx_checksum_disabled == 0) {
2690                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2691                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2692                                                                   nesvnic->netdev->name); */
2693                                                 }
2694                                         } else
2695                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2696                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2697                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2698                                         }
2699                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2700                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2701
2702                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2703                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2704                                 } else {
2705                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2706                                                 vlan_tag = (u16)(le32_to_cpu(
2707                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2708                                                                 >> 16);
2709                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2710                                                                 nesvnic->netdev->name, vlan_tag);
2711                                                 if (nes_use_lro)
2712                                                         lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2713                                                                         nesvnic->vlan_grp, vlan_tag, NULL);
2714                                                 else
2715                                                         nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2716                                         } else {
2717                                                 if (nes_use_lro)
2718                                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2719                                                 else
2720                                                         nes_netif_rx(rx_skb);
2721                                         }
2722                                 }
2723
2724                                 nesvnic->netdev->last_rx = jiffies;
2725                                 /* nesvnic->netstats.rx_packets++; */
2726                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2727                         }
2728
2729                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2730                         /* Accounting... */
2731                         cqe_count++;
2732                         if (++head >= cq_size)
2733                                 head = 0;
2734                         if (cqe_count == 255) {
2735                                 /* Replenish Nic CQ */
2736                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2737                                                 cq->cq_number | (cqe_count << 16));
2738                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2739                                 nesdev->currcq_count += cqe_count;
2740                                 cqe_count = 0;
2741                         }
2742
2743                         if (cq->rx_cqes_completed >= nesvnic->budget)
2744                                 break;
2745                 } else {
2746                         cq->cqes_pending = 0;
2747                         break;
2748                 }
2749
2750         } while (1);
2751
2752         if (nes_use_lro)
2753                 lro_flush_all(&nesvnic->lro_mgr);
2754         if (sq_cqes) {
2755                 barrier();
2756                 /* restart the queue if it had been stopped */
2757                 if (netif_queue_stopped(nesvnic->netdev))
2758                         netif_wake_queue(nesvnic->netdev);
2759         }
2760         cq->cq_head = head;
2761         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2762                         cq->cq_number, cqe_count, cq->cq_head); */
2763         cq->cqe_allocs_pending = cqe_count;
2764         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2765         {
2766                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2767                 nesdev->currcq_count += cqe_count;
2768                 nes_nic_tune_timer(nesdev);
2769         }
2770         if (atomic_read(&nesvnic->rx_skbs_needed))
2771                 nes_replenish_nic_rq(nesvnic);
2772 }
2773
2774
2775 /**
2776  * nes_cqp_ce_handler
2777  */
2778 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2779 {
2780         u64 u64temp;
2781         unsigned long flags;
2782         struct nes_hw_cqp *cqp = NULL;
2783         struct nes_cqp_request *cqp_request;
2784         struct nes_hw_cqp_wqe *cqp_wqe;
2785         u32 head;
2786         u32 cq_size;
2787         u32 cqe_count=0;
2788         u32 error_code;
2789         /* u32 counter; */
2790
2791         head = cq->cq_head;
2792         cq_size = cq->cq_size;
2793
2794         do {
2795                 /* process the CQE */
2796                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2797                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2798
2799                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2800                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2801                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2802                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2803                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2804                         cqp = *((struct nes_hw_cqp **)&u64temp);
2805
2806                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2807                         if (error_code) {
2808                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2809                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2810                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2811                                                 (u16)(error_code >> 16),
2812                                                 (u16)error_code);
2813                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2814                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2815                         }
2816
2817                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2818                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2819                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2820                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2821                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2822                         if (cqp_request) {
2823                                 if (cqp_request->waiting) {
2824                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2825                                         cqp_request->major_code = (u16)(error_code >> 16);
2826                                         cqp_request->minor_code = (u16)error_code;
2827                                         barrier();
2828                                         cqp_request->request_done = 1;
2829                                         wake_up(&cqp_request->waitq);
2830                                         nes_put_cqp_request(nesdev, cqp_request);
2831                                 } else {
2832                                         if (cqp_request->callback)
2833                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2834                                         nes_free_cqp_request(nesdev, cqp_request);
2835                                 }
2836                         } else {
2837                                 wake_up(&nesdev->cqp.waitq);
2838                         }
2839
2840                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2841                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2842                         if (++cqp->sq_tail >= cqp->sq_size)
2843                                 cqp->sq_tail = 0;
2844
2845                         /* Accounting... */
2846                         cqe_count++;
2847                         if (++head >= cq_size)
2848                                 head = 0;
2849                 } else {
2850                         break;
2851                 }
2852         } while (1);
2853         cq->cq_head = head;
2854
2855         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2856         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2857                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2858                         (nesdev->cqp.sq_size - 1)) != 1)) {
2859                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2860                                 struct nes_cqp_request, list);
2861                 list_del_init(&cqp_request->list);
2862                 head = nesdev->cqp.sq_head++;
2863                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2864                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2865                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2866                 barrier();
2867                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2868                         cpu_to_le32((u32)((unsigned long)cqp_request));
2869                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2870                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2871                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2872                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2873                 /* Ring doorbell (1 WQEs) */
2874                 barrier();
2875                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2876         }
2877         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2878
2879         /* Arm the CCQ */
2880         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2881                         cq->cq_number);
2882         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2883 }
2884
2885
2886 /**
2887  * nes_process_iwarp_aeqe
2888  */
2889 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2890                                    struct nes_hw_aeqe *aeqe)
2891 {
2892         u64 context;
2893         u64 aeqe_context = 0;
2894         unsigned long flags;
2895         struct nes_qp *nesqp;
2896         int resource_allocated;
2897         /* struct iw_cm_id *cm_id; */
2898         struct nes_adapter *nesadapter = nesdev->nesadapter;
2899         struct ib_event ibevent;
2900         /* struct iw_cm_event cm_event; */
2901         u32 aeq_info;
2902         u32 next_iwarp_state = 0;
2903         u16 async_event_id;
2904         u8 tcp_state;
2905         u8 iwarp_state;
2906
2907         nes_debug(NES_DBG_AEQ, "\n");
2908         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2909         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2910                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2911                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2912         } else {
2913                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2914                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2915                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2916                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2917                 BUG_ON(!context);
2918         }
2919
2920         async_event_id = (u16)aeq_info;
2921         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2922         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2923         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2924                         " Tcp state = %s, iWARP state = %s\n",
2925                         async_event_id,
2926                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2927                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2928
2929         switch (async_event_id) {
2930                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2931                         nesqp = *((struct nes_qp **)&context);
2932                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2933                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2934                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2935                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2936                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2937                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2938                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2939                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2940                                                 async_event_id, nesqp->last_aeq, tcp_state);
2941                         }
2942                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2943                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2944                                 /* FIN Received but tcp state or IB state moved on,
2945                                                 should expect a close complete */
2946                                 return;
2947                         }
2948                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2949                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2950                 case NES_AEQE_AEID_TERMINATE_SENT:
2951                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2952                 case NES_AEQE_AEID_RESET_SENT:
2953                         nesqp = *((struct nes_qp **)&context);
2954                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2955                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2956                         }
2957                         spin_lock_irqsave(&nesqp->lock, flags);
2958                         nesqp->hw_iwarp_state = iwarp_state;
2959                         nesqp->hw_tcp_state = tcp_state;
2960                         nesqp->last_aeq = async_event_id;
2961
2962                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2963                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2964                                 nesqp->hte_added = 0;
2965                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2966                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2967                                                 nesqp->hwqp.qp_id);
2968                                 nes_hw_modify_qp(nesdev, nesqp,
2969                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2970                                 spin_lock_irqsave(&nesqp->lock, flags);
2971                         }
2972
2973                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2974                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2975                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2976                                 switch (nesqp->hw_iwarp_state) {
2977                                         case NES_AEQE_IWARP_STATE_RTS:
2978                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2979                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2980                                                 break;
2981                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2982                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2983                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2984                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2985                                                         next_iwarp_state |= 0x02000000;
2986                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2987                                                 }
2988                                                 break;
2989                                         default:
2990                                                 next_iwarp_state = 0;
2991                                 }
2992                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2993                                 if (next_iwarp_state) {
2994                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2995                                                         " also added another reference\n",
2996                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2997                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2998                                 }
2999                                 nes_cm_disconn(nesqp);
3000                         } else {
3001                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3002                                         /* FIN Received but ib state not RTS,
3003                                                         close complete will be on its way */
3004                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3005                                         return;
3006                                 }
3007                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3008                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3009                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3010                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3011                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3012                                                         " also added another reference\n",
3013                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3014                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3015                                 }
3016                                 nes_cm_disconn(nesqp);
3017                         }
3018                         break;
3019                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3020                         nesqp = *((struct nes_qp **)&context);
3021                         spin_lock_irqsave(&nesqp->lock, flags);
3022                         nesqp->hw_iwarp_state = iwarp_state;
3023                         nesqp->hw_tcp_state = tcp_state;
3024                         nesqp->last_aeq = async_event_id;
3025                         spin_unlock_irqrestore(&nesqp->lock, flags);
3026                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3027                                         " event on QP%u \n  Q2 Data:\n",
3028                                         nesqp->hwqp.qp_id);
3029                         if (nesqp->ibqp.event_handler) {
3030                                 ibevent.device = nesqp->ibqp.device;
3031                                 ibevent.element.qp = &nesqp->ibqp;
3032                                 ibevent.event = IB_EVENT_QP_FATAL;
3033                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3034                         }
3035                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3036                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3037                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3038                                 nes_cm_disconn(nesqp);
3039                         } else {
3040                                 nesqp->in_disconnect = 0;
3041                                 wake_up(&nesqp->kick_waitq);
3042                         }
3043                         break;
3044                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3045                         nesqp = *((struct nes_qp **)&context);
3046                         spin_lock_irqsave(&nesqp->lock, flags);
3047                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3048                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3049                         nesqp->last_aeq = async_event_id;
3050                         if (nesqp->cm_id) {
3051                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3052                                                 " event on QP%u, remote IP = 0x%08X \n",
3053                                                 nesqp->hwqp.qp_id,
3054                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3055                         } else {
3056                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3057                                                 " event on QP%u \n",
3058                                                 nesqp->hwqp.qp_id);
3059                         }
3060                         spin_unlock_irqrestore(&nesqp->lock, flags);
3061                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3062                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3063                         if (nesqp->ibqp.event_handler) {
3064                                 ibevent.device = nesqp->ibqp.device;
3065                                 ibevent.element.qp = &nesqp->ibqp;
3066                                 ibevent.event = IB_EVENT_QP_FATAL;
3067                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3068                         }
3069                         break;
3070                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3071                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3072                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3073                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3074                         } else {
3075                                 /* TODO: get the actual WQE and mask off wqe index */
3076                                 context &= ~((u64)511);
3077                                 nesqp = *((struct nes_qp **)&context);
3078                         }
3079                         spin_lock_irqsave(&nesqp->lock, flags);
3080                         nesqp->hw_iwarp_state = iwarp_state;
3081                         nesqp->hw_tcp_state = tcp_state;
3082                         nesqp->last_aeq = async_event_id;
3083                         spin_unlock_irqrestore(&nesqp->lock, flags);
3084                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3085                                         nesqp->hwqp.qp_id);
3086                         if (nesqp->ibqp.event_handler) {
3087                                 ibevent.device = nesqp->ibqp.device;
3088                                 ibevent.element.qp = &nesqp->ibqp;
3089                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3090                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3091                         }
3092                         break;
3093                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3094                         nesqp = *((struct nes_qp **)&context);
3095                         spin_lock_irqsave(&nesqp->lock, flags);
3096                         nesqp->hw_iwarp_state = iwarp_state;
3097                         nesqp->hw_tcp_state = tcp_state;
3098                         nesqp->last_aeq = async_event_id;
3099                         spin_unlock_irqrestore(&nesqp->lock, flags);
3100                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3101                                         nesqp->hwqp.qp_id);
3102                         if (nesqp->ibqp.event_handler) {
3103                                 ibevent.device = nesqp->ibqp.device;
3104                                 ibevent.element.qp = &nesqp->ibqp;
3105                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3106                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3107                         }
3108                         break;
3109                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3110                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3111                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3112                         spin_lock_irqsave(&nesqp->lock, flags);
3113                         nesqp->hw_iwarp_state = iwarp_state;
3114                         nesqp->hw_tcp_state = tcp_state;
3115                         nesqp->last_aeq = async_event_id;
3116                         spin_unlock_irqrestore(&nesqp->lock, flags);
3117                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3118                                         " nesqp = %p, AE reported %p\n",
3119                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3120                         if (nesqp->ibqp.event_handler) {
3121                                 ibevent.device = nesqp->ibqp.device;
3122                                 ibevent.element.qp = &nesqp->ibqp;
3123                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3124                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3125                         }
3126                         break;
3127                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3128                         context <<= 1;
3129                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3130                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3131                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3132                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3133                         if (resource_allocated) {
3134                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3135                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3136                         }
3137                         break;
3138                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3139                         nesqp = nesadapter->qp_table[le32_to_cpu(
3140                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3141                         spin_lock_irqsave(&nesqp->lock, flags);
3142                         nesqp->hw_iwarp_state = iwarp_state;
3143                         nesqp->hw_tcp_state = tcp_state;
3144                         nesqp->last_aeq = async_event_id;
3145                         spin_unlock_irqrestore(&nesqp->lock, flags);
3146                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3147                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3148                                         nesqp->hwqp.qp_id);
3149                         if (nesqp->ibqp.event_handler) {
3150                                 ibevent.device = nesqp->ibqp.device;
3151                                 ibevent.element.qp = &nesqp->ibqp;
3152                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3153                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3154                         }
3155                         /* tell cm to disconnect, cm will queue work to thread */
3156                         nes_cm_disconn(nesqp);
3157                         break;
3158                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3159                         nesqp = *((struct nes_qp **)&context);
3160                         spin_lock_irqsave(&nesqp->lock, flags);
3161                         nesqp->hw_iwarp_state = iwarp_state;
3162                         nesqp->hw_tcp_state = tcp_state;
3163                         nesqp->last_aeq = async_event_id;
3164                         spin_unlock_irqrestore(&nesqp->lock, flags);
3165                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3166                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3167                                         nesqp->hwqp.qp_id);
3168                         if (nesqp->ibqp.event_handler) {
3169                                 ibevent.device = nesqp->ibqp.device;
3170                                 ibevent.element.qp = &nesqp->ibqp;
3171                                 ibevent.event = IB_EVENT_QP_FATAL;
3172                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3173                         }
3174                         /* tell cm to disconnect, cm will queue work to thread */
3175                         nes_cm_disconn(nesqp);
3176                         break;
3177                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3178                         nesqp = *((struct nes_qp **)&context);
3179                         spin_lock_irqsave(&nesqp->lock, flags);
3180                         nesqp->hw_iwarp_state = iwarp_state;
3181                         nesqp->hw_tcp_state = tcp_state;
3182                         nesqp->last_aeq = async_event_id;
3183                         spin_unlock_irqrestore(&nesqp->lock, flags);
3184                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3185                                         " event on QP%u \n  Q2 Data:\n",
3186                                         nesqp->hwqp.qp_id);
3187                         if (nesqp->ibqp.event_handler) {
3188                                 ibevent.device = nesqp->ibqp.device;
3189                                 ibevent.element.qp = &nesqp->ibqp;
3190                                 ibevent.event = IB_EVENT_QP_FATAL;
3191                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3192                         }
3193                         /* tell cm to disconnect, cm will queue work to thread */
3194                         nes_cm_disconn(nesqp);
3195                         break;
3196                         /* TODO: additional AEs need to be here */
3197                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3198                         nesqp = *((struct nes_qp **)&context);
3199                         spin_lock_irqsave(&nesqp->lock, flags);
3200                         nesqp->hw_iwarp_state = iwarp_state;
3201                         nesqp->hw_tcp_state = tcp_state;
3202                         nesqp->last_aeq = async_event_id;
3203                         spin_unlock_irqrestore(&nesqp->lock, flags);
3204                         if (nesqp->ibqp.event_handler) {
3205                                 ibevent.device = nesqp->ibqp.device;
3206                                 ibevent.element.qp = &nesqp->ibqp;
3207                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3208                                 nesqp->ibqp.event_handler(&ibevent,
3209                                                 nesqp->ibqp.qp_context);
3210                         }
3211                         nes_cm_disconn(nesqp);
3212                         break;
3213                 default:
3214                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3215                                         async_event_id);
3216                         break;
3217         }
3218
3219 }
3220
3221
3222 /**
3223  * nes_iwarp_ce_handler
3224  */
3225 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3226 {
3227         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3228
3229         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3230                         nescq->hw_cq.cq_number); */
3231         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3232
3233         if (nescq->ibcq.comp_handler)
3234                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3235
3236         return;
3237 }
3238
3239
3240 /**
3241  * nes_manage_apbvt()
3242  */
3243 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3244                 u32 nic_index, u32 add_port)
3245 {
3246         struct nes_device *nesdev = nesvnic->nesdev;
3247         struct nes_hw_cqp_wqe *cqp_wqe;
3248         struct nes_cqp_request *cqp_request;
3249         int ret = 0;
3250         u16 major_code;
3251
3252         /* Send manage APBVT request to CQP */
3253         cqp_request = nes_get_cqp_request(nesdev);
3254         if (cqp_request == NULL) {
3255                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3256                 return -ENOMEM;
3257         }
3258         cqp_request->waiting = 1;
3259         cqp_wqe = &cqp_request->cqp_wqe;
3260
3261         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3262                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3263                         accel_local_port, accel_local_port, nic_index);
3264
3265         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3266         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3267                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3268         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3269                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3270
3271         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3272
3273         atomic_set(&cqp_request->refcount, 2);
3274         nes_post_cqp_request(nesdev, cqp_request);
3275
3276         if (add_port == NES_MANAGE_APBVT_ADD)
3277                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3278                                 NES_EVENT_TIMEOUT);
3279         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3280                         ret, cqp_request->major_code, cqp_request->minor_code);
3281         major_code = cqp_request->major_code;
3282
3283         nes_put_cqp_request(nesdev, cqp_request);
3284
3285         if (!ret)
3286                 return -ETIME;
3287         else if (major_code)
3288                 return -EIO;
3289         else
3290                 return 0;
3291 }
3292
3293
3294 /**
3295  * nes_manage_arp_cache
3296  */
3297 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3298                 u32 ip_addr, u32 action)
3299 {
3300         struct nes_hw_cqp_wqe *cqp_wqe;
3301         struct nes_vnic *nesvnic = netdev_priv(netdev);
3302         struct nes_device *nesdev;
3303         struct nes_cqp_request *cqp_request;
3304         int arp_index;
3305
3306         nesdev = nesvnic->nesdev;
3307         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3308         if (arp_index == -1) {
3309                 return;
3310         }
3311
3312         /* update the ARP entry */
3313         cqp_request = nes_get_cqp_request(nesdev);
3314         if (cqp_request == NULL) {
3315                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3316                 return;
3317         }
3318         cqp_request->waiting = 0;
3319         cqp_wqe = &cqp_request->cqp_wqe;
3320         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3321
3322         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3323                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3324         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3325                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3326         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3327
3328         if (action == NES_ARP_ADD) {
3329                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3330                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3331                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3332                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3333                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3334                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3335         } else {
3336                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3337                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3338         }
3339
3340         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3341                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3342
3343         atomic_set(&cqp_request->refcount, 1);
3344         nes_post_cqp_request(nesdev, cqp_request);
3345 }
3346
3347
3348 /**
3349  * flush_wqes
3350  */
3351 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3352                 u32 which_wq, u32 wait_completion)
3353 {
3354         struct nes_cqp_request *cqp_request;
3355         struct nes_hw_cqp_wqe *cqp_wqe;
3356         int ret;
3357
3358         cqp_request = nes_get_cqp_request(nesdev);
3359         if (cqp_request == NULL) {
3360                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3361                 return;
3362         }
3363         if (wait_completion) {
3364                 cqp_request->waiting = 1;
3365                 atomic_set(&cqp_request->refcount, 2);
3366         } else {
3367                 cqp_request->waiting = 0;
3368         }
3369         cqp_wqe = &cqp_request->cqp_wqe;
3370         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3371
3372         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3373                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3374         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3375
3376         nes_post_cqp_request(nesdev, cqp_request);
3377
3378         if (wait_completion) {
3379                 /* Wait for CQP */
3380                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3381                                 NES_EVENT_TIMEOUT);
3382                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3383                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3384                                 ret, cqp_request->major_code, cqp_request->minor_code);
3385                 nes_put_cqp_request(nesdev, cqp_request);
3386         }
3387 }