git.oblomov.eu Git - linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41
  42 #include "nes.h"
  43
  44 u32 crit_err_count = 0;
  45 u32 int_mod_timer_init;
  46 u32 int_mod_cq_depth_256;
  47 u32 int_mod_cq_depth_128;
  48 u32 int_mod_cq_depth_32;
  49 u32 int_mod_cq_depth_24;
  50 u32 int_mod_cq_depth_16;
  51 u32 int_mod_cq_depth_4;
  52 u32 int_mod_cq_depth_1;
  53
  54 #include "nes_cm.h"
  55
  56
  57 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  58 static unsigned char *nes_iwarp_state_str[] = {
  59         "Non-Existant",
  60         "Idle",
  61         "RTS",
  62         "Closing",
  63         "RSVD1",
  64         "Terminate",
  65         "Error",
  66         "RSVD2",
  67 };
  68
  69 static unsigned char *nes_tcp_state_str[] = {
  70         "Non-Existant",
  71         "Closed",
  72         "Listen",
  73         "SYN Sent",
  74         "SYN Rcvd",
  75         "Established",
  76         "Close Wait",
  77         "FIN Wait 1",
  78         "Closing",
  79         "Last Ack",
  80         "FIN Wait 2",
  81         "Time Wait",
  82         "RSVD1",
  83         "RSVD2",
  84         "RSVD3",
  85         "RSVD4",
  86 };
  87 #endif
  88
  89
  90 /**
  91  * nes_nic_init_timer_defaults
  92  */
  93 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
  94 {
  95         unsigned long flags;
  96         struct nes_adapter *nesadapter = nesdev->nesadapter;
  97         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
  98
  99         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 100
 101         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 102         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 103         if (jumbomode) {
 104                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 105                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 106                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 107         } else {
 108                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 109                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 110                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 111         }
 112
 113         /* todo use netdev->mtu to set thresholds */
 114         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 115 }
 116
 117
 118 /**
 119  * nes_nic_init_timer
 120  */
 121 static void  nes_nic_init_timer(struct nes_device *nesdev)
 122 {
 123         unsigned long flags;
 124         struct nes_adapter *nesadapter = nesdev->nesadapter;
 125         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 126
 127         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 128
 129         if (shared_timer->timer_in_use_old == 0) {
 130                 nesdev->deepcq_count = 0;
 131                 shared_timer->timer_direction_upward = 0;
 132                 shared_timer->timer_direction_downward = 0;
 133                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 134                 shared_timer->timer_in_use_old = 0;
 135
 136         }
 137         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 138                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 139                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 140                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 141         }
 142         /* todo use netdev->mtu to set thresholds */
 143         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 144 }
 145
 146
 147 /**
 148  * nes_nic_tune_timer
 149  */
 150 static void nes_nic_tune_timer(struct nes_device *nesdev)
 151 {
 152         unsigned long flags;
 153         struct nes_adapter *nesadapter = nesdev->nesadapter;
 154         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 155         u16 cq_count = nesdev->currcq_count;
 156
 157         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 158
 159         if (shared_timer->cq_count_old <= cq_count)
 160                 shared_timer->cq_direction_downward = 0;
 161         else
 162                 shared_timer->cq_direction_downward++;
 163         shared_timer->cq_count_old = cq_count;
 164         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 165                 if (cq_count <= shared_timer->threshold_low &&
 166                     shared_timer->threshold_low > 4) {
 167                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 168                         shared_timer->cq_direction_downward=0;
 169                         nesdev->currcq_count = 0;
 170                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 171                         return;
 172                 }
 173         }
 174
 175         if (cq_count > 1) {
 176                 nesdev->deepcq_count += cq_count;
 177                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 178                         shared_timer->timer_direction_upward++;
 179                         shared_timer->timer_direction_downward = 0;
 180                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 181                         shared_timer->timer_direction_upward = 0;
 182                         shared_timer->timer_direction_downward = 0;
 183                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 184                         shared_timer->timer_direction_downward++;
 185                         shared_timer->timer_direction_upward = 0;
 186                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 187                         shared_timer->timer_in_use -= 2;
 188                         shared_timer->timer_direction_upward = 0;
 189                         shared_timer->timer_direction_downward++;
 190                 } else {
 191                         shared_timer->timer_in_use -= 4;
 192                         shared_timer->timer_direction_upward = 0;
 193                         shared_timer->timer_direction_downward++;
 194                 }
 195
 196                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 197                         shared_timer->timer_in_use += 3;
 198                         shared_timer->timer_direction_upward = 0;
 199                         shared_timer->timer_direction_downward = 0;
 200                 }
 201                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 202                         shared_timer->timer_in_use -= 4 ;
 203                         shared_timer->timer_direction_downward = 0;
 204                         shared_timer->timer_direction_upward = 0;
 205                 }
 206         }
 207
 208         /* boundary checking */
 209         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 210                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 211         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 212                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 213         }
 214
 215         nesdev->currcq_count = 0;
 216
 217         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 218 }
 219
 220
 221 /**
 222  * nes_init_adapter - initialize adapter
 223  */
 224 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 225         struct nes_adapter *nesadapter = NULL;
 226         unsigned long num_pds;
 227         u32 u32temp;
 228         u32 port_count;
 229         u16 max_rq_wrs;
 230         u16 max_sq_wrs;
 231         u32 max_mr;
 232         u32 max_256pbl;
 233         u32 max_4kpbl;
 234         u32 max_qp;
 235         u32 max_irrq;
 236         u32 max_cq;
 237         u32 hte_index_mask;
 238         u32 adapter_size;
 239         u32 arp_table_size;
 240         u16 vendor_id;
 241         u8  OneG_Mode;
 242         u8  func_index;
 243
 244         /* search the list of existing adapters */
 245         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 246                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 247                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 248                                 nesdev->pcidev->devfn,
 249                                 PCI_SLOT(nesadapter->devfn),
 250                                 nesadapter->bus_number,
 251                                 PCI_SLOT(nesdev->pcidev->devfn),
 252                                 nesdev->pcidev->bus->number );
 253                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 254                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 255                         nesadapter->ref_count++;
 256                         return nesadapter;
 257                 }
 258         }
 259
 260         /* no adapter found */
 261         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 262         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 263                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 264                                 hw_rev);
 265                 return NULL;
 266         }
 267
 268         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 269                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 270                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 271                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 272                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 273
 274         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 275
 276
 277         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 278                 return NULL;
 279         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 280                 return NULL;
 281         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 282
 283         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 284         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 285
 286         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 287         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 288                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 289                                 max_qp, u32temp);
 290                 max_qp = (u32)1 << (u32temp & 0x001f);
 291         }
 292
 293         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 294         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 295                         max_qp, hte_index_mask);
 296
 297         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 298
 299         max_irrq = 1 << (u32temp & 0x001f);
 300
 301         if (max_qp > max_irrq) {
 302                 max_qp = max_irrq;
 303                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 304                                 max_qp);
 305         }
 306
 307         /* there should be no reason to allocate more pds than qps */
 308         if (num_pds > max_qp)
 309                 num_pds = max_qp;
 310
 311         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 312         max_mr = (u32)8192 << (u32temp & 0x7);
 313
 314         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 315         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 316         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 317         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 318
 319         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 320         arp_table_size = 1 << u32temp;
 321
 322         adapter_size = (sizeof(struct nes_adapter) +
 323                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 324         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 325         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 326         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 327         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 328         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 329         adapter_size += sizeof(struct nes_qp **) * max_qp;
 330
 331         /* allocate a new adapter struct */
 332         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 333         if (nesadapter == NULL) {
 334                 return NULL;
 335         }
 336
 337         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 338                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 339
 340         /* populate the new nesadapter */
 341         nesadapter->devfn = nesdev->pcidev->devfn;
 342         nesadapter->bus_number = nesdev->pcidev->bus->number;
 343         nesadapter->ref_count = 1;
 344         nesadapter->timer_int_req = 0xffff0000;
 345         nesadapter->OneG_Mode = OneG_Mode;
 346         nesadapter->doorbell_start = nesdev->doorbell_region;
 347
 348         /* nesadapter->tick_delta = clk_divisor; */
 349         nesadapter->hw_rev = hw_rev;
 350         nesadapter->port_count = port_count;
 351
 352         nesadapter->max_qp = max_qp;
 353         nesadapter->hte_index_mask = hte_index_mask;
 354         nesadapter->max_irrq = max_irrq;
 355         nesadapter->max_mr = max_mr;
 356         nesadapter->max_256pbl = max_256pbl - 1;
 357         nesadapter->max_4kpbl = max_4kpbl - 1;
 358         nesadapter->max_cq = max_cq;
 359         nesadapter->free_256pbl = max_256pbl - 1;
 360         nesadapter->free_4kpbl = max_4kpbl - 1;
 361         nesadapter->max_pd = num_pds;
 362         nesadapter->arp_table_size = arp_table_size;
 363
 364         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 365         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 366                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 367                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 368                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 369         } else {
 370                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 371                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 372                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 373                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __FUNCTION__);
 374         }
 375         /* Setup and enable the periodic timer */
 376         if (nesadapter->et_rx_coalesce_usecs_irq)
 377                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 378                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 379         else
 380                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 381
 382         nesadapter->base_pd = 1;
 383
 384         nesadapter->device_cap_flags =
 385                         IB_DEVICE_ZERO_STAG | IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW;
 386
 387         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 388                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 389         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 390         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 391         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 392         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 393         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 394
 395
 396         /* mark the usual suspect QPs and CQs as in use */
 397         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 398                 set_bit(u32temp, nesadapter->allocated_qps);
 399                 set_bit(u32temp, nesadapter->allocated_cqs);
 400         }
 401
 402         for (u32temp = 0; u32temp < 20; u32temp++)
 403                 set_bit(u32temp, nesadapter->allocated_pds);
 404         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 405
 406         max_rq_wrs = ((u32temp >> 8) & 3);
 407         switch (max_rq_wrs) {
 408                 case 0:
 409                         max_rq_wrs = 4;
 410                         break;
 411                 case 1:
 412                         max_rq_wrs = 16;
 413                         break;
 414                 case 2:
 415                         max_rq_wrs = 32;
 416                         break;
 417                 case 3:
 418                         max_rq_wrs = 512;
 419                         break;
 420         }
 421
 422         max_sq_wrs = (u32temp & 3);
 423         switch (max_sq_wrs) {
 424                 case 0:
 425                         max_sq_wrs = 4;
 426                         break;
 427                 case 1:
 428                         max_sq_wrs = 16;
 429                         break;
 430                 case 2:
 431                         max_sq_wrs = 32;
 432                         break;
 433                 case 3:
 434                         max_sq_wrs = 512;
 435                         break;
 436         }
 437         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 438         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 439
 440         nesadapter->max_sge = 4;
 441         nesadapter->max_cqe = 32767;
 442
 443         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 444                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 445                 kfree(nesadapter);
 446                 return NULL;
 447         }
 448
 449         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 450         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 451                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 452
 453         /* setup port configuration */
 454         if (nesadapter->port_count == 1) {
 455                 u32temp = 0x00000000;
 456                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 457                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 458                 else
 459                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 460         } else {
 461                 if (nesadapter->port_count == 2)
 462                         u32temp = 0x00000044;
 463                 else
 464                         u32temp = 0x000000e4;
 465                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 466         }
 467
 468         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 469         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 470                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 471
 472         spin_lock_init(&nesadapter->resource_lock);
 473         spin_lock_init(&nesadapter->phy_lock);
 474         spin_lock_init(&nesadapter->pbl_lock);
 475         spin_lock_init(&nesadapter->periodic_timer_lock);
 476
 477         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 478         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 479         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 480         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 481
 482         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 483                 u32 pcs_control_status0, pcs_control_status1;
 484                 u32 reset_value;
 485                 u32 i = 0;
 486                 u32 int_cnt = 0;
 487                 u32 ext_cnt = 0;
 488                 unsigned long flags;
 489                 u32 j = 0;
 490
 491                 pcs_control_status0 = nes_read_indexed(nesdev,
 492                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 493                 pcs_control_status1 = nes_read_indexed(nesdev,
 494                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 495
 496                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 497                         pcs_control_status0 = nes_read_indexed(nesdev,
 498                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 499                         pcs_control_status1 = nes_read_indexed(nesdev,
 500                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 501                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 502                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 503                                 int_cnt++;
 504                         msleep(1);
 505                 }
 506                 if (int_cnt > 1) {
 507                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 508                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 509                         mh_detected++;
 510                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 511                         reset_value |= 0x0000003d;
 512                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 513
 514                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 515                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 516                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 517
 518                         pcs_control_status0 = nes_read_indexed(nesdev,
 519                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 520                         pcs_control_status1 = nes_read_indexed(nesdev,
 521                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 522
 523                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 524                                 pcs_control_status0 = nes_read_indexed(nesdev,
 525                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 526                                 pcs_control_status1 = nes_read_indexed(nesdev,
 527                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 528                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 529                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 530                                         if (++ext_cnt > int_cnt) {
 531                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 532                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 533                                                                 0x0000F0C8);
 534                                                 mh_detected++;
 535                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 536                                                 reset_value |= 0x0000003d;
 537                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 538
 539                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 540                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 541                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 542                                                 break;
 543                                         }
 544                                 }
 545                                 msleep(1);
 546                         }
 547                 }
 548         }
 549
 550         if (nesadapter->hw_rev == NE020_REV) {
 551                 init_timer(&nesadapter->mh_timer);
 552                 nesadapter->mh_timer.function = nes_mh_fix;
 553                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 554                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 555                 add_timer(&nesadapter->mh_timer);
 556         } else {
 557                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 558         }
 559
 560         init_timer(&nesadapter->lc_timer);
 561         nesadapter->lc_timer.function = nes_clc;
 562         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 563         nesadapter->lc_timer.data = (unsigned long)nesdev;
 564         add_timer(&nesadapter->lc_timer);
 565
 566         list_add_tail(&nesadapter->list, &nes_adapter_list);
 567
 568         for (func_index = 0; func_index < 8; func_index++) {
 569                 pci_bus_read_config_word(nesdev->pcidev->bus,
 570                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 571                                         func_index), 0, &vendor_id);
 572                 if (vendor_id == 0xffff)
 573                         break;
 574         }
 575         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __FUNCTION__,
 576                 func_index, pci_name(nesdev->pcidev));
 577         nesadapter->adapter_fcn_count = func_index;
 578
 579         return nesadapter;
 580 }
 581
 582
 583 /**
 584  * nes_reset_adapter_ne020
 585  */
 586 unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 587 {
 588         u32 port_count;
 589         u32 u32temp;
 590         u32 i;
 591
 592         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 593         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 594         /* TODO: assuming that both SERDES are set the same for now */
 595         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 596         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 597                         u32temp, port_count);
 598         if (*OneG_Mode)
 599                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 600         u32temp &= 0xff00ffc0;
 601         switch (port_count) {
 602                 case 1:
 603                         u32temp |= 0x00ee0000;
 604                         break;
 605                 case 2:
 606                         u32temp |= 0x00cc0000;
 607                         break;
 608                 case 4:
 609                         u32temp |= 0x00000000;
 610                         break;
 611                 default:
 612                         return 0;
 613                         break;
 614         }
 615
 616         /* check and do full reset if needed */
 617         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 618                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 619                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 620
 621                 i = 0;
 622                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 623                         mdelay(1);
 624                 if (i >= 10000) {
 625                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 626                         return 0;
 627                 }
 628         }
 629
 630         /* port reset */
 631         switch (port_count) {
 632                 case 1:
 633                         u32temp |= 0x00ee0010;
 634                         break;
 635                 case 2:
 636                         u32temp |= 0x00cc0030;
 637                         break;
 638                 case 4:
 639                         u32temp |= 0x00000030;
 640                         break;
 641         }
 642
 643         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 644         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 645
 646         i = 0;
 647         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 648                 mdelay(1);
 649         if (i >= 10000) {
 650                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 651                 return 0;
 652         }
 653
 654         /* serdes 0 */
 655         i = 0;
 656         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 657                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 658                 mdelay(1);
 659         if (i >= 5000) {
 660                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 661                 return 0;
 662         }
 663
 664         /* serdes 1 */
 665         if (port_count > 1) {
 666                 i = 0;
 667                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 668                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 669                         mdelay(1);
 670                 if (i >= 5000) {
 671                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 672                         return 0;
 673                 }
 674         }
 675
 676
 677
 678         i = 0;
 679         while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 680                 mdelay(1);
 681         if (i >= 10000) {
 682                 printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 683                                 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 684                 return 0;
 685         }
 686
 687         return port_count;
 688 }
 689
 690
 691 /**
 692  * nes_init_serdes
 693  */
 694 int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count, u8  OneG_Mode)
 695 {
 696         int i;
 697         u32 u32temp;
 698
 699         if (hw_rev != NE020_REV) {
 700                 /* init serdes 0 */
 701
 702                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 703                 if (!OneG_Mode)
 704                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 705                 if (port_count > 1) {
 706                         /* init serdes 1 */
 707                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 708                         if (!OneG_Mode)
 709                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 710                         }
 711         } else {
 712                 /* init serdes 0 */
 713                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 714                 i = 0;
 715                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 716                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 717                         mdelay(1);
 718                 if (i >= 5000) {
 719                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 720                         return 1;
 721                 }
 722                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 723                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 724                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 725                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 726                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 727                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 728                 if (OneG_Mode)
 729                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 730                 else
 731                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 732
 733                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 734                 if (port_count > 1) {
 735                         /* init serdes 1 */
 736                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 737                         i = 0;
 738                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 739                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 740                                 mdelay(1);
 741                         if (i >= 5000) {
 742                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __FUNCTION__, u32temp);
 743                                 /* return 1; */
 744                         }
 745                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 746                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 747                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 748                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 749                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 750                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 751                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 752                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 753                 }
 754         }
 755         return 0;
 756 }
 757
 758
 759 /**
 760  * nes_init_csr_ne020
 761  * Initialize registers for ne020 hardware
 762  */
 763 void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 764 {
 765         u32 u32temp;
 766
 767         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 768
 769         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 770         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 771         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 772         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 773         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 774         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 775         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 776         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 777
 778         /* TODO: move these MAC register settings to NIC bringup */
 779         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 780         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 781         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 782         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 783         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 784         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 785         if (port_count > 1) {
 786                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 787                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 788                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 789                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 790                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 791                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 792                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 793         }
 794         if (port_count > 2) {
 795                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 796                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 797                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 798                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 799                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 800                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 801                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 802
 803                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 804                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 805                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 806                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 807                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 808                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 809                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 810         }
 811
 812         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 813         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 814         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 815         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 816         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 817         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 818         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 819         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 820
 821         /* TODO: move this to code, get from EEPROM */
 822         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 823         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 824         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 825                                                                                                                 //
 826         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 827         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 828
 829         if (hw_rev != NE020_REV) {
 830                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 831                 u32temp |= 0x80000000;
 832                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 833                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 834                 u32temp &= 0x7fffffff;
 835                 u32temp |= 0x7fff0010;
 836                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 837         }
 838 }
 839
 840
 841 /**
 842  * nes_destroy_adapter - destroy the adapter structure
 843  */
 844 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 845 {
 846         struct nes_adapter *tmp_adapter;
 847
 848         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 849                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 850                                 tmp_adapter);
 851         }
 852
 853         nesadapter->ref_count--;
 854         if (!nesadapter->ref_count) {
 855                 if (nesadapter->hw_rev == NE020_REV) {
 856                         del_timer(&nesadapter->mh_timer);
 857                 }
 858                 del_timer(&nesadapter->lc_timer);
 859
 860                 list_del(&nesadapter->list);
 861                 kfree(nesadapter);
 862         }
 863 }
 864
 865
 866 /**
 867  * nes_init_cqp
 868  */
 869 int nes_init_cqp(struct nes_device *nesdev)
 870 {
 871         struct nes_adapter *nesadapter = nesdev->nesadapter;
 872         struct nes_hw_cqp_qp_context *cqp_qp_context;
 873         struct nes_hw_cqp_wqe *cqp_wqe;
 874         struct nes_hw_ceq *ceq;
 875         struct nes_hw_ceq *nic_ceq;
 876         struct nes_hw_aeq *aeq;
 877         void *vmem;
 878         dma_addr_t pmem;
 879         u32 count=0;
 880         u32 cqp_head;
 881         u64 u64temp;
 882         u32 u32temp;
 883
 884         /* allocate CQP memory */
 885         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 886         /* SQ is 512 byte aligned, others are 256 byte aligned */
 887         nesdev->cqp_mem_size = 512 +
 888                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 889                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 890                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 891                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 892                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 893                         sizeof(struct nes_hw_cqp_qp_context);
 894
 895         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 896                         &nesdev->cqp_pbase);
 897         if (!nesdev->cqp_vbase) {
 898                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 899                 return -ENOMEM;
 900         }
 901         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 902
 903         /* Allocate a twice the number of CQP requests as the SQ size */
 904         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 905                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 906         if (nesdev->nes_cqp_requests == NULL) {
 907                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 908                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 909                                 nesdev->cqp.sq_pbase);
 910                 return -ENOMEM;
 911         }
 912
 913         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 914                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 915
 916         spin_lock_init(&nesdev->cqp.lock);
 917         init_waitqueue_head(&nesdev->cqp.waitq);
 918
 919         /* Setup Various Structures */
 920         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 921                         ~(unsigned long)(512 - 1));
 922         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 923                         ~(unsigned long long)(512 - 1));
 924
 925         nesdev->cqp.sq_vbase = vmem;
 926         nesdev->cqp.sq_pbase = pmem;
 927         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 928         nesdev->cqp.sq_head = 0;
 929         nesdev->cqp.sq_tail = 0;
 930         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 931
 932         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 933         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 934
 935         nesdev->ccq.cq_vbase = vmem;
 936         nesdev->ccq.cq_pbase = pmem;
 937         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 938         nesdev->ccq.cq_head = 0;
 939         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 940         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 941
 942         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 943         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 944
 945         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 946         ceq = &nesadapter->ceq[nesdev->ceq_index];
 947         ceq->ceq_vbase = vmem;
 948         ceq->ceq_pbase = pmem;
 949         ceq->ceq_size = NES_CCEQ_SIZE;
 950         ceq->ceq_head = 0;
 951
 952         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 953         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 954
 955         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 956         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 957         nic_ceq->ceq_vbase = vmem;
 958         nic_ceq->ceq_pbase = pmem;
 959         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 960         nic_ceq->ceq_head = 0;
 961
 962         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 963         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 964
 965         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 966         aeq->aeq_vbase = vmem;
 967         aeq->aeq_pbase = pmem;
 968         aeq->aeq_size = nesadapter->max_qp;
 969         aeq->aeq_head = 0;
 970
 971         /* Setup QP Context */
 972         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 973         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 974
 975         cqp_qp_context = vmem;
 976         cqp_qp_context->context_words[0] =
 977                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 978         cqp_qp_context->context_words[1] = 0;
 979         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 980         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 981
 982
 983         /* Write the address to Create CQP */
 984         if ((sizeof(dma_addr_t) > 4)) {
 985                 nes_write_indexed(nesdev,
 986                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 987                                 ((u64)pmem) >> 32);
 988         } else {
 989                 nes_write_indexed(nesdev,
 990                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
 991         }
 992         nes_write_indexed(nesdev,
 993                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 994                         (u32)pmem);
 995
 996         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
 997         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
 998
 999         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1000                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1001                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1002         }
1003
1004         /* Write Create CCQ WQE */
1005         cqp_head = nesdev->cqp.sq_head++;
1006         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1007         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1008         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1009                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1010                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1011         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1012                             (nesdev->ccq.cq_number |
1013                              ((u32)nesdev->ceq_index << 16)));
1014         u64temp = (u64)nesdev->ccq.cq_pbase;
1015         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1016         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1017         u64temp = (unsigned long)&nesdev->ccq;
1018         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1019                         cpu_to_le32((u32)(u64temp >> 1));
1020         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1021                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1022         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1023
1024         /* Write Create CEQ WQE */
1025         cqp_head = nesdev->cqp.sq_head++;
1026         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1027         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1028         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1029                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1030         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1031         u64temp = (u64)ceq->ceq_pbase;
1032         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1033
1034         /* Write Create AEQ WQE */
1035         cqp_head = nesdev->cqp.sq_head++;
1036         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1037         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1038         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1039                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1040         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1041         u64temp = (u64)aeq->aeq_pbase;
1042         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1043
1044         /* Write Create NIC CEQ WQE */
1045         cqp_head = nesdev->cqp.sq_head++;
1046         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1047         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1048         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1049                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1050         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1051         u64temp = (u64)nic_ceq->ceq_pbase;
1052         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1053
1054         /* Poll until CCQP done */
1055         count = 0;
1056         do {
1057                 if (count++ > 1000) {
1058                         printk(KERN_ERR PFX "Error creating CQP\n");
1059                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1060                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1061                         return -1;
1062                 }
1063                 udelay(10);
1064         } while (!(nes_read_indexed(nesdev,
1065                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1066
1067         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1068                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1069
1070         u32temp = 0x04800000;
1071         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1072
1073         /* wait for the CCQ, CEQ, and AEQ to get created */
1074         count = 0;
1075         do {
1076                 if (count++ > 1000) {
1077                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1078                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1079                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1080                         return -1;
1081                 }
1082                 udelay(10);
1083         } while (((nes_read_indexed(nesdev,
1084                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1085
1086         /* dump the QP status value */
1087         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1088                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1089
1090         nesdev->cqp.sq_tail++;
1091
1092         return 0;
1093 }
1094
1095
1096 /**
1097  * nes_destroy_cqp
1098  */
1099 int nes_destroy_cqp(struct nes_device *nesdev)
1100 {
1101         struct nes_hw_cqp_wqe *cqp_wqe;
1102         u32 count = 0;
1103         u32 cqp_head;
1104         unsigned long flags;
1105
1106         do {
1107                 if (count++ > 1000)
1108                         break;
1109                 udelay(10);
1110         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1111
1112         /* Reset CCQ */
1113         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1114                         nesdev->ccq.cq_number);
1115
1116         /* Disable device interrupts */
1117         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1118
1119         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1120
1121         /* Destroy the AEQ */
1122         cqp_head = nesdev->cqp.sq_head++;
1123         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1124         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1125         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1126                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1127         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1128
1129         /* Destroy the NIC CEQ */
1130         cqp_head = nesdev->cqp.sq_head++;
1131         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1132         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1133         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1134                         ((u32)nesdev->nic_ceq_index << 8));
1135
1136         /* Destroy the CEQ */
1137         cqp_head = nesdev->cqp.sq_head++;
1138         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1139         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1140         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1141                         (nesdev->ceq_index << 8));
1142
1143         /* Destroy the CCQ */
1144         cqp_head = nesdev->cqp.sq_head++;
1145         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1146         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1147         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1148         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1149                         ((u32)nesdev->ceq_index << 16));
1150
1151         /* Destroy CQP */
1152         cqp_head = nesdev->cqp.sq_head++;
1153         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1154         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1155         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1156                         NES_CQP_QP_TYPE_CQP);
1157         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1158
1159         barrier();
1160         /* Ring doorbell (5 WQEs) */
1161         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1162
1163         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1164
1165         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1166         count = 0;
1167         do {
1168                 if (count++ > 1000) {
1169                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1170                                         PCI_FUNC(nesdev->pcidev->devfn));
1171                         break;
1172                 }
1173                 udelay(10);
1174         } while (((nes_read_indexed(nesdev,
1175                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1176
1177         /* dump the QP status value */
1178         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1179                         PCI_FUNC(nesdev->pcidev->devfn),
1180                         nes_read_indexed(nesdev,
1181                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1182
1183         kfree(nesdev->nes_cqp_requests);
1184
1185         /* Free the control structures */
1186         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1187                         nesdev->cqp.sq_pbase);
1188
1189         return 0;
1190 }
1191
1192
1193 /**
1194  * nes_init_phy
1195  */
1196 int nes_init_phy(struct nes_device *nesdev)
1197 {
1198         struct nes_adapter *nesadapter = nesdev->nesadapter;
1199         u32 counter = 0;
1200         u32 mac_index = nesdev->mac_index;
1201         u32 tx_config;
1202         u16 phy_data;
1203
1204         if (nesadapter->OneG_Mode) {
1205                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1206                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1207                         printk(PFX "%s: Programming mdc config for 1G\n", __FUNCTION__);
1208                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1209                         tx_config |= 0x04;
1210                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1211                 }
1212
1213                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1214                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1215                                 nesadapter->phy_index[mac_index], phy_data);
1216                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index],  0xb000);
1217
1218                 /* Reset the PHY */
1219                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1220                 udelay(100);
1221                 counter = 0;
1222                 do {
1223                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1224                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1225                         if (counter++ > 100) break;
1226                 } while (phy_data & 0x8000);
1227
1228                 /* Setting no phy loopback */
1229                 phy_data &= 0xbfff;
1230                 phy_data |= 0x1140;
1231                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1232                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1233                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1234
1235                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1236                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1237
1238                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1239                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1240
1241                 /* Setting the interrupt mask */
1242                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1243                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1244                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1245
1246                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1247                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1248
1249                 /* turning on flow control */
1250                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1251                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1252                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1253                                 (phy_data & ~(0x03E0)) | 0xc00);
1254                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1255                                 phy_data | 0xc00); */
1256                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1257                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1258
1259                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1260                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1261                 /* Clear Half duplex */
1262                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1263                                 phy_data & ~(0x0100));
1264                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1265                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1266
1267                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1268                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1269         } else {
1270                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
1271                         /* setup 10G MDIO operation */
1272                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1273                         tx_config |= 0x14;
1274                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1275                 }
1276         }
1277         return 0;
1278 }
1279
1280
1281 /**
1282  * nes_replenish_nic_rq
1283  */
1284 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1285 {
1286         unsigned long flags;
1287         dma_addr_t bus_address;
1288         struct sk_buff *skb;
1289         struct nes_hw_nic_rq_wqe *nic_rqe;
1290         struct nes_hw_nic *nesnic;
1291         struct nes_device *nesdev;
1292         u32 rx_wqes_posted = 0;
1293
1294         nesnic = &nesvnic->nic;
1295         nesdev = nesvnic->nesdev;
1296         spin_lock_irqsave(&nesnic->rq_lock, flags);
1297         if (nesnic->replenishing_rq !=0) {
1298                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1299                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1300                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1301                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1302                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1303                         add_timer(&nesvnic->rq_wqes_timer);
1304                 } else
1305                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1306                 return;
1307         }
1308         nesnic->replenishing_rq = 1;
1309         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1310         do {
1311                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1312                 if (skb) {
1313                         skb->dev = nesvnic->netdev;
1314
1315                         bus_address = pci_map_single(nesdev->pcidev,
1316                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1317
1318                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1319                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1320                                         cpu_to_le32(nesvnic->max_frame_size);
1321                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1322                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1323                                         cpu_to_le32((u32)bus_address);
1324                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1325                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1326                         nesnic->rx_skb[nesnic->rq_head] = skb;
1327                         nesnic->rq_head++;
1328                         nesnic->rq_head &= nesnic->rq_size - 1;
1329                         atomic_dec(&nesvnic->rx_skbs_needed);
1330                         barrier();
1331                         if (++rx_wqes_posted == 255) {
1332                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1333                                 rx_wqes_posted = 0;
1334                         }
1335                 } else {
1336                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1337                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1338                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1339                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1340                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1341                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1342                                 add_timer(&nesvnic->rq_wqes_timer);
1343                         } else
1344                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1345                         break;
1346                 }
1347         } while (atomic_read(&nesvnic->rx_skbs_needed));
1348         barrier();
1349         if (rx_wqes_posted)
1350                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1351         nesnic->replenishing_rq = 0;
1352 }
1353
1354
1355 /**
1356  * nes_rq_wqes_timeout
1357  */
1358 static void nes_rq_wqes_timeout(unsigned long parm)
1359 {
1360         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1361         printk("%s: Timer fired.\n", __FUNCTION__);
1362         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1363         if (atomic_read(&nesvnic->rx_skbs_needed))
1364                 nes_replenish_nic_rq(nesvnic);
1365 }
1366
1367
1368 /**
1369  * nes_init_nic_qp
1370  */
1371 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1372 {
1373         struct nes_hw_cqp_wqe *cqp_wqe;
1374         struct nes_hw_nic_sq_wqe *nic_sqe;
1375         struct nes_hw_nic_qp_context *nic_context;
1376         struct sk_buff *skb;
1377         struct nes_hw_nic_rq_wqe *nic_rqe;
1378         struct nes_vnic *nesvnic = netdev_priv(netdev);
1379         unsigned long flags;
1380         void *vmem;
1381         dma_addr_t pmem;
1382         u64 u64temp;
1383         int ret;
1384         u32 cqp_head;
1385         u32 counter;
1386         u32 wqe_count;
1387         u8 jumbomode=0;
1388
1389         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1390         nesvnic->nic_mem_size = 256 +
1391                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1392                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1393                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1394                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1395                         sizeof(struct nes_hw_nic_qp_context);
1396
1397         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1398                         &nesvnic->nic_pbase);
1399         if (!nesvnic->nic_vbase) {
1400                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1401                 return -ENOMEM;
1402         }
1403         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1404         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1405                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1406
1407         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1408                         ~(unsigned long)(256 - 1));
1409         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1410                         ~(unsigned long long)(256 - 1));
1411
1412         /* Setup the first Fragment buffers */
1413         nesvnic->nic.first_frag_vbase = vmem;
1414
1415         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1416                 nesvnic->nic.frag_paddr[counter] = pmem;
1417                 pmem += sizeof(struct nes_first_frag);
1418         }
1419
1420         /* setup the SQ */
1421         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1422
1423         nesvnic->nic.sq_vbase = (void *)vmem;
1424         nesvnic->nic.sq_pbase = pmem;
1425         nesvnic->nic.sq_head = 0;
1426         nesvnic->nic.sq_tail = 0;
1427         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1428         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1429                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1430                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1431                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1432                                 NES_NIC_SQ_WQE_COMPLETION);
1433                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1434                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1435                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1436                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1437                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1438                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1439         }
1440
1441         nesvnic->get_cqp_request = nes_get_cqp_request;
1442         nesvnic->post_cqp_request = nes_post_cqp_request;
1443         nesvnic->mcrq_mcast_filter = NULL;
1444
1445         spin_lock_init(&nesvnic->nic.sq_lock);
1446         spin_lock_init(&nesvnic->nic.rq_lock);
1447
1448         /* setup the RQ */
1449         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1450         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1451
1452
1453         nesvnic->nic.rq_vbase = vmem;
1454         nesvnic->nic.rq_pbase = pmem;
1455         nesvnic->nic.rq_head = 0;
1456         nesvnic->nic.rq_tail = 0;
1457         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1458
1459         /* setup the CQ */
1460         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1461         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1462
1463         if (nesdev->nesadapter->netdev_count > 2)
1464                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1465         else
1466                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1467
1468         nesvnic->nic_cq.cq_vbase = vmem;
1469         nesvnic->nic_cq.cq_pbase = pmem;
1470         nesvnic->nic_cq.cq_head = 0;
1471         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1472
1473         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1474
1475         /* Send CreateCQ request to CQP */
1476         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1477         cqp_head = nesdev->cqp.sq_head;
1478
1479         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1480         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1481
1482         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1483                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1484                         ((u32)nesvnic->nic_cq.cq_size << 16));
1485         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1486                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1487         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1488         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1489         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1490         u64temp = (unsigned long)&nesvnic->nic_cq;
1491         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1492         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1493                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1494         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1495         if (++cqp_head >= nesdev->cqp.sq_size)
1496                 cqp_head = 0;
1497         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1498         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1499
1500         /* Send CreateQP request to CQP */
1501         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1502         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1503                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1504                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1505         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1506                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1507                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1508         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1509                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1510         }
1511
1512         u64temp = (u64)nesvnic->nic.sq_pbase;
1513         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1514         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1515         u64temp = (u64)nesvnic->nic.rq_pbase;
1516         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1517         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1518
1519         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1520                         NES_CQP_QP_TYPE_NIC);
1521         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1522         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1523                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1524         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1525
1526         if (++cqp_head >= nesdev->cqp.sq_size)
1527                 cqp_head = 0;
1528         nesdev->cqp.sq_head = cqp_head;
1529
1530         barrier();
1531
1532         /* Ring doorbell (2 WQEs) */
1533         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1534
1535         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1536         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1537                         nesvnic->nic.qp_id);
1538
1539         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1540                         NES_EVENT_TIMEOUT);
1541         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1542                         nesvnic->nic.qp_id, ret);
1543         if (!ret) {
1544                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1545                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1546                                 nesvnic->nic_pbase);
1547                 return -EIO;
1548         }
1549
1550         /* Populate the RQ */
1551         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1552                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1553                 if (!skb) {
1554                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1555
1556                         nes_destroy_nic_qp(nesvnic);
1557                         return -ENOMEM;
1558                 }
1559
1560                 skb->dev = netdev;
1561
1562                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1563                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1564
1565                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1566                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1567                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1568                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1569                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1570                 nesvnic->nic.rx_skb[counter] = skb;
1571         }
1572
1573         wqe_count = NES_NIC_WQ_SIZE - 1;
1574         nesvnic->nic.rq_head = wqe_count;
1575         barrier();
1576         do {
1577                 counter = min(wqe_count, ((u32)255));
1578                 wqe_count -= counter;
1579                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1580         } while (wqe_count);
1581         init_timer(&nesvnic->rq_wqes_timer);
1582         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1583         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1584         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1585
1586         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1587         {
1588                 nes_nic_init_timer(nesdev);
1589                 if (netdev->mtu > 1500)
1590                         jumbomode = 1;
1591                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1592         }
1593
1594         return 0;
1595 }
1596
1597
1598 /**
1599  * nes_destroy_nic_qp
1600  */
1601 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1602 {
1603         struct nes_device *nesdev = nesvnic->nesdev;
1604         struct nes_hw_cqp_wqe *cqp_wqe;
1605         struct nes_hw_nic_rq_wqe *nic_rqe;
1606         u64 wqe_frag;
1607         u32 cqp_head;
1608         unsigned long flags;
1609         int ret;
1610
1611         /* Free remaining NIC receive buffers */
1612         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1613                 nic_rqe = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1614                 wqe_frag = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1615                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1616                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1617                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1618                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1619                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1620         }
1621
1622         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1623
1624         /* Destroy NIC QP */
1625         cqp_head = nesdev->cqp.sq_head;
1626         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1627         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1628
1629         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1630                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1631         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1632                 nesvnic->nic.qp_id);
1633
1634         if (++cqp_head >= nesdev->cqp.sq_size)
1635                 cqp_head = 0;
1636
1637         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1638
1639         /* Destroy NIC CQ */
1640         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1641         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1642                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1643         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1644                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1645
1646         if (++cqp_head >= nesdev->cqp.sq_size)
1647                 cqp_head = 0;
1648
1649         nesdev->cqp.sq_head = cqp_head;
1650         barrier();
1651
1652         /* Ring doorbell (2 WQEs) */
1653         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1654
1655         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1656         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1657                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1658                         cqp_head, nesdev->cqp.sq_head,
1659                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1660
1661         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1662                         NES_EVENT_TIMEOUT);
1663
1664         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1665                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1666                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1667         if (!ret) {
1668                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1669                                 nesvnic->nic.qp_id);
1670         }
1671
1672         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1673                         nesvnic->nic_pbase);
1674 }
1675
1676 /**
1677  * nes_napi_isr
1678  */
1679 int nes_napi_isr(struct nes_device *nesdev)
1680 {
1681         struct nes_adapter *nesadapter = nesdev->nesadapter;
1682         u32 int_stat;
1683
1684         if (nesdev->napi_isr_ran) {
1685                 /* interrupt status has already been read in ISR */
1686                 int_stat = nesdev->int_stat;
1687         } else {
1688                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1689                 nesdev->int_stat = int_stat;
1690                 nesdev->napi_isr_ran = 1;
1691         }
1692
1693         int_stat &= nesdev->int_req;
1694         /* iff NIC, process here, else wait for DPC */
1695         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1696                 nesdev->napi_isr_ran = 0;
1697                 nes_write32(nesdev->regs+NES_INT_STAT,
1698                                 (int_stat &
1699                                 ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1700
1701                 /* Process the CEQs */
1702                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1703
1704                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1705                                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1706                                           ((nesadapter->et_use_adaptive_rx_coalesce) &&
1707                                            (nesdev->deepcq_count > nesadapter->et_pkt_rate_low)))) ) {
1708                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1709                                 /* Enable Periodic timer interrupts */
1710                                 nesdev->int_req |= NES_INT_TIMER;
1711                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1712                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1713                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1714                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1715                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1716                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1717                         }
1718
1719                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1720                         {
1721                                 nes_nic_init_timer(nesdev);
1722                         }
1723                         /* Enable interrupts, except CEQs */
1724                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1725                 } else {
1726                         /* Enable interrupts, make sure timer is off */
1727                         nesdev->int_req &= ~NES_INT_TIMER;
1728                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1729                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1730                 }
1731                 nesdev->deepcq_count = 0;
1732                 return 1;
1733         } else {
1734                 return 0;
1735         }
1736 }
1737
1738
1739 /**
1740  * nes_dpc
1741  */
1742 void nes_dpc(unsigned long param)
1743 {
1744         struct nes_device *nesdev = (struct nes_device *)param;
1745         struct nes_adapter *nesadapter = nesdev->nesadapter;
1746         u32 counter;
1747         u32 loop_counter = 0;
1748         u32 int_status_bit;
1749         u32 int_stat;
1750         u32 timer_stat;
1751         u32 temp_int_stat;
1752         u32 intf_int_stat;
1753         u32 debug_error;
1754         u32 processed_intf_int = 0;
1755         u16 processed_timer_int = 0;
1756         u16 completion_ints = 0;
1757         u16 timer_ints = 0;
1758
1759         /* nes_debug(NES_DBG_ISR, "\n"); */
1760
1761         do {
1762                 timer_stat = 0;
1763                 if (nesdev->napi_isr_ran) {
1764                         nesdev->napi_isr_ran = 0;
1765                         int_stat = nesdev->int_stat;
1766                 } else
1767                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1768                 if (processed_intf_int != 0)
1769                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1770                 else
1771                         int_stat &= nesdev->int_req;
1772                 if (processed_timer_int == 0) {
1773                         processed_timer_int = 1;
1774                         if (int_stat & NES_INT_TIMER) {
1775                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1776                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1777                                         int_stat &= ~NES_INT_TIMER;
1778                                 }
1779                         }
1780                 } else {
1781                         int_stat &= ~NES_INT_TIMER;
1782                 }
1783
1784                 if (int_stat) {
1785                         if (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1786                                         NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)) {
1787                                 /* Ack the interrupts */
1788                                 nes_write32(nesdev->regs+NES_INT_STAT,
1789                                                 (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1790                                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1791                         }
1792
1793                         temp_int_stat = int_stat;
1794                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1795                                 if (int_stat & int_status_bit) {
1796                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1797                                         temp_int_stat &= ~int_status_bit;
1798                                         completion_ints = 1;
1799                                 }
1800                                 if (!(temp_int_stat & 0x0000ffff))
1801                                         break;
1802                                 int_status_bit <<= 1;
1803                         }
1804
1805                         /* Process the AEQ for this pci function */
1806                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1807                         if (int_stat & int_status_bit) {
1808                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1809                         }
1810
1811                         /* Process the MAC interrupt for this pci function */
1812                         int_status_bit = 1 << (24 + nesdev->mac_index);
1813                         if (int_stat & int_status_bit) {
1814                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1815                         }
1816
1817                         if (int_stat & NES_INT_TIMER) {
1818                                 if (timer_stat & nesdev->timer_int_req) {
1819                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1820                                                         (timer_stat & nesdev->timer_int_req) |
1821                                                         ~(nesdev->nesadapter->timer_int_req));
1822                                         timer_ints = 1;
1823                                 }
1824                         }
1825
1826                         if (int_stat & NES_INT_INTF) {
1827                                 processed_intf_int = 1;
1828                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1829                                 intf_int_stat &= nesdev->intf_int_req;
1830                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1831                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1832                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1833                                                         (u16)debug_error);
1834                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1835                                                         0x01010000 | (debug_error & 0x0000ffff));
1836                                         /* BUG(); */
1837                                         if (crit_err_count++ > 10)
1838                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1839                                 }
1840                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1841                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1842                                         BUG();
1843                                 }
1844                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
1845                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
1846                                         BUG();
1847                                 }
1848                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
1849                         }
1850
1851                         if (int_stat & NES_INT_TSW) {
1852                         }
1853                 }
1854                 /* Don't use the interface interrupt bit stay in loop */
1855                 int_stat &= ~NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1856                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3;
1857         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
1858
1859         if (timer_ints == 1) {
1860                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
1861                         if (completion_ints == 0) {
1862                                 nesdev->timer_only_int_count++;
1863                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
1864                                         nesdev->timer_only_int_count = 0;
1865                                         nesdev->int_req &= ~NES_INT_TIMER;
1866                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1867                                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1868                                 } else {
1869                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1870                                 }
1871                         } else {
1872                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1873                                 {
1874                                         nes_nic_init_timer(nesdev);
1875                                 }
1876                                 nesdev->timer_only_int_count = 0;
1877                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1878                         }
1879                 } else {
1880                         nesdev->timer_only_int_count = 0;
1881                         nesdev->int_req &= ~NES_INT_TIMER;
1882                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1883                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1884                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1885                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1886                 }
1887         } else {
1888                 if ( (completion_ints == 1) &&
1889                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
1890                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1891                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
1892                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
1893                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
1894                         nesdev->timer_only_int_count = 0;
1895                         nesdev->int_req |= NES_INT_TIMER;
1896                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1897                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1898                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1899                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1900                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1901                 } else {
1902                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1903                 }
1904         }
1905         nesdev->deepcq_count = 0;
1906 }
1907
1908
1909 /**
1910  * nes_process_ceq
1911  */
1912 void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
1913 {
1914         u64 u64temp;
1915         struct nes_hw_cq *cq;
1916         u32 head;
1917         u32 ceq_size;
1918
1919         /* nes_debug(NES_DBG_CQ, "\n"); */
1920         head = ceq->ceq_head;
1921         ceq_size = ceq->ceq_size;
1922
1923         do {
1924                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
1925                                 NES_CEQE_VALID) {
1926                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX])))<<32) |
1927                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
1928                         u64temp <<= 1;
1929                         cq = *((struct nes_hw_cq **)&u64temp);
1930                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
1931                         barrier();
1932                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
1933
1934                         /* call the event handler */
1935                         cq->ce_handler(nesdev, cq);
1936
1937                         if (++head >= ceq_size)
1938                                 head = 0;
1939                 } else {
1940                         break;
1941                 }
1942
1943         } while (1);
1944
1945         ceq->ceq_head = head;
1946 }
1947
1948
1949 /**
1950  * nes_process_aeq
1951  */
1952 void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
1953 {
1954 //      u64 u64temp;
1955         u32 head;
1956         u32 aeq_size;
1957         u32 aeqe_misc;
1958         u32 aeqe_cq_id;
1959         struct nes_hw_aeqe volatile *aeqe;
1960
1961         head = aeq->aeq_head;
1962         aeq_size = aeq->aeq_size;
1963
1964         do {
1965                 aeqe = &aeq->aeq_vbase[head];
1966                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
1967                         break;
1968                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
1969                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
1970                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
1971                         if (aeqe_cq_id >= NES_FIRST_QPN) {
1972                                 /* dealing with an accelerated QP related AE */
1973 //                              u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])))<<32) |
1974 //                                      ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
1975                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
1976                         } else {
1977                                 /* TODO: dealing with a CQP related AE */
1978                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
1979                                                 (u16)(aeqe_misc >> 16));
1980                         }
1981                 }
1982
1983                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
1984
1985                 if (++head >= aeq_size)
1986                         head = 0;
1987         }
1988         while (1);
1989         aeq->aeq_head = head;
1990 }
1991
1992 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
1993 {
1994         struct nes_adapter *nesadapter = nesdev->nesadapter;
1995         u32 reset_value;
1996         u32 i=0;
1997         u32 u32temp;
1998
1999         if (nesadapter->hw_rev == NE020_REV) {
2000                 return;
2001         }
2002         mh_detected++;
2003
2004         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2005
2006         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2007                 reset_value |= 0x0000001d;
2008         else
2009                 reset_value |= 0x0000002d;
2010
2011         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2012                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2013                         nesadapter->link_interrupt_count[0] = 0;
2014                         nesadapter->link_interrupt_count[1] = 0;
2015                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2016                         if (0x00000040 & u32temp)
2017                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2018                         else
2019                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2020
2021                         reset_value |= 0x0000003d;
2022                 }
2023                 nesadapter->link_interrupt_count[mac_index] = 0;
2024         }
2025
2026         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2027
2028         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2029                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2030
2031         if (0x0000003d == (reset_value & 0x0000003d)) {
2032                 u32 pcs_control_status0, pcs_control_status1;
2033
2034                 for (i = 0; i < 10; i++) {
2035                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2036                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2037                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2038                              && (pcs_control_status0 & 0x00100000))
2039                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2040                                 && (pcs_control_status1 & 0x00100000)))
2041                                 continue;
2042                         else
2043                                 break;
2044                 }
2045                 if (10 == i) {
2046                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2047                         if (0x00000040 & u32temp)
2048                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2049                         else
2050                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2051
2052                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2053
2054                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2055                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2056                 }
2057         }
2058 }
2059
2060 /**
2061  * nes_process_mac_intr
2062  */
2063 void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2064 {
2065         unsigned long flags;
2066         u32 pcs_control_status;
2067         struct nes_adapter *nesadapter = nesdev->nesadapter;
2068         struct nes_vnic *nesvnic;
2069         u32 mac_status;
2070         u32 mac_index = nesdev->mac_index;
2071         u32 u32temp;
2072         u16 phy_data;
2073         u16 temp_phy_data;
2074
2075         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2076         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2077                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2078                 return;
2079         }
2080         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2081         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2082
2083         /* ack the MAC interrupt */
2084         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2085         /* Clear the interrupt */
2086         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2087
2088         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2089
2090         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2091                 nesdev->link_status_interrupts++;
2092                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2093                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2094                         nes_reset_link(nesdev, mac_index);
2095                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2096                 }
2097                 /* read the PHY interrupt status register */
2098                 if (nesadapter->OneG_Mode) {
2099                         do {
2100                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2101                                                 nesadapter->phy_index[mac_index], &phy_data);
2102                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2103                                                 nesadapter->phy_index[mac_index], phy_data);
2104                         } while (phy_data&0x8000);
2105
2106                         temp_phy_data = 0;
2107                         do {
2108                                 nes_read_1G_phy_reg(nesdev, 0x11,
2109                                                 nesadapter->phy_index[mac_index], &phy_data);
2110                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2111                                                 nesadapter->phy_index[mac_index], phy_data);
2112                                 if (temp_phy_data == phy_data)
2113                                         break;
2114                                 temp_phy_data = phy_data;
2115                         } while (1);
2116
2117                         nes_read_1G_phy_reg(nesdev, 0x1e,
2118                                         nesadapter->phy_index[mac_index], &phy_data);
2119                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2120                                         nesadapter->phy_index[mac_index], phy_data);
2121
2122                         nes_read_1G_phy_reg(nesdev, 1,
2123                                         nesadapter->phy_index[mac_index], &phy_data);
2124                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2125                                         nesadapter->phy_index[mac_index], phy_data);
2126
2127                         if (temp_phy_data & 0x1000) {
2128                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2129                                 phy_data = 4;
2130                         } else {
2131                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2132                         }
2133                 }
2134                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2135                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2136                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2137                 pcs_control_status = nes_read_indexed(nesdev,
2138                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2139                 pcs_control_status = nes_read_indexed(nesdev,
2140                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2141                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2142                                 mac_index, pcs_control_status);
2143                 if (nesadapter->OneG_Mode) {
2144                         u32temp = 0x01010000;
2145                         if (nesadapter->port_count > 2) {
2146                                 u32temp |= 0x02020000;
2147                         }
2148                         if ((pcs_control_status & u32temp)!= u32temp) {
2149                                 phy_data = 0;
2150                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2151                         }
2152                 } else if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
2153                         nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2154                         temp_phy_data = (u16)nes_read_indexed(nesdev,
2155                                                                 NES_IDX_MAC_MDIO_CONTROL);
2156                         u32temp = 20;
2157                         do {
2158                                 nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2159                                 phy_data = (u16)nes_read_indexed(nesdev,
2160                                                                 NES_IDX_MAC_MDIO_CONTROL);
2161                                 if ((phy_data == temp_phy_data) || (!(--u32temp)))
2162                                         break;
2163                                 temp_phy_data = phy_data;
2164                         } while (1);
2165                         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2166                                 __FUNCTION__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2167
2168                 } else {
2169                         phy_data = (0x0f0f0000 == (pcs_control_status & 0x0f1f0000)) ? 4 : 0;
2170                 }
2171
2172                 if (phy_data & 0x0004) {
2173                         nesadapter->mac_link_down[mac_index] = 0;
2174                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2175                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2176                                                 nesvnic->linkup);
2177                                 if (nesvnic->linkup == 0) {
2178                                         printk(PFX "The Link is now up for port %u, netdev %p.\n",
2179                                                         mac_index, nesvnic->netdev);
2180                                         if (netif_queue_stopped(nesvnic->netdev))
2181                                                 netif_start_queue(nesvnic->netdev);
2182                                         nesvnic->linkup = 1;
2183                                         netif_carrier_on(nesvnic->netdev);
2184                                 }
2185                         }
2186                 } else {
2187                         nesadapter->mac_link_down[mac_index] = 1;
2188                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2189                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2190                                                 nesvnic->linkup);
2191                                 if (nesvnic->linkup == 1) {
2192                                         printk(PFX "The Link is now down for port %u, netdev %p.\n",
2193                                                         mac_index, nesvnic->netdev);
2194                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2195                                                 netif_stop_queue(nesvnic->netdev);
2196                                         nesvnic->linkup = 0;
2197                                         netif_carrier_off(nesvnic->netdev);
2198                                 }
2199                         }
2200                 }
2201         }
2202
2203         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2204 }
2205
2206
2207
2208 void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2209 {
2210         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2211
2212         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2213 }
2214
2215
2216 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2217 * getting out of nic_ce_handler
2218 */
2219 #define MAX_RQES_TO_PROCESS     384
2220
2221 /**
2222  * nes_nic_ce_handler
2223  */
2224 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2225 {
2226         u64 u64temp;
2227         dma_addr_t bus_address;
2228         struct nes_hw_nic *nesnic;
2229         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2230         struct nes_adapter *nesadapter = nesdev->nesadapter;
2231         struct nes_hw_nic_rq_wqe *nic_rqe;
2232         struct nes_hw_nic_sq_wqe *nic_sqe;
2233         struct sk_buff *skb;
2234         struct sk_buff *rx_skb;
2235         __le16 *wqe_fragment_length;
2236         u32 head;
2237         u32 cq_size;
2238         u32 rx_pkt_size;
2239         u32 cqe_count=0;
2240         u32 cqe_errv;
2241         u32 cqe_misc;
2242         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2243         u16 vlan_tag;
2244         u16 pkt_type;
2245         u16 rqes_processed = 0;
2246         u8 sq_cqes = 0;
2247
2248         head = cq->cq_head;
2249         cq_size = cq->cq_size;
2250         cq->cqes_pending = 1;
2251         do {
2252                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2253                                 NES_NIC_CQE_VALID) {
2254                         nesnic = &nesvnic->nic;
2255                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2256                         if (cqe_misc & NES_NIC_CQE_SQ) {
2257                                 sq_cqes++;
2258                                 wqe_fragment_index = 1;
2259                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2260                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2261                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2262                                 /* bump past the vlan tag */
2263                                 wqe_fragment_length++;
2264                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2265                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2266                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2267                                         bus_address = (dma_addr_t)u64temp;
2268                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2269                                                 pci_unmap_single(nesdev->pcidev,
2270                                                                 bus_address,
2271                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2272                                                                 PCI_DMA_TODEVICE);
2273                                         }
2274                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2275                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2276                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2277                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2278                                                         bus_address = (dma_addr_t)u64temp;
2279                                                         pci_unmap_page(nesdev->pcidev,
2280                                                                         bus_address,
2281                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2282                                                                         PCI_DMA_TODEVICE);
2283                                                 } else
2284                                                         break;
2285                                         }
2286                                         if (skb)
2287                                                 dev_kfree_skb_any(skb);
2288                                 }
2289                                 nesnic->sq_tail++;
2290                                 nesnic->sq_tail &= nesnic->sq_size-1;
2291                                 if (sq_cqes > 128) {
2292                                         barrier();
2293                                 /* restart the queue if it had been stopped */
2294                                 if (netif_queue_stopped(nesvnic->netdev))
2295                                         netif_wake_queue(nesvnic->netdev);
2296                                         sq_cqes = 0;
2297                                 }
2298                         } else {
2299                                 rqes_processed ++;
2300
2301                                 cq->rx_cqes_completed++;
2302                                 cq->rx_pkts_indicated++;
2303                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2304                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2305                                 /* Get the skb */
2306                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2307                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2308                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2309                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2310                                 pci_unmap_single(nesdev->pcidev, bus_address,
2311                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2312                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2313                                 /* rx_skb->len = rx_pkt_size; */
2314                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2315                                 skb_put(rx_skb, rx_pkt_size);
2316                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2317                                 nesnic->rq_tail++;
2318                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2319
2320                                 atomic_inc(&nesvnic->rx_skbs_needed);
2321                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2322                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2323                                                         cq->cq_number | (cqe_count << 16));
2324 //                                      nesadapter->tune_timer.cq_count += cqe_count;
2325                                         nesdev->currcq_count += cqe_count;
2326                                         cqe_count = 0;
2327                                         nes_replenish_nic_rq(nesvnic);
2328                                 }
2329                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2330                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2331                                 rx_skb->ip_summed = CHECKSUM_NONE;
2332
2333                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2334                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2335                                         if ((cqe_errv &
2336                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2337                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2338                                                 if (nesvnic->rx_checksum_disabled == 0) {
2339                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2340                                                 }
2341                                         } else
2342                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2343                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2344                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2345
2346                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2347                                         if ((cqe_errv &
2348                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2349                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2350                                                 if (nesvnic->rx_checksum_disabled == 0) {
2351                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2352                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2353                                                                   nesvnic->netdev->name); */
2354                                                 }
2355                                         } else
2356                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2357                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2358                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2359                                         }
2360                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2361                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2362
2363                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2364                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2365                                 } else {
2366                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2367                                                 vlan_tag = (u16)(le32_to_cpu(
2368                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2369                                                                 >> 16);
2370                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2371                                                                 nesvnic->netdev->name, vlan_tag);
2372                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2373                                         } else {
2374                                                 nes_netif_rx(rx_skb);
2375                                         }
2376                                 }
2377
2378                                 nesvnic->netdev->last_rx = jiffies;
2379                                 /* nesvnic->netstats.rx_packets++; */
2380                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2381                         }
2382
2383                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2384                         /* Accounting... */
2385                         cqe_count++;
2386                         if (++head >= cq_size)
2387                                 head = 0;
2388                         if (cqe_count == 255) {
2389                                 /* Replenish Nic CQ */
2390                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2391                                                 cq->cq_number | (cqe_count << 16));
2392 //                              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2393                                 nesdev->currcq_count += cqe_count;
2394                                 cqe_count = 0;
2395                         }
2396
2397                         if (cq->rx_cqes_completed >= nesvnic->budget)
2398                                 break;
2399                 } else {
2400                         cq->cqes_pending = 0;
2401                         break;
2402                 }
2403
2404         } while (1);
2405
2406         if (sq_cqes) {
2407                 barrier();
2408                 /* restart the queue if it had been stopped */
2409                 if (netif_queue_stopped(nesvnic->netdev))
2410                         netif_wake_queue(nesvnic->netdev);
2411         }
2412
2413         cq->cq_head = head;
2414         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2415                         cq->cq_number, cqe_count, cq->cq_head); */
2416         cq->cqe_allocs_pending = cqe_count;
2417         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2418         {
2419 //              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2420                 nesdev->currcq_count += cqe_count;
2421                 nes_nic_tune_timer(nesdev);
2422         }
2423         if (atomic_read(&nesvnic->rx_skbs_needed))
2424                 nes_replenish_nic_rq(nesvnic);
2425         }
2426
2427
2428 /**
2429  * nes_cqp_ce_handler
2430  */
2431 void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2432 {
2433         u64 u64temp;
2434         unsigned long flags;
2435         struct nes_hw_cqp *cqp = NULL;
2436         struct nes_cqp_request *cqp_request;
2437         struct nes_hw_cqp_wqe *cqp_wqe;
2438         u32 head;
2439         u32 cq_size;
2440         u32 cqe_count=0;
2441         u32 error_code;
2442         /* u32 counter; */
2443
2444         head = cq->cq_head;
2445         cq_size = cq->cq_size;
2446
2447         do {
2448                 /* process the CQE */
2449                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2450                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2451
2452                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2453                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2454                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
2455                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2456                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2457                         cqp = *((struct nes_hw_cqp **)&u64temp);
2458
2459                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2460                         if (error_code) {
2461                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2462                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2463                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2464                                                 (u16)(error_code >> 16),
2465                                                 (u16)error_code);
2466                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2467                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2468                         }
2469
2470                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2471                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX])))<<32) |
2472                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2473                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2474                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2475                         if (cqp_request) {
2476                                 if (cqp_request->waiting) {
2477                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2478                                         cqp_request->major_code = (u16)(error_code >> 16);
2479                                         cqp_request->minor_code = (u16)error_code;
2480                                         barrier();
2481                                         cqp_request->request_done = 1;
2482                                         wake_up(&cqp_request->waitq);
2483                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2484                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2485                                                                 cqp_request,
2486                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2487                                                 if (cqp_request->dynamic) {
2488                                                         kfree(cqp_request);
2489                                                 } else {
2490                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2491                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2492                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2493                                                 }
2494                                         }
2495                                 } else if (cqp_request->callback) {
2496                                         /* Envoke the callback routine */
2497                                         cqp_request->cqp_callback(nesdev, cqp_request);
2498                                         if (cqp_request->dynamic) {
2499                                                 kfree(cqp_request);
2500                                         } else {
2501                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2502                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2503                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2504                                         }
2505                                 } else {
2506                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2507                                                         cqp_request,
2508                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2509                                         if (cqp_request->dynamic) {
2510                                                 kfree(cqp_request);
2511                                         } else {
2512                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2513                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2514                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2515                                         }
2516                                 }
2517                         } else {
2518                                 wake_up(&nesdev->cqp.waitq);
2519                         }
2520
2521                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2522                         nes_write32(nesdev->regs+NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2523                         if (++cqp->sq_tail >= cqp->sq_size)
2524                                 cqp->sq_tail = 0;
2525
2526                         /* Accounting... */
2527                         cqe_count++;
2528                         if (++head >= cq_size)
2529                                 head = 0;
2530                 } else {
2531                         break;
2532                 }
2533         } while (1);
2534         cq->cq_head = head;
2535
2536         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2537         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2538                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2539                         (nesdev->cqp.sq_size - 1)) != 1)) {
2540                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2541                                 struct nes_cqp_request, list);
2542                 list_del_init(&cqp_request->list);
2543                 head = nesdev->cqp.sq_head++;
2544                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2545                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2546                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2547                 barrier();
2548                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2549                         cpu_to_le32((u32)((unsigned long)cqp_request));
2550                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2551                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2552                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2553                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2554                 /* Ring doorbell (1 WQEs) */
2555                 barrier();
2556                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2557         }
2558         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2559
2560         /* Arm the CCQ */
2561         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2562                         cq->cq_number);
2563         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2564 }
2565
2566
2567 /**
2568  * nes_process_iwarp_aeqe
2569  */
2570 void nes_process_iwarp_aeqe(struct nes_device *nesdev, struct nes_hw_aeqe *aeqe)
2571 {
2572         u64 context;
2573         u64 aeqe_context = 0;
2574         unsigned long flags;
2575         struct nes_qp *nesqp;
2576         int resource_allocated;
2577         /* struct iw_cm_id *cm_id; */
2578         struct nes_adapter *nesadapter = nesdev->nesadapter;
2579         struct ib_event ibevent;
2580         /* struct iw_cm_event cm_event; */
2581         u32 aeq_info;
2582         u32 next_iwarp_state = 0;
2583         u16 async_event_id;
2584         u8 tcp_state;
2585         u8 iwarp_state;
2586
2587         nes_debug(NES_DBG_AEQ, "\n");
2588         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2589         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2590                 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2591                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2592         } else {
2593                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2594                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2595                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2596                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2597                 BUG_ON(!context);
2598         }
2599
2600         async_event_id = (u16)aeq_info;
2601         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2602         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2603         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2604                         " Tcp state = %s, iWARP state = %s\n",
2605                         async_event_id,
2606                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2607                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2608
2609
2610         switch (async_event_id) {
2611                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2612                         nesqp = *((struct nes_qp **)&context);
2613                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2614                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2615                                 nes_add_ref(&nesqp->ibqp);
2616                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2617                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2618                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2619                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2620                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2621                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2622                                                 async_event_id, nesqp->last_aeq, tcp_state);
2623                         }
2624                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2625                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2626                                 /* FIN Received but tcp state or IB state moved on,
2627                                                 should expect a close complete */
2628                                 return;
2629                         }
2630                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2631                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2632                 case NES_AEQE_AEID_TERMINATE_SENT:
2633                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2634                 case NES_AEQE_AEID_RESET_SENT:
2635                         nesqp = *((struct nes_qp **)&context);
2636                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2637                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2638                         }
2639                         nes_add_ref(&nesqp->ibqp);
2640                         spin_lock_irqsave(&nesqp->lock, flags);
2641                         nesqp->hw_iwarp_state = iwarp_state;
2642                         nesqp->hw_tcp_state = tcp_state;
2643                         nesqp->last_aeq = async_event_id;
2644
2645                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2646                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2647                                 nesqp->hte_added = 0;
2648                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2649                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2650                                                 nesqp->hwqp.qp_id);
2651                                 nes_hw_modify_qp(nesdev, nesqp,
2652                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2653                                 spin_lock_irqsave(&nesqp->lock, flags);
2654                         }
2655
2656                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2657                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2658                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2659                                 switch (nesqp->hw_iwarp_state) {
2660                                         case NES_AEQE_IWARP_STATE_RTS:
2661                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2662                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2663                                                 break;
2664                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2665                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2666                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2667                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2668                                                         next_iwarp_state |= 0x02000000;
2669                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2670                                                 }
2671                                                 break;
2672                                         default:
2673                                                 next_iwarp_state = 0;
2674                                 }
2675                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2676                                 if (next_iwarp_state) {
2677                                         nes_add_ref(&nesqp->ibqp);
2678                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2679                                                         " also added another reference\n",
2680                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2681                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2682                                 }
2683                                 nes_cm_disconn(nesqp);
2684                         } else {
2685                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2686                                         /* FIN Received but ib state not RTS,
2687                                                         close complete will be on its way */
2688                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2689                                         nes_rem_ref(&nesqp->ibqp);
2690                                         return;
2691                                 }
2692                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2693                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2694                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2695                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2696                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2697                                                         " also added another reference\n",
2698                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2699                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2700                                 }
2701                                 nes_cm_disconn(nesqp);
2702                         }
2703                         break;
2704                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2705                         nesqp = *((struct nes_qp **)&context);
2706                         spin_lock_irqsave(&nesqp->lock, flags);
2707                         nesqp->hw_iwarp_state = iwarp_state;
2708                         nesqp->hw_tcp_state = tcp_state;
2709                         nesqp->last_aeq = async_event_id;
2710                         spin_unlock_irqrestore(&nesqp->lock, flags);
2711                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2712                                         " event on QP%u \n  Q2 Data:\n",
2713                                         nesqp->hwqp.qp_id);
2714                         if (nesqp->ibqp.event_handler) {
2715                                 ibevent.device = nesqp->ibqp.device;
2716                                 ibevent.element.qp = &nesqp->ibqp;
2717                                 ibevent.event = IB_EVENT_QP_FATAL;
2718                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2719                         }
2720                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2721                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2722                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2723                                 nes_add_ref(&nesqp->ibqp);
2724                                 nes_cm_disconn(nesqp);
2725                         } else {
2726                                 nesqp->in_disconnect = 0;
2727                                 wake_up(&nesqp->kick_waitq);
2728                         }
2729                         break;
2730                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2731                         nesqp = *((struct nes_qp **)&context);
2732                         nes_add_ref(&nesqp->ibqp);
2733                         spin_lock_irqsave(&nesqp->lock, flags);
2734                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2735                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2736                         nesqp->last_aeq = async_event_id;
2737                         if (nesqp->cm_id) {
2738                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2739                                                 " event on QP%u, remote IP = 0x%08X \n",
2740                                                 nesqp->hwqp.qp_id,
2741                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2742                         } else {
2743                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2744                                                 " event on QP%u \n",
2745                                                 nesqp->hwqp.qp_id);
2746                         }
2747                         spin_unlock_irqrestore(&nesqp->lock, flags);
2748                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2749                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2750                         if (nesqp->ibqp.event_handler) {
2751                                 ibevent.device = nesqp->ibqp.device;
2752                                 ibevent.element.qp = &nesqp->ibqp;
2753                                 ibevent.event = IB_EVENT_QP_FATAL;
2754                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2755                         }
2756                         break;
2757                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2758                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2759                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2760                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2761                         } else {
2762                                 /* TODO: get the actual WQE and mask off wqe index */
2763                                 context &= ~((u64)511);
2764                                 nesqp = *((struct nes_qp **)&context);
2765                         }
2766                         spin_lock_irqsave(&nesqp->lock, flags);
2767                         nesqp->hw_iwarp_state = iwarp_state;
2768                         nesqp->hw_tcp_state = tcp_state;
2769                         nesqp->last_aeq = async_event_id;
2770                         spin_unlock_irqrestore(&nesqp->lock, flags);
2771                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2772                                         nesqp->hwqp.qp_id);
2773                         if (nesqp->ibqp.event_handler) {
2774                                 ibevent.device = nesqp->ibqp.device;
2775                                 ibevent.element.qp = &nesqp->ibqp;
2776                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2777                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2778                         }
2779                         break;
2780                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
2781                         nesqp = *((struct nes_qp **)&context);
2782                         spin_lock_irqsave(&nesqp->lock, flags);
2783                         nesqp->hw_iwarp_state = iwarp_state;
2784                         nesqp->hw_tcp_state = tcp_state;
2785                         nesqp->last_aeq = async_event_id;
2786                         spin_unlock_irqrestore(&nesqp->lock, flags);
2787                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
2788                                         nesqp->hwqp.qp_id);
2789                         if (nesqp->ibqp.event_handler) {
2790                                 ibevent.device = nesqp->ibqp.device;
2791                                 ibevent.element.qp = &nesqp->ibqp;
2792                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2793                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2794                         }
2795                         break;
2796                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
2797                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
2798                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2799                         spin_lock_irqsave(&nesqp->lock, flags);
2800                         nesqp->hw_iwarp_state = iwarp_state;
2801                         nesqp->hw_tcp_state = tcp_state;
2802                         nesqp->last_aeq = async_event_id;
2803                         spin_unlock_irqrestore(&nesqp->lock, flags);
2804                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
2805                                         " nesqp = %p, AE reported %p\n",
2806                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
2807                         if (nesqp->ibqp.event_handler) {
2808                                 ibevent.device = nesqp->ibqp.device;
2809                                 ibevent.element.qp = &nesqp->ibqp;
2810                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2811                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2812                         }
2813                         break;
2814                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
2815                         context <<= 1;
2816                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
2817                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
2818                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
2819                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2820                         if (resource_allocated) {
2821                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
2822                                                 __FUNCTION__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2823                         }
2824                         break;
2825                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
2826                         nesqp = nesadapter->qp_table[le32_to_cpu(
2827                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2828                         spin_lock_irqsave(&nesqp->lock, flags);
2829                         nesqp->hw_iwarp_state = iwarp_state;
2830                         nesqp->hw_tcp_state = tcp_state;
2831                         nesqp->last_aeq = async_event_id;
2832                         spin_unlock_irqrestore(&nesqp->lock, flags);
2833                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
2834                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
2835                                         nesqp->hwqp.qp_id);
2836                         if (nesqp->ibqp.event_handler) {
2837                                 ibevent.device = nesqp->ibqp.device;
2838                                 ibevent.element.qp = &nesqp->ibqp;
2839                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2840                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2841                         }
2842                         /* tell cm to disconnect, cm will queue work to thread */
2843                         nes_add_ref(&nesqp->ibqp);
2844                         nes_cm_disconn(nesqp);
2845                         break;
2846                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
2847                         nesqp = *((struct nes_qp **)&context);
2848                         spin_lock_irqsave(&nesqp->lock, flags);
2849                         nesqp->hw_iwarp_state = iwarp_state;
2850                         nesqp->hw_tcp_state = tcp_state;
2851                         nesqp->last_aeq = async_event_id;
2852                         spin_unlock_irqrestore(&nesqp->lock, flags);
2853                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
2854                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
2855                                         nesqp->hwqp.qp_id);
2856                         if (nesqp->ibqp.event_handler) {
2857                                 ibevent.device = nesqp->ibqp.device;
2858                                 ibevent.element.qp = &nesqp->ibqp;
2859                                 ibevent.event = IB_EVENT_QP_FATAL;
2860                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2861                         }
2862                         /* tell cm to disconnect, cm will queue work to thread */
2863                         nes_add_ref(&nesqp->ibqp);
2864                         nes_cm_disconn(nesqp);
2865                         break;
2866                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
2867                         nesqp = *((struct nes_qp **)&context);
2868                         spin_lock_irqsave(&nesqp->lock, flags);
2869                         nesqp->hw_iwarp_state = iwarp_state;
2870                         nesqp->hw_tcp_state = tcp_state;
2871                         nesqp->last_aeq = async_event_id;
2872                         spin_unlock_irqrestore(&nesqp->lock, flags);
2873                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
2874                                         " event on QP%u \n  Q2 Data:\n",
2875                                         nesqp->hwqp.qp_id);
2876                         if (nesqp->ibqp.event_handler) {
2877                                 ibevent.device = nesqp->ibqp.device;
2878                                 ibevent.element.qp = &nesqp->ibqp;
2879                                 ibevent.event = IB_EVENT_QP_FATAL;
2880                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2881                         }
2882                         /* tell cm to disconnect, cm will queue work to thread */
2883                         nes_add_ref(&nesqp->ibqp);
2884                         nes_cm_disconn(nesqp);
2885                         break;
2886                         /* TODO: additional AEs need to be here */
2887                 default:
2888                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
2889                                         async_event_id);
2890                         break;
2891         }
2892
2893 }
2894
2895
2896 /**
2897  * nes_iwarp_ce_handler
2898  */
2899 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
2900 {
2901         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
2902
2903         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
2904                         nescq->hw_cq.cq_number); */
2905         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
2906
2907         if (nescq->ibcq.comp_handler)
2908                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
2909
2910         return;
2911 }
2912
2913
2914 /**
2915  * nes_manage_apbvt()
2916  */
2917 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
2918                 u32 nic_index, u32 add_port)
2919 {
2920         struct nes_device *nesdev = nesvnic->nesdev;
2921         struct nes_hw_cqp_wqe *cqp_wqe;
2922         unsigned long flags;
2923         struct nes_cqp_request *cqp_request;
2924         int ret = 0;
2925         u16 major_code;
2926
2927         /* Send manage APBVT request to CQP */
2928         cqp_request = nes_get_cqp_request(nesdev);
2929         if (cqp_request == NULL) {
2930                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
2931                 return -ENOMEM;
2932         }
2933         cqp_request->waiting = 1;
2934         cqp_wqe = &cqp_request->cqp_wqe;
2935
2936         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
2937                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
2938                         accel_local_port, accel_local_port, nic_index);
2939
2940         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2941         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
2942                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
2943         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2944                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
2945
2946         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
2947
2948         atomic_set(&cqp_request->refcount, 2);
2949         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
2950
2951         if (add_port == NES_MANAGE_APBVT_ADD)
2952                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
2953                                 NES_EVENT_TIMEOUT);
2954         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
2955                         ret, cqp_request->major_code, cqp_request->minor_code);
2956         major_code = cqp_request->major_code;
2957         if (atomic_dec_and_test(&cqp_request->refcount)) {
2958                 if (cqp_request->dynamic) {
2959                         kfree(cqp_request);
2960                 } else {
2961                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2962                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2963                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2964                 }
2965         }
2966         if (!ret)
2967                 return -ETIME;
2968         else if (major_code)
2969                 return -EIO;
2970         else
2971                 return 0;
2972 }
2973
2974
2975 /**
2976  * nes_manage_arp_cache
2977  */
2978 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
2979                 u32 ip_addr, u32 action)
2980 {
2981         struct nes_hw_cqp_wqe *cqp_wqe;
2982         struct nes_vnic *nesvnic = netdev_priv(netdev);
2983         struct nes_device *nesdev;
2984         struct nes_cqp_request *cqp_request;
2985         int arp_index;
2986
2987         nesdev = nesvnic->nesdev;
2988         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
2989         if (arp_index == -1) {
2990                 return;
2991         }
2992
2993         /* update the ARP entry */
2994         cqp_request = nes_get_cqp_request(nesdev);
2995         if (cqp_request == NULL) {
2996                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
2997                 return;
2998         }
2999         cqp_request->waiting = 0;
3000         cqp_wqe = &cqp_request->cqp_wqe;
3001         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3002
3003         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3004                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3005         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3006                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3007         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3008
3009         if (action == NES_ARP_ADD) {
3010                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3011                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3012                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3013                                 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3014                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3015                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3016         } else {
3017                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3018                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3019         }
3020
3021         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3022                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3023
3024         atomic_set(&cqp_request->refcount, 1);
3025         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3026 }
3027
3028
3029 /**
3030  * flush_wqes
3031  */
3032 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3033                 u32 which_wq, u32 wait_completion)
3034 {
3035         unsigned long flags;
3036         struct nes_cqp_request *cqp_request;
3037         struct nes_hw_cqp_wqe *cqp_wqe;
3038         int ret;
3039
3040         cqp_request = nes_get_cqp_request(nesdev);
3041         if (cqp_request == NULL) {
3042                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3043                 return;
3044         }
3045         if (wait_completion) {
3046                 cqp_request->waiting = 1;
3047                 atomic_set(&cqp_request->refcount, 2);
3048         } else {
3049                 cqp_request->waiting = 0;
3050         }
3051         cqp_wqe = &cqp_request->cqp_wqe;
3052         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3053
3054         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3055                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3056         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3057
3058         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3059
3060         if (wait_completion) {
3061                 /* Wait for CQP */
3062                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3063                                 NES_EVENT_TIMEOUT);
3064                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3065                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3066                                 ret, cqp_request->major_code, cqp_request->minor_code);
3067                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3068                         if (cqp_request->dynamic) {
3069                                 kfree(cqp_request);
3070                         } else {
3071                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3072                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3073                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3074                         }
3075                 }
3076         }
3077 }