git.oblomov.eu Git - linux-2.6/blob - drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41
  42 #include "nes.h"
  43
  44 static u32 crit_err_count;
  45 u32 int_mod_timer_init;
  46 u32 int_mod_cq_depth_256;
  47 u32 int_mod_cq_depth_128;
  48 u32 int_mod_cq_depth_32;
  49 u32 int_mod_cq_depth_24;
  50 u32 int_mod_cq_depth_16;
  51 u32 int_mod_cq_depth_4;
  52 u32 int_mod_cq_depth_1;
  53
  54 #include "nes_cm.h"
  55
  56 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  57 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  58 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  59                            u8 OneG_Mode);
  60 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  61 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  62 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  63 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  64                                    struct nes_hw_aeqe *aeqe);
  65 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  66 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  67
  68 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  69 static unsigned char *nes_iwarp_state_str[] = {
  70         "Non-Existant",
  71         "Idle",
  72         "RTS",
  73         "Closing",
  74         "RSVD1",
  75         "Terminate",
  76         "Error",
  77         "RSVD2",
  78 };
  79
  80 static unsigned char *nes_tcp_state_str[] = {
  81         "Non-Existant",
  82         "Closed",
  83         "Listen",
  84         "SYN Sent",
  85         "SYN Rcvd",
  86         "Established",
  87         "Close Wait",
  88         "FIN Wait 1",
  89         "Closing",
  90         "Last Ack",
  91         "FIN Wait 2",
  92         "Time Wait",
  93         "RSVD1",
  94         "RSVD2",
  95         "RSVD3",
  96         "RSVD4",
  97 };
  98 #endif
  99
 100
 101 /**
 102  * nes_nic_init_timer_defaults
 103  */
 104 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 105 {
 106         unsigned long flags;
 107         struct nes_adapter *nesadapter = nesdev->nesadapter;
 108         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 109
 110         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 111
 112         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 113         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 114         if (jumbomode) {
 115                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 116                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 117                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 118         } else {
 119                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 120                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 121                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 122         }
 123
 124         /* todo use netdev->mtu to set thresholds */
 125         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 126 }
 127
 128
 129 /**
 130  * nes_nic_init_timer
 131  */
 132 static void  nes_nic_init_timer(struct nes_device *nesdev)
 133 {
 134         unsigned long flags;
 135         struct nes_adapter *nesadapter = nesdev->nesadapter;
 136         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 137
 138         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 139
 140         if (shared_timer->timer_in_use_old == 0) {
 141                 nesdev->deepcq_count = 0;
 142                 shared_timer->timer_direction_upward = 0;
 143                 shared_timer->timer_direction_downward = 0;
 144                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 145                 shared_timer->timer_in_use_old = 0;
 146
 147         }
 148         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 149                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 150                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 151                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 152         }
 153         /* todo use netdev->mtu to set thresholds */
 154         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 155 }
 156
 157
 158 /**
 159  * nes_nic_tune_timer
 160  */
 161 static void nes_nic_tune_timer(struct nes_device *nesdev)
 162 {
 163         unsigned long flags;
 164         struct nes_adapter *nesadapter = nesdev->nesadapter;
 165         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 166         u16 cq_count = nesdev->currcq_count;
 167
 168         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 169
 170         if (shared_timer->cq_count_old <= cq_count)
 171                 shared_timer->cq_direction_downward = 0;
 172         else
 173                 shared_timer->cq_direction_downward++;
 174         shared_timer->cq_count_old = cq_count;
 175         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 176                 if (cq_count <= shared_timer->threshold_low &&
 177                     shared_timer->threshold_low > 4) {
 178                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 179                         shared_timer->cq_direction_downward=0;
 180                         nesdev->currcq_count = 0;
 181                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 182                         return;
 183                 }
 184         }
 185
 186         if (cq_count > 1) {
 187                 nesdev->deepcq_count += cq_count;
 188                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 189                         shared_timer->timer_direction_upward++;
 190                         shared_timer->timer_direction_downward = 0;
 191                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 192                         shared_timer->timer_direction_upward = 0;
 193                         shared_timer->timer_direction_downward = 0;
 194                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 195                         shared_timer->timer_direction_downward++;
 196                         shared_timer->timer_direction_upward = 0;
 197                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 198                         shared_timer->timer_in_use -= 2;
 199                         shared_timer->timer_direction_upward = 0;
 200                         shared_timer->timer_direction_downward++;
 201                 } else {
 202                         shared_timer->timer_in_use -= 4;
 203                         shared_timer->timer_direction_upward = 0;
 204                         shared_timer->timer_direction_downward++;
 205                 }
 206
 207                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 208                         shared_timer->timer_in_use += 3;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward = 0;
 211                 }
 212                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 213                         shared_timer->timer_in_use -= 4 ;
 214                         shared_timer->timer_direction_downward = 0;
 215                         shared_timer->timer_direction_upward = 0;
 216                 }
 217         }
 218
 219         /* boundary checking */
 220         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 221                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 222         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 223                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 224         }
 225
 226         nesdev->currcq_count = 0;
 227
 228         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 229 }
 230
 231
 232 /**
 233  * nes_init_adapter - initialize adapter
 234  */
 235 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 236         struct nes_adapter *nesadapter = NULL;
 237         unsigned long num_pds;
 238         u32 u32temp;
 239         u32 port_count;
 240         u16 max_rq_wrs;
 241         u16 max_sq_wrs;
 242         u32 max_mr;
 243         u32 max_256pbl;
 244         u32 max_4kpbl;
 245         u32 max_qp;
 246         u32 max_irrq;
 247         u32 max_cq;
 248         u32 hte_index_mask;
 249         u32 adapter_size;
 250         u32 arp_table_size;
 251         u16 vendor_id;
 252         u8  OneG_Mode;
 253         u8  func_index;
 254
 255         /* search the list of existing adapters */
 256         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 257                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 258                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 259                                 nesdev->pcidev->devfn,
 260                                 PCI_SLOT(nesadapter->devfn),
 261                                 nesadapter->bus_number,
 262                                 PCI_SLOT(nesdev->pcidev->devfn),
 263                                 nesdev->pcidev->bus->number );
 264                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 265                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 266                         nesadapter->ref_count++;
 267                         return nesadapter;
 268                 }
 269         }
 270
 271         /* no adapter found */
 272         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 273         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 274                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 275                                 hw_rev);
 276                 return NULL;
 277         }
 278
 279         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 280                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 281                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 282                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 283                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 284
 285         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 286
 287
 288         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 289                 return NULL;
 290         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 291                 return NULL;
 292         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 293
 294         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 295         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 296
 297         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 298         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 299                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 300                                 max_qp, u32temp);
 301                 max_qp = (u32)1 << (u32temp & 0x001f);
 302         }
 303
 304         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 305         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 306                         max_qp, hte_index_mask);
 307
 308         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 309
 310         max_irrq = 1 << (u32temp & 0x001f);
 311
 312         if (max_qp > max_irrq) {
 313                 max_qp = max_irrq;
 314                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 315                                 max_qp);
 316         }
 317
 318         /* there should be no reason to allocate more pds than qps */
 319         if (num_pds > max_qp)
 320                 num_pds = max_qp;
 321
 322         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 323         max_mr = (u32)8192 << (u32temp & 0x7);
 324
 325         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 326         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 327         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 328         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 329
 330         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 331         arp_table_size = 1 << u32temp;
 332
 333         adapter_size = (sizeof(struct nes_adapter) +
 334                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 335         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 336         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 337         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 340         adapter_size += sizeof(struct nes_qp **) * max_qp;
 341
 342         /* allocate a new adapter struct */
 343         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 344         if (nesadapter == NULL) {
 345                 return NULL;
 346         }
 347
 348         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 349                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 350
 351         /* populate the new nesadapter */
 352         nesadapter->devfn = nesdev->pcidev->devfn;
 353         nesadapter->bus_number = nesdev->pcidev->bus->number;
 354         nesadapter->ref_count = 1;
 355         nesadapter->timer_int_req = 0xffff0000;
 356         nesadapter->OneG_Mode = OneG_Mode;
 357         nesadapter->doorbell_start = nesdev->doorbell_region;
 358
 359         /* nesadapter->tick_delta = clk_divisor; */
 360         nesadapter->hw_rev = hw_rev;
 361         nesadapter->port_count = port_count;
 362
 363         nesadapter->max_qp = max_qp;
 364         nesadapter->hte_index_mask = hte_index_mask;
 365         nesadapter->max_irrq = max_irrq;
 366         nesadapter->max_mr = max_mr;
 367         nesadapter->max_256pbl = max_256pbl - 1;
 368         nesadapter->max_4kpbl = max_4kpbl - 1;
 369         nesadapter->max_cq = max_cq;
 370         nesadapter->free_256pbl = max_256pbl - 1;
 371         nesadapter->free_4kpbl = max_4kpbl - 1;
 372         nesadapter->max_pd = num_pds;
 373         nesadapter->arp_table_size = arp_table_size;
 374
 375         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 376         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 377                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 378                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 379                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 380         } else {
 381                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 382                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 383                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 384                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 385         }
 386         /* Setup and enable the periodic timer */
 387         if (nesadapter->et_rx_coalesce_usecs_irq)
 388                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 389                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 390         else
 391                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 392
 393         nesadapter->base_pd = 1;
 394
 395         nesadapter->device_cap_flags =
 396                 IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
 397
 398         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 399                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 400         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 401         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 402         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 403         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 404         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 405
 406
 407         /* mark the usual suspect QPs and CQs as in use */
 408         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 409                 set_bit(u32temp, nesadapter->allocated_qps);
 410                 set_bit(u32temp, nesadapter->allocated_cqs);
 411         }
 412
 413         for (u32temp = 0; u32temp < 20; u32temp++)
 414                 set_bit(u32temp, nesadapter->allocated_pds);
 415         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 416
 417         max_rq_wrs = ((u32temp >> 8) & 3);
 418         switch (max_rq_wrs) {
 419                 case 0:
 420                         max_rq_wrs = 4;
 421                         break;
 422                 case 1:
 423                         max_rq_wrs = 16;
 424                         break;
 425                 case 2:
 426                         max_rq_wrs = 32;
 427                         break;
 428                 case 3:
 429                         max_rq_wrs = 512;
 430                         break;
 431         }
 432
 433         max_sq_wrs = (u32temp & 3);
 434         switch (max_sq_wrs) {
 435                 case 0:
 436                         max_sq_wrs = 4;
 437                         break;
 438                 case 1:
 439                         max_sq_wrs = 16;
 440                         break;
 441                 case 2:
 442                         max_sq_wrs = 32;
 443                         break;
 444                 case 3:
 445                         max_sq_wrs = 512;
 446                         break;
 447         }
 448         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 449         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 450
 451         nesadapter->max_sge = 4;
 452         nesadapter->max_cqe = 32767;
 453
 454         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 455                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 456                 kfree(nesadapter);
 457                 return NULL;
 458         }
 459
 460         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 461         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 462                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 463
 464         /* setup port configuration */
 465         if (nesadapter->port_count == 1) {
 466                 u32temp = 0x00000000;
 467                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 468                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 469                 else
 470                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 471         } else {
 472                 if (nesadapter->port_count == 2)
 473                         u32temp = 0x00000044;
 474                 else
 475                         u32temp = 0x000000e4;
 476                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 477         }
 478
 479         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 480         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 481                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 482
 483         spin_lock_init(&nesadapter->resource_lock);
 484         spin_lock_init(&nesadapter->phy_lock);
 485         spin_lock_init(&nesadapter->pbl_lock);
 486         spin_lock_init(&nesadapter->periodic_timer_lock);
 487
 488         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 489         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 490         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 491         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 492
 493         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 494                 u32 pcs_control_status0, pcs_control_status1;
 495                 u32 reset_value;
 496                 u32 i = 0;
 497                 u32 int_cnt = 0;
 498                 u32 ext_cnt = 0;
 499                 unsigned long flags;
 500                 u32 j = 0;
 501
 502                 pcs_control_status0 = nes_read_indexed(nesdev,
 503                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 504                 pcs_control_status1 = nes_read_indexed(nesdev,
 505                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 506
 507                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 508                         pcs_control_status0 = nes_read_indexed(nesdev,
 509                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 510                         pcs_control_status1 = nes_read_indexed(nesdev,
 511                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 512                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 513                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 514                                 int_cnt++;
 515                         msleep(1);
 516                 }
 517                 if (int_cnt > 1) {
 518                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 519                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 520                         mh_detected++;
 521                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 522                         reset_value |= 0x0000003d;
 523                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 524
 525                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 526                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 527                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 528
 529                         pcs_control_status0 = nes_read_indexed(nesdev,
 530                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 531                         pcs_control_status1 = nes_read_indexed(nesdev,
 532                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 533
 534                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 535                                 pcs_control_status0 = nes_read_indexed(nesdev,
 536                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 537                                 pcs_control_status1 = nes_read_indexed(nesdev,
 538                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 539                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 540                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 541                                         if (++ext_cnt > int_cnt) {
 542                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 543                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 544                                                                 0x0000F0C8);
 545                                                 mh_detected++;
 546                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 547                                                 reset_value |= 0x0000003d;
 548                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 549
 550                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 551                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 552                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 553                                                 break;
 554                                         }
 555                                 }
 556                                 msleep(1);
 557                         }
 558                 }
 559         }
 560
 561         if (nesadapter->hw_rev == NE020_REV) {
 562                 init_timer(&nesadapter->mh_timer);
 563                 nesadapter->mh_timer.function = nes_mh_fix;
 564                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 565                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 566                 add_timer(&nesadapter->mh_timer);
 567         } else {
 568                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 569         }
 570
 571         init_timer(&nesadapter->lc_timer);
 572         nesadapter->lc_timer.function = nes_clc;
 573         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 574         nesadapter->lc_timer.data = (unsigned long)nesdev;
 575         add_timer(&nesadapter->lc_timer);
 576
 577         list_add_tail(&nesadapter->list, &nes_adapter_list);
 578
 579         for (func_index = 0; func_index < 8; func_index++) {
 580                 pci_bus_read_config_word(nesdev->pcidev->bus,
 581                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 582                                         func_index), 0, &vendor_id);
 583                 if (vendor_id == 0xffff)
 584                         break;
 585         }
 586         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 587                 func_index, pci_name(nesdev->pcidev));
 588         nesadapter->adapter_fcn_count = func_index;
 589
 590         return nesadapter;
 591 }
 592
 593
 594 /**
 595  * nes_reset_adapter_ne020
 596  */
 597 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 598 {
 599         u32 port_count;
 600         u32 u32temp;
 601         u32 i;
 602
 603         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 604         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 605         /* TODO: assuming that both SERDES are set the same for now */
 606         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 607         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 608                         u32temp, port_count);
 609         if (*OneG_Mode)
 610                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 611         u32temp &= 0xff00ffc0;
 612         switch (port_count) {
 613                 case 1:
 614                         u32temp |= 0x00ee0000;
 615                         break;
 616                 case 2:
 617                         u32temp |= 0x00cc0000;
 618                         break;
 619                 case 4:
 620                         u32temp |= 0x00000000;
 621                         break;
 622                 default:
 623                         return 0;
 624                         break;
 625         }
 626
 627         /* check and do full reset if needed */
 628         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 629                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 630                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 631
 632                 i = 0;
 633                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 634                         mdelay(1);
 635                 if (i >= 10000) {
 636                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 637                         return 0;
 638                 }
 639
 640                 i = 0;
 641                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 642                         mdelay(1);
 643                 if (i >= 10000) {
 644                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 645                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 646                         return 0;
 647                 }
 648         }
 649
 650         /* port reset */
 651         switch (port_count) {
 652                 case 1:
 653                         u32temp |= 0x00ee0010;
 654                         break;
 655                 case 2:
 656                         u32temp |= 0x00cc0030;
 657                         break;
 658                 case 4:
 659                         u32temp |= 0x00000030;
 660                         break;
 661         }
 662
 663         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 664         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 665
 666         i = 0;
 667         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 668                 mdelay(1);
 669         if (i >= 10000) {
 670                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 671                 return 0;
 672         }
 673
 674         /* serdes 0 */
 675         i = 0;
 676         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 677                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 678                 mdelay(1);
 679         if (i >= 5000) {
 680                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 681                 return 0;
 682         }
 683
 684         /* serdes 1 */
 685         if (port_count > 1) {
 686                 i = 0;
 687                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 688                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 689                         mdelay(1);
 690                 if (i >= 5000) {
 691                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 692                         return 0;
 693                 }
 694         }
 695
 696         return port_count;
 697 }
 698
 699
 700 /**
 701  * nes_init_serdes
 702  */
 703 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 704                            u8 OneG_Mode)
 705 {
 706         int i;
 707         u32 u32temp;
 708
 709         if (hw_rev != NE020_REV) {
 710                 /* init serdes 0 */
 711
 712                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 713                 if (!OneG_Mode)
 714                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 715                 if (port_count > 1) {
 716                         /* init serdes 1 */
 717                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 718                         if (!OneG_Mode)
 719                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 720                         }
 721         } else {
 722                 /* init serdes 0 */
 723                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 724                 i = 0;
 725                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 726                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 727                         mdelay(1);
 728                 if (i >= 5000) {
 729                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 730                         return 1;
 731                 }
 732                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 733                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 734                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 735                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 736                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 737                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 738                 if (OneG_Mode)
 739                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 740                 else
 741                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 742
 743                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 744                 if (port_count > 1) {
 745                         /* init serdes 1 */
 746                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 747                         i = 0;
 748                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 749                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 750                                 mdelay(1);
 751                         if (i >= 5000) {
 752                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 753                                 /* return 1; */
 754                         }
 755                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 756                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 757                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 762                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 763                 }
 764         }
 765         return 0;
 766 }
 767
 768
 769 /**
 770  * nes_init_csr_ne020
 771  * Initialize registers for ne020 hardware
 772  */
 773 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 774 {
 775         u32 u32temp;
 776
 777         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 778
 779         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 780         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 781         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 782         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 783         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 784         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 785         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 786         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 787
 788         /* TODO: move these MAC register settings to NIC bringup */
 789         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 790         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 791         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 792         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 793         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 794         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 795         if (port_count > 1) {
 796                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 797                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 798                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 799                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 800                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 801                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 802                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 803         }
 804         if (port_count > 2) {
 805                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 806                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 807                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 808                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 809                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 810                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 811                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 812
 813                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 814                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 815                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 816                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 817                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 818                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 819                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 820         }
 821
 822         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 823         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 824         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 825         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 826         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 827         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 828         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 829         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 830
 831         /* TODO: move this to code, get from EEPROM */
 832         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 833         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 834         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 835                                                                                                                 //
 836         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 837         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 838
 839         if (hw_rev != NE020_REV) {
 840                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 841                 u32temp |= 0x80000000;
 842                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 843                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 844                 u32temp &= 0x7fffffff;
 845                 u32temp |= 0x7fff0010;
 846                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 847         }
 848 }
 849
 850
 851 /**
 852  * nes_destroy_adapter - destroy the adapter structure
 853  */
 854 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 855 {
 856         struct nes_adapter *tmp_adapter;
 857
 858         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 859                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 860                                 tmp_adapter);
 861         }
 862
 863         nesadapter->ref_count--;
 864         if (!nesadapter->ref_count) {
 865                 if (nesadapter->hw_rev == NE020_REV) {
 866                         del_timer(&nesadapter->mh_timer);
 867                 }
 868                 del_timer(&nesadapter->lc_timer);
 869
 870                 list_del(&nesadapter->list);
 871                 kfree(nesadapter);
 872         }
 873 }
 874
 875
 876 /**
 877  * nes_init_cqp
 878  */
 879 int nes_init_cqp(struct nes_device *nesdev)
 880 {
 881         struct nes_adapter *nesadapter = nesdev->nesadapter;
 882         struct nes_hw_cqp_qp_context *cqp_qp_context;
 883         struct nes_hw_cqp_wqe *cqp_wqe;
 884         struct nes_hw_ceq *ceq;
 885         struct nes_hw_ceq *nic_ceq;
 886         struct nes_hw_aeq *aeq;
 887         void *vmem;
 888         dma_addr_t pmem;
 889         u32 count=0;
 890         u32 cqp_head;
 891         u64 u64temp;
 892         u32 u32temp;
 893
 894         /* allocate CQP memory */
 895         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 896         /* SQ is 512 byte aligned, others are 256 byte aligned */
 897         nesdev->cqp_mem_size = 512 +
 898                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 899                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 900                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 901                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 902                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 903                         sizeof(struct nes_hw_cqp_qp_context);
 904
 905         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 906                         &nesdev->cqp_pbase);
 907         if (!nesdev->cqp_vbase) {
 908                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 909                 return -ENOMEM;
 910         }
 911         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 912
 913         /* Allocate a twice the number of CQP requests as the SQ size */
 914         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 915                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 916         if (nesdev->nes_cqp_requests == NULL) {
 917                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 918                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 919                                 nesdev->cqp.sq_pbase);
 920                 return -ENOMEM;
 921         }
 922
 923         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 924                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 925
 926         spin_lock_init(&nesdev->cqp.lock);
 927         init_waitqueue_head(&nesdev->cqp.waitq);
 928
 929         /* Setup Various Structures */
 930         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 931                         ~(unsigned long)(512 - 1));
 932         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 933                         ~(unsigned long long)(512 - 1));
 934
 935         nesdev->cqp.sq_vbase = vmem;
 936         nesdev->cqp.sq_pbase = pmem;
 937         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 938         nesdev->cqp.sq_head = 0;
 939         nesdev->cqp.sq_tail = 0;
 940         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 941
 942         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 943         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 944
 945         nesdev->ccq.cq_vbase = vmem;
 946         nesdev->ccq.cq_pbase = pmem;
 947         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 948         nesdev->ccq.cq_head = 0;
 949         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 950         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 951
 952         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 953         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 954
 955         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 956         ceq = &nesadapter->ceq[nesdev->ceq_index];
 957         ceq->ceq_vbase = vmem;
 958         ceq->ceq_pbase = pmem;
 959         ceq->ceq_size = NES_CCEQ_SIZE;
 960         ceq->ceq_head = 0;
 961
 962         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 963         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 964
 965         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 966         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 967         nic_ceq->ceq_vbase = vmem;
 968         nic_ceq->ceq_pbase = pmem;
 969         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 970         nic_ceq->ceq_head = 0;
 971
 972         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 973         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 974
 975         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 976         aeq->aeq_vbase = vmem;
 977         aeq->aeq_pbase = pmem;
 978         aeq->aeq_size = nesadapter->max_qp;
 979         aeq->aeq_head = 0;
 980
 981         /* Setup QP Context */
 982         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 983         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 984
 985         cqp_qp_context = vmem;
 986         cqp_qp_context->context_words[0] =
 987                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 988         cqp_qp_context->context_words[1] = 0;
 989         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 990         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 991
 992
 993         /* Write the address to Create CQP */
 994         if ((sizeof(dma_addr_t) > 4)) {
 995                 nes_write_indexed(nesdev,
 996                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 997                                 ((u64)pmem) >> 32);
 998         } else {
 999                 nes_write_indexed(nesdev,
1000                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1001         }
1002         nes_write_indexed(nesdev,
1003                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1004                         (u32)pmem);
1005
1006         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1007         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1008
1009         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1010                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1011                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1012         }
1013
1014         /* Write Create CCQ WQE */
1015         cqp_head = nesdev->cqp.sq_head++;
1016         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1017         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1018         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1019                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1020                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1021         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1022                             (nesdev->ccq.cq_number |
1023                              ((u32)nesdev->ceq_index << 16)));
1024         u64temp = (u64)nesdev->ccq.cq_pbase;
1025         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1026         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1027         u64temp = (unsigned long)&nesdev->ccq;
1028         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1029                         cpu_to_le32((u32)(u64temp >> 1));
1030         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1031                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1032         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1033
1034         /* Write Create CEQ WQE */
1035         cqp_head = nesdev->cqp.sq_head++;
1036         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1037         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1038         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1039                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1040         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1041         u64temp = (u64)ceq->ceq_pbase;
1042         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1043
1044         /* Write Create AEQ WQE */
1045         cqp_head = nesdev->cqp.sq_head++;
1046         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1047         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1048         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1049                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1050         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1051         u64temp = (u64)aeq->aeq_pbase;
1052         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1053
1054         /* Write Create NIC CEQ WQE */
1055         cqp_head = nesdev->cqp.sq_head++;
1056         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1057         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1058         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1059                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1060         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1061         u64temp = (u64)nic_ceq->ceq_pbase;
1062         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1063
1064         /* Poll until CCQP done */
1065         count = 0;
1066         do {
1067                 if (count++ > 1000) {
1068                         printk(KERN_ERR PFX "Error creating CQP\n");
1069                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1070                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1071                         return -1;
1072                 }
1073                 udelay(10);
1074         } while (!(nes_read_indexed(nesdev,
1075                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1076
1077         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1078                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1079
1080         u32temp = 0x04800000;
1081         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1082
1083         /* wait for the CCQ, CEQ, and AEQ to get created */
1084         count = 0;
1085         do {
1086                 if (count++ > 1000) {
1087                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1088                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1089                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1090                         return -1;
1091                 }
1092                 udelay(10);
1093         } while (((nes_read_indexed(nesdev,
1094                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1095
1096         /* dump the QP status value */
1097         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1098                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1099
1100         nesdev->cqp.sq_tail++;
1101
1102         return 0;
1103 }
1104
1105
1106 /**
1107  * nes_destroy_cqp
1108  */
1109 int nes_destroy_cqp(struct nes_device *nesdev)
1110 {
1111         struct nes_hw_cqp_wqe *cqp_wqe;
1112         u32 count = 0;
1113         u32 cqp_head;
1114         unsigned long flags;
1115
1116         do {
1117                 if (count++ > 1000)
1118                         break;
1119                 udelay(10);
1120         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1121
1122         /* Reset CCQ */
1123         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1124                         nesdev->ccq.cq_number);
1125
1126         /* Disable device interrupts */
1127         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1128
1129         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1130
1131         /* Destroy the AEQ */
1132         cqp_head = nesdev->cqp.sq_head++;
1133         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1134         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1135         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1136                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1137         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1138
1139         /* Destroy the NIC CEQ */
1140         cqp_head = nesdev->cqp.sq_head++;
1141         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1142         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1143         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1144                         ((u32)nesdev->nic_ceq_index << 8));
1145
1146         /* Destroy the CEQ */
1147         cqp_head = nesdev->cqp.sq_head++;
1148         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1149         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1150         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1151                         (nesdev->ceq_index << 8));
1152
1153         /* Destroy the CCQ */
1154         cqp_head = nesdev->cqp.sq_head++;
1155         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1156         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1157         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1158         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1159                         ((u32)nesdev->ceq_index << 16));
1160
1161         /* Destroy CQP */
1162         cqp_head = nesdev->cqp.sq_head++;
1163         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1164         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1165         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1166                         NES_CQP_QP_TYPE_CQP);
1167         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1168
1169         barrier();
1170         /* Ring doorbell (5 WQEs) */
1171         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1172
1173         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1174
1175         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1176         count = 0;
1177         do {
1178                 if (count++ > 1000) {
1179                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1180                                         PCI_FUNC(nesdev->pcidev->devfn));
1181                         break;
1182                 }
1183                 udelay(10);
1184         } while (((nes_read_indexed(nesdev,
1185                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1186
1187         /* dump the QP status value */
1188         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1189                         PCI_FUNC(nesdev->pcidev->devfn),
1190                         nes_read_indexed(nesdev,
1191                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1192
1193         kfree(nesdev->nes_cqp_requests);
1194
1195         /* Free the control structures */
1196         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1197                         nesdev->cqp.sq_pbase);
1198
1199         return 0;
1200 }
1201
1202
1203 /**
1204  * nes_init_phy
1205  */
1206 int nes_init_phy(struct nes_device *nesdev)
1207 {
1208         struct nes_adapter *nesadapter = nesdev->nesadapter;
1209         u32 counter = 0;
1210         u32 mac_index = nesdev->mac_index;
1211         u32 tx_config;
1212         u16 phy_data;
1213
1214         if (nesadapter->OneG_Mode) {
1215                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1216                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1217                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1218                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1219                         tx_config |= 0x04;
1220                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1221                 }
1222
1223                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1224                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1225                                 nesadapter->phy_index[mac_index], phy_data);
1226                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index],  0xb000);
1227
1228                 /* Reset the PHY */
1229                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1230                 udelay(100);
1231                 counter = 0;
1232                 do {
1233                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1234                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1235                         if (counter++ > 100) break;
1236                 } while (phy_data & 0x8000);
1237
1238                 /* Setting no phy loopback */
1239                 phy_data &= 0xbfff;
1240                 phy_data |= 0x1140;
1241                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1242                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1243                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1244
1245                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1246                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1247
1248                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1249                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1250
1251                 /* Setting the interrupt mask */
1252                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1253                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1254                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1255
1256                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1257                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1258
1259                 /* turning on flow control */
1260                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1261                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1262                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1263                                 (phy_data & ~(0x03E0)) | 0xc00);
1264                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1265                                 phy_data | 0xc00); */
1266                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1267                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1268
1269                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1270                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1271                 /* Clear Half duplex */
1272                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1273                                 phy_data & ~(0x0100));
1274                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1275                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1276
1277                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1278                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1279         } else {
1280                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
1281                         /* setup 10G MDIO operation */
1282                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1283                         tx_config |= 0x14;
1284                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1285                 }
1286         }
1287         return 0;
1288 }
1289
1290
1291 /**
1292  * nes_replenish_nic_rq
1293  */
1294 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1295 {
1296         unsigned long flags;
1297         dma_addr_t bus_address;
1298         struct sk_buff *skb;
1299         struct nes_hw_nic_rq_wqe *nic_rqe;
1300         struct nes_hw_nic *nesnic;
1301         struct nes_device *nesdev;
1302         u32 rx_wqes_posted = 0;
1303
1304         nesnic = &nesvnic->nic;
1305         nesdev = nesvnic->nesdev;
1306         spin_lock_irqsave(&nesnic->rq_lock, flags);
1307         if (nesnic->replenishing_rq !=0) {
1308                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1309                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1310                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1311                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1312                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1313                         add_timer(&nesvnic->rq_wqes_timer);
1314                 } else
1315                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1316                 return;
1317         }
1318         nesnic->replenishing_rq = 1;
1319         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1320         do {
1321                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1322                 if (skb) {
1323                         skb->dev = nesvnic->netdev;
1324
1325                         bus_address = pci_map_single(nesdev->pcidev,
1326                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1327
1328                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1329                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1330                                         cpu_to_le32(nesvnic->max_frame_size);
1331                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1332                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1333                                         cpu_to_le32((u32)bus_address);
1334                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1335                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1336                         nesnic->rx_skb[nesnic->rq_head] = skb;
1337                         nesnic->rq_head++;
1338                         nesnic->rq_head &= nesnic->rq_size - 1;
1339                         atomic_dec(&nesvnic->rx_skbs_needed);
1340                         barrier();
1341                         if (++rx_wqes_posted == 255) {
1342                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1343                                 rx_wqes_posted = 0;
1344                         }
1345                 } else {
1346                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1347                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1348                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1349                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1350                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1351                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1352                                 add_timer(&nesvnic->rq_wqes_timer);
1353                         } else
1354                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1355                         break;
1356                 }
1357         } while (atomic_read(&nesvnic->rx_skbs_needed));
1358         barrier();
1359         if (rx_wqes_posted)
1360                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1361         nesnic->replenishing_rq = 0;
1362 }
1363
1364
1365 /**
1366  * nes_rq_wqes_timeout
1367  */
1368 static void nes_rq_wqes_timeout(unsigned long parm)
1369 {
1370         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1371         printk("%s: Timer fired.\n", __func__);
1372         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1373         if (atomic_read(&nesvnic->rx_skbs_needed))
1374                 nes_replenish_nic_rq(nesvnic);
1375 }
1376
1377
1378 /**
1379  * nes_init_nic_qp
1380  */
1381 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1382 {
1383         struct nes_hw_cqp_wqe *cqp_wqe;
1384         struct nes_hw_nic_sq_wqe *nic_sqe;
1385         struct nes_hw_nic_qp_context *nic_context;
1386         struct sk_buff *skb;
1387         struct nes_hw_nic_rq_wqe *nic_rqe;
1388         struct nes_vnic *nesvnic = netdev_priv(netdev);
1389         unsigned long flags;
1390         void *vmem;
1391         dma_addr_t pmem;
1392         u64 u64temp;
1393         int ret;
1394         u32 cqp_head;
1395         u32 counter;
1396         u32 wqe_count;
1397         u8 jumbomode=0;
1398
1399         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1400         nesvnic->nic_mem_size = 256 +
1401                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1402                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1403                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1404                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1405                         sizeof(struct nes_hw_nic_qp_context);
1406
1407         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1408                         &nesvnic->nic_pbase);
1409         if (!nesvnic->nic_vbase) {
1410                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1411                 return -ENOMEM;
1412         }
1413         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1414         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1415                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1416
1417         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1418                         ~(unsigned long)(256 - 1));
1419         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1420                         ~(unsigned long long)(256 - 1));
1421
1422         /* Setup the first Fragment buffers */
1423         nesvnic->nic.first_frag_vbase = vmem;
1424
1425         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1426                 nesvnic->nic.frag_paddr[counter] = pmem;
1427                 pmem += sizeof(struct nes_first_frag);
1428         }
1429
1430         /* setup the SQ */
1431         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1432
1433         nesvnic->nic.sq_vbase = (void *)vmem;
1434         nesvnic->nic.sq_pbase = pmem;
1435         nesvnic->nic.sq_head = 0;
1436         nesvnic->nic.sq_tail = 0;
1437         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1438         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1439                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1440                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1441                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1442                                 NES_NIC_SQ_WQE_COMPLETION);
1443                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1444                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1445                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1446                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1447                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1448                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1449         }
1450
1451         nesvnic->get_cqp_request = nes_get_cqp_request;
1452         nesvnic->post_cqp_request = nes_post_cqp_request;
1453         nesvnic->mcrq_mcast_filter = NULL;
1454
1455         spin_lock_init(&nesvnic->nic.sq_lock);
1456         spin_lock_init(&nesvnic->nic.rq_lock);
1457
1458         /* setup the RQ */
1459         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1460         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1461
1462
1463         nesvnic->nic.rq_vbase = vmem;
1464         nesvnic->nic.rq_pbase = pmem;
1465         nesvnic->nic.rq_head = 0;
1466         nesvnic->nic.rq_tail = 0;
1467         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1468
1469         /* setup the CQ */
1470         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1471         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1472
1473         if (nesdev->nesadapter->netdev_count > 2)
1474                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1475         else
1476                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1477
1478         nesvnic->nic_cq.cq_vbase = vmem;
1479         nesvnic->nic_cq.cq_pbase = pmem;
1480         nesvnic->nic_cq.cq_head = 0;
1481         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1482
1483         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1484
1485         /* Send CreateCQ request to CQP */
1486         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1487         cqp_head = nesdev->cqp.sq_head;
1488
1489         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1490         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1491
1492         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1493                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1494                         ((u32)nesvnic->nic_cq.cq_size << 16));
1495         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1496                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1497         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1498         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1499         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1500         u64temp = (unsigned long)&nesvnic->nic_cq;
1501         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1502         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1503                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1504         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1505         if (++cqp_head >= nesdev->cqp.sq_size)
1506                 cqp_head = 0;
1507         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1508         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1509
1510         /* Send CreateQP request to CQP */
1511         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1512         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1513                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1514                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1515         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1516                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1517                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1518         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1519                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1520         }
1521
1522         u64temp = (u64)nesvnic->nic.sq_pbase;
1523         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1524         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1525         u64temp = (u64)nesvnic->nic.rq_pbase;
1526         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1527         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1528
1529         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1530                         NES_CQP_QP_TYPE_NIC);
1531         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1532         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1533                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1534         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1535
1536         if (++cqp_head >= nesdev->cqp.sq_size)
1537                 cqp_head = 0;
1538         nesdev->cqp.sq_head = cqp_head;
1539
1540         barrier();
1541
1542         /* Ring doorbell (2 WQEs) */
1543         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1544
1545         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1546         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1547                         nesvnic->nic.qp_id);
1548
1549         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1550                         NES_EVENT_TIMEOUT);
1551         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1552                         nesvnic->nic.qp_id, ret);
1553         if (!ret) {
1554                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1555                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1556                                 nesvnic->nic_pbase);
1557                 return -EIO;
1558         }
1559
1560         /* Populate the RQ */
1561         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1562                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1563                 if (!skb) {
1564                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1565
1566                         nes_destroy_nic_qp(nesvnic);
1567                         return -ENOMEM;
1568                 }
1569
1570                 skb->dev = netdev;
1571
1572                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1573                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1574
1575                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1576                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1577                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1578                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1579                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1580                 nesvnic->nic.rx_skb[counter] = skb;
1581         }
1582
1583         wqe_count = NES_NIC_WQ_SIZE - 1;
1584         nesvnic->nic.rq_head = wqe_count;
1585         barrier();
1586         do {
1587                 counter = min(wqe_count, ((u32)255));
1588                 wqe_count -= counter;
1589                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1590         } while (wqe_count);
1591         init_timer(&nesvnic->rq_wqes_timer);
1592         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1593         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1594         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1595
1596         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1597         {
1598                 nes_nic_init_timer(nesdev);
1599                 if (netdev->mtu > 1500)
1600                         jumbomode = 1;
1601                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1602         }
1603
1604         return 0;
1605 }
1606
1607
1608 /**
1609  * nes_destroy_nic_qp
1610  */
1611 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1612 {
1613         struct nes_device *nesdev = nesvnic->nesdev;
1614         struct nes_hw_cqp_wqe *cqp_wqe;
1615         struct nes_hw_nic_rq_wqe *nic_rqe;
1616         u64 wqe_frag;
1617         u32 cqp_head;
1618         unsigned long flags;
1619         int ret;
1620
1621         /* Free remaining NIC receive buffers */
1622         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1623                 nic_rqe = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1624                 wqe_frag = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1625                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1626                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1627                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1628                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1629                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1630         }
1631
1632         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1633
1634         /* Destroy NIC QP */
1635         cqp_head = nesdev->cqp.sq_head;
1636         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1637         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1638
1639         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1640                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1641         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1642                 nesvnic->nic.qp_id);
1643
1644         if (++cqp_head >= nesdev->cqp.sq_size)
1645                 cqp_head = 0;
1646
1647         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1648
1649         /* Destroy NIC CQ */
1650         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1651         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1652                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1653         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1654                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1655
1656         if (++cqp_head >= nesdev->cqp.sq_size)
1657                 cqp_head = 0;
1658
1659         nesdev->cqp.sq_head = cqp_head;
1660         barrier();
1661
1662         /* Ring doorbell (2 WQEs) */
1663         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1664
1665         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1666         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1667                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1668                         cqp_head, nesdev->cqp.sq_head,
1669                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1670
1671         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1672                         NES_EVENT_TIMEOUT);
1673
1674         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1675                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1676                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1677         if (!ret) {
1678                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1679                                 nesvnic->nic.qp_id);
1680         }
1681
1682         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1683                         nesvnic->nic_pbase);
1684 }
1685
1686 /**
1687  * nes_napi_isr
1688  */
1689 int nes_napi_isr(struct nes_device *nesdev)
1690 {
1691         struct nes_adapter *nesadapter = nesdev->nesadapter;
1692         u32 int_stat;
1693
1694         if (nesdev->napi_isr_ran) {
1695                 /* interrupt status has already been read in ISR */
1696                 int_stat = nesdev->int_stat;
1697         } else {
1698                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1699                 nesdev->int_stat = int_stat;
1700                 nesdev->napi_isr_ran = 1;
1701         }
1702
1703         int_stat &= nesdev->int_req;
1704         /* iff NIC, process here, else wait for DPC */
1705         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1706                 nesdev->napi_isr_ran = 0;
1707                 nes_write32(nesdev->regs+NES_INT_STAT,
1708                                 (int_stat &
1709                                 ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1710
1711                 /* Process the CEQs */
1712                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1713
1714                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1715                                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1716                                           ((nesadapter->et_use_adaptive_rx_coalesce) &&
1717                                            (nesdev->deepcq_count > nesadapter->et_pkt_rate_low)))) ) {
1718                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1719                                 /* Enable Periodic timer interrupts */
1720                                 nesdev->int_req |= NES_INT_TIMER;
1721                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1722                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1723                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1724                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1725                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1726                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1727                         }
1728
1729                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1730                         {
1731                                 nes_nic_init_timer(nesdev);
1732                         }
1733                         /* Enable interrupts, except CEQs */
1734                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1735                 } else {
1736                         /* Enable interrupts, make sure timer is off */
1737                         nesdev->int_req &= ~NES_INT_TIMER;
1738                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1739                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1740                 }
1741                 nesdev->deepcq_count = 0;
1742                 return 1;
1743         } else {
1744                 return 0;
1745         }
1746 }
1747
1748
1749 /**
1750  * nes_dpc
1751  */
1752 void nes_dpc(unsigned long param)
1753 {
1754         struct nes_device *nesdev = (struct nes_device *)param;
1755         struct nes_adapter *nesadapter = nesdev->nesadapter;
1756         u32 counter;
1757         u32 loop_counter = 0;
1758         u32 int_status_bit;
1759         u32 int_stat;
1760         u32 timer_stat;
1761         u32 temp_int_stat;
1762         u32 intf_int_stat;
1763         u32 debug_error;
1764         u32 processed_intf_int = 0;
1765         u16 processed_timer_int = 0;
1766         u16 completion_ints = 0;
1767         u16 timer_ints = 0;
1768
1769         /* nes_debug(NES_DBG_ISR, "\n"); */
1770
1771         do {
1772                 timer_stat = 0;
1773                 if (nesdev->napi_isr_ran) {
1774                         nesdev->napi_isr_ran = 0;
1775                         int_stat = nesdev->int_stat;
1776                 } else
1777                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1778                 if (processed_intf_int != 0)
1779                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1780                 else
1781                         int_stat &= nesdev->int_req;
1782                 if (processed_timer_int == 0) {
1783                         processed_timer_int = 1;
1784                         if (int_stat & NES_INT_TIMER) {
1785                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1786                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1787                                         int_stat &= ~NES_INT_TIMER;
1788                                 }
1789                         }
1790                 } else {
1791                         int_stat &= ~NES_INT_TIMER;
1792                 }
1793
1794                 if (int_stat) {
1795                         if (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1796                                         NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)) {
1797                                 /* Ack the interrupts */
1798                                 nes_write32(nesdev->regs+NES_INT_STAT,
1799                                                 (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1800                                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1801                         }
1802
1803                         temp_int_stat = int_stat;
1804                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1805                                 if (int_stat & int_status_bit) {
1806                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1807                                         temp_int_stat &= ~int_status_bit;
1808                                         completion_ints = 1;
1809                                 }
1810                                 if (!(temp_int_stat & 0x0000ffff))
1811                                         break;
1812                                 int_status_bit <<= 1;
1813                         }
1814
1815                         /* Process the AEQ for this pci function */
1816                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1817                         if (int_stat & int_status_bit) {
1818                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1819                         }
1820
1821                         /* Process the MAC interrupt for this pci function */
1822                         int_status_bit = 1 << (24 + nesdev->mac_index);
1823                         if (int_stat & int_status_bit) {
1824                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1825                         }
1826
1827                         if (int_stat & NES_INT_TIMER) {
1828                                 if (timer_stat & nesdev->timer_int_req) {
1829                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1830                                                         (timer_stat & nesdev->timer_int_req) |
1831                                                         ~(nesdev->nesadapter->timer_int_req));
1832                                         timer_ints = 1;
1833                                 }
1834                         }
1835
1836                         if (int_stat & NES_INT_INTF) {
1837                                 processed_intf_int = 1;
1838                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1839                                 intf_int_stat &= nesdev->intf_int_req;
1840                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1841                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1842                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1843                                                         (u16)debug_error);
1844                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1845                                                         0x01010000 | (debug_error & 0x0000ffff));
1846                                         /* BUG(); */
1847                                         if (crit_err_count++ > 10)
1848                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1849                                 }
1850                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1851                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1852                                         BUG();
1853                                 }
1854                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
1855                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
1856                                         BUG();
1857                                 }
1858                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
1859                         }
1860
1861                         if (int_stat & NES_INT_TSW) {
1862                         }
1863                 }
1864                 /* Don't use the interface interrupt bit stay in loop */
1865                 int_stat &= ~NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1866                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3;
1867         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
1868
1869         if (timer_ints == 1) {
1870                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
1871                         if (completion_ints == 0) {
1872                                 nesdev->timer_only_int_count++;
1873                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
1874                                         nesdev->timer_only_int_count = 0;
1875                                         nesdev->int_req &= ~NES_INT_TIMER;
1876                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1877                                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1878                                 } else {
1879                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1880                                 }
1881                         } else {
1882                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1883                                 {
1884                                         nes_nic_init_timer(nesdev);
1885                                 }
1886                                 nesdev->timer_only_int_count = 0;
1887                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1888                         }
1889                 } else {
1890                         nesdev->timer_only_int_count = 0;
1891                         nesdev->int_req &= ~NES_INT_TIMER;
1892                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1893                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1894                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1895                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1896                 }
1897         } else {
1898                 if ( (completion_ints == 1) &&
1899                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
1900                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1901                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
1902                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
1903                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
1904                         nesdev->timer_only_int_count = 0;
1905                         nesdev->int_req |= NES_INT_TIMER;
1906                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1907                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1908                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1909                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1910                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1911                 } else {
1912                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1913                 }
1914         }
1915         nesdev->deepcq_count = 0;
1916 }
1917
1918
1919 /**
1920  * nes_process_ceq
1921  */
1922 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
1923 {
1924         u64 u64temp;
1925         struct nes_hw_cq *cq;
1926         u32 head;
1927         u32 ceq_size;
1928
1929         /* nes_debug(NES_DBG_CQ, "\n"); */
1930         head = ceq->ceq_head;
1931         ceq_size = ceq->ceq_size;
1932
1933         do {
1934                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
1935                                 NES_CEQE_VALID) {
1936                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX])))<<32) |
1937                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
1938                         u64temp <<= 1;
1939                         cq = *((struct nes_hw_cq **)&u64temp);
1940                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
1941                         barrier();
1942                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
1943
1944                         /* call the event handler */
1945                         cq->ce_handler(nesdev, cq);
1946
1947                         if (++head >= ceq_size)
1948                                 head = 0;
1949                 } else {
1950                         break;
1951                 }
1952
1953         } while (1);
1954
1955         ceq->ceq_head = head;
1956 }
1957
1958
1959 /**
1960  * nes_process_aeq
1961  */
1962 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
1963 {
1964 //      u64 u64temp;
1965         u32 head;
1966         u32 aeq_size;
1967         u32 aeqe_misc;
1968         u32 aeqe_cq_id;
1969         struct nes_hw_aeqe volatile *aeqe;
1970
1971         head = aeq->aeq_head;
1972         aeq_size = aeq->aeq_size;
1973
1974         do {
1975                 aeqe = &aeq->aeq_vbase[head];
1976                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
1977                         break;
1978                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
1979                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
1980                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
1981                         if (aeqe_cq_id >= NES_FIRST_QPN) {
1982                                 /* dealing with an accelerated QP related AE */
1983 //                              u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])))<<32) |
1984 //                                      ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
1985                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
1986                         } else {
1987                                 /* TODO: dealing with a CQP related AE */
1988                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
1989                                                 (u16)(aeqe_misc >> 16));
1990                         }
1991                 }
1992
1993                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
1994
1995                 if (++head >= aeq_size)
1996                         head = 0;
1997         }
1998         while (1);
1999         aeq->aeq_head = head;
2000 }
2001
2002 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2003 {
2004         struct nes_adapter *nesadapter = nesdev->nesadapter;
2005         u32 reset_value;
2006         u32 i=0;
2007         u32 u32temp;
2008
2009         if (nesadapter->hw_rev == NE020_REV) {
2010                 return;
2011         }
2012         mh_detected++;
2013
2014         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2015
2016         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2017                 reset_value |= 0x0000001d;
2018         else
2019                 reset_value |= 0x0000002d;
2020
2021         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2022                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2023                         nesadapter->link_interrupt_count[0] = 0;
2024                         nesadapter->link_interrupt_count[1] = 0;
2025                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2026                         if (0x00000040 & u32temp)
2027                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2028                         else
2029                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2030
2031                         reset_value |= 0x0000003d;
2032                 }
2033                 nesadapter->link_interrupt_count[mac_index] = 0;
2034         }
2035
2036         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2037
2038         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2039                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2040
2041         if (0x0000003d == (reset_value & 0x0000003d)) {
2042                 u32 pcs_control_status0, pcs_control_status1;
2043
2044                 for (i = 0; i < 10; i++) {
2045                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2046                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2047                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2048                              && (pcs_control_status0 & 0x00100000))
2049                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2050                                 && (pcs_control_status1 & 0x00100000)))
2051                                 continue;
2052                         else
2053                                 break;
2054                 }
2055                 if (10 == i) {
2056                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2057                         if (0x00000040 & u32temp)
2058                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2059                         else
2060                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2061
2062                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2063
2064                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2065                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2066                 }
2067         }
2068 }
2069
2070 /**
2071  * nes_process_mac_intr
2072  */
2073 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2074 {
2075         unsigned long flags;
2076         u32 pcs_control_status;
2077         struct nes_adapter *nesadapter = nesdev->nesadapter;
2078         struct nes_vnic *nesvnic;
2079         u32 mac_status;
2080         u32 mac_index = nesdev->mac_index;
2081         u32 u32temp;
2082         u16 phy_data;
2083         u16 temp_phy_data;
2084
2085         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2086         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2087                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2088                 return;
2089         }
2090         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2091         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2092
2093         /* ack the MAC interrupt */
2094         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2095         /* Clear the interrupt */
2096         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2097
2098         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2099
2100         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2101                 nesdev->link_status_interrupts++;
2102                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2103                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2104                         nes_reset_link(nesdev, mac_index);
2105                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2106                 }
2107                 /* read the PHY interrupt status register */
2108                 if (nesadapter->OneG_Mode) {
2109                         do {
2110                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2111                                                 nesadapter->phy_index[mac_index], &phy_data);
2112                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2113                                                 nesadapter->phy_index[mac_index], phy_data);
2114                         } while (phy_data&0x8000);
2115
2116                         temp_phy_data = 0;
2117                         do {
2118                                 nes_read_1G_phy_reg(nesdev, 0x11,
2119                                                 nesadapter->phy_index[mac_index], &phy_data);
2120                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2121                                                 nesadapter->phy_index[mac_index], phy_data);
2122                                 if (temp_phy_data == phy_data)
2123                                         break;
2124                                 temp_phy_data = phy_data;
2125                         } while (1);
2126
2127                         nes_read_1G_phy_reg(nesdev, 0x1e,
2128                                         nesadapter->phy_index[mac_index], &phy_data);
2129                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2130                                         nesadapter->phy_index[mac_index], phy_data);
2131
2132                         nes_read_1G_phy_reg(nesdev, 1,
2133                                         nesadapter->phy_index[mac_index], &phy_data);
2134                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2135                                         nesadapter->phy_index[mac_index], phy_data);
2136
2137                         if (temp_phy_data & 0x1000) {
2138                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2139                                 phy_data = 4;
2140                         } else {
2141                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2142                         }
2143                 }
2144                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2145                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2146                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2147                 pcs_control_status = nes_read_indexed(nesdev,
2148                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2149                 pcs_control_status = nes_read_indexed(nesdev,
2150                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2151                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2152                                 mac_index, pcs_control_status);
2153                 if (nesadapter->OneG_Mode) {
2154                         u32temp = 0x01010000;
2155                         if (nesadapter->port_count > 2) {
2156                                 u32temp |= 0x02020000;
2157                         }
2158                         if ((pcs_control_status & u32temp)!= u32temp) {
2159                                 phy_data = 0;
2160                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2161                         }
2162                 } else if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
2163                         nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2164                         temp_phy_data = (u16)nes_read_indexed(nesdev,
2165                                                                 NES_IDX_MAC_MDIO_CONTROL);
2166                         u32temp = 20;
2167                         do {
2168                                 nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2169                                 phy_data = (u16)nes_read_indexed(nesdev,
2170                                                                 NES_IDX_MAC_MDIO_CONTROL);
2171                                 if ((phy_data == temp_phy_data) || (!(--u32temp)))
2172                                         break;
2173                                 temp_phy_data = phy_data;
2174                         } while (1);
2175                         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2176                                 __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2177
2178                 } else {
2179                         phy_data = (0x0f0f0000 == (pcs_control_status & 0x0f1f0000)) ? 4 : 0;
2180                 }
2181
2182                 if (phy_data & 0x0004) {
2183                         nesadapter->mac_link_down[mac_index] = 0;
2184                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2185                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2186                                                 nesvnic->linkup);
2187                                 if (nesvnic->linkup == 0) {
2188                                         printk(PFX "The Link is now up for port %u, netdev %p.\n",
2189                                                         mac_index, nesvnic->netdev);
2190                                         if (netif_queue_stopped(nesvnic->netdev))
2191                                                 netif_start_queue(nesvnic->netdev);
2192                                         nesvnic->linkup = 1;
2193                                         netif_carrier_on(nesvnic->netdev);
2194                                 }
2195                         }
2196                 } else {
2197                         nesadapter->mac_link_down[mac_index] = 1;
2198                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2199                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2200                                                 nesvnic->linkup);
2201                                 if (nesvnic->linkup == 1) {
2202                                         printk(PFX "The Link is now down for port %u, netdev %p.\n",
2203                                                         mac_index, nesvnic->netdev);
2204                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2205                                                 netif_stop_queue(nesvnic->netdev);
2206                                         nesvnic->linkup = 0;
2207                                         netif_carrier_off(nesvnic->netdev);
2208                                 }
2209                         }
2210                 }
2211         }
2212
2213         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2214 }
2215
2216
2217
2218 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2219 {
2220         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2221
2222         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2223 }
2224
2225
2226 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2227 * getting out of nic_ce_handler
2228 */
2229 #define MAX_RQES_TO_PROCESS     384
2230
2231 /**
2232  * nes_nic_ce_handler
2233  */
2234 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2235 {
2236         u64 u64temp;
2237         dma_addr_t bus_address;
2238         struct nes_hw_nic *nesnic;
2239         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2240         struct nes_adapter *nesadapter = nesdev->nesadapter;
2241         struct nes_hw_nic_rq_wqe *nic_rqe;
2242         struct nes_hw_nic_sq_wqe *nic_sqe;
2243         struct sk_buff *skb;
2244         struct sk_buff *rx_skb;
2245         __le16 *wqe_fragment_length;
2246         u32 head;
2247         u32 cq_size;
2248         u32 rx_pkt_size;
2249         u32 cqe_count=0;
2250         u32 cqe_errv;
2251         u32 cqe_misc;
2252         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2253         u16 vlan_tag;
2254         u16 pkt_type;
2255         u16 rqes_processed = 0;
2256         u8 sq_cqes = 0;
2257
2258         head = cq->cq_head;
2259         cq_size = cq->cq_size;
2260         cq->cqes_pending = 1;
2261         do {
2262                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2263                                 NES_NIC_CQE_VALID) {
2264                         nesnic = &nesvnic->nic;
2265                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2266                         if (cqe_misc & NES_NIC_CQE_SQ) {
2267                                 sq_cqes++;
2268                                 wqe_fragment_index = 1;
2269                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2270                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2271                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2272                                 /* bump past the vlan tag */
2273                                 wqe_fragment_length++;
2274                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2275                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2276                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2277                                         bus_address = (dma_addr_t)u64temp;
2278                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2279                                                 pci_unmap_single(nesdev->pcidev,
2280                                                                 bus_address,
2281                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2282                                                                 PCI_DMA_TODEVICE);
2283                                         }
2284                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2285                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2286                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2287                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2288                                                         bus_address = (dma_addr_t)u64temp;
2289                                                         pci_unmap_page(nesdev->pcidev,
2290                                                                         bus_address,
2291                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2292                                                                         PCI_DMA_TODEVICE);
2293                                                 } else
2294                                                         break;
2295                                         }
2296                                         if (skb)
2297                                                 dev_kfree_skb_any(skb);
2298                                 }
2299                                 nesnic->sq_tail++;
2300                                 nesnic->sq_tail &= nesnic->sq_size-1;
2301                                 if (sq_cqes > 128) {
2302                                         barrier();
2303                                 /* restart the queue if it had been stopped */
2304                                 if (netif_queue_stopped(nesvnic->netdev))
2305                                         netif_wake_queue(nesvnic->netdev);
2306                                         sq_cqes = 0;
2307                                 }
2308                         } else {
2309                                 rqes_processed ++;
2310
2311                                 cq->rx_cqes_completed++;
2312                                 cq->rx_pkts_indicated++;
2313                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2314                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2315                                 /* Get the skb */
2316                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2317                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2318                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2319                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2320                                 pci_unmap_single(nesdev->pcidev, bus_address,
2321                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2322                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2323                                 /* rx_skb->len = rx_pkt_size; */
2324                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2325                                 skb_put(rx_skb, rx_pkt_size);
2326                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2327                                 nesnic->rq_tail++;
2328                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2329
2330                                 atomic_inc(&nesvnic->rx_skbs_needed);
2331                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2332                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2333                                                         cq->cq_number | (cqe_count << 16));
2334 //                                      nesadapter->tune_timer.cq_count += cqe_count;
2335                                         nesdev->currcq_count += cqe_count;
2336                                         cqe_count = 0;
2337                                         nes_replenish_nic_rq(nesvnic);
2338                                 }
2339                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2340                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2341                                 rx_skb->ip_summed = CHECKSUM_NONE;
2342
2343                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2344                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2345                                         if ((cqe_errv &
2346                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2347                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2348                                                 if (nesvnic->rx_checksum_disabled == 0) {
2349                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2350                                                 }
2351                                         } else
2352                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2353                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2354                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2355
2356                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2357                                         if ((cqe_errv &
2358                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2359                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2360                                                 if (nesvnic->rx_checksum_disabled == 0) {
2361                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2362                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2363                                                                   nesvnic->netdev->name); */
2364                                                 }
2365                                         } else
2366                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2367                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2368                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2369                                         }
2370                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2371                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2372
2373                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2374                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2375                                 } else {
2376                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2377                                                 vlan_tag = (u16)(le32_to_cpu(
2378                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2379                                                                 >> 16);
2380                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2381                                                                 nesvnic->netdev->name, vlan_tag);
2382                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2383                                         } else {
2384                                                 nes_netif_rx(rx_skb);
2385                                         }
2386                                 }
2387
2388                                 nesvnic->netdev->last_rx = jiffies;
2389                                 /* nesvnic->netstats.rx_packets++; */
2390                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2391                         }
2392
2393                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2394                         /* Accounting... */
2395                         cqe_count++;
2396                         if (++head >= cq_size)
2397                                 head = 0;
2398                         if (cqe_count == 255) {
2399                                 /* Replenish Nic CQ */
2400                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2401                                                 cq->cq_number | (cqe_count << 16));
2402 //                              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2403                                 nesdev->currcq_count += cqe_count;
2404                                 cqe_count = 0;
2405                         }
2406
2407                         if (cq->rx_cqes_completed >= nesvnic->budget)
2408                                 break;
2409                 } else {
2410                         cq->cqes_pending = 0;
2411                         break;
2412                 }
2413
2414         } while (1);
2415
2416         if (sq_cqes) {
2417                 barrier();
2418                 /* restart the queue if it had been stopped */
2419                 if (netif_queue_stopped(nesvnic->netdev))
2420                         netif_wake_queue(nesvnic->netdev);
2421         }
2422
2423         cq->cq_head = head;
2424         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2425                         cq->cq_number, cqe_count, cq->cq_head); */
2426         cq->cqe_allocs_pending = cqe_count;
2427         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2428         {
2429 //              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2430                 nesdev->currcq_count += cqe_count;
2431                 nes_nic_tune_timer(nesdev);
2432         }
2433         if (atomic_read(&nesvnic->rx_skbs_needed))
2434                 nes_replenish_nic_rq(nesvnic);
2435         }
2436
2437
2438 /**
2439  * nes_cqp_ce_handler
2440  */
2441 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2442 {
2443         u64 u64temp;
2444         unsigned long flags;
2445         struct nes_hw_cqp *cqp = NULL;
2446         struct nes_cqp_request *cqp_request;
2447         struct nes_hw_cqp_wqe *cqp_wqe;
2448         u32 head;
2449         u32 cq_size;
2450         u32 cqe_count=0;
2451         u32 error_code;
2452         /* u32 counter; */
2453
2454         head = cq->cq_head;
2455         cq_size = cq->cq_size;
2456
2457         do {
2458                 /* process the CQE */
2459                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2460                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2461
2462                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2463                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2464                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
2465                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2466                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2467                         cqp = *((struct nes_hw_cqp **)&u64temp);
2468
2469                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2470                         if (error_code) {
2471                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2472                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2473                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2474                                                 (u16)(error_code >> 16),
2475                                                 (u16)error_code);
2476                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2477                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2478                         }
2479
2480                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2481                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX])))<<32) |
2482                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2483                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2484                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2485                         if (cqp_request) {
2486                                 if (cqp_request->waiting) {
2487                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2488                                         cqp_request->major_code = (u16)(error_code >> 16);
2489                                         cqp_request->minor_code = (u16)error_code;
2490                                         barrier();
2491                                         cqp_request->request_done = 1;
2492                                         wake_up(&cqp_request->waitq);
2493                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2494                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2495                                                                 cqp_request,
2496                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2497                                                 if (cqp_request->dynamic) {
2498                                                         kfree(cqp_request);
2499                                                 } else {
2500                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2501                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2502                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2503                                                 }
2504                                         }
2505                                 } else if (cqp_request->callback) {
2506                                         /* Envoke the callback routine */
2507                                         cqp_request->cqp_callback(nesdev, cqp_request);
2508                                         if (cqp_request->dynamic) {
2509                                                 kfree(cqp_request);
2510                                         } else {
2511                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2512                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2513                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2514                                         }
2515                                 } else {
2516                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2517                                                         cqp_request,
2518                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2519                                         if (cqp_request->dynamic) {
2520                                                 kfree(cqp_request);
2521                                         } else {
2522                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2523                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2524                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2525                                         }
2526                                 }
2527                         } else {
2528                                 wake_up(&nesdev->cqp.waitq);
2529                         }
2530
2531                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2532                         nes_write32(nesdev->regs+NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2533                         if (++cqp->sq_tail >= cqp->sq_size)
2534                                 cqp->sq_tail = 0;
2535
2536                         /* Accounting... */
2537                         cqe_count++;
2538                         if (++head >= cq_size)
2539                                 head = 0;
2540                 } else {
2541                         break;
2542                 }
2543         } while (1);
2544         cq->cq_head = head;
2545
2546         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2547         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2548                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2549                         (nesdev->cqp.sq_size - 1)) != 1)) {
2550                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2551                                 struct nes_cqp_request, list);
2552                 list_del_init(&cqp_request->list);
2553                 head = nesdev->cqp.sq_head++;
2554                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2555                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2556                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2557                 barrier();
2558                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2559                         cpu_to_le32((u32)((unsigned long)cqp_request));
2560                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2561                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2562                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2563                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2564                 /* Ring doorbell (1 WQEs) */
2565                 barrier();
2566                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2567         }
2568         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2569
2570         /* Arm the CCQ */
2571         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2572                         cq->cq_number);
2573         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2574 }
2575
2576
2577 /**
2578  * nes_process_iwarp_aeqe
2579  */
2580 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2581                                    struct nes_hw_aeqe *aeqe)
2582 {
2583         u64 context;
2584         u64 aeqe_context = 0;
2585         unsigned long flags;
2586         struct nes_qp *nesqp;
2587         int resource_allocated;
2588         /* struct iw_cm_id *cm_id; */
2589         struct nes_adapter *nesadapter = nesdev->nesadapter;
2590         struct ib_event ibevent;
2591         /* struct iw_cm_event cm_event; */
2592         u32 aeq_info;
2593         u32 next_iwarp_state = 0;
2594         u16 async_event_id;
2595         u8 tcp_state;
2596         u8 iwarp_state;
2597
2598         nes_debug(NES_DBG_AEQ, "\n");
2599         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2600         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2601                 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2602                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2603         } else {
2604                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2605                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2606                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2607                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2608                 BUG_ON(!context);
2609         }
2610
2611         async_event_id = (u16)aeq_info;
2612         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2613         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2614         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2615                         " Tcp state = %s, iWARP state = %s\n",
2616                         async_event_id,
2617                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2618                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2619
2620
2621         switch (async_event_id) {
2622                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2623                         nesqp = *((struct nes_qp **)&context);
2624                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2625                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2626                                 nes_add_ref(&nesqp->ibqp);
2627                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2628                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2629                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2630                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2631                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2632                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2633                                                 async_event_id, nesqp->last_aeq, tcp_state);
2634                         }
2635                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2636                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2637                                 /* FIN Received but tcp state or IB state moved on,
2638                                                 should expect a close complete */
2639                                 return;
2640                         }
2641                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2642                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2643                 case NES_AEQE_AEID_TERMINATE_SENT:
2644                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2645                 case NES_AEQE_AEID_RESET_SENT:
2646                         nesqp = *((struct nes_qp **)&context);
2647                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2648                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2649                         }
2650                         nes_add_ref(&nesqp->ibqp);
2651                         spin_lock_irqsave(&nesqp->lock, flags);
2652                         nesqp->hw_iwarp_state = iwarp_state;
2653                         nesqp->hw_tcp_state = tcp_state;
2654                         nesqp->last_aeq = async_event_id;
2655
2656                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2657                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2658                                 nesqp->hte_added = 0;
2659                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2660                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2661                                                 nesqp->hwqp.qp_id);
2662                                 nes_hw_modify_qp(nesdev, nesqp,
2663                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2664                                 spin_lock_irqsave(&nesqp->lock, flags);
2665                         }
2666
2667                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2668                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2669                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2670                                 switch (nesqp->hw_iwarp_state) {
2671                                         case NES_AEQE_IWARP_STATE_RTS:
2672                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2673                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2674                                                 break;
2675                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2676                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2677                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2678                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2679                                                         next_iwarp_state |= 0x02000000;
2680                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2681                                                 }
2682                                                 break;
2683                                         default:
2684                                                 next_iwarp_state = 0;
2685                                 }
2686                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2687                                 if (next_iwarp_state) {
2688                                         nes_add_ref(&nesqp->ibqp);
2689                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2690                                                         " also added another reference\n",
2691                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2692                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2693                                 }
2694                                 nes_cm_disconn(nesqp);
2695                         } else {
2696                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2697                                         /* FIN Received but ib state not RTS,
2698                                                         close complete will be on its way */
2699                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2700                                         nes_rem_ref(&nesqp->ibqp);
2701                                         return;
2702                                 }
2703                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2704                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2705                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2706                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2707                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2708                                                         " also added another reference\n",
2709                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2710                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2711                                 }
2712                                 nes_cm_disconn(nesqp);
2713                         }
2714                         break;
2715                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2716                         nesqp = *((struct nes_qp **)&context);
2717                         spin_lock_irqsave(&nesqp->lock, flags);
2718                         nesqp->hw_iwarp_state = iwarp_state;
2719                         nesqp->hw_tcp_state = tcp_state;
2720                         nesqp->last_aeq = async_event_id;
2721                         spin_unlock_irqrestore(&nesqp->lock, flags);
2722                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2723                                         " event on QP%u \n  Q2 Data:\n",
2724                                         nesqp->hwqp.qp_id);
2725                         if (nesqp->ibqp.event_handler) {
2726                                 ibevent.device = nesqp->ibqp.device;
2727                                 ibevent.element.qp = &nesqp->ibqp;
2728                                 ibevent.event = IB_EVENT_QP_FATAL;
2729                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2730                         }
2731                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2732                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2733                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2734                                 nes_add_ref(&nesqp->ibqp);
2735                                 nes_cm_disconn(nesqp);
2736                         } else {
2737                                 nesqp->in_disconnect = 0;
2738                                 wake_up(&nesqp->kick_waitq);
2739                         }
2740                         break;
2741                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2742                         nesqp = *((struct nes_qp **)&context);
2743                         nes_add_ref(&nesqp->ibqp);
2744                         spin_lock_irqsave(&nesqp->lock, flags);
2745                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2746                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2747                         nesqp->last_aeq = async_event_id;
2748                         if (nesqp->cm_id) {
2749                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2750                                                 " event on QP%u, remote IP = 0x%08X \n",
2751                                                 nesqp->hwqp.qp_id,
2752                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2753                         } else {
2754                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2755                                                 " event on QP%u \n",
2756                                                 nesqp->hwqp.qp_id);
2757                         }
2758                         spin_unlock_irqrestore(&nesqp->lock, flags);
2759                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2760                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2761                         if (nesqp->ibqp.event_handler) {
2762                                 ibevent.device = nesqp->ibqp.device;
2763                                 ibevent.element.qp = &nesqp->ibqp;
2764                                 ibevent.event = IB_EVENT_QP_FATAL;
2765                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2766                         }
2767                         break;
2768                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2769                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2770                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2771                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2772                         } else {
2773                                 /* TODO: get the actual WQE and mask off wqe index */
2774                                 context &= ~((u64)511);
2775                                 nesqp = *((struct nes_qp **)&context);
2776                         }
2777                         spin_lock_irqsave(&nesqp->lock, flags);
2778                         nesqp->hw_iwarp_state = iwarp_state;
2779                         nesqp->hw_tcp_state = tcp_state;
2780                         nesqp->last_aeq = async_event_id;
2781                         spin_unlock_irqrestore(&nesqp->lock, flags);
2782                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2783                                         nesqp->hwqp.qp_id);
2784                         if (nesqp->ibqp.event_handler) {
2785                                 ibevent.device = nesqp->ibqp.device;
2786                                 ibevent.element.qp = &nesqp->ibqp;
2787                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2788                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2789                         }
2790                         break;
2791                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
2792                         nesqp = *((struct nes_qp **)&context);
2793                         spin_lock_irqsave(&nesqp->lock, flags);
2794                         nesqp->hw_iwarp_state = iwarp_state;
2795                         nesqp->hw_tcp_state = tcp_state;
2796                         nesqp->last_aeq = async_event_id;
2797                         spin_unlock_irqrestore(&nesqp->lock, flags);
2798                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
2799                                         nesqp->hwqp.qp_id);
2800                         if (nesqp->ibqp.event_handler) {
2801                                 ibevent.device = nesqp->ibqp.device;
2802                                 ibevent.element.qp = &nesqp->ibqp;
2803                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2804                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2805                         }
2806                         break;
2807                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
2808                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
2809                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2810                         spin_lock_irqsave(&nesqp->lock, flags);
2811                         nesqp->hw_iwarp_state = iwarp_state;
2812                         nesqp->hw_tcp_state = tcp_state;
2813                         nesqp->last_aeq = async_event_id;
2814                         spin_unlock_irqrestore(&nesqp->lock, flags);
2815                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
2816                                         " nesqp = %p, AE reported %p\n",
2817                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
2818                         if (nesqp->ibqp.event_handler) {
2819                                 ibevent.device = nesqp->ibqp.device;
2820                                 ibevent.element.qp = &nesqp->ibqp;
2821                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2822                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2823                         }
2824                         break;
2825                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
2826                         context <<= 1;
2827                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
2828                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
2829                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
2830                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2831                         if (resource_allocated) {
2832                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
2833                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2834                         }
2835                         break;
2836                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
2837                         nesqp = nesadapter->qp_table[le32_to_cpu(
2838                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2839                         spin_lock_irqsave(&nesqp->lock, flags);
2840                         nesqp->hw_iwarp_state = iwarp_state;
2841                         nesqp->hw_tcp_state = tcp_state;
2842                         nesqp->last_aeq = async_event_id;
2843                         spin_unlock_irqrestore(&nesqp->lock, flags);
2844                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
2845                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
2846                                         nesqp->hwqp.qp_id);
2847                         if (nesqp->ibqp.event_handler) {
2848                                 ibevent.device = nesqp->ibqp.device;
2849                                 ibevent.element.qp = &nesqp->ibqp;
2850                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2851                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2852                         }
2853                         /* tell cm to disconnect, cm will queue work to thread */
2854                         nes_add_ref(&nesqp->ibqp);
2855                         nes_cm_disconn(nesqp);
2856                         break;
2857                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
2858                         nesqp = *((struct nes_qp **)&context);
2859                         spin_lock_irqsave(&nesqp->lock, flags);
2860                         nesqp->hw_iwarp_state = iwarp_state;
2861                         nesqp->hw_tcp_state = tcp_state;
2862                         nesqp->last_aeq = async_event_id;
2863                         spin_unlock_irqrestore(&nesqp->lock, flags);
2864                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
2865                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
2866                                         nesqp->hwqp.qp_id);
2867                         if (nesqp->ibqp.event_handler) {
2868                                 ibevent.device = nesqp->ibqp.device;
2869                                 ibevent.element.qp = &nesqp->ibqp;
2870                                 ibevent.event = IB_EVENT_QP_FATAL;
2871                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2872                         }
2873                         /* tell cm to disconnect, cm will queue work to thread */
2874                         nes_add_ref(&nesqp->ibqp);
2875                         nes_cm_disconn(nesqp);
2876                         break;
2877                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
2878                         nesqp = *((struct nes_qp **)&context);
2879                         spin_lock_irqsave(&nesqp->lock, flags);
2880                         nesqp->hw_iwarp_state = iwarp_state;
2881                         nesqp->hw_tcp_state = tcp_state;
2882                         nesqp->last_aeq = async_event_id;
2883                         spin_unlock_irqrestore(&nesqp->lock, flags);
2884                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
2885                                         " event on QP%u \n  Q2 Data:\n",
2886                                         nesqp->hwqp.qp_id);
2887                         if (nesqp->ibqp.event_handler) {
2888                                 ibevent.device = nesqp->ibqp.device;
2889                                 ibevent.element.qp = &nesqp->ibqp;
2890                                 ibevent.event = IB_EVENT_QP_FATAL;
2891                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2892                         }
2893                         /* tell cm to disconnect, cm will queue work to thread */
2894                         nes_add_ref(&nesqp->ibqp);
2895                         nes_cm_disconn(nesqp);
2896                         break;
2897                         /* TODO: additional AEs need to be here */
2898                 default:
2899                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
2900                                         async_event_id);
2901                         break;
2902         }
2903
2904 }
2905
2906
2907 /**
2908  * nes_iwarp_ce_handler
2909  */
2910 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
2911 {
2912         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
2913
2914         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
2915                         nescq->hw_cq.cq_number); */
2916         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
2917
2918         if (nescq->ibcq.comp_handler)
2919                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
2920
2921         return;
2922 }
2923
2924
2925 /**
2926  * nes_manage_apbvt()
2927  */
2928 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
2929                 u32 nic_index, u32 add_port)
2930 {
2931         struct nes_device *nesdev = nesvnic->nesdev;
2932         struct nes_hw_cqp_wqe *cqp_wqe;
2933         unsigned long flags;
2934         struct nes_cqp_request *cqp_request;
2935         int ret = 0;
2936         u16 major_code;
2937
2938         /* Send manage APBVT request to CQP */
2939         cqp_request = nes_get_cqp_request(nesdev);
2940         if (cqp_request == NULL) {
2941                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
2942                 return -ENOMEM;
2943         }
2944         cqp_request->waiting = 1;
2945         cqp_wqe = &cqp_request->cqp_wqe;
2946
2947         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
2948                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
2949                         accel_local_port, accel_local_port, nic_index);
2950
2951         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2952         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
2953                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
2954         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2955                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
2956
2957         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
2958
2959         atomic_set(&cqp_request->refcount, 2);
2960         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
2961
2962         if (add_port == NES_MANAGE_APBVT_ADD)
2963                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
2964                                 NES_EVENT_TIMEOUT);
2965         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
2966                         ret, cqp_request->major_code, cqp_request->minor_code);
2967         major_code = cqp_request->major_code;
2968         if (atomic_dec_and_test(&cqp_request->refcount)) {
2969                 if (cqp_request->dynamic) {
2970                         kfree(cqp_request);
2971                 } else {
2972                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2973                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2974                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2975                 }
2976         }
2977         if (!ret)
2978                 return -ETIME;
2979         else if (major_code)
2980                 return -EIO;
2981         else
2982                 return 0;
2983 }
2984
2985
2986 /**
2987  * nes_manage_arp_cache
2988  */
2989 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
2990                 u32 ip_addr, u32 action)
2991 {
2992         struct nes_hw_cqp_wqe *cqp_wqe;
2993         struct nes_vnic *nesvnic = netdev_priv(netdev);
2994         struct nes_device *nesdev;
2995         struct nes_cqp_request *cqp_request;
2996         int arp_index;
2997
2998         nesdev = nesvnic->nesdev;
2999         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3000         if (arp_index == -1) {
3001                 return;
3002         }
3003
3004         /* update the ARP entry */
3005         cqp_request = nes_get_cqp_request(nesdev);
3006         if (cqp_request == NULL) {
3007                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3008                 return;
3009         }
3010         cqp_request->waiting = 0;
3011         cqp_wqe = &cqp_request->cqp_wqe;
3012         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3013
3014         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3015                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3016         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3017                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3018         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3019
3020         if (action == NES_ARP_ADD) {
3021                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3022                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3023                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3024                                 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3025                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3026                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3027         } else {
3028                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3029                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3030         }
3031
3032         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3033                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3034
3035         atomic_set(&cqp_request->refcount, 1);
3036         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3037 }
3038
3039
3040 /**
3041  * flush_wqes
3042  */
3043 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3044                 u32 which_wq, u32 wait_completion)
3045 {
3046         unsigned long flags;
3047         struct nes_cqp_request *cqp_request;
3048         struct nes_hw_cqp_wqe *cqp_wqe;
3049         int ret;
3050
3051         cqp_request = nes_get_cqp_request(nesdev);
3052         if (cqp_request == NULL) {
3053                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3054                 return;
3055         }
3056         if (wait_completion) {
3057                 cqp_request->waiting = 1;
3058                 atomic_set(&cqp_request->refcount, 2);
3059         } else {
3060                 cqp_request->waiting = 0;
3061         }
3062         cqp_wqe = &cqp_request->cqp_wqe;
3063         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3064
3065         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3066                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3067         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3068
3069         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3070
3071         if (wait_completion) {
3072                 /* Wait for CQP */
3073                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3074                                 NES_EVENT_TIMEOUT);
3075                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3076                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3077                                 ret, cqp_request->major_code, cqp_request->minor_code);
3078                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3079                         if (cqp_request->dynamic) {
3080                                 kfree(cqp_request);
3081                         } else {
3082                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3083                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3084                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3085                         }
3086                 }
3087         }
3088 }