git.oblomov.eu Git - linux-2.6/blob - drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mii.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 0),     /* PE9000 */
  84         CH_DEVICE(0x21, 1),     /* T302E */
  85         CH_DEVICE(0x22, 2),     /* T310E */
  86         CH_DEVICE(0x23, 3),     /* T320X */
  87         CH_DEVICE(0x24, 1),     /* T302X */
  88         CH_DEVICE(0x25, 3),     /* T320E */
  89         CH_DEVICE(0x26, 2),     /* T310X */
  90         CH_DEVICE(0x30, 2),     /* T3B10 */
  91         CH_DEVICE(0x31, 3),     /* T3B20 */
  92         CH_DEVICE(0x32, 1),     /* T3B02 */
  93         {0,}
  94 };
  95
  96 MODULE_DESCRIPTION(DRV_DESC);
  97 MODULE_AUTHOR("Chelsio Communications");
  98 MODULE_LICENSE("Dual BSD/GPL");
  99 MODULE_VERSION(DRV_VERSION);
 100 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 101
 102 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 103
 104 module_param(dflt_msg_enable, int, 0644);
 105 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 106
 107 /*
 108  * The driver uses the best interrupt scheme available on a platform in the
 109  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 110  * of these schemes the driver may consider as follows:
 111  *
 112  * msi = 2: choose from among all three options
 113  * msi = 1: only consider MSI and pin interrupts
 114  * msi = 0: force pin interrupts
 115  */
 116 static int msi = 2;
 117
 118 module_param(msi, int, 0644);
 119 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 120
 121 /*
 122  * The driver enables offload as a default.
 123  * To disable it, use ofld_disable = 1.
 124  */
 125
 126 static int ofld_disable = 0;
 127
 128 module_param(ofld_disable, int, 0644);
 129 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 130
 131 /*
 132  * We have work elements that we need to cancel when an interface is taken
 133  * down.  Normally the work elements would be executed by keventd but that
 134  * can deadlock because of linkwatch.  If our close method takes the rtnl
 135  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 136  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 137  * for our work to complete.  Get our own work queue to solve this.
 138  */
 139 static struct workqueue_struct *cxgb3_wq;
 140
 141 /**
 142  *      link_report - show link status and link speed/duplex
 143  *      @p: the port whose settings are to be reported
 144  *
 145  *      Shows the link status, speed, and duplex of a port.
 146  */
 147 static void link_report(struct net_device *dev)
 148 {
 149         if (!netif_carrier_ok(dev))
 150                 printk(KERN_INFO "%s: link down\n", dev->name);
 151         else {
 152                 const char *s = "10Mbps";
 153                 const struct port_info *p = netdev_priv(dev);
 154
 155                 switch (p->link_config.speed) {
 156                 case SPEED_10000:
 157                         s = "10Gbps";
 158                         break;
 159                 case SPEED_1000:
 160                         s = "1000Mbps";
 161                         break;
 162                 case SPEED_100:
 163                         s = "100Mbps";
 164                         break;
 165                 }
 166
 167                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 168                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 169         }
 170 }
 171
 172 /**
 173  *      t3_os_link_changed - handle link status changes
 174  *      @adapter: the adapter associated with the link change
 175  *      @port_id: the port index whose limk status has changed
 176  *      @link_stat: the new status of the link
 177  *      @speed: the new speed setting
 178  *      @duplex: the new duplex setting
 179  *      @pause: the new flow-control setting
 180  *
 181  *      This is the OS-dependent handler for link status changes.  The OS
 182  *      neutral handler takes care of most of the processing for these events,
 183  *      then calls this handler for any OS-specific processing.
 184  */
 185 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 186                         int speed, int duplex, int pause)
 187 {
 188         struct net_device *dev = adapter->port[port_id];
 189         struct port_info *pi = netdev_priv(dev);
 190         struct cmac *mac = &pi->mac;
 191
 192         /* Skip changes from disabled ports. */
 193         if (!netif_running(dev))
 194                 return;
 195
 196         if (link_stat != netif_carrier_ok(dev)) {
 197                 if (link_stat) {
 198                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 199                         netif_carrier_on(dev);
 200                 } else {
 201                         netif_carrier_off(dev);
 202                         pi->phy.ops->power_down(&pi->phy, 1);
 203                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 204                         t3_link_start(&pi->phy, mac, &pi->link_config);
 205                 }
 206
 207                 link_report(dev);
 208         }
 209 }
 210
 211 /**
 212  *      t3_os_phymod_changed - handle PHY module changes
 213  *      @phy: the PHY reporting the module change
 214  *      @mod_type: new module type
 215  *
 216  *      This is the OS-dependent handler for PHY module changes.  It is
 217  *      invoked when a PHY module is removed or inserted for any OS-specific
 218  *      processing.
 219  */
 220 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 221 {
 222         static const char *mod_str[] = {
 223                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 224         };
 225
 226         const struct net_device *dev = adap->port[port_id];
 227         const struct port_info *pi = netdev_priv(dev);
 228
 229         if (pi->phy.modtype == phy_modtype_none)
 230                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 231         else
 232                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 233                        mod_str[pi->phy.modtype]);
 234 }
 235
 236 static void cxgb_set_rxmode(struct net_device *dev)
 237 {
 238         struct t3_rx_mode rm;
 239         struct port_info *pi = netdev_priv(dev);
 240
 241         init_rx_mode(&rm, dev, dev->mc_list);
 242         t3_mac_set_rx_mode(&pi->mac, &rm);
 243 }
 244
 245 /**
 246  *      link_start - enable a port
 247  *      @dev: the device to enable
 248  *
 249  *      Performs the MAC and PHY actions needed to enable a port.
 250  */
 251 static void link_start(struct net_device *dev)
 252 {
 253         struct t3_rx_mode rm;
 254         struct port_info *pi = netdev_priv(dev);
 255         struct cmac *mac = &pi->mac;
 256
 257         init_rx_mode(&rm, dev, dev->mc_list);
 258         t3_mac_reset(mac);
 259         t3_mac_set_mtu(mac, dev->mtu);
 260         t3_mac_set_address(mac, 0, dev->dev_addr);
 261         t3_mac_set_rx_mode(mac, &rm);
 262         t3_link_start(&pi->phy, mac, &pi->link_config);
 263         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 264 }
 265
 266 static inline void cxgb_disable_msi(struct adapter *adapter)
 267 {
 268         if (adapter->flags & USING_MSIX) {
 269                 pci_disable_msix(adapter->pdev);
 270                 adapter->flags &= ~USING_MSIX;
 271         } else if (adapter->flags & USING_MSI) {
 272                 pci_disable_msi(adapter->pdev);
 273                 adapter->flags &= ~USING_MSI;
 274         }
 275 }
 276
 277 /*
 278  * Interrupt handler for asynchronous events used with MSI-X.
 279  */
 280 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 281 {
 282         t3_slow_intr_handler(cookie);
 283         return IRQ_HANDLED;
 284 }
 285
 286 /*
 287  * Name the MSI-X interrupts.
 288  */
 289 static void name_msix_vecs(struct adapter *adap)
 290 {
 291         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 292
 293         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 294         adap->msix_info[0].desc[n] = 0;
 295
 296         for_each_port(adap, j) {
 297                 struct net_device *d = adap->port[j];
 298                 const struct port_info *pi = netdev_priv(d);
 299
 300                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 301                         snprintf(adap->msix_info[msi_idx].desc, n,
 302                                  "%s-%d", d->name, pi->first_qset + i);
 303                         adap->msix_info[msi_idx].desc[n] = 0;
 304                 }
 305         }
 306 }
 307
 308 static int request_msix_data_irqs(struct adapter *adap)
 309 {
 310         int i, j, err, qidx = 0;
 311
 312         for_each_port(adap, i) {
 313                 int nqsets = adap2pinfo(adap, i)->nqsets;
 314
 315                 for (j = 0; j < nqsets; ++j) {
 316                         err = request_irq(adap->msix_info[qidx + 1].vec,
 317                                           t3_intr_handler(adap,
 318                                                           adap->sge.qs[qidx].
 319                                                           rspq.polling), 0,
 320                                           adap->msix_info[qidx + 1].desc,
 321                                           &adap->sge.qs[qidx]);
 322                         if (err) {
 323                                 while (--qidx >= 0)
 324                                         free_irq(adap->msix_info[qidx + 1].vec,
 325                                                  &adap->sge.qs[qidx]);
 326                                 return err;
 327                         }
 328                         qidx++;
 329                 }
 330         }
 331         return 0;
 332 }
 333
 334 static void free_irq_resources(struct adapter *adapter)
 335 {
 336         if (adapter->flags & USING_MSIX) {
 337                 int i, n = 0;
 338
 339                 free_irq(adapter->msix_info[0].vec, adapter);
 340                 for_each_port(adapter, i)
 341                         n += adap2pinfo(adapter, i)->nqsets;
 342
 343                 for (i = 0; i < n; ++i)
 344                         free_irq(adapter->msix_info[i + 1].vec,
 345                                  &adapter->sge.qs[i]);
 346         } else
 347                 free_irq(adapter->pdev->irq, adapter);
 348 }
 349
 350 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 351                               unsigned long n)
 352 {
 353         int attempts = 5;
 354
 355         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 356                 if (!--attempts)
 357                         return -ETIMEDOUT;
 358                 msleep(10);
 359         }
 360         return 0;
 361 }
 362
 363 static int init_tp_parity(struct adapter *adap)
 364 {
 365         int i;
 366         struct sk_buff *skb;
 367         struct cpl_set_tcb_field *greq;
 368         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 369
 370         t3_tp_set_offload_mode(adap, 1);
 371
 372         for (i = 0; i < 16; i++) {
 373                 struct cpl_smt_write_req *req;
 374
 375                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 376                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 377                 memset(req, 0, sizeof(*req));
 378                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 379                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 380                 req->iff = i;
 381                 t3_mgmt_tx(adap, skb);
 382         }
 383
 384         for (i = 0; i < 2048; i++) {
 385                 struct cpl_l2t_write_req *req;
 386
 387                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 388                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 389                 memset(req, 0, sizeof(*req));
 390                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 391                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 392                 req->params = htonl(V_L2T_W_IDX(i));
 393                 t3_mgmt_tx(adap, skb);
 394         }
 395
 396         for (i = 0; i < 2048; i++) {
 397                 struct cpl_rte_write_req *req;
 398
 399                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 400                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 401                 memset(req, 0, sizeof(*req));
 402                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 403                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 404                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 405                 t3_mgmt_tx(adap, skb);
 406         }
 407
 408         skb = alloc_skb(sizeof(*greq), GFP_KERNEL | __GFP_NOFAIL);
 409         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 410         memset(greq, 0, sizeof(*greq));
 411         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 412         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 413         greq->mask = cpu_to_be64(1);
 414         t3_mgmt_tx(adap, skb);
 415
 416         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 417         t3_tp_set_offload_mode(adap, 0);
 418         return i;
 419 }
 420
 421 /**
 422  *      setup_rss - configure RSS
 423  *      @adap: the adapter
 424  *
 425  *      Sets up RSS to distribute packets to multiple receive queues.  We
 426  *      configure the RSS CPU lookup table to distribute to the number of HW
 427  *      receive queues, and the response queue lookup table to narrow that
 428  *      down to the response queues actually configured for each port.
 429  *      We always configure the RSS mapping for two ports since the mapping
 430  *      table has plenty of entries.
 431  */
 432 static void setup_rss(struct adapter *adap)
 433 {
 434         int i;
 435         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 436         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 437         u8 cpus[SGE_QSETS + 1];
 438         u16 rspq_map[RSS_TABLE_SIZE];
 439
 440         for (i = 0; i < SGE_QSETS; ++i)
 441                 cpus[i] = i;
 442         cpus[SGE_QSETS] = 0xff; /* terminator */
 443
 444         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 445                 rspq_map[i] = i % nq0;
 446                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 447         }
 448
 449         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 450                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 451                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 452 }
 453
 454 static void init_napi(struct adapter *adap)
 455 {
 456         int i;
 457
 458         for (i = 0; i < SGE_QSETS; i++) {
 459                 struct sge_qset *qs = &adap->sge.qs[i];
 460
 461                 if (qs->adap)
 462                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 463                                        64);
 464         }
 465
 466         /*
 467          * netif_napi_add() can be called only once per napi_struct because it
 468          * adds each new napi_struct to a list.  Be careful not to call it a
 469          * second time, e.g., during EEH recovery, by making a note of it.
 470          */
 471         adap->flags |= NAPI_INIT;
 472 }
 473
 474 /*
 475  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 476  * both netdevices representing interfaces and the dummy ones for the extra
 477  * queues.
 478  */
 479 static void quiesce_rx(struct adapter *adap)
 480 {
 481         int i;
 482
 483         for (i = 0; i < SGE_QSETS; i++)
 484                 if (adap->sge.qs[i].adap)
 485                         napi_disable(&adap->sge.qs[i].napi);
 486 }
 487
 488 static void enable_all_napi(struct adapter *adap)
 489 {
 490         int i;
 491         for (i = 0; i < SGE_QSETS; i++)
 492                 if (adap->sge.qs[i].adap)
 493                         napi_enable(&adap->sge.qs[i].napi);
 494 }
 495
 496 /**
 497  *      set_qset_lro - Turn a queue set's LRO capability on and off
 498  *      @dev: the device the qset is attached to
 499  *      @qset_idx: the queue set index
 500  *      @val: the LRO switch
 501  *
 502  *      Sets LRO on or off for a particular queue set.
 503  *      the device's features flag is updated to reflect the LRO
 504  *      capability when all queues belonging to the device are
 505  *      in the same state.
 506  */
 507 static void set_qset_lro(struct net_device *dev, int qset_idx, int val)
 508 {
 509         struct port_info *pi = netdev_priv(dev);
 510         struct adapter *adapter = pi->adapter;
 511
 512         adapter->params.sge.qset[qset_idx].lro = !!val;
 513         adapter->sge.qs[qset_idx].lro_enabled = !!val;
 514 }
 515
 516 /**
 517  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 518  *      @adap: the adapter
 519  *
 520  *      Determines how many sets of SGE queues to use and initializes them.
 521  *      We support multiple queue sets per port if we have MSI-X, otherwise
 522  *      just one queue set per port.
 523  */
 524 static int setup_sge_qsets(struct adapter *adap)
 525 {
 526         int i, j, err, irq_idx = 0, qset_idx = 0;
 527         unsigned int ntxq = SGE_TXQ_PER_SET;
 528
 529         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 530                 irq_idx = -1;
 531
 532         for_each_port(adap, i) {
 533                 struct net_device *dev = adap->port[i];
 534                 struct port_info *pi = netdev_priv(dev);
 535
 536                 pi->qs = &adap->sge.qs[pi->first_qset];
 537                 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets;
 538                      ++j, ++qset_idx) {
 539                         set_qset_lro(dev, qset_idx, pi->rx_offload & T3_LRO);
 540                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 541                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 542                                                              irq_idx,
 543                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 544                                 netdev_get_tx_queue(dev, j));
 545                         if (err) {
 546                                 t3_stop_sge_timers(adap);
 547                                 t3_free_sge_resources(adap);
 548                                 return err;
 549                         }
 550                 }
 551         }
 552
 553         return 0;
 554 }
 555
 556 static ssize_t attr_show(struct device *d, char *buf,
 557                          ssize_t(*format) (struct net_device *, char *))
 558 {
 559         ssize_t len;
 560
 561         /* Synchronize with ioctls that may shut down the device */
 562         rtnl_lock();
 563         len = (*format) (to_net_dev(d), buf);
 564         rtnl_unlock();
 565         return len;
 566 }
 567
 568 static ssize_t attr_store(struct device *d,
 569                           const char *buf, size_t len,
 570                           ssize_t(*set) (struct net_device *, unsigned int),
 571                           unsigned int min_val, unsigned int max_val)
 572 {
 573         char *endp;
 574         ssize_t ret;
 575         unsigned int val;
 576
 577         if (!capable(CAP_NET_ADMIN))
 578                 return -EPERM;
 579
 580         val = simple_strtoul(buf, &endp, 0);
 581         if (endp == buf || val < min_val || val > max_val)
 582                 return -EINVAL;
 583
 584         rtnl_lock();
 585         ret = (*set) (to_net_dev(d), val);
 586         if (!ret)
 587                 ret = len;
 588         rtnl_unlock();
 589         return ret;
 590 }
 591
 592 #define CXGB3_SHOW(name, val_expr) \
 593 static ssize_t format_##name(struct net_device *dev, char *buf) \
 594 { \
 595         struct port_info *pi = netdev_priv(dev); \
 596         struct adapter *adap = pi->adapter; \
 597         return sprintf(buf, "%u\n", val_expr); \
 598 } \
 599 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 600                            char *buf) \
 601 { \
 602         return attr_show(d, buf, format_##name); \
 603 }
 604
 605 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 606 {
 607         struct port_info *pi = netdev_priv(dev);
 608         struct adapter *adap = pi->adapter;
 609         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 610
 611         if (adap->flags & FULL_INIT_DONE)
 612                 return -EBUSY;
 613         if (val && adap->params.rev == 0)
 614                 return -EINVAL;
 615         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 616             min_tids)
 617                 return -EINVAL;
 618         adap->params.mc5.nfilters = val;
 619         return 0;
 620 }
 621
 622 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 623                               const char *buf, size_t len)
 624 {
 625         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 626 }
 627
 628 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 629 {
 630         struct port_info *pi = netdev_priv(dev);
 631         struct adapter *adap = pi->adapter;
 632
 633         if (adap->flags & FULL_INIT_DONE)
 634                 return -EBUSY;
 635         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 636             MC5_MIN_TIDS)
 637                 return -EINVAL;
 638         adap->params.mc5.nservers = val;
 639         return 0;
 640 }
 641
 642 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 643                               const char *buf, size_t len)
 644 {
 645         return attr_store(d, buf, len, set_nservers, 0, ~0);
 646 }
 647
 648 #define CXGB3_ATTR_R(name, val_expr) \
 649 CXGB3_SHOW(name, val_expr) \
 650 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 651
 652 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 653 CXGB3_SHOW(name, val_expr) \
 654 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 655
 656 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 657 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 658 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 659
 660 static struct attribute *cxgb3_attrs[] = {
 661         &dev_attr_cam_size.attr,
 662         &dev_attr_nfilters.attr,
 663         &dev_attr_nservers.attr,
 664         NULL
 665 };
 666
 667 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 668
 669 static ssize_t tm_attr_show(struct device *d,
 670                             char *buf, int sched)
 671 {
 672         struct port_info *pi = netdev_priv(to_net_dev(d));
 673         struct adapter *adap = pi->adapter;
 674         unsigned int v, addr, bpt, cpt;
 675         ssize_t len;
 676
 677         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 678         rtnl_lock();
 679         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 680         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 681         if (sched & 1)
 682                 v >>= 16;
 683         bpt = (v >> 8) & 0xff;
 684         cpt = v & 0xff;
 685         if (!cpt)
 686                 len = sprintf(buf, "disabled\n");
 687         else {
 688                 v = (adap->params.vpd.cclk * 1000) / cpt;
 689                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 690         }
 691         rtnl_unlock();
 692         return len;
 693 }
 694
 695 static ssize_t tm_attr_store(struct device *d,
 696                              const char *buf, size_t len, int sched)
 697 {
 698         struct port_info *pi = netdev_priv(to_net_dev(d));
 699         struct adapter *adap = pi->adapter;
 700         unsigned int val;
 701         char *endp;
 702         ssize_t ret;
 703
 704         if (!capable(CAP_NET_ADMIN))
 705                 return -EPERM;
 706
 707         val = simple_strtoul(buf, &endp, 0);
 708         if (endp == buf || val > 10000000)
 709                 return -EINVAL;
 710
 711         rtnl_lock();
 712         ret = t3_config_sched(adap, val, sched);
 713         if (!ret)
 714                 ret = len;
 715         rtnl_unlock();
 716         return ret;
 717 }
 718
 719 #define TM_ATTR(name, sched) \
 720 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 721                            char *buf) \
 722 { \
 723         return tm_attr_show(d, buf, sched); \
 724 } \
 725 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 726                             const char *buf, size_t len) \
 727 { \
 728         return tm_attr_store(d, buf, len, sched); \
 729 } \
 730 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 731
 732 TM_ATTR(sched0, 0);
 733 TM_ATTR(sched1, 1);
 734 TM_ATTR(sched2, 2);
 735 TM_ATTR(sched3, 3);
 736 TM_ATTR(sched4, 4);
 737 TM_ATTR(sched5, 5);
 738 TM_ATTR(sched6, 6);
 739 TM_ATTR(sched7, 7);
 740
 741 static struct attribute *offload_attrs[] = {
 742         &dev_attr_sched0.attr,
 743         &dev_attr_sched1.attr,
 744         &dev_attr_sched2.attr,
 745         &dev_attr_sched3.attr,
 746         &dev_attr_sched4.attr,
 747         &dev_attr_sched5.attr,
 748         &dev_attr_sched6.attr,
 749         &dev_attr_sched7.attr,
 750         NULL
 751 };
 752
 753 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 754
 755 /*
 756  * Sends an sk_buff to an offload queue driver
 757  * after dealing with any active network taps.
 758  */
 759 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 760 {
 761         int ret;
 762
 763         local_bh_disable();
 764         ret = t3_offload_tx(tdev, skb);
 765         local_bh_enable();
 766         return ret;
 767 }
 768
 769 static int write_smt_entry(struct adapter *adapter, int idx)
 770 {
 771         struct cpl_smt_write_req *req;
 772         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 773
 774         if (!skb)
 775                 return -ENOMEM;
 776
 777         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 778         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 779         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 780         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 781         req->iff = idx;
 782         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 783         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 784         skb->priority = 1;
 785         offload_tx(&adapter->tdev, skb);
 786         return 0;
 787 }
 788
 789 static int init_smt(struct adapter *adapter)
 790 {
 791         int i;
 792
 793         for_each_port(adapter, i)
 794             write_smt_entry(adapter, i);
 795         return 0;
 796 }
 797
 798 static void init_port_mtus(struct adapter *adapter)
 799 {
 800         unsigned int mtus = adapter->port[0]->mtu;
 801
 802         if (adapter->port[1])
 803                 mtus |= adapter->port[1]->mtu << 16;
 804         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 805 }
 806
 807 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 808                               int hi, int port)
 809 {
 810         struct sk_buff *skb;
 811         struct mngt_pktsched_wr *req;
 812         int ret;
 813
 814         skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 815         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 816         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 817         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 818         req->sched = sched;
 819         req->idx = qidx;
 820         req->min = lo;
 821         req->max = hi;
 822         req->binding = port;
 823         ret = t3_mgmt_tx(adap, skb);
 824
 825         return ret;
 826 }
 827
 828 static int bind_qsets(struct adapter *adap)
 829 {
 830         int i, j, err = 0;
 831
 832         for_each_port(adap, i) {
 833                 const struct port_info *pi = adap2pinfo(adap, i);
 834
 835                 for (j = 0; j < pi->nqsets; ++j) {
 836                         int ret = send_pktsched_cmd(adap, 1,
 837                                                     pi->first_qset + j, -1,
 838                                                     -1, i);
 839                         if (ret)
 840                                 err = ret;
 841                 }
 842         }
 843
 844         return err;
 845 }
 846
 847 #define FW_FNAME "cxgb3/t3fw-%d.%d.%d.bin"
 848 #define TPSRAM_NAME "cxgb3/t3%c_psram-%d.%d.%d.bin"
 849
 850 static int upgrade_fw(struct adapter *adap)
 851 {
 852         int ret;
 853         char buf[64];
 854         const struct firmware *fw;
 855         struct device *dev = &adap->pdev->dev;
 856
 857         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
 858                  FW_VERSION_MINOR, FW_VERSION_MICRO);
 859         ret = request_firmware(&fw, buf, dev);
 860         if (ret < 0) {
 861                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
 862                         buf);
 863                 return ret;
 864         }
 865         ret = t3_load_fw(adap, fw->data, fw->size);
 866         release_firmware(fw);
 867
 868         if (ret == 0)
 869                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
 870                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 871         else
 872                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
 873                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 874
 875         return ret;
 876 }
 877
 878 static inline char t3rev2char(struct adapter *adapter)
 879 {
 880         char rev = 0;
 881
 882         switch(adapter->params.rev) {
 883         case T3_REV_B:
 884         case T3_REV_B2:
 885                 rev = 'b';
 886                 break;
 887         case T3_REV_C:
 888                 rev = 'c';
 889                 break;
 890         }
 891         return rev;
 892 }
 893
 894 static int update_tpsram(struct adapter *adap)
 895 {
 896         const struct firmware *tpsram;
 897         char buf[64];
 898         struct device *dev = &adap->pdev->dev;
 899         int ret;
 900         char rev;
 901
 902         rev = t3rev2char(adap);
 903         if (!rev)
 904                 return 0;
 905
 906         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
 907                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 908
 909         ret = request_firmware(&tpsram, buf, dev);
 910         if (ret < 0) {
 911                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
 912                         buf);
 913                 return ret;
 914         }
 915
 916         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
 917         if (ret)
 918                 goto release_tpsram;
 919
 920         ret = t3_set_proto_sram(adap, tpsram->data);
 921         if (ret == 0)
 922                 dev_info(dev,
 923                          "successful update of protocol engine "
 924                          "to %d.%d.%d\n",
 925                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 926         else
 927                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
 928                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 929         if (ret)
 930                 dev_err(dev, "loading protocol SRAM failed\n");
 931
 932 release_tpsram:
 933         release_firmware(tpsram);
 934
 935         return ret;
 936 }
 937
 938 /**
 939  *      cxgb_up - enable the adapter
 940  *      @adapter: adapter being enabled
 941  *
 942  *      Called when the first port is enabled, this function performs the
 943  *      actions necessary to make an adapter operational, such as completing
 944  *      the initialization of HW modules, and enabling interrupts.
 945  *
 946  *      Must be called with the rtnl lock held.
 947  */
 948 static int cxgb_up(struct adapter *adap)
 949 {
 950         int err;
 951
 952         if (!(adap->flags & FULL_INIT_DONE)) {
 953                 err = t3_check_fw_version(adap);
 954                 if (err == -EINVAL) {
 955                         err = upgrade_fw(adap);
 956                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
 957                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
 958                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
 959                 }
 960
 961                 err = t3_check_tpsram_version(adap);
 962                 if (err == -EINVAL) {
 963                         err = update_tpsram(adap);
 964                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
 965                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
 966                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
 967                 }
 968
 969                 /*
 970                  * Clear interrupts now to catch errors if t3_init_hw fails.
 971                  * We clear them again later as initialization may trigger
 972                  * conditions that can interrupt.
 973                  */
 974                 t3_intr_clear(adap);
 975
 976                 err = t3_init_hw(adap, 0);
 977                 if (err)
 978                         goto out;
 979
 980                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 981                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 982
 983                 err = setup_sge_qsets(adap);
 984                 if (err)
 985                         goto out;
 986
 987                 setup_rss(adap);
 988                 if (!(adap->flags & NAPI_INIT))
 989                         init_napi(adap);
 990                 adap->flags |= FULL_INIT_DONE;
 991         }
 992
 993         t3_intr_clear(adap);
 994
 995         if (adap->flags & USING_MSIX) {
 996                 name_msix_vecs(adap);
 997                 err = request_irq(adap->msix_info[0].vec,
 998                                   t3_async_intr_handler, 0,
 999                                   adap->msix_info[0].desc, adap);
1000                 if (err)
1001                         goto irq_err;
1002
1003                 err = request_msix_data_irqs(adap);
1004                 if (err) {
1005                         free_irq(adap->msix_info[0].vec, adap);
1006                         goto irq_err;
1007                 }
1008         } else if ((err = request_irq(adap->pdev->irq,
1009                                       t3_intr_handler(adap,
1010                                                       adap->sge.qs[0].rspq.
1011                                                       polling),
1012                                       (adap->flags & USING_MSI) ?
1013                                        0 : IRQF_SHARED,
1014                                       adap->name, adap)))
1015                 goto irq_err;
1016
1017         enable_all_napi(adap);
1018         t3_sge_start(adap);
1019         t3_intr_enable(adap);
1020
1021         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1022             is_offload(adap) && init_tp_parity(adap) == 0)
1023                 adap->flags |= TP_PARITY_INIT;
1024
1025         if (adap->flags & TP_PARITY_INIT) {
1026                 t3_write_reg(adap, A_TP_INT_CAUSE,
1027                              F_CMCACHEPERR | F_ARPLUTPERR);
1028                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1029         }
1030
1031         if (!(adap->flags & QUEUES_BOUND)) {
1032                 err = bind_qsets(adap);
1033                 if (err) {
1034                         CH_ERR(adap, "failed to bind qsets, err %d\n", err);
1035                         t3_intr_disable(adap);
1036                         free_irq_resources(adap);
1037                         goto out;
1038                 }
1039                 adap->flags |= QUEUES_BOUND;
1040         }
1041
1042 out:
1043         return err;
1044 irq_err:
1045         CH_ERR(adap, "request_irq failed, err %d\n", err);
1046         goto out;
1047 }
1048
1049 /*
1050  * Release resources when all the ports and offloading have been stopped.
1051  */
1052 static void cxgb_down(struct adapter *adapter)
1053 {
1054         t3_sge_stop(adapter);
1055         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1056         t3_intr_disable(adapter);
1057         spin_unlock_irq(&adapter->work_lock);
1058
1059         free_irq_resources(adapter);
1060         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
1061         quiesce_rx(adapter);
1062 }
1063
1064 static void schedule_chk_task(struct adapter *adap)
1065 {
1066         unsigned int timeo;
1067
1068         timeo = adap->params.linkpoll_period ?
1069             (HZ * adap->params.linkpoll_period) / 10 :
1070             adap->params.stats_update_period * HZ;
1071         if (timeo)
1072                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1073 }
1074
1075 static int offload_open(struct net_device *dev)
1076 {
1077         struct port_info *pi = netdev_priv(dev);
1078         struct adapter *adapter = pi->adapter;
1079         struct t3cdev *tdev = dev2t3cdev(dev);
1080         int adap_up = adapter->open_device_map & PORT_MASK;
1081         int err;
1082
1083         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1084                 return 0;
1085
1086         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1087                 goto out;
1088
1089         t3_tp_set_offload_mode(adapter, 1);
1090         tdev->lldev = adapter->port[0];
1091         err = cxgb3_offload_activate(adapter);
1092         if (err)
1093                 goto out;
1094
1095         init_port_mtus(adapter);
1096         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1097                      adapter->params.b_wnd,
1098                      adapter->params.rev == 0 ?
1099                      adapter->port[0]->mtu : 0xffff);
1100         init_smt(adapter);
1101
1102         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1103                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1104
1105         /* Call back all registered clients */
1106         cxgb3_add_clients(tdev);
1107
1108 out:
1109         /* restore them in case the offload module has changed them */
1110         if (err) {
1111                 t3_tp_set_offload_mode(adapter, 0);
1112                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1113                 cxgb3_set_dummy_ops(tdev);
1114         }
1115         return err;
1116 }
1117
1118 static int offload_close(struct t3cdev *tdev)
1119 {
1120         struct adapter *adapter = tdev2adap(tdev);
1121
1122         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1123                 return 0;
1124
1125         /* Call back all registered clients */
1126         cxgb3_remove_clients(tdev);
1127
1128         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1129
1130         tdev->lldev = NULL;
1131         cxgb3_set_dummy_ops(tdev);
1132         t3_tp_set_offload_mode(adapter, 0);
1133         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1134
1135         if (!adapter->open_device_map)
1136                 cxgb_down(adapter);
1137
1138         cxgb3_offload_deactivate(adapter);
1139         return 0;
1140 }
1141
1142 static int cxgb_open(struct net_device *dev)
1143 {
1144         struct port_info *pi = netdev_priv(dev);
1145         struct adapter *adapter = pi->adapter;
1146         int other_ports = adapter->open_device_map & PORT_MASK;
1147         int err;
1148
1149         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1150                 return err;
1151
1152         set_bit(pi->port_id, &adapter->open_device_map);
1153         if (is_offload(adapter) && !ofld_disable) {
1154                 err = offload_open(dev);
1155                 if (err)
1156                         printk(KERN_WARNING
1157                                "Could not initialize offload capabilities\n");
1158         }
1159
1160         dev->real_num_tx_queues = pi->nqsets;
1161         link_start(dev);
1162         t3_port_intr_enable(adapter, pi->port_id);
1163         netif_tx_start_all_queues(dev);
1164         if (!other_ports)
1165                 schedule_chk_task(adapter);
1166
1167         return 0;
1168 }
1169
1170 static int cxgb_close(struct net_device *dev)
1171 {
1172         struct port_info *pi = netdev_priv(dev);
1173         struct adapter *adapter = pi->adapter;
1174
1175         t3_port_intr_disable(adapter, pi->port_id);
1176         netif_tx_stop_all_queues(dev);
1177         pi->phy.ops->power_down(&pi->phy, 1);
1178         netif_carrier_off(dev);
1179         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1180
1181         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1182         clear_bit(pi->port_id, &adapter->open_device_map);
1183         spin_unlock_irq(&adapter->work_lock);
1184
1185         if (!(adapter->open_device_map & PORT_MASK))
1186                 cancel_rearming_delayed_workqueue(cxgb3_wq,
1187                                                   &adapter->adap_check_task);
1188
1189         if (!adapter->open_device_map)
1190                 cxgb_down(adapter);
1191
1192         return 0;
1193 }
1194
1195 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1196 {
1197         struct port_info *pi = netdev_priv(dev);
1198         struct adapter *adapter = pi->adapter;
1199         struct net_device_stats *ns = &pi->netstats;
1200         const struct mac_stats *pstats;
1201
1202         spin_lock(&adapter->stats_lock);
1203         pstats = t3_mac_update_stats(&pi->mac);
1204         spin_unlock(&adapter->stats_lock);
1205
1206         ns->tx_bytes = pstats->tx_octets;
1207         ns->tx_packets = pstats->tx_frames;
1208         ns->rx_bytes = pstats->rx_octets;
1209         ns->rx_packets = pstats->rx_frames;
1210         ns->multicast = pstats->rx_mcast_frames;
1211
1212         ns->tx_errors = pstats->tx_underrun;
1213         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1214             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1215             pstats->rx_fifo_ovfl;
1216
1217         /* detailed rx_errors */
1218         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1219         ns->rx_over_errors = 0;
1220         ns->rx_crc_errors = pstats->rx_fcs_errs;
1221         ns->rx_frame_errors = pstats->rx_symbol_errs;
1222         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1223         ns->rx_missed_errors = pstats->rx_cong_drops;
1224
1225         /* detailed tx_errors */
1226         ns->tx_aborted_errors = 0;
1227         ns->tx_carrier_errors = 0;
1228         ns->tx_fifo_errors = pstats->tx_underrun;
1229         ns->tx_heartbeat_errors = 0;
1230         ns->tx_window_errors = 0;
1231         return ns;
1232 }
1233
1234 static u32 get_msglevel(struct net_device *dev)
1235 {
1236         struct port_info *pi = netdev_priv(dev);
1237         struct adapter *adapter = pi->adapter;
1238
1239         return adapter->msg_enable;
1240 }
1241
1242 static void set_msglevel(struct net_device *dev, u32 val)
1243 {
1244         struct port_info *pi = netdev_priv(dev);
1245         struct adapter *adapter = pi->adapter;
1246
1247         adapter->msg_enable = val;
1248 }
1249
1250 static char stats_strings[][ETH_GSTRING_LEN] = {
1251         "TxOctetsOK         ",
1252         "TxFramesOK         ",
1253         "TxMulticastFramesOK",
1254         "TxBroadcastFramesOK",
1255         "TxPauseFrames      ",
1256         "TxUnderrun         ",
1257         "TxExtUnderrun      ",
1258
1259         "TxFrames64         ",
1260         "TxFrames65To127    ",
1261         "TxFrames128To255   ",
1262         "TxFrames256To511   ",
1263         "TxFrames512To1023  ",
1264         "TxFrames1024To1518 ",
1265         "TxFrames1519ToMax  ",
1266
1267         "RxOctetsOK         ",
1268         "RxFramesOK         ",
1269         "RxMulticastFramesOK",
1270         "RxBroadcastFramesOK",
1271         "RxPauseFrames      ",
1272         "RxFCSErrors        ",
1273         "RxSymbolErrors     ",
1274         "RxShortErrors      ",
1275         "RxJabberErrors     ",
1276         "RxLengthErrors     ",
1277         "RxFIFOoverflow     ",
1278
1279         "RxFrames64         ",
1280         "RxFrames65To127    ",
1281         "RxFrames128To255   ",
1282         "RxFrames256To511   ",
1283         "RxFrames512To1023  ",
1284         "RxFrames1024To1518 ",
1285         "RxFrames1519ToMax  ",
1286
1287         "PhyFIFOErrors      ",
1288         "TSO                ",
1289         "VLANextractions    ",
1290         "VLANinsertions     ",
1291         "TxCsumOffload      ",
1292         "RxCsumGood         ",
1293         "LroAggregated      ",
1294         "LroFlushed         ",
1295         "LroNoDesc          ",
1296         "RxDrops            ",
1297
1298         "CheckTXEnToggled   ",
1299         "CheckResets        ",
1300
1301 };
1302
1303 static int get_sset_count(struct net_device *dev, int sset)
1304 {
1305         switch (sset) {
1306         case ETH_SS_STATS:
1307                 return ARRAY_SIZE(stats_strings);
1308         default:
1309                 return -EOPNOTSUPP;
1310         }
1311 }
1312
1313 #define T3_REGMAP_SIZE (3 * 1024)
1314
1315 static int get_regs_len(struct net_device *dev)
1316 {
1317         return T3_REGMAP_SIZE;
1318 }
1319
1320 static int get_eeprom_len(struct net_device *dev)
1321 {
1322         return EEPROMSIZE;
1323 }
1324
1325 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1326 {
1327         struct port_info *pi = netdev_priv(dev);
1328         struct adapter *adapter = pi->adapter;
1329         u32 fw_vers = 0;
1330         u32 tp_vers = 0;
1331
1332         spin_lock(&adapter->stats_lock);
1333         t3_get_fw_version(adapter, &fw_vers);
1334         t3_get_tp_version(adapter, &tp_vers);
1335         spin_unlock(&adapter->stats_lock);
1336
1337         strcpy(info->driver, DRV_NAME);
1338         strcpy(info->version, DRV_VERSION);
1339         strcpy(info->bus_info, pci_name(adapter->pdev));
1340         if (!fw_vers)
1341                 strcpy(info->fw_version, "N/A");
1342         else {
1343                 snprintf(info->fw_version, sizeof(info->fw_version),
1344                          "%s %u.%u.%u TP %u.%u.%u",
1345                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1346                          G_FW_VERSION_MAJOR(fw_vers),
1347                          G_FW_VERSION_MINOR(fw_vers),
1348                          G_FW_VERSION_MICRO(fw_vers),
1349                          G_TP_VERSION_MAJOR(tp_vers),
1350                          G_TP_VERSION_MINOR(tp_vers),
1351                          G_TP_VERSION_MICRO(tp_vers));
1352         }
1353 }
1354
1355 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1356 {
1357         if (stringset == ETH_SS_STATS)
1358                 memcpy(data, stats_strings, sizeof(stats_strings));
1359 }
1360
1361 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1362                                             struct port_info *p, int idx)
1363 {
1364         int i;
1365         unsigned long tot = 0;
1366
1367         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1368                 tot += adapter->sge.qs[i].port_stats[idx];
1369         return tot;
1370 }
1371
1372 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1373                       u64 *data)
1374 {
1375         struct port_info *pi = netdev_priv(dev);
1376         struct adapter *adapter = pi->adapter;
1377         const struct mac_stats *s;
1378
1379         spin_lock(&adapter->stats_lock);
1380         s = t3_mac_update_stats(&pi->mac);
1381         spin_unlock(&adapter->stats_lock);
1382
1383         *data++ = s->tx_octets;
1384         *data++ = s->tx_frames;
1385         *data++ = s->tx_mcast_frames;
1386         *data++ = s->tx_bcast_frames;
1387         *data++ = s->tx_pause;
1388         *data++ = s->tx_underrun;
1389         *data++ = s->tx_fifo_urun;
1390
1391         *data++ = s->tx_frames_64;
1392         *data++ = s->tx_frames_65_127;
1393         *data++ = s->tx_frames_128_255;
1394         *data++ = s->tx_frames_256_511;
1395         *data++ = s->tx_frames_512_1023;
1396         *data++ = s->tx_frames_1024_1518;
1397         *data++ = s->tx_frames_1519_max;
1398
1399         *data++ = s->rx_octets;
1400         *data++ = s->rx_frames;
1401         *data++ = s->rx_mcast_frames;
1402         *data++ = s->rx_bcast_frames;
1403         *data++ = s->rx_pause;
1404         *data++ = s->rx_fcs_errs;
1405         *data++ = s->rx_symbol_errs;
1406         *data++ = s->rx_short;
1407         *data++ = s->rx_jabber;
1408         *data++ = s->rx_too_long;
1409         *data++ = s->rx_fifo_ovfl;
1410
1411         *data++ = s->rx_frames_64;
1412         *data++ = s->rx_frames_65_127;
1413         *data++ = s->rx_frames_128_255;
1414         *data++ = s->rx_frames_256_511;
1415         *data++ = s->rx_frames_512_1023;
1416         *data++ = s->rx_frames_1024_1518;
1417         *data++ = s->rx_frames_1519_max;
1418
1419         *data++ = pi->phy.fifo_errors;
1420
1421         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1422         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1423         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1424         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1425         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1426         *data++ = 0;
1427         *data++ = 0;
1428         *data++ = 0;
1429         *data++ = s->rx_cong_drops;
1430
1431         *data++ = s->num_toggled;
1432         *data++ = s->num_resets;
1433 }
1434
1435 static inline void reg_block_dump(struct adapter *ap, void *buf,
1436                                   unsigned int start, unsigned int end)
1437 {
1438         u32 *p = buf + start;
1439
1440         for (; start <= end; start += sizeof(u32))
1441                 *p++ = t3_read_reg(ap, start);
1442 }
1443
1444 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1445                      void *buf)
1446 {
1447         struct port_info *pi = netdev_priv(dev);
1448         struct adapter *ap = pi->adapter;
1449
1450         /*
1451          * Version scheme:
1452          * bits 0..9: chip version
1453          * bits 10..15: chip revision
1454          * bit 31: set for PCIe cards
1455          */
1456         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1457
1458         /*
1459          * We skip the MAC statistics registers because they are clear-on-read.
1460          * Also reading multi-register stats would need to synchronize with the
1461          * periodic mac stats accumulation.  Hard to justify the complexity.
1462          */
1463         memset(buf, 0, T3_REGMAP_SIZE);
1464         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1465         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1466         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1467         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1468         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1469         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1470                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1471         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1472                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1473 }
1474
1475 static int restart_autoneg(struct net_device *dev)
1476 {
1477         struct port_info *p = netdev_priv(dev);
1478
1479         if (!netif_running(dev))
1480                 return -EAGAIN;
1481         if (p->link_config.autoneg != AUTONEG_ENABLE)
1482                 return -EINVAL;
1483         p->phy.ops->autoneg_restart(&p->phy);
1484         return 0;
1485 }
1486
1487 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1488 {
1489         struct port_info *pi = netdev_priv(dev);
1490         struct adapter *adapter = pi->adapter;
1491         int i;
1492
1493         if (data == 0)
1494                 data = 2;
1495
1496         for (i = 0; i < data * 2; i++) {
1497                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1498                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1499                 if (msleep_interruptible(500))
1500                         break;
1501         }
1502         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1503                          F_GPIO0_OUT_VAL);
1504         return 0;
1505 }
1506
1507 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1508 {
1509         struct port_info *p = netdev_priv(dev);
1510
1511         cmd->supported = p->link_config.supported;
1512         cmd->advertising = p->link_config.advertising;
1513
1514         if (netif_carrier_ok(dev)) {
1515                 cmd->speed = p->link_config.speed;
1516                 cmd->duplex = p->link_config.duplex;
1517         } else {
1518                 cmd->speed = -1;
1519                 cmd->duplex = -1;
1520         }
1521
1522         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1523         cmd->phy_address = p->phy.addr;
1524         cmd->transceiver = XCVR_EXTERNAL;
1525         cmd->autoneg = p->link_config.autoneg;
1526         cmd->maxtxpkt = 0;
1527         cmd->maxrxpkt = 0;
1528         return 0;
1529 }
1530
1531 static int speed_duplex_to_caps(int speed, int duplex)
1532 {
1533         int cap = 0;
1534
1535         switch (speed) {
1536         case SPEED_10:
1537                 if (duplex == DUPLEX_FULL)
1538                         cap = SUPPORTED_10baseT_Full;
1539                 else
1540                         cap = SUPPORTED_10baseT_Half;
1541                 break;
1542         case SPEED_100:
1543                 if (duplex == DUPLEX_FULL)
1544                         cap = SUPPORTED_100baseT_Full;
1545                 else
1546                         cap = SUPPORTED_100baseT_Half;
1547                 break;
1548         case SPEED_1000:
1549                 if (duplex == DUPLEX_FULL)
1550                         cap = SUPPORTED_1000baseT_Full;
1551                 else
1552                         cap = SUPPORTED_1000baseT_Half;
1553                 break;
1554         case SPEED_10000:
1555                 if (duplex == DUPLEX_FULL)
1556                         cap = SUPPORTED_10000baseT_Full;
1557         }
1558         return cap;
1559 }
1560
1561 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1562                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1563                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1564                       ADVERTISED_10000baseT_Full)
1565
1566 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1567 {
1568         struct port_info *p = netdev_priv(dev);
1569         struct link_config *lc = &p->link_config;
1570
1571         if (!(lc->supported & SUPPORTED_Autoneg)) {
1572                 /*
1573                  * PHY offers a single speed/duplex.  See if that's what's
1574                  * being requested.
1575                  */
1576                 if (cmd->autoneg == AUTONEG_DISABLE) {
1577                         int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1578                         if (lc->supported & cap)
1579                                 return 0;
1580                 }
1581                 return -EINVAL;
1582         }
1583
1584         if (cmd->autoneg == AUTONEG_DISABLE) {
1585                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1586
1587                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1588                         return -EINVAL;
1589                 lc->requested_speed = cmd->speed;
1590                 lc->requested_duplex = cmd->duplex;
1591                 lc->advertising = 0;
1592         } else {
1593                 cmd->advertising &= ADVERTISED_MASK;
1594                 cmd->advertising &= lc->supported;
1595                 if (!cmd->advertising)
1596                         return -EINVAL;
1597                 lc->requested_speed = SPEED_INVALID;
1598                 lc->requested_duplex = DUPLEX_INVALID;
1599                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1600         }
1601         lc->autoneg = cmd->autoneg;
1602         if (netif_running(dev))
1603                 t3_link_start(&p->phy, &p->mac, lc);
1604         return 0;
1605 }
1606
1607 static void get_pauseparam(struct net_device *dev,
1608                            struct ethtool_pauseparam *epause)
1609 {
1610         struct port_info *p = netdev_priv(dev);
1611
1612         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1613         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1614         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1615 }
1616
1617 static int set_pauseparam(struct net_device *dev,
1618                           struct ethtool_pauseparam *epause)
1619 {
1620         struct port_info *p = netdev_priv(dev);
1621         struct link_config *lc = &p->link_config;
1622
1623         if (epause->autoneg == AUTONEG_DISABLE)
1624                 lc->requested_fc = 0;
1625         else if (lc->supported & SUPPORTED_Autoneg)
1626                 lc->requested_fc = PAUSE_AUTONEG;
1627         else
1628                 return -EINVAL;
1629
1630         if (epause->rx_pause)
1631                 lc->requested_fc |= PAUSE_RX;
1632         if (epause->tx_pause)
1633                 lc->requested_fc |= PAUSE_TX;
1634         if (lc->autoneg == AUTONEG_ENABLE) {
1635                 if (netif_running(dev))
1636                         t3_link_start(&p->phy, &p->mac, lc);
1637         } else {
1638                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1639                 if (netif_running(dev))
1640                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1641         }
1642         return 0;
1643 }
1644
1645 static u32 get_rx_csum(struct net_device *dev)
1646 {
1647         struct port_info *p = netdev_priv(dev);
1648
1649         return p->rx_offload & T3_RX_CSUM;
1650 }
1651
1652 static int set_rx_csum(struct net_device *dev, u32 data)
1653 {
1654         struct port_info *p = netdev_priv(dev);
1655
1656         if (data) {
1657                 p->rx_offload |= T3_RX_CSUM;
1658         } else {
1659                 int i;
1660
1661                 p->rx_offload &= ~(T3_RX_CSUM | T3_LRO);
1662                 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++)
1663                         set_qset_lro(dev, i, 0);
1664         }
1665         return 0;
1666 }
1667
1668 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1669 {
1670         struct port_info *pi = netdev_priv(dev);
1671         struct adapter *adapter = pi->adapter;
1672         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1673
1674         e->rx_max_pending = MAX_RX_BUFFERS;
1675         e->rx_mini_max_pending = 0;
1676         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1677         e->tx_max_pending = MAX_TXQ_ENTRIES;
1678
1679         e->rx_pending = q->fl_size;
1680         e->rx_mini_pending = q->rspq_size;
1681         e->rx_jumbo_pending = q->jumbo_size;
1682         e->tx_pending = q->txq_size[0];
1683 }
1684
1685 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1686 {
1687         struct port_info *pi = netdev_priv(dev);
1688         struct adapter *adapter = pi->adapter;
1689         struct qset_params *q;
1690         int i;
1691
1692         if (e->rx_pending > MAX_RX_BUFFERS ||
1693             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1694             e->tx_pending > MAX_TXQ_ENTRIES ||
1695             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1696             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1697             e->rx_pending < MIN_FL_ENTRIES ||
1698             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1699             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1700                 return -EINVAL;
1701
1702         if (adapter->flags & FULL_INIT_DONE)
1703                 return -EBUSY;
1704
1705         q = &adapter->params.sge.qset[pi->first_qset];
1706         for (i = 0; i < pi->nqsets; ++i, ++q) {
1707                 q->rspq_size = e->rx_mini_pending;
1708                 q->fl_size = e->rx_pending;
1709                 q->jumbo_size = e->rx_jumbo_pending;
1710                 q->txq_size[0] = e->tx_pending;
1711                 q->txq_size[1] = e->tx_pending;
1712                 q->txq_size[2] = e->tx_pending;
1713         }
1714         return 0;
1715 }
1716
1717 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1718 {
1719         struct port_info *pi = netdev_priv(dev);
1720         struct adapter *adapter = pi->adapter;
1721         struct qset_params *qsp = &adapter->params.sge.qset[0];
1722         struct sge_qset *qs = &adapter->sge.qs[0];
1723
1724         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1725                 return -EINVAL;
1726
1727         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1728         t3_update_qset_coalesce(qs, qsp);
1729         return 0;
1730 }
1731
1732 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1733 {
1734         struct port_info *pi = netdev_priv(dev);
1735         struct adapter *adapter = pi->adapter;
1736         struct qset_params *q = adapter->params.sge.qset;
1737
1738         c->rx_coalesce_usecs = q->coalesce_usecs;
1739         return 0;
1740 }
1741
1742 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1743                       u8 * data)
1744 {
1745         struct port_info *pi = netdev_priv(dev);
1746         struct adapter *adapter = pi->adapter;
1747         int i, err = 0;
1748
1749         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1750         if (!buf)
1751                 return -ENOMEM;
1752
1753         e->magic = EEPROM_MAGIC;
1754         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1755                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
1756
1757         if (!err)
1758                 memcpy(data, buf + e->offset, e->len);
1759         kfree(buf);
1760         return err;
1761 }
1762
1763 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1764                       u8 * data)
1765 {
1766         struct port_info *pi = netdev_priv(dev);
1767         struct adapter *adapter = pi->adapter;
1768         u32 aligned_offset, aligned_len;
1769         __le32 *p;
1770         u8 *buf;
1771         int err;
1772
1773         if (eeprom->magic != EEPROM_MAGIC)
1774                 return -EINVAL;
1775
1776         aligned_offset = eeprom->offset & ~3;
1777         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1778
1779         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1780                 buf = kmalloc(aligned_len, GFP_KERNEL);
1781                 if (!buf)
1782                         return -ENOMEM;
1783                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
1784                 if (!err && aligned_len > 4)
1785                         err = t3_seeprom_read(adapter,
1786                                               aligned_offset + aligned_len - 4,
1787                                               (__le32 *) & buf[aligned_len - 4]);
1788                 if (err)
1789                         goto out;
1790                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1791         } else
1792                 buf = data;
1793
1794         err = t3_seeprom_wp(adapter, 0);
1795         if (err)
1796                 goto out;
1797
1798         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
1799                 err = t3_seeprom_write(adapter, aligned_offset, *p);
1800                 aligned_offset += 4;
1801         }
1802
1803         if (!err)
1804                 err = t3_seeprom_wp(adapter, 1);
1805 out:
1806         if (buf != data)
1807                 kfree(buf);
1808         return err;
1809 }
1810
1811 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1812 {
1813         wol->supported = 0;
1814         wol->wolopts = 0;
1815         memset(&wol->sopass, 0, sizeof(wol->sopass));
1816 }
1817
1818 static const struct ethtool_ops cxgb_ethtool_ops = {
1819         .get_settings = get_settings,
1820         .set_settings = set_settings,
1821         .get_drvinfo = get_drvinfo,
1822         .get_msglevel = get_msglevel,
1823         .set_msglevel = set_msglevel,
1824         .get_ringparam = get_sge_param,
1825         .set_ringparam = set_sge_param,
1826         .get_coalesce = get_coalesce,
1827         .set_coalesce = set_coalesce,
1828         .get_eeprom_len = get_eeprom_len,
1829         .get_eeprom = get_eeprom,
1830         .set_eeprom = set_eeprom,
1831         .get_pauseparam = get_pauseparam,
1832         .set_pauseparam = set_pauseparam,
1833         .get_rx_csum = get_rx_csum,
1834         .set_rx_csum = set_rx_csum,
1835         .set_tx_csum = ethtool_op_set_tx_csum,
1836         .set_sg = ethtool_op_set_sg,
1837         .get_link = ethtool_op_get_link,
1838         .get_strings = get_strings,
1839         .phys_id = cxgb3_phys_id,
1840         .nway_reset = restart_autoneg,
1841         .get_sset_count = get_sset_count,
1842         .get_ethtool_stats = get_stats,
1843         .get_regs_len = get_regs_len,
1844         .get_regs = get_regs,
1845         .get_wol = get_wol,
1846         .set_tso = ethtool_op_set_tso,
1847 };
1848
1849 static int in_range(int val, int lo, int hi)
1850 {
1851         return val < 0 || (val <= hi && val >= lo);
1852 }
1853
1854 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
1855 {
1856         struct port_info *pi = netdev_priv(dev);
1857         struct adapter *adapter = pi->adapter;
1858         u32 cmd;
1859         int ret;
1860
1861         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
1862                 return -EFAULT;
1863
1864         switch (cmd) {
1865         case CHELSIO_SET_QSET_PARAMS:{
1866                 int i;
1867                 struct qset_params *q;
1868                 struct ch_qset_params t;
1869                 int q1 = pi->first_qset;
1870                 int nqsets = pi->nqsets;
1871
1872                 if (!capable(CAP_NET_ADMIN))
1873                         return -EPERM;
1874                 if (copy_from_user(&t, useraddr, sizeof(t)))
1875                         return -EFAULT;
1876                 if (t.qset_idx >= SGE_QSETS)
1877                         return -EINVAL;
1878                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
1879                         !in_range(t.cong_thres, 0, 255) ||
1880                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
1881                                 MAX_TXQ_ENTRIES) ||
1882                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
1883                                 MAX_TXQ_ENTRIES) ||
1884                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1885                                 MAX_CTRL_TXQ_ENTRIES) ||
1886                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
1887                                 MAX_RX_BUFFERS)
1888                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
1889                                         MAX_RX_JUMBO_BUFFERS)
1890                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
1891                                         MAX_RSPQ_ENTRIES))
1892                         return -EINVAL;
1893
1894                 if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
1895                         for_each_port(adapter, i) {
1896                                 pi = adap2pinfo(adapter, i);
1897                                 if (t.qset_idx >= pi->first_qset &&
1898                                     t.qset_idx < pi->first_qset + pi->nqsets &&
1899                                     !(pi->rx_offload & T3_RX_CSUM))
1900                                         return -EINVAL;
1901                         }
1902
1903                 if ((adapter->flags & FULL_INIT_DONE) &&
1904                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
1905                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
1906                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
1907                         t.polling >= 0 || t.cong_thres >= 0))
1908                         return -EBUSY;
1909
1910                 /* Allow setting of any available qset when offload enabled */
1911                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
1912                         q1 = 0;
1913                         for_each_port(adapter, i) {
1914                                 pi = adap2pinfo(adapter, i);
1915                                 nqsets += pi->first_qset + pi->nqsets;
1916                         }
1917                 }
1918
1919                 if (t.qset_idx < q1)
1920                         return -EINVAL;
1921                 if (t.qset_idx > q1 + nqsets - 1)
1922                         return -EINVAL;
1923
1924                 q = &adapter->params.sge.qset[t.qset_idx];
1925
1926                 if (t.rspq_size >= 0)
1927                         q->rspq_size = t.rspq_size;
1928                 if (t.fl_size[0] >= 0)
1929                         q->fl_size = t.fl_size[0];
1930                 if (t.fl_size[1] >= 0)
1931                         q->jumbo_size = t.fl_size[1];
1932                 if (t.txq_size[0] >= 0)
1933                         q->txq_size[0] = t.txq_size[0];
1934                 if (t.txq_size[1] >= 0)
1935                         q->txq_size[1] = t.txq_size[1];
1936                 if (t.txq_size[2] >= 0)
1937                         q->txq_size[2] = t.txq_size[2];
1938                 if (t.cong_thres >= 0)
1939                         q->cong_thres = t.cong_thres;
1940                 if (t.intr_lat >= 0) {
1941                         struct sge_qset *qs =
1942                                 &adapter->sge.qs[t.qset_idx];
1943
1944                         q->coalesce_usecs = t.intr_lat;
1945                         t3_update_qset_coalesce(qs, q);
1946                 }
1947                 if (t.polling >= 0) {
1948                         if (adapter->flags & USING_MSIX)
1949                                 q->polling = t.polling;
1950                         else {
1951                                 /* No polling with INTx for T3A */
1952                                 if (adapter->params.rev == 0 &&
1953                                         !(adapter->flags & USING_MSI))
1954                                         t.polling = 0;
1955
1956                                 for (i = 0; i < SGE_QSETS; i++) {
1957                                         q = &adapter->params.sge.
1958                                                 qset[i];
1959                                         q->polling = t.polling;
1960                                 }
1961                         }
1962                 }
1963                 if (t.lro >= 0)
1964                         set_qset_lro(dev, t.qset_idx, t.lro);
1965
1966                 break;
1967         }
1968         case CHELSIO_GET_QSET_PARAMS:{
1969                 struct qset_params *q;
1970                 struct ch_qset_params t;
1971                 int q1 = pi->first_qset;
1972                 int nqsets = pi->nqsets;
1973                 int i;
1974
1975                 if (copy_from_user(&t, useraddr, sizeof(t)))
1976                         return -EFAULT;
1977
1978                 /* Display qsets for all ports when offload enabled */
1979                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
1980                         q1 = 0;
1981                         for_each_port(adapter, i) {
1982                                 pi = adap2pinfo(adapter, i);
1983                                 nqsets = pi->first_qset + pi->nqsets;
1984                         }
1985                 }
1986
1987                 if (t.qset_idx >= nqsets)
1988                         return -EINVAL;
1989
1990                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
1991                 t.rspq_size = q->rspq_size;
1992                 t.txq_size[0] = q->txq_size[0];
1993                 t.txq_size[1] = q->txq_size[1];
1994                 t.txq_size[2] = q->txq_size[2];
1995                 t.fl_size[0] = q->fl_size;
1996                 t.fl_size[1] = q->jumbo_size;
1997                 t.polling = q->polling;
1998                 t.lro = q->lro;
1999                 t.intr_lat = q->coalesce_usecs;
2000                 t.cong_thres = q->cong_thres;
2001                 t.qnum = q1;
2002
2003                 if (adapter->flags & USING_MSIX)
2004                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2005                 else
2006                         t.vector = adapter->pdev->irq;
2007
2008                 if (copy_to_user(useraddr, &t, sizeof(t)))
2009                         return -EFAULT;
2010                 break;
2011         }
2012         case CHELSIO_SET_QSET_NUM:{
2013                 struct ch_reg edata;
2014                 unsigned int i, first_qset = 0, other_qsets = 0;
2015
2016                 if (!capable(CAP_NET_ADMIN))
2017                         return -EPERM;
2018                 if (adapter->flags & FULL_INIT_DONE)
2019                         return -EBUSY;
2020                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2021                         return -EFAULT;
2022                 if (edata.val < 1 ||
2023                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2024                         return -EINVAL;
2025
2026                 for_each_port(adapter, i)
2027                         if (adapter->port[i] && adapter->port[i] != dev)
2028                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2029
2030                 if (edata.val + other_qsets > SGE_QSETS)
2031                         return -EINVAL;
2032
2033                 pi->nqsets = edata.val;
2034
2035                 for_each_port(adapter, i)
2036                         if (adapter->port[i]) {
2037                                 pi = adap2pinfo(adapter, i);
2038                                 pi->first_qset = first_qset;
2039                                 first_qset += pi->nqsets;
2040                         }
2041                 break;
2042         }
2043         case CHELSIO_GET_QSET_NUM:{
2044                 struct ch_reg edata;
2045
2046                 edata.cmd = CHELSIO_GET_QSET_NUM;
2047                 edata.val = pi->nqsets;
2048                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2049                         return -EFAULT;
2050                 break;
2051         }
2052         case CHELSIO_LOAD_FW:{
2053                 u8 *fw_data;
2054                 struct ch_mem_range t;
2055
2056                 if (!capable(CAP_SYS_RAWIO))
2057                         return -EPERM;
2058                 if (copy_from_user(&t, useraddr, sizeof(t)))
2059                         return -EFAULT;
2060                 /* Check t.len sanity ? */
2061                 fw_data = kmalloc(t.len, GFP_KERNEL);
2062                 if (!fw_data)
2063                         return -ENOMEM;
2064
2065                 if (copy_from_user
2066                         (fw_data, useraddr + sizeof(t), t.len)) {
2067                         kfree(fw_data);
2068                         return -EFAULT;
2069                 }
2070
2071                 ret = t3_load_fw(adapter, fw_data, t.len);
2072                 kfree(fw_data);
2073                 if (ret)
2074                         return ret;
2075                 break;
2076         }
2077         case CHELSIO_SETMTUTAB:{
2078                 struct ch_mtus m;
2079                 int i;
2080
2081                 if (!is_offload(adapter))
2082                         return -EOPNOTSUPP;
2083                 if (!capable(CAP_NET_ADMIN))
2084                         return -EPERM;
2085                 if (offload_running(adapter))
2086                         return -EBUSY;
2087                 if (copy_from_user(&m, useraddr, sizeof(m)))
2088                         return -EFAULT;
2089                 if (m.nmtus != NMTUS)
2090                         return -EINVAL;
2091                 if (m.mtus[0] < 81)     /* accommodate SACK */
2092                         return -EINVAL;
2093
2094                 /* MTUs must be in ascending order */
2095                 for (i = 1; i < NMTUS; ++i)
2096                         if (m.mtus[i] < m.mtus[i - 1])
2097                                 return -EINVAL;
2098
2099                 memcpy(adapter->params.mtus, m.mtus,
2100                         sizeof(adapter->params.mtus));
2101                 break;
2102         }
2103         case CHELSIO_GET_PM:{
2104                 struct tp_params *p = &adapter->params.tp;
2105                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2106
2107                 if (!is_offload(adapter))
2108                         return -EOPNOTSUPP;
2109                 m.tx_pg_sz = p->tx_pg_size;
2110                 m.tx_num_pg = p->tx_num_pgs;
2111                 m.rx_pg_sz = p->rx_pg_size;
2112                 m.rx_num_pg = p->rx_num_pgs;
2113                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2114                 if (copy_to_user(useraddr, &m, sizeof(m)))
2115                         return -EFAULT;
2116                 break;
2117         }
2118         case CHELSIO_SET_PM:{
2119                 struct ch_pm m;
2120                 struct tp_params *p = &adapter->params.tp;
2121
2122                 if (!is_offload(adapter))
2123                         return -EOPNOTSUPP;
2124                 if (!capable(CAP_NET_ADMIN))
2125                         return -EPERM;
2126                 if (adapter->flags & FULL_INIT_DONE)
2127                         return -EBUSY;
2128                 if (copy_from_user(&m, useraddr, sizeof(m)))
2129                         return -EFAULT;
2130                 if (!is_power_of_2(m.rx_pg_sz) ||
2131                         !is_power_of_2(m.tx_pg_sz))
2132                         return -EINVAL; /* not power of 2 */
2133                 if (!(m.rx_pg_sz & 0x14000))
2134                         return -EINVAL; /* not 16KB or 64KB */
2135                 if (!(m.tx_pg_sz & 0x1554000))
2136                         return -EINVAL;
2137                 if (m.tx_num_pg == -1)
2138                         m.tx_num_pg = p->tx_num_pgs;
2139                 if (m.rx_num_pg == -1)
2140                         m.rx_num_pg = p->rx_num_pgs;
2141                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2142                         return -EINVAL;
2143                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2144                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2145                         return -EINVAL;
2146                 p->rx_pg_size = m.rx_pg_sz;
2147                 p->tx_pg_size = m.tx_pg_sz;
2148                 p->rx_num_pgs = m.rx_num_pg;
2149                 p->tx_num_pgs = m.tx_num_pg;
2150                 break;
2151         }
2152         case CHELSIO_GET_MEM:{
2153                 struct ch_mem_range t;
2154                 struct mc7 *mem;
2155                 u64 buf[32];
2156
2157                 if (!is_offload(adapter))
2158                         return -EOPNOTSUPP;
2159                 if (!(adapter->flags & FULL_INIT_DONE))
2160                         return -EIO;    /* need the memory controllers */
2161                 if (copy_from_user(&t, useraddr, sizeof(t)))
2162                         return -EFAULT;
2163                 if ((t.addr & 7) || (t.len & 7))
2164                         return -EINVAL;
2165                 if (t.mem_id == MEM_CM)
2166                         mem = &adapter->cm;
2167                 else if (t.mem_id == MEM_PMRX)
2168                         mem = &adapter->pmrx;
2169                 else if (t.mem_id == MEM_PMTX)
2170                         mem = &adapter->pmtx;
2171                 else
2172                         return -EINVAL;
2173
2174                 /*
2175                  * Version scheme:
2176                  * bits 0..9: chip version
2177                  * bits 10..15: chip revision
2178                  */
2179                 t.version = 3 | (adapter->params.rev << 10);
2180                 if (copy_to_user(useraddr, &t, sizeof(t)))
2181                         return -EFAULT;
2182
2183                 /*
2184                  * Read 256 bytes at a time as len can be large and we don't
2185                  * want to use huge intermediate buffers.
2186                  */
2187                 useraddr += sizeof(t);  /* advance to start of buffer */
2188                 while (t.len) {
2189                         unsigned int chunk =
2190                                 min_t(unsigned int, t.len, sizeof(buf));
2191
2192                         ret =
2193                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2194                                                 buf);
2195                         if (ret)
2196                                 return ret;
2197                         if (copy_to_user(useraddr, buf, chunk))
2198                                 return -EFAULT;
2199                         useraddr += chunk;
2200                         t.addr += chunk;
2201                         t.len -= chunk;
2202                 }
2203                 break;
2204         }
2205         case CHELSIO_SET_TRACE_FILTER:{
2206                 struct ch_trace t;
2207                 const struct trace_params *tp;
2208
2209                 if (!capable(CAP_NET_ADMIN))
2210                         return -EPERM;
2211                 if (!offload_running(adapter))
2212                         return -EAGAIN;
2213                 if (copy_from_user(&t, useraddr, sizeof(t)))
2214                         return -EFAULT;
2215
2216                 tp = (const struct trace_params *)&t.sip;
2217                 if (t.config_tx)
2218                         t3_config_trace_filter(adapter, tp, 0,
2219                                                 t.invert_match,
2220                                                 t.trace_tx);
2221                 if (t.config_rx)
2222                         t3_config_trace_filter(adapter, tp, 1,
2223                                                 t.invert_match,
2224                                                 t.trace_rx);
2225                 break;
2226         }
2227         default:
2228                 return -EOPNOTSUPP;
2229         }
2230         return 0;
2231 }
2232
2233 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2234 {
2235         struct mii_ioctl_data *data = if_mii(req);
2236         struct port_info *pi = netdev_priv(dev);
2237         struct adapter *adapter = pi->adapter;
2238         int ret, mmd;
2239
2240         switch (cmd) {
2241         case SIOCGMIIPHY:
2242                 data->phy_id = pi->phy.addr;
2243                 /* FALLTHRU */
2244         case SIOCGMIIREG:{
2245                 u32 val;
2246                 struct cphy *phy = &pi->phy;
2247
2248                 if (!phy->mdio_read)
2249                         return -EOPNOTSUPP;
2250                 if (is_10G(adapter)) {
2251                         mmd = data->phy_id >> 8;
2252                         if (!mmd)
2253                                 mmd = MDIO_DEV_PCS;
2254                         else if (mmd > MDIO_DEV_VEND2)
2255                                 return -EINVAL;
2256
2257                         ret =
2258                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2259                                                 mmd, data->reg_num, &val);
2260                 } else
2261                         ret =
2262                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2263                                                 0, data->reg_num & 0x1f,
2264                                                 &val);
2265                 if (!ret)
2266                         data->val_out = val;
2267                 break;
2268         }
2269         case SIOCSMIIREG:{
2270                 struct cphy *phy = &pi->phy;
2271
2272                 if (!capable(CAP_NET_ADMIN))
2273                         return -EPERM;
2274                 if (!phy->mdio_write)
2275                         return -EOPNOTSUPP;
2276                 if (is_10G(adapter)) {
2277                         mmd = data->phy_id >> 8;
2278                         if (!mmd)
2279                                 mmd = MDIO_DEV_PCS;
2280                         else if (mmd > MDIO_DEV_VEND2)
2281                                 return -EINVAL;
2282
2283                         ret =
2284                                 phy->mdio_write(adapter,
2285                                                 data->phy_id & 0x1f, mmd,
2286                                                 data->reg_num,
2287                                                 data->val_in);
2288                 } else
2289                         ret =
2290                                 phy->mdio_write(adapter,
2291                                                 data->phy_id & 0x1f, 0,
2292                                                 data->reg_num & 0x1f,
2293                                                 data->val_in);
2294                 break;
2295         }
2296         case SIOCCHIOCTL:
2297                 return cxgb_extension_ioctl(dev, req->ifr_data);
2298         default:
2299                 return -EOPNOTSUPP;
2300         }
2301         return ret;
2302 }
2303
2304 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2305 {
2306         struct port_info *pi = netdev_priv(dev);
2307         struct adapter *adapter = pi->adapter;
2308         int ret;
2309
2310         if (new_mtu < 81)       /* accommodate SACK */
2311                 return -EINVAL;
2312         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2313                 return ret;
2314         dev->mtu = new_mtu;
2315         init_port_mtus(adapter);
2316         if (adapter->params.rev == 0 && offload_running(adapter))
2317                 t3_load_mtus(adapter, adapter->params.mtus,
2318                              adapter->params.a_wnd, adapter->params.b_wnd,
2319                              adapter->port[0]->mtu);
2320         return 0;
2321 }
2322
2323 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2324 {
2325         struct port_info *pi = netdev_priv(dev);
2326         struct adapter *adapter = pi->adapter;
2327         struct sockaddr *addr = p;
2328
2329         if (!is_valid_ether_addr(addr->sa_data))
2330                 return -EINVAL;
2331
2332         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2333         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2334         if (offload_running(adapter))
2335                 write_smt_entry(adapter, pi->port_id);
2336         return 0;
2337 }
2338
2339 /**
2340  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2341  * @adap: the adapter
2342  * @p: the port
2343  *
2344  * Ensures that current Rx processing on any of the queues associated with
2345  * the given port completes before returning.  We do this by acquiring and
2346  * releasing the locks of the response queues associated with the port.
2347  */
2348 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2349 {
2350         int i;
2351
2352         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2353                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2354
2355                 spin_lock_irq(&q->lock);
2356                 spin_unlock_irq(&q->lock);
2357         }
2358 }
2359
2360 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2361 {
2362         struct port_info *pi = netdev_priv(dev);
2363         struct adapter *adapter = pi->adapter;
2364
2365         pi->vlan_grp = grp;
2366         if (adapter->params.rev > 0)
2367                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2368         else {
2369                 /* single control for all ports */
2370                 unsigned int i, have_vlans = 0;
2371                 for_each_port(adapter, i)
2372                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2373
2374                 t3_set_vlan_accel(adapter, 1, have_vlans);
2375         }
2376         t3_synchronize_rx(adapter, pi);
2377 }
2378
2379 #ifdef CONFIG_NET_POLL_CONTROLLER
2380 static void cxgb_netpoll(struct net_device *dev)
2381 {
2382         struct port_info *pi = netdev_priv(dev);
2383         struct adapter *adapter = pi->adapter;
2384         int qidx;
2385
2386         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2387                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2388                 void *source;
2389
2390                 if (adapter->flags & USING_MSIX)
2391                         source = qs;
2392                 else
2393                         source = adapter;
2394
2395                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2396         }
2397 }
2398 #endif
2399
2400 /*
2401  * Periodic accumulation of MAC statistics.
2402  */
2403 static void mac_stats_update(struct adapter *adapter)
2404 {
2405         int i;
2406
2407         for_each_port(adapter, i) {
2408                 struct net_device *dev = adapter->port[i];
2409                 struct port_info *p = netdev_priv(dev);
2410
2411                 if (netif_running(dev)) {
2412                         spin_lock(&adapter->stats_lock);
2413                         t3_mac_update_stats(&p->mac);
2414                         spin_unlock(&adapter->stats_lock);
2415                 }
2416         }
2417 }
2418
2419 static void check_link_status(struct adapter *adapter)
2420 {
2421         int i;
2422
2423         for_each_port(adapter, i) {
2424                 struct net_device *dev = adapter->port[i];
2425                 struct port_info *p = netdev_priv(dev);
2426
2427                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev))
2428                         t3_link_changed(adapter, i);
2429         }
2430 }
2431
2432 static void check_t3b2_mac(struct adapter *adapter)
2433 {
2434         int i;
2435
2436         if (!rtnl_trylock())    /* synchronize with ifdown */
2437                 return;
2438
2439         for_each_port(adapter, i) {
2440                 struct net_device *dev = adapter->port[i];
2441                 struct port_info *p = netdev_priv(dev);
2442                 int status;
2443
2444                 if (!netif_running(dev))
2445                         continue;
2446
2447                 status = 0;
2448                 if (netif_running(dev) && netif_carrier_ok(dev))
2449                         status = t3b2_mac_watchdog_task(&p->mac);
2450                 if (status == 1)
2451                         p->mac.stats.num_toggled++;
2452                 else if (status == 2) {
2453                         struct cmac *mac = &p->mac;
2454
2455                         t3_mac_set_mtu(mac, dev->mtu);
2456                         t3_mac_set_address(mac, 0, dev->dev_addr);
2457                         cxgb_set_rxmode(dev);
2458                         t3_link_start(&p->phy, mac, &p->link_config);
2459                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2460                         t3_port_intr_enable(adapter, p->port_id);
2461                         p->mac.stats.num_resets++;
2462                 }
2463         }
2464         rtnl_unlock();
2465 }
2466
2467
2468 static void t3_adap_check_task(struct work_struct *work)
2469 {
2470         struct adapter *adapter = container_of(work, struct adapter,
2471                                                adap_check_task.work);
2472         const struct adapter_params *p = &adapter->params;
2473
2474         adapter->check_task_cnt++;
2475
2476         /* Check link status for PHYs without interrupts */
2477         if (p->linkpoll_period)
2478                 check_link_status(adapter);
2479
2480         /* Accumulate MAC stats if needed */
2481         if (!p->linkpoll_period ||
2482             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2483             p->stats_update_period) {
2484                 mac_stats_update(adapter);
2485                 adapter->check_task_cnt = 0;
2486         }
2487
2488         if (p->rev == T3_REV_B2)
2489                 check_t3b2_mac(adapter);
2490
2491         /* Schedule the next check update if any port is active. */
2492         spin_lock_irq(&adapter->work_lock);
2493         if (adapter->open_device_map & PORT_MASK)
2494                 schedule_chk_task(adapter);
2495         spin_unlock_irq(&adapter->work_lock);
2496 }
2497
2498 /*
2499  * Processes external (PHY) interrupts in process context.
2500  */
2501 static void ext_intr_task(struct work_struct *work)
2502 {
2503         struct adapter *adapter = container_of(work, struct adapter,
2504                                                ext_intr_handler_task);
2505
2506         t3_phy_intr_handler(adapter);
2507
2508         /* Now reenable external interrupts */
2509         spin_lock_irq(&adapter->work_lock);
2510         if (adapter->slow_intr_mask) {
2511                 adapter->slow_intr_mask |= F_T3DBG;
2512                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2513                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2514                              adapter->slow_intr_mask);
2515         }
2516         spin_unlock_irq(&adapter->work_lock);
2517 }
2518
2519 /*
2520  * Interrupt-context handler for external (PHY) interrupts.
2521  */
2522 void t3_os_ext_intr_handler(struct adapter *adapter)
2523 {
2524         /*
2525          * Schedule a task to handle external interrupts as they may be slow
2526          * and we use a mutex to protect MDIO registers.  We disable PHY
2527          * interrupts in the meantime and let the task reenable them when
2528          * it's done.
2529          */
2530         spin_lock(&adapter->work_lock);
2531         if (adapter->slow_intr_mask) {
2532                 adapter->slow_intr_mask &= ~F_T3DBG;
2533                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2534                              adapter->slow_intr_mask);
2535                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2536         }
2537         spin_unlock(&adapter->work_lock);
2538 }
2539
2540 static int t3_adapter_error(struct adapter *adapter, int reset)
2541 {
2542         int i, ret = 0;
2543
2544         if (is_offload(adapter) &&
2545             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2546                 cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2547                 offload_close(&adapter->tdev);
2548         }
2549
2550         /* Stop all ports */
2551         for_each_port(adapter, i) {
2552                 struct net_device *netdev = adapter->port[i];
2553
2554                 if (netif_running(netdev))
2555                         cxgb_close(netdev);
2556         }
2557
2558         /* Stop SGE timers */
2559         t3_stop_sge_timers(adapter);
2560
2561         adapter->flags &= ~FULL_INIT_DONE;
2562
2563         if (reset)
2564                 ret = t3_reset_adapter(adapter);
2565
2566         pci_disable_device(adapter->pdev);
2567
2568         return ret;
2569 }
2570
2571 static int t3_reenable_adapter(struct adapter *adapter)
2572 {
2573         if (pci_enable_device(adapter->pdev)) {
2574                 dev_err(&adapter->pdev->dev,
2575                         "Cannot re-enable PCI device after reset.\n");
2576                 goto err;
2577         }
2578         pci_set_master(adapter->pdev);
2579         pci_restore_state(adapter->pdev);
2580
2581         /* Free sge resources */
2582         t3_free_sge_resources(adapter);
2583
2584         if (t3_replay_prep_adapter(adapter))
2585                 goto err;
2586
2587         return 0;
2588 err:
2589         return -1;
2590 }
2591
2592 static void t3_resume_ports(struct adapter *adapter)
2593 {
2594         int i;
2595
2596         /* Restart the ports */
2597         for_each_port(adapter, i) {
2598                 struct net_device *netdev = adapter->port[i];
2599
2600                 if (netif_running(netdev)) {
2601                         if (cxgb_open(netdev)) {
2602                                 dev_err(&adapter->pdev->dev,
2603                                         "can't bring device back up"
2604                                         " after reset\n");
2605                                 continue;
2606                         }
2607                 }
2608         }
2609
2610         if (is_offload(adapter) && !ofld_disable)
2611                 cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2612 }
2613
2614 /*
2615  * processes a fatal error.
2616  * Bring the ports down, reset the chip, bring the ports back up.
2617  */
2618 static void fatal_error_task(struct work_struct *work)
2619 {
2620         struct adapter *adapter = container_of(work, struct adapter,
2621                                                fatal_error_handler_task);
2622         int err = 0;
2623
2624         rtnl_lock();
2625         err = t3_adapter_error(adapter, 1);
2626         if (!err)
2627                 err = t3_reenable_adapter(adapter);
2628         if (!err)
2629                 t3_resume_ports(adapter);
2630
2631         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2632         rtnl_unlock();
2633 }
2634
2635 void t3_fatal_err(struct adapter *adapter)
2636 {
2637         unsigned int fw_status[4];
2638
2639         if (adapter->flags & FULL_INIT_DONE) {
2640                 t3_sge_stop(adapter);
2641                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2642                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2643                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2644                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2645
2646                 spin_lock(&adapter->work_lock);
2647                 t3_intr_disable(adapter);
2648                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2649                 spin_unlock(&adapter->work_lock);
2650         }
2651         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2652         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2653                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2654                          fw_status[0], fw_status[1],
2655                          fw_status[2], fw_status[3]);
2656
2657 }
2658
2659 /**
2660  * t3_io_error_detected - called when PCI error is detected
2661  * @pdev: Pointer to PCI device
2662  * @state: The current pci connection state
2663  *
2664  * This function is called after a PCI bus error affecting
2665  * this device has been detected.
2666  */
2667 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2668                                              pci_channel_state_t state)
2669 {
2670         struct adapter *adapter = pci_get_drvdata(pdev);
2671         int ret;
2672
2673         ret = t3_adapter_error(adapter, 0);
2674
2675         /* Request a slot reset. */
2676         return PCI_ERS_RESULT_NEED_RESET;
2677 }
2678
2679 /**
2680  * t3_io_slot_reset - called after the pci bus has been reset.
2681  * @pdev: Pointer to PCI device
2682  *
2683  * Restart the card from scratch, as if from a cold-boot.
2684  */
2685 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
2686 {
2687         struct adapter *adapter = pci_get_drvdata(pdev);
2688
2689         if (!t3_reenable_adapter(adapter))
2690                 return PCI_ERS_RESULT_RECOVERED;
2691
2692         return PCI_ERS_RESULT_DISCONNECT;
2693 }
2694
2695 /**
2696  * t3_io_resume - called when traffic can start flowing again.
2697  * @pdev: Pointer to PCI device
2698  *
2699  * This callback is called when the error recovery driver tells us that
2700  * its OK to resume normal operation.
2701  */
2702 static void t3_io_resume(struct pci_dev *pdev)
2703 {
2704         struct adapter *adapter = pci_get_drvdata(pdev);
2705
2706         t3_resume_ports(adapter);
2707 }
2708
2709 static struct pci_error_handlers t3_err_handler = {
2710         .error_detected = t3_io_error_detected,
2711         .slot_reset = t3_io_slot_reset,
2712         .resume = t3_io_resume,
2713 };
2714
2715 /*
2716  * Set the number of qsets based on the number of CPUs and the number of ports,
2717  * not to exceed the number of available qsets, assuming there are enough qsets
2718  * per port in HW.
2719  */
2720 static void set_nqsets(struct adapter *adap)
2721 {
2722         int i, j = 0;
2723         int num_cpus = num_online_cpus();
2724         int hwports = adap->params.nports;
2725         int nqsets = adap->msix_nvectors - 1;
2726
2727         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
2728                 if (hwports == 2 &&
2729                     (hwports * nqsets > SGE_QSETS ||
2730                      num_cpus >= nqsets / hwports))
2731                         nqsets /= hwports;
2732                 if (nqsets > num_cpus)
2733                         nqsets = num_cpus;
2734                 if (nqsets < 1 || hwports == 4)
2735                         nqsets = 1;
2736         } else
2737                 nqsets = 1;
2738
2739         for_each_port(adap, i) {
2740                 struct port_info *pi = adap2pinfo(adap, i);
2741
2742                 pi->first_qset = j;
2743                 pi->nqsets = nqsets;
2744                 j = pi->first_qset + nqsets;
2745
2746                 dev_info(&adap->pdev->dev,
2747                          "Port %d using %d queue sets.\n", i, nqsets);
2748         }
2749 }
2750
2751 static int __devinit cxgb_enable_msix(struct adapter *adap)
2752 {
2753         struct msix_entry entries[SGE_QSETS + 1];
2754         int vectors;
2755         int i, err;
2756
2757         vectors = ARRAY_SIZE(entries);
2758         for (i = 0; i < vectors; ++i)
2759                 entries[i].entry = i;
2760
2761         while ((err = pci_enable_msix(adap->pdev, entries, vectors)) > 0)
2762                 vectors = err;
2763
2764         if (!err && vectors < (adap->params.nports + 1))
2765                 err = -1;
2766
2767         if (!err) {
2768                 for (i = 0; i < vectors; ++i)
2769                         adap->msix_info[i].vec = entries[i].vector;
2770                 adap->msix_nvectors = vectors;
2771         }
2772
2773         return err;
2774 }
2775
2776 static void __devinit print_port_info(struct adapter *adap,
2777                                       const struct adapter_info *ai)
2778 {
2779         static const char *pci_variant[] = {
2780                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
2781         };
2782
2783         int i;
2784         char buf[80];
2785
2786         if (is_pcie(adap))
2787                 snprintf(buf, sizeof(buf), "%s x%d",
2788                          pci_variant[adap->params.pci.variant],
2789                          adap->params.pci.width);
2790         else
2791                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
2792                          pci_variant[adap->params.pci.variant],
2793                          adap->params.pci.speed, adap->params.pci.width);
2794
2795         for_each_port(adap, i) {
2796                 struct net_device *dev = adap->port[i];
2797                 const struct port_info *pi = netdev_priv(dev);
2798
2799                 if (!test_bit(i, &adap->registered_device_map))
2800                         continue;
2801                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
2802                        dev->name, ai->desc, pi->phy.desc,
2803                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
2804                        (adap->flags & USING_MSIX) ? " MSI-X" :
2805                        (adap->flags & USING_MSI) ? " MSI" : "");
2806                 if (adap->name == dev->name && adap->params.vpd.mclk)
2807                         printk(KERN_INFO
2808                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
2809                                adap->name, t3_mc7_size(&adap->cm) >> 20,
2810                                t3_mc7_size(&adap->pmtx) >> 20,
2811                                t3_mc7_size(&adap->pmrx) >> 20,
2812                                adap->params.vpd.sn);
2813         }
2814 }
2815
2816 static const struct net_device_ops cxgb_netdev_ops = {
2817         .ndo_open               = cxgb_open,
2818         .ndo_stop               = cxgb_close,
2819         .ndo_start_xmit         = t3_eth_xmit,
2820         .ndo_get_stats          = cxgb_get_stats,
2821         .ndo_validate_addr      = eth_validate_addr,
2822         .ndo_set_multicast_list = cxgb_set_rxmode,
2823         .ndo_do_ioctl           = cxgb_ioctl,
2824         .ndo_change_mtu         = cxgb_change_mtu,
2825         .ndo_set_mac_address    = cxgb_set_mac_addr,
2826         .ndo_vlan_rx_register   = vlan_rx_register,
2827 #ifdef CONFIG_NET_POLL_CONTROLLER
2828         .ndo_poll_controller    = cxgb_netpoll,
2829 #endif
2830 };
2831
2832 static int __devinit init_one(struct pci_dev *pdev,
2833                               const struct pci_device_id *ent)
2834 {
2835         static int version_printed;
2836
2837         int i, err, pci_using_dac = 0;
2838         unsigned long mmio_start, mmio_len;
2839         const struct adapter_info *ai;
2840         struct adapter *adapter = NULL;
2841         struct port_info *pi;
2842
2843         if (!version_printed) {
2844                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
2845                 ++version_printed;
2846         }
2847
2848         if (!cxgb3_wq) {
2849                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
2850                 if (!cxgb3_wq) {
2851                         printk(KERN_ERR DRV_NAME
2852                                ": cannot initialize work queue\n");
2853                         return -ENOMEM;
2854                 }
2855         }
2856
2857         err = pci_request_regions(pdev, DRV_NAME);
2858         if (err) {
2859                 /* Just info, some other driver may have claimed the device. */
2860                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
2861                 return err;
2862         }
2863
2864         err = pci_enable_device(pdev);
2865         if (err) {
2866                 dev_err(&pdev->dev, "cannot enable PCI device\n");
2867                 goto out_release_regions;
2868         }
2869
2870         if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
2871                 pci_using_dac = 1;
2872                 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2873                 if (err) {
2874                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
2875                                "coherent allocations\n");
2876                         goto out_disable_device;
2877                 }
2878         } else if ((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK)) != 0) {
2879                 dev_err(&pdev->dev, "no usable DMA configuration\n");
2880                 goto out_disable_device;
2881         }
2882
2883         pci_set_master(pdev);
2884         pci_save_state(pdev);
2885
2886         mmio_start = pci_resource_start(pdev, 0);
2887         mmio_len = pci_resource_len(pdev, 0);
2888         ai = t3_get_adapter_info(ent->driver_data);
2889
2890         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2891         if (!adapter) {
2892                 err = -ENOMEM;
2893                 goto out_disable_device;
2894         }
2895
2896         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
2897         if (!adapter->regs) {
2898                 dev_err(&pdev->dev, "cannot map device registers\n");
2899                 err = -ENOMEM;
2900                 goto out_free_adapter;
2901         }
2902
2903         adapter->pdev = pdev;
2904         adapter->name = pci_name(pdev);
2905         adapter->msg_enable = dflt_msg_enable;
2906         adapter->mmio_len = mmio_len;
2907
2908         mutex_init(&adapter->mdio_lock);
2909         spin_lock_init(&adapter->work_lock);
2910         spin_lock_init(&adapter->stats_lock);
2911
2912         INIT_LIST_HEAD(&adapter->adapter_list);
2913         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
2914         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
2915         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
2916
2917         for (i = 0; i < ai->nports; ++i) {
2918                 struct net_device *netdev;
2919
2920                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
2921                 if (!netdev) {
2922                         err = -ENOMEM;
2923                         goto out_free_dev;
2924                 }
2925
2926                 SET_NETDEV_DEV(netdev, &pdev->dev);
2927
2928                 adapter->port[i] = netdev;
2929                 pi = netdev_priv(netdev);
2930                 pi->adapter = adapter;
2931                 pi->rx_offload = T3_RX_CSUM | T3_LRO;
2932                 pi->port_id = i;
2933                 netif_carrier_off(netdev);
2934                 netif_tx_stop_all_queues(netdev);
2935                 netdev->irq = pdev->irq;
2936                 netdev->mem_start = mmio_start;
2937                 netdev->mem_end = mmio_start + mmio_len - 1;
2938                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
2939                 netdev->features |= NETIF_F_LLTX;
2940                 netdev->features |= NETIF_F_GRO;
2941                 if (pci_using_dac)
2942                         netdev->features |= NETIF_F_HIGHDMA;
2943
2944                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
2945                 netdev->netdev_ops = &cxgb_netdev_ops;
2946                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
2947         }
2948
2949         pci_set_drvdata(pdev, adapter);
2950         if (t3_prep_adapter(adapter, ai, 1) < 0) {
2951                 err = -ENODEV;
2952                 goto out_free_dev;
2953         }
2954
2955         /*
2956          * The card is now ready to go.  If any errors occur during device
2957          * registration we do not fail the whole card but rather proceed only
2958          * with the ports we manage to register successfully.  However we must
2959          * register at least one net device.
2960          */
2961         for_each_port(adapter, i) {
2962                 err = register_netdev(adapter->port[i]);
2963                 if (err)
2964                         dev_warn(&pdev->dev,
2965                                  "cannot register net device %s, skipping\n",
2966                                  adapter->port[i]->name);
2967                 else {
2968                         /*
2969                          * Change the name we use for messages to the name of
2970                          * the first successfully registered interface.
2971                          */
2972                         if (!adapter->registered_device_map)
2973                                 adapter->name = adapter->port[i]->name;
2974
2975                         __set_bit(i, &adapter->registered_device_map);
2976                 }
2977         }
2978         if (!adapter->registered_device_map) {
2979                 dev_err(&pdev->dev, "could not register any net devices\n");
2980                 goto out_free_dev;
2981         }
2982
2983         /* Driver's ready. Reflect it on LEDs */
2984         t3_led_ready(adapter);
2985
2986         if (is_offload(adapter)) {
2987                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
2988                 cxgb3_adapter_ofld(adapter);
2989         }
2990
2991         /* See what interrupts we'll be using */
2992         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
2993                 adapter->flags |= USING_MSIX;
2994         else if (msi > 0 && pci_enable_msi(pdev) == 0)
2995                 adapter->flags |= USING_MSI;
2996
2997         set_nqsets(adapter);
2998
2999         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3000                                  &cxgb3_attr_group);
3001
3002         print_port_info(adapter, ai);
3003         return 0;
3004
3005 out_free_dev:
3006         iounmap(adapter->regs);
3007         for (i = ai->nports - 1; i >= 0; --i)
3008                 if (adapter->port[i])
3009                         free_netdev(adapter->port[i]);
3010
3011 out_free_adapter:
3012         kfree(adapter);
3013
3014 out_disable_device:
3015         pci_disable_device(pdev);
3016 out_release_regions:
3017         pci_release_regions(pdev);
3018         pci_set_drvdata(pdev, NULL);
3019         return err;
3020 }
3021
3022 static void __devexit remove_one(struct pci_dev *pdev)
3023 {
3024         struct adapter *adapter = pci_get_drvdata(pdev);
3025
3026         if (adapter) {
3027                 int i;
3028
3029                 t3_sge_stop(adapter);
3030                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3031                                    &cxgb3_attr_group);
3032
3033                 if (is_offload(adapter)) {
3034                         cxgb3_adapter_unofld(adapter);
3035                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3036                                      &adapter->open_device_map))
3037                                 offload_close(&adapter->tdev);
3038                 }
3039
3040                 for_each_port(adapter, i)
3041                     if (test_bit(i, &adapter->registered_device_map))
3042                         unregister_netdev(adapter->port[i]);
3043
3044                 t3_stop_sge_timers(adapter);
3045                 t3_free_sge_resources(adapter);
3046                 cxgb_disable_msi(adapter);
3047
3048                 for_each_port(adapter, i)
3049                         if (adapter->port[i])
3050                                 free_netdev(adapter->port[i]);
3051
3052                 iounmap(adapter->regs);
3053                 kfree(adapter);
3054                 pci_release_regions(pdev);
3055                 pci_disable_device(pdev);
3056                 pci_set_drvdata(pdev, NULL);
3057         }
3058 }
3059
3060 static struct pci_driver driver = {
3061         .name = DRV_NAME,
3062         .id_table = cxgb3_pci_tbl,
3063         .probe = init_one,
3064         .remove = __devexit_p(remove_one),
3065         .err_handler = &t3_err_handler,
3066 };
3067
3068 static int __init cxgb3_init_module(void)
3069 {
3070         int ret;
3071
3072         cxgb3_offload_init();
3073
3074         ret = pci_register_driver(&driver);
3075         return ret;
3076 }
3077
3078 static void __exit cxgb3_cleanup_module(void)
3079 {
3080         pci_unregister_driver(&driver);
3081         if (cxgb3_wq)
3082                 destroy_workqueue(cxgb3_wq);
3083 }
3084
3085 module_init(cxgb3_init_module);
3086 module_exit(cxgb3_cleanup_module);