git.oblomov.eu Git - linux-2.6/blob - drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kernel.h>
  22 #include <linux/types.h>
  23 #include <linux/compiler.h>
  24 #include <linux/slab.h>
  25 #include <linux/delay.h>
  26 #include <linux/in.h>
  27 #include <linux/init.h>
  28 #include <linux/ioport.h>
  29 #include <linux/pci.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/mii.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/workqueue.h>
  39 #include <linux/prefetch.h>
  40 #include <linux/dma-mapping.h>
  41
  42 #include <net/checksum.h>
  43
  44 #include <asm/system.h>
  45 #include <asm/io.h>
  46 #include <asm/byteorder.h>
  47 #include <asm/uaccess.h>
  48
  49 #ifdef CONFIG_SPARC64
  50 #include <asm/idprom.h>
  51 #include <asm/oplib.h>
  52 #include <asm/pbm.h>
  53 #endif
  54
  55 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
  56 #define TG3_VLAN_TAG_USED 1
  57 #else
  58 #define TG3_VLAN_TAG_USED 0
  59 #endif
  60
  61 #ifdef NETIF_F_TSO
  62 #define TG3_TSO_SUPPORT 1
  63 #else
  64 #define TG3_TSO_SUPPORT 0
  65 #endif
  66
  67 #include "tg3.h"
  68
  69 #define DRV_MODULE_NAME         "tg3"
  70 #define PFX DRV_MODULE_NAME     ": "
  71 #define DRV_MODULE_VERSION      "3.62"
  72 #define DRV_MODULE_RELDATE      "June 30, 2006"
  73
  74 #define TG3_DEF_MAC_MODE        0
  75 #define TG3_DEF_RX_MODE         0
  76 #define TG3_DEF_TX_MODE         0
  77 #define TG3_DEF_MSG_ENABLE        \
  78         (NETIF_MSG_DRV          | \
  79          NETIF_MSG_PROBE        | \
  80          NETIF_MSG_LINK         | \
  81          NETIF_MSG_TIMER        | \
  82          NETIF_MSG_IFDOWN       | \
  83          NETIF_MSG_IFUP         | \
  84          NETIF_MSG_RX_ERR       | \
  85          NETIF_MSG_TX_ERR)
  86
  87 /* length of time before we decide the hardware is borked,
  88  * and dev->tx_timeout() should be called to fix the problem
  89  */
  90 #define TG3_TX_TIMEOUT                  (5 * HZ)
  91
  92 /* hardware minimum and maximum for a single frame's data payload */
  93 #define TG3_MIN_MTU                     60
  94 #define TG3_MAX_MTU(tp) \
  95         ((tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) ? 9000 : 1500)
  96
  97 /* These numbers seem to be hard coded in the NIC firmware somehow.
  98  * You can't change the ring sizes, but you can change where you place
  99  * them in the NIC onboard memory.
 100  */
 101 #define TG3_RX_RING_SIZE                512
 102 #define TG3_DEF_RX_RING_PENDING         200
 103 #define TG3_RX_JUMBO_RING_SIZE          256
 104 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 105
 106 /* Do not place this n-ring entries value into the tp struct itself,
 107  * we really want to expose these constants to GCC so that modulo et
 108  * al.  operations are done with shifts and masks instead of with
 109  * hw multiply/modulo instructions.  Another solution would be to
 110  * replace things like '% foo' with '& (foo - 1)'.
 111  */
 112 #define TG3_RX_RCB_RING_SIZE(tp)        \
 113         ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ?  512 : 1024)
 114
 115 #define TG3_TX_RING_SIZE                512
 116 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 117
 118 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
 119                                  TG3_RX_RING_SIZE)
 120 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_rx_buffer_desc) * \
 121                                  TG3_RX_JUMBO_RING_SIZE)
 122 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
 123                                    TG3_RX_RCB_RING_SIZE(tp))
 124 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 125                                  TG3_TX_RING_SIZE)
 126 #define TX_BUFFS_AVAIL(TP)                                              \
 127         ((TP)->tx_pending -                                             \
 128          (((TP)->tx_prod - (TP)->tx_cons) & (TG3_TX_RING_SIZE - 1)))
 129 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 130
 131 #define RX_PKT_BUF_SZ           (1536 + tp->rx_offset + 64)
 132 #define RX_JUMBO_PKT_BUF_SZ     (9046 + tp->rx_offset + 64)
 133
 134 /* minimum number of free TX descriptors required to wake up TX process */
 135 #define TG3_TX_WAKEUP_THRESH            (TG3_TX_RING_SIZE / 4)
 136
 137 /* number of ETHTOOL_GSTATS u64's */
 138 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
 139
 140 #define TG3_NUM_TEST            6
 141
 142 static char version[] __devinitdata =
 143         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 144
 145 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 146 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 147 MODULE_LICENSE("GPL");
 148 MODULE_VERSION(DRV_MODULE_VERSION);
 149
 150 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 151 module_param(tg3_debug, int, 0);
 152 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 153
 154 static struct pci_device_id tg3_pci_tbl[] = {
 155         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700,
 156           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 157         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701,
 158           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 159         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702,
 160           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 161         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703,
 162           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 163         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704,
 164           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 165         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE,
 166           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 167         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705,
 168           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 169         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2,
 170           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 171         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M,
 172           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 173         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2,
 174           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 175         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X,
 176           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 177         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X,
 178           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 179         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S,
 180           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 181         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3,
 182           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 183         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3,
 184           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 185         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782,
 186           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 187         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788,
 188           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 189         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789,
 190           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 191         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901,
 192           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 193         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2,
 194           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 195         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2,
 196           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 197         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F,
 198           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 199         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720,
 200           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 201         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721,
 202           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 203         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750,
 204           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 205         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751,
 206           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 207         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M,
 208           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 209         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M,
 210           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 211         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F,
 212           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 213         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752,
 214           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 215         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M,
 216           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 217         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753,
 218           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 219         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M,
 220           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 221         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F,
 222           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 223         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754,
 224           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 225         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M,
 226           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 227         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755,
 228           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 229         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M,
 230           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 231         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786,
 232           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 233         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787,
 234           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 235         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M,
 236           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 237         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714,
 238           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 239         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S,
 240           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 241         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715,
 242           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 243         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S,
 244           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 245         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780,
 246           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 247         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S,
 248           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 249         { PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781,
 250           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 251         { PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX,
 252           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 253         { PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX,
 254           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 255         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000,
 256           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 257         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001,
 258           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 259         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003,
 260           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 261         { PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100,
 262           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 263         { PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3,
 264           PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
 265         { 0, }
 266 };
 267
 268 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 269
 270 static struct {
 271         const char string[ETH_GSTRING_LEN];
 272 } ethtool_stats_keys[TG3_NUM_STATS] = {
 273         { "rx_octets" },
 274         { "rx_fragments" },
 275         { "rx_ucast_packets" },
 276         { "rx_mcast_packets" },
 277         { "rx_bcast_packets" },
 278         { "rx_fcs_errors" },
 279         { "rx_align_errors" },
 280         { "rx_xon_pause_rcvd" },
 281         { "rx_xoff_pause_rcvd" },
 282         { "rx_mac_ctrl_rcvd" },
 283         { "rx_xoff_entered" },
 284         { "rx_frame_too_long_errors" },
 285         { "rx_jabbers" },
 286         { "rx_undersize_packets" },
 287         { "rx_in_length_errors" },
 288         { "rx_out_length_errors" },
 289         { "rx_64_or_less_octet_packets" },
 290         { "rx_65_to_127_octet_packets" },
 291         { "rx_128_to_255_octet_packets" },
 292         { "rx_256_to_511_octet_packets" },
 293         { "rx_512_to_1023_octet_packets" },
 294         { "rx_1024_to_1522_octet_packets" },
 295         { "rx_1523_to_2047_octet_packets" },
 296         { "rx_2048_to_4095_octet_packets" },
 297         { "rx_4096_to_8191_octet_packets" },
 298         { "rx_8192_to_9022_octet_packets" },
 299
 300         { "tx_octets" },
 301         { "tx_collisions" },
 302
 303         { "tx_xon_sent" },
 304         { "tx_xoff_sent" },
 305         { "tx_flow_control" },
 306         { "tx_mac_errors" },
 307         { "tx_single_collisions" },
 308         { "tx_mult_collisions" },
 309         { "tx_deferred" },
 310         { "tx_excessive_collisions" },
 311         { "tx_late_collisions" },
 312         { "tx_collide_2times" },
 313         { "tx_collide_3times" },
 314         { "tx_collide_4times" },
 315         { "tx_collide_5times" },
 316         { "tx_collide_6times" },
 317         { "tx_collide_7times" },
 318         { "tx_collide_8times" },
 319         { "tx_collide_9times" },
 320         { "tx_collide_10times" },
 321         { "tx_collide_11times" },
 322         { "tx_collide_12times" },
 323         { "tx_collide_13times" },
 324         { "tx_collide_14times" },
 325         { "tx_collide_15times" },
 326         { "tx_ucast_packets" },
 327         { "tx_mcast_packets" },
 328         { "tx_bcast_packets" },
 329         { "tx_carrier_sense_errors" },
 330         { "tx_discards" },
 331         { "tx_errors" },
 332
 333         { "dma_writeq_full" },
 334         { "dma_write_prioq_full" },
 335         { "rxbds_empty" },
 336         { "rx_discards" },
 337         { "rx_errors" },
 338         { "rx_threshold_hit" },
 339
 340         { "dma_readq_full" },
 341         { "dma_read_prioq_full" },
 342         { "tx_comp_queue_full" },
 343
 344         { "ring_set_send_prod_index" },
 345         { "ring_status_update" },
 346         { "nic_irqs" },
 347         { "nic_avoided_irqs" },
 348         { "nic_tx_threshold_hit" }
 349 };
 350
 351 static struct {
 352         const char string[ETH_GSTRING_LEN];
 353 } ethtool_test_keys[TG3_NUM_TEST] = {
 354         { "nvram test     (online) " },
 355         { "link test      (online) " },
 356         { "register test  (offline)" },
 357         { "memory test    (offline)" },
 358         { "loopback test  (offline)" },
 359         { "interrupt test (offline)" },
 360 };
 361
 362 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 363 {
 364         writel(val, tp->regs + off);
 365 }
 366
 367 static u32 tg3_read32(struct tg3 *tp, u32 off)
 368 {
 369         return (readl(tp->regs + off));
 370 }
 371
 372 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 373 {
 374         unsigned long flags;
 375
 376         spin_lock_irqsave(&tp->indirect_lock, flags);
 377         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 378         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 379         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 380 }
 381
 382 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 383 {
 384         writel(val, tp->regs + off);
 385         readl(tp->regs + off);
 386 }
 387
 388 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 389 {
 390         unsigned long flags;
 391         u32 val;
 392
 393         spin_lock_irqsave(&tp->indirect_lock, flags);
 394         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 395         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 396         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 397         return val;
 398 }
 399
 400 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 401 {
 402         unsigned long flags;
 403
 404         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 405                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 406                                        TG3_64BIT_REG_LOW, val);
 407                 return;
 408         }
 409         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
 410                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 411                                        TG3_64BIT_REG_LOW, val);
 412                 return;
 413         }
 414
 415         spin_lock_irqsave(&tp->indirect_lock, flags);
 416         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 417         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 418         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 419
 420         /* In indirect mode when disabling interrupts, we also need
 421          * to clear the interrupt bit in the GRC local ctrl register.
 422          */
 423         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 424             (val == 0x1)) {
 425                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 426                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 427         }
 428 }
 429
 430 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 431 {
 432         unsigned long flags;
 433         u32 val;
 434
 435         spin_lock_irqsave(&tp->indirect_lock, flags);
 436         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 437         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 438         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 439         return val;
 440 }
 441
 442 /* usec_wait specifies the wait time in usec when writing to certain registers
 443  * where it is unsafe to read back the register without some delay.
 444  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 445  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 446  */
 447 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 448 {
 449         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
 450             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 451                 /* Non-posted methods */
 452                 tp->write32(tp, off, val);
 453         else {
 454                 /* Posted method */
 455                 tg3_write32(tp, off, val);
 456                 if (usec_wait)
 457                         udelay(usec_wait);
 458                 tp->read32(tp, off);
 459         }
 460         /* Wait again after the read for the posted method to guarantee that
 461          * the wait time is met.
 462          */
 463         if (usec_wait)
 464                 udelay(usec_wait);
 465 }
 466
 467 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 468 {
 469         tp->write32_mbox(tp, off, val);
 470         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
 471             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 472                 tp->read32_mbox(tp, off);
 473 }
 474
 475 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 476 {
 477         void __iomem *mbox = tp->regs + off;
 478         writel(val, mbox);
 479         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
 480                 writel(val, mbox);
 481         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
 482                 readl(mbox);
 483 }
 484
 485 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
 486 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 487 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
 488 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
 489 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
 490
 491 #define tw32(reg,val)           tp->write32(tp, reg, val)
 492 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
 493 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
 494 #define tr32(reg)               tp->read32(tp, reg)
 495
 496 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 497 {
 498         unsigned long flags;
 499
 500         spin_lock_irqsave(&tp->indirect_lock, flags);
 501         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 502                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 503                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 504
 505                 /* Always leave this as zero. */
 506                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 507         } else {
 508                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 509                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 510
 511                 /* Always leave this as zero. */
 512                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 513         }
 514         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 515 }
 516
 517 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 518 {
 519         unsigned long flags;
 520
 521         spin_lock_irqsave(&tp->indirect_lock, flags);
 522         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 523                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 524                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 525
 526                 /* Always leave this as zero. */
 527                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 528         } else {
 529                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 530                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 531
 532                 /* Always leave this as zero. */
 533                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 534         }
 535         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 536 }
 537
 538 static void tg3_disable_ints(struct tg3 *tp)
 539 {
 540         tw32(TG3PCI_MISC_HOST_CTRL,
 541              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 542         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
 543 }
 544
 545 static inline void tg3_cond_int(struct tg3 *tp)
 546 {
 547         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 548             (tp->hw_status->status & SD_STATUS_UPDATED))
 549                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 550 }
 551
 552 static void tg3_enable_ints(struct tg3 *tp)
 553 {
 554         tp->irq_sync = 0;
 555         wmb();
 556
 557         tw32(TG3PCI_MISC_HOST_CTRL,
 558              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 559         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 560                        (tp->last_tag << 24));
 561         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
 562                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 563                                (tp->last_tag << 24));
 564         tg3_cond_int(tp);
 565 }
 566
 567 static inline unsigned int tg3_has_work(struct tg3 *tp)
 568 {
 569         struct tg3_hw_status *sblk = tp->hw_status;
 570         unsigned int work_exists = 0;
 571
 572         /* check for phy events */
 573         if (!(tp->tg3_flags &
 574               (TG3_FLAG_USE_LINKCHG_REG |
 575                TG3_FLAG_POLL_SERDES))) {
 576                 if (sblk->status & SD_STATUS_LINK_CHG)
 577                         work_exists = 1;
 578         }
 579         /* check for RX/TX work to do */
 580         if (sblk->idx[0].tx_consumer != tp->tx_cons ||
 581             sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
 582                 work_exists = 1;
 583
 584         return work_exists;
 585 }
 586
 587 /* tg3_restart_ints
 588  *  similar to tg3_enable_ints, but it accurately determines whether there
 589  *  is new work pending and can return without flushing the PIO write
 590  *  which reenables interrupts
 591  */
 592 static void tg3_restart_ints(struct tg3 *tp)
 593 {
 594         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
 595                      tp->last_tag << 24);
 596         mmiowb();
 597
 598         /* When doing tagged status, this work check is unnecessary.
 599          * The last_tag we write above tells the chip which piece of
 600          * work we've completed.
 601          */
 602         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 603             tg3_has_work(tp))
 604                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 605                      (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
 606 }
 607
 608 static inline void tg3_netif_stop(struct tg3 *tp)
 609 {
 610         tp->dev->trans_start = jiffies; /* prevent tx timeout */
 611         netif_poll_disable(tp->dev);
 612         netif_tx_disable(tp->dev);
 613 }
 614
 615 static inline void tg3_netif_start(struct tg3 *tp)
 616 {
 617         netif_wake_queue(tp->dev);
 618         /* NOTE: unconditional netif_wake_queue is only appropriate
 619          * so long as all callers are assured to have free tx slots
 620          * (such as after tg3_init_hw)
 621          */
 622         netif_poll_enable(tp->dev);
 623         tp->hw_status->status |= SD_STATUS_UPDATED;
 624         tg3_enable_ints(tp);
 625 }
 626
 627 static void tg3_switch_clocks(struct tg3 *tp)
 628 {
 629         u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 630         u32 orig_clock_ctrl;
 631
 632         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
 633                 return;
 634
 635         orig_clock_ctrl = clock_ctrl;
 636         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 637                        CLOCK_CTRL_CLKRUN_OENABLE |
 638                        0x1f);
 639         tp->pci_clock_ctrl = clock_ctrl;
 640
 641         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
 642                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 643                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 644                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 645                 }
 646         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 647                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 648                             clock_ctrl |
 649                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 650                             40);
 651                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 652                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 653                             40);
 654         }
 655         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 656 }
 657
 658 #define PHY_BUSY_LOOPS  5000
 659
 660 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 661 {
 662         u32 frame_val;
 663         unsigned int loops;
 664         int ret;
 665
 666         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 667                 tw32_f(MAC_MI_MODE,
 668                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 669                 udelay(80);
 670         }
 671
 672         *val = 0x0;
 673
 674         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 675                       MI_COM_PHY_ADDR_MASK);
 676         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 677                       MI_COM_REG_ADDR_MASK);
 678         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 679
 680         tw32_f(MAC_MI_COM, frame_val);
 681
 682         loops = PHY_BUSY_LOOPS;
 683         while (loops != 0) {
 684                 udelay(10);
 685                 frame_val = tr32(MAC_MI_COM);
 686
 687                 if ((frame_val & MI_COM_BUSY) == 0) {
 688                         udelay(5);
 689                         frame_val = tr32(MAC_MI_COM);
 690                         break;
 691                 }
 692                 loops -= 1;
 693         }
 694
 695         ret = -EBUSY;
 696         if (loops != 0) {
 697                 *val = frame_val & MI_COM_DATA_MASK;
 698                 ret = 0;
 699         }
 700
 701         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 702                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 703                 udelay(80);
 704         }
 705
 706         return ret;
 707 }
 708
 709 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 710 {
 711         u32 frame_val;
 712         unsigned int loops;
 713         int ret;
 714
 715         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 716                 tw32_f(MAC_MI_MODE,
 717                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 718                 udelay(80);
 719         }
 720
 721         frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
 722                       MI_COM_PHY_ADDR_MASK);
 723         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 724                       MI_COM_REG_ADDR_MASK);
 725         frame_val |= (val & MI_COM_DATA_MASK);
 726         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 727
 728         tw32_f(MAC_MI_COM, frame_val);
 729
 730         loops = PHY_BUSY_LOOPS;
 731         while (loops != 0) {
 732                 udelay(10);
 733                 frame_val = tr32(MAC_MI_COM);
 734                 if ((frame_val & MI_COM_BUSY) == 0) {
 735                         udelay(5);
 736                         frame_val = tr32(MAC_MI_COM);
 737                         break;
 738                 }
 739                 loops -= 1;
 740         }
 741
 742         ret = -EBUSY;
 743         if (loops != 0)
 744                 ret = 0;
 745
 746         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 747                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 748                 udelay(80);
 749         }
 750
 751         return ret;
 752 }
 753
 754 static void tg3_phy_set_wirespeed(struct tg3 *tp)
 755 {
 756         u32 val;
 757
 758         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
 759                 return;
 760
 761         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
 762             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
 763                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
 764                              (val | (1 << 15) | (1 << 4)));
 765 }
 766
 767 static int tg3_bmcr_reset(struct tg3 *tp)
 768 {
 769         u32 phy_control;
 770         int limit, err;
 771
 772         /* OK, reset it, and poll the BMCR_RESET bit until it
 773          * clears or we time out.
 774          */
 775         phy_control = BMCR_RESET;
 776         err = tg3_writephy(tp, MII_BMCR, phy_control);
 777         if (err != 0)
 778                 return -EBUSY;
 779
 780         limit = 5000;
 781         while (limit--) {
 782                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
 783                 if (err != 0)
 784                         return -EBUSY;
 785
 786                 if ((phy_control & BMCR_RESET) == 0) {
 787                         udelay(40);
 788                         break;
 789                 }
 790                 udelay(10);
 791         }
 792         if (limit <= 0)
 793                 return -EBUSY;
 794
 795         return 0;
 796 }
 797
 798 static int tg3_wait_macro_done(struct tg3 *tp)
 799 {
 800         int limit = 100;
 801
 802         while (limit--) {
 803                 u32 tmp32;
 804
 805                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
 806                         if ((tmp32 & 0x1000) == 0)
 807                                 break;
 808                 }
 809         }
 810         if (limit <= 0)
 811                 return -EBUSY;
 812
 813         return 0;
 814 }
 815
 816 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
 817 {
 818         static const u32 test_pat[4][6] = {
 819         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
 820         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
 821         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
 822         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
 823         };
 824         int chan;
 825
 826         for (chan = 0; chan < 4; chan++) {
 827                 int i;
 828
 829                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 830                              (chan * 0x2000) | 0x0200);
 831                 tg3_writephy(tp, 0x16, 0x0002);
 832
 833                 for (i = 0; i < 6; i++)
 834                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
 835                                      test_pat[chan][i]);
 836
 837                 tg3_writephy(tp, 0x16, 0x0202);
 838                 if (tg3_wait_macro_done(tp)) {
 839                         *resetp = 1;
 840                         return -EBUSY;
 841                 }
 842
 843                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 844                              (chan * 0x2000) | 0x0200);
 845                 tg3_writephy(tp, 0x16, 0x0082);
 846                 if (tg3_wait_macro_done(tp)) {
 847                         *resetp = 1;
 848                         return -EBUSY;
 849                 }
 850
 851                 tg3_writephy(tp, 0x16, 0x0802);
 852                 if (tg3_wait_macro_done(tp)) {
 853                         *resetp = 1;
 854                         return -EBUSY;
 855                 }
 856
 857                 for (i = 0; i < 6; i += 2) {
 858                         u32 low, high;
 859
 860                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
 861                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
 862                             tg3_wait_macro_done(tp)) {
 863                                 *resetp = 1;
 864                                 return -EBUSY;
 865                         }
 866                         low &= 0x7fff;
 867                         high &= 0x000f;
 868                         if (low != test_pat[chan][i] ||
 869                             high != test_pat[chan][i+1]) {
 870                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
 871                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
 872                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
 873
 874                                 return -EBUSY;
 875                         }
 876                 }
 877         }
 878
 879         return 0;
 880 }
 881
 882 static int tg3_phy_reset_chanpat(struct tg3 *tp)
 883 {
 884         int chan;
 885
 886         for (chan = 0; chan < 4; chan++) {
 887                 int i;
 888
 889                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
 890                              (chan * 0x2000) | 0x0200);
 891                 tg3_writephy(tp, 0x16, 0x0002);
 892                 for (i = 0; i < 6; i++)
 893                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
 894                 tg3_writephy(tp, 0x16, 0x0202);
 895                 if (tg3_wait_macro_done(tp))
 896                         return -EBUSY;
 897         }
 898
 899         return 0;
 900 }
 901
 902 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
 903 {
 904         u32 reg32, phy9_orig;
 905         int retries, do_phy_reset, err;
 906
 907         retries = 10;
 908         do_phy_reset = 1;
 909         do {
 910                 if (do_phy_reset) {
 911                         err = tg3_bmcr_reset(tp);
 912                         if (err)
 913                                 return err;
 914                         do_phy_reset = 0;
 915                 }
 916
 917                 /* Disable transmitter and interrupt.  */
 918                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
 919                         continue;
 920
 921                 reg32 |= 0x3000;
 922                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
 923
 924                 /* Set full-duplex, 1000 mbps.  */
 925                 tg3_writephy(tp, MII_BMCR,
 926                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
 927
 928                 /* Set to master mode.  */
 929                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
 930                         continue;
 931
 932                 tg3_writephy(tp, MII_TG3_CTRL,
 933                              (MII_TG3_CTRL_AS_MASTER |
 934                               MII_TG3_CTRL_ENABLE_AS_MASTER));
 935
 936                 /* Enable SM_DSP_CLOCK and 6dB.  */
 937                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
 938
 939                 /* Block the PHY control access.  */
 940                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
 941                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
 942
 943                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
 944                 if (!err)
 945                         break;
 946         } while (--retries);
 947
 948         err = tg3_phy_reset_chanpat(tp);
 949         if (err)
 950                 return err;
 951
 952         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
 953         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
 954
 955         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
 956         tg3_writephy(tp, 0x16, 0x0000);
 957
 958         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
 959             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
 960                 /* Set Extended packet length bit for jumbo frames */
 961                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
 962         }
 963         else {
 964                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
 965         }
 966
 967         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
 968
 969         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
 970                 reg32 &= ~0x3000;
 971                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
 972         } else if (!err)
 973                 err = -EBUSY;
 974
 975         return err;
 976 }
 977
 978 static void tg3_link_report(struct tg3 *);
 979
 980 /* This will reset the tigon3 PHY if there is no valid
 981  * link unless the FORCE argument is non-zero.
 982  */
 983 static int tg3_phy_reset(struct tg3 *tp)
 984 {
 985         u32 phy_status;
 986         int err;
 987
 988         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
 989         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
 990         if (err != 0)
 991                 return -EBUSY;
 992
 993         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
 994                 netif_carrier_off(tp->dev);
 995                 tg3_link_report(tp);
 996         }
 997
 998         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
 999             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1000             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1001                 err = tg3_phy_reset_5703_4_5(tp);
1002                 if (err)
1003                         return err;
1004                 goto out;
1005         }
1006
1007         err = tg3_bmcr_reset(tp);
1008         if (err)
1009                 return err;
1010
1011 out:
1012         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1013                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1014                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1015                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1016                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1017                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1018                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1019         }
1020         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1021                 tg3_writephy(tp, 0x1c, 0x8d68);
1022                 tg3_writephy(tp, 0x1c, 0x8d68);
1023         }
1024         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1025                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1026                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1027                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1028                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1029                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1030                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1031                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1032                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1033         }
1034         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1035                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1036                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1037                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1038                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1039         }
1040         /* Set Extended packet length bit (bit 14) on all chips that */
1041         /* support jumbo frames */
1042         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1043                 /* Cannot do read-modify-write on 5401 */
1044                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1045         } else if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1046                 u32 phy_reg;
1047
1048                 /* Set bit 14 with read-modify-write to preserve other bits */
1049                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1050                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1051                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1052         }
1053
1054         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1055          * jumbo frames transmission.
1056          */
1057         if (tp->tg3_flags2 & TG3_FLG2_JUMBO_CAPABLE) {
1058                 u32 phy_reg;
1059
1060                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1061                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1062                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1063         }
1064
1065         tg3_phy_set_wirespeed(tp);
1066         return 0;
1067 }
1068
1069 static void tg3_frob_aux_power(struct tg3 *tp)
1070 {
1071         struct tg3 *tp_peer = tp;
1072
1073         if ((tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) != 0)
1074                 return;
1075
1076         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
1077             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
1078                 struct net_device *dev_peer;
1079
1080                 dev_peer = pci_get_drvdata(tp->pdev_peer);
1081                 /* remove_one() may have been run on the peer. */
1082                 if (!dev_peer)
1083                         tp_peer = tp;
1084                 else
1085                         tp_peer = netdev_priv(dev_peer);
1086         }
1087
1088         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1089             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
1090             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
1091             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
1092                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1093                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1094                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1095                                     (GRC_LCLCTRL_GPIO_OE0 |
1096                                      GRC_LCLCTRL_GPIO_OE1 |
1097                                      GRC_LCLCTRL_GPIO_OE2 |
1098                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
1099                                      GRC_LCLCTRL_GPIO_OUTPUT1),
1100                                     100);
1101                 } else {
1102                         u32 no_gpio2;
1103                         u32 grc_local_ctrl = 0;
1104
1105                         if (tp_peer != tp &&
1106                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1107                                 return;
1108
1109                         /* Workaround to prevent overdrawing Amps. */
1110                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
1111                             ASIC_REV_5714) {
1112                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
1113                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1114                                             grc_local_ctrl, 100);
1115                         }
1116
1117                         /* On 5753 and variants, GPIO2 cannot be used. */
1118                         no_gpio2 = tp->nic_sram_data_cfg &
1119                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
1120
1121                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
1122                                          GRC_LCLCTRL_GPIO_OE1 |
1123                                          GRC_LCLCTRL_GPIO_OE2 |
1124                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
1125                                          GRC_LCLCTRL_GPIO_OUTPUT2;
1126                         if (no_gpio2) {
1127                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
1128                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
1129                         }
1130                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1131                                                     grc_local_ctrl, 100);
1132
1133                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
1134
1135                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1136                                                     grc_local_ctrl, 100);
1137
1138                         if (!no_gpio2) {
1139                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
1140                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1141                                             grc_local_ctrl, 100);
1142                         }
1143                 }
1144         } else {
1145                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
1146                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
1147                         if (tp_peer != tp &&
1148                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
1149                                 return;
1150
1151                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1152                                     (GRC_LCLCTRL_GPIO_OE1 |
1153                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1154
1155                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1156                                     GRC_LCLCTRL_GPIO_OE1, 100);
1157
1158                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
1159                                     (GRC_LCLCTRL_GPIO_OE1 |
1160                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
1161                 }
1162         }
1163 }
1164
1165 static int tg3_setup_phy(struct tg3 *, int);
1166
1167 #define RESET_KIND_SHUTDOWN     0
1168 #define RESET_KIND_INIT         1
1169 #define RESET_KIND_SUSPEND      2
1170
1171 static void tg3_write_sig_post_reset(struct tg3 *, int);
1172 static int tg3_halt_cpu(struct tg3 *, u32);
1173 static int tg3_nvram_lock(struct tg3 *);
1174 static void tg3_nvram_unlock(struct tg3 *);
1175
1176 static void tg3_power_down_phy(struct tg3 *tp)
1177 {
1178         /* The PHY should not be powered down on some chips because
1179          * of bugs.
1180          */
1181         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1182             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1183             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
1184              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
1185                 return;
1186         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
1187 }
1188
1189 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
1190 {
1191         u32 misc_host_ctrl;
1192         u16 power_control, power_caps;
1193         int pm = tp->pm_cap;
1194
1195         /* Make sure register accesses (indirect or otherwise)
1196          * will function correctly.
1197          */
1198         pci_write_config_dword(tp->pdev,
1199                                TG3PCI_MISC_HOST_CTRL,
1200                                tp->misc_host_ctrl);
1201
1202         pci_read_config_word(tp->pdev,
1203                              pm + PCI_PM_CTRL,
1204                              &power_control);
1205         power_control |= PCI_PM_CTRL_PME_STATUS;
1206         power_control &= ~(PCI_PM_CTRL_STATE_MASK);
1207         switch (state) {
1208         case PCI_D0:
1209                 power_control |= 0;
1210                 pci_write_config_word(tp->pdev,
1211                                       pm + PCI_PM_CTRL,
1212                                       power_control);
1213                 udelay(100);    /* Delay after power state change */
1214
1215                 /* Switch out of Vaux if it is not a LOM */
1216                 if (!(tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
1217                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
1218
1219                 return 0;
1220
1221         case PCI_D1:
1222                 power_control |= 1;
1223                 break;
1224
1225         case PCI_D2:
1226                 power_control |= 2;
1227                 break;
1228
1229         case PCI_D3hot:
1230                 power_control |= 3;
1231                 break;
1232
1233         default:
1234                 printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
1235                        "requested.\n",
1236                        tp->dev->name, state);
1237                 return -EINVAL;
1238         };
1239
1240         power_control |= PCI_PM_CTRL_PME_ENABLE;
1241
1242         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
1243         tw32(TG3PCI_MISC_HOST_CTRL,
1244              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
1245
1246         if (tp->link_config.phy_is_low_power == 0) {
1247                 tp->link_config.phy_is_low_power = 1;
1248                 tp->link_config.orig_speed = tp->link_config.speed;
1249                 tp->link_config.orig_duplex = tp->link_config.duplex;
1250                 tp->link_config.orig_autoneg = tp->link_config.autoneg;
1251         }
1252
1253         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
1254                 tp->link_config.speed = SPEED_10;
1255                 tp->link_config.duplex = DUPLEX_HALF;
1256                 tp->link_config.autoneg = AUTONEG_ENABLE;
1257                 tg3_setup_phy(tp, 0);
1258         }
1259
1260         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1261                 int i;
1262                 u32 val;
1263
1264                 for (i = 0; i < 200; i++) {
1265                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
1266                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
1267                                 break;
1268                         msleep(1);
1269                 }
1270         }
1271         tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
1272                                              WOL_DRV_STATE_SHUTDOWN |
1273                                              WOL_DRV_WOL | WOL_SET_MAGIC_PKT);
1274
1275         pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);
1276
1277         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
1278                 u32 mac_mode;
1279
1280                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1281                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
1282                         udelay(40);
1283
1284                         mac_mode = MAC_MODE_PORT_MODE_MII;
1285
1286                         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 ||
1287                             !(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB))
1288                                 mac_mode |= MAC_MODE_LINK_POLARITY;
1289                 } else {
1290                         mac_mode = MAC_MODE_PORT_MODE_TBI;
1291                 }
1292
1293                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
1294                         tw32(MAC_LED_CTRL, tp->led_ctrl);
1295
1296                 if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
1297                      (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
1298                         mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
1299
1300                 tw32_f(MAC_MODE, mac_mode);
1301                 udelay(100);
1302
1303                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
1304                 udelay(10);
1305         }
1306
1307         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
1308             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1309              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
1310                 u32 base_val;
1311
1312                 base_val = tp->pci_clock_ctrl;
1313                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
1314                              CLOCK_CTRL_TXCLK_DISABLE);
1315
1316                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
1317                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
1318         } else if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
1319                 /* do nothing */
1320         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
1321                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
1322                 u32 newbits1, newbits2;
1323
1324                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1325                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1326                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
1327                                     CLOCK_CTRL_TXCLK_DISABLE |
1328                                     CLOCK_CTRL_ALTCLK);
1329                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1330                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
1331                         newbits1 = CLOCK_CTRL_625_CORE;
1332                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
1333                 } else {
1334                         newbits1 = CLOCK_CTRL_ALTCLK;
1335                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
1336                 }
1337
1338                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
1339                             40);
1340
1341                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
1342                             40);
1343
1344                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
1345                         u32 newbits3;
1346
1347                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1348                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1349                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
1350                                             CLOCK_CTRL_TXCLK_DISABLE |
1351                                             CLOCK_CTRL_44MHZ_CORE);
1352                         } else {
1353                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
1354                         }
1355
1356                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
1357                                     tp->pci_clock_ctrl | newbits3, 40);
1358                 }
1359         }
1360
1361         if (!(tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
1362             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1363                 /* Turn off the PHY */
1364                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
1365                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
1366                                      MII_TG3_EXT_CTRL_FORCE_LED_OFF);
1367                         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x01b2);
1368                         tg3_power_down_phy(tp);
1369                 }
1370         }
1371
1372         tg3_frob_aux_power(tp);
1373
1374         /* Workaround for unstable PLL clock */
1375         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
1376             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
1377                 u32 val = tr32(0x7d00);
1378
1379                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
1380                 tw32(0x7d00, val);
1381                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
1382                         int err;
1383
1384                         err = tg3_nvram_lock(tp);
1385                         tg3_halt_cpu(tp, RX_CPU_BASE);
1386                         if (!err)
1387                                 tg3_nvram_unlock(tp);
1388                 }
1389         }
1390
1391         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
1392
1393         /* Finally, set the new power state. */
1394         pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);
1395         udelay(100);    /* Delay after power state change */
1396
1397         return 0;
1398 }
1399
1400 static void tg3_link_report(struct tg3 *tp)
1401 {
1402         if (!netif_carrier_ok(tp->dev)) {
1403                 printk(KERN_INFO PFX "%s: Link is down.\n", tp->dev->name);
1404         } else {
1405                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1406                        tp->dev->name,
1407                        (tp->link_config.active_speed == SPEED_1000 ?
1408                         1000 :
1409                         (tp->link_config.active_speed == SPEED_100 ?
1410                          100 : 10)),
1411                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1412                         "full" : "half"));
1413
1414                 printk(KERN_INFO PFX "%s: Flow control is %s for TX and "
1415                        "%s for RX.\n",
1416                        tp->dev->name,
1417                        (tp->tg3_flags & TG3_FLAG_TX_PAUSE) ? "on" : "off",
1418                        (tp->tg3_flags & TG3_FLAG_RX_PAUSE) ? "on" : "off");
1419         }
1420 }
1421
1422 static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
1423 {
1424         u32 new_tg3_flags = 0;
1425         u32 old_rx_mode = tp->rx_mode;
1426         u32 old_tx_mode = tp->tx_mode;
1427
1428         if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) {
1429
1430                 /* Convert 1000BaseX flow control bits to 1000BaseT
1431                  * bits before resolving flow control.
1432                  */
1433                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
1434                         local_adv &= ~(ADVERTISE_PAUSE_CAP |
1435                                        ADVERTISE_PAUSE_ASYM);
1436                         remote_adv &= ~(LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1437
1438                         if (local_adv & ADVERTISE_1000XPAUSE)
1439                                 local_adv |= ADVERTISE_PAUSE_CAP;
1440                         if (local_adv & ADVERTISE_1000XPSE_ASYM)
1441                                 local_adv |= ADVERTISE_PAUSE_ASYM;
1442                         if (remote_adv & LPA_1000XPAUSE)
1443                                 remote_adv |= LPA_PAUSE_CAP;
1444                         if (remote_adv & LPA_1000XPAUSE_ASYM)
1445                                 remote_adv |= LPA_PAUSE_ASYM;
1446                 }
1447
1448                 if (local_adv & ADVERTISE_PAUSE_CAP) {
1449                         if (local_adv & ADVERTISE_PAUSE_ASYM) {
1450                                 if (remote_adv & LPA_PAUSE_CAP)
1451                                         new_tg3_flags |=
1452                                                 (TG3_FLAG_RX_PAUSE |
1453                                                 TG3_FLAG_TX_PAUSE);
1454                                 else if (remote_adv & LPA_PAUSE_ASYM)
1455                                         new_tg3_flags |=
1456                                                 (TG3_FLAG_RX_PAUSE);
1457                         } else {
1458                                 if (remote_adv & LPA_PAUSE_CAP)
1459                                         new_tg3_flags |=
1460                                                 (TG3_FLAG_RX_PAUSE |
1461                                                 TG3_FLAG_TX_PAUSE);
1462                         }
1463                 } else if (local_adv & ADVERTISE_PAUSE_ASYM) {
1464                         if ((remote_adv & LPA_PAUSE_CAP) &&
1465                         (remote_adv & LPA_PAUSE_ASYM))
1466                                 new_tg3_flags |= TG3_FLAG_TX_PAUSE;
1467                 }
1468
1469                 tp->tg3_flags &= ~(TG3_FLAG_RX_PAUSE | TG3_FLAG_TX_PAUSE);
1470                 tp->tg3_flags |= new_tg3_flags;
1471         } else {
1472                 new_tg3_flags = tp->tg3_flags;
1473         }
1474
1475         if (new_tg3_flags & TG3_FLAG_RX_PAUSE)
1476                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1477         else
1478                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1479
1480         if (old_rx_mode != tp->rx_mode) {
1481                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1482         }
1483
1484         if (new_tg3_flags & TG3_FLAG_TX_PAUSE)
1485                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1486         else
1487                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1488
1489         if (old_tx_mode != tp->tx_mode) {
1490                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1491         }
1492 }
1493
1494 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
1495 {
1496         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
1497         case MII_TG3_AUX_STAT_10HALF:
1498                 *speed = SPEED_10;
1499                 *duplex = DUPLEX_HALF;
1500                 break;
1501
1502         case MII_TG3_AUX_STAT_10FULL:
1503                 *speed = SPEED_10;
1504                 *duplex = DUPLEX_FULL;
1505                 break;
1506
1507         case MII_TG3_AUX_STAT_100HALF:
1508                 *speed = SPEED_100;
1509                 *duplex = DUPLEX_HALF;
1510                 break;
1511
1512         case MII_TG3_AUX_STAT_100FULL:
1513                 *speed = SPEED_100;
1514                 *duplex = DUPLEX_FULL;
1515                 break;
1516
1517         case MII_TG3_AUX_STAT_1000HALF:
1518                 *speed = SPEED_1000;
1519                 *duplex = DUPLEX_HALF;
1520                 break;
1521
1522         case MII_TG3_AUX_STAT_1000FULL:
1523                 *speed = SPEED_1000;
1524                 *duplex = DUPLEX_FULL;
1525                 break;
1526
1527         default:
1528                 *speed = SPEED_INVALID;
1529                 *duplex = DUPLEX_INVALID;
1530                 break;
1531         };
1532 }
1533
1534 static void tg3_phy_copper_begin(struct tg3 *tp)
1535 {
1536         u32 new_adv;
1537         int i;
1538
1539         if (tp->link_config.phy_is_low_power) {
1540                 /* Entering low power mode.  Disable gigabit and
1541                  * 100baseT advertisements.
1542                  */
1543                 tg3_writephy(tp, MII_TG3_CTRL, 0);
1544
1545                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1546                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1547                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
1548                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
1549
1550                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1551         } else if (tp->link_config.speed == SPEED_INVALID) {
1552                 tp->link_config.advertising =
1553                         (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
1554                          ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
1555                          ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
1556                          ADVERTISED_Autoneg | ADVERTISED_MII);
1557
1558                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
1559                         tp->link_config.advertising &=
1560                                 ~(ADVERTISED_1000baseT_Half |
1561                                   ADVERTISED_1000baseT_Full);
1562
1563                 new_adv = (ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
1564                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
1565                         new_adv |= ADVERTISE_10HALF;
1566                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
1567                         new_adv |= ADVERTISE_10FULL;
1568                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
1569                         new_adv |= ADVERTISE_100HALF;
1570                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
1571                         new_adv |= ADVERTISE_100FULL;
1572                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
1573
1574                 if (tp->link_config.advertising &
1575                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
1576                         new_adv = 0;
1577                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
1578                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
1579                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
1580                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
1581                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
1582                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1583                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
1584                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1585                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1586                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1587                 } else {
1588                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1589                 }
1590         } else {
1591                 /* Asking for a specific link mode. */
1592                 if (tp->link_config.speed == SPEED_1000) {
1593                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1594                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1595
1596                         if (tp->link_config.duplex == DUPLEX_FULL)
1597                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
1598                         else
1599                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
1600                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1601                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
1602                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
1603                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
1604                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
1605                 } else {
1606                         tg3_writephy(tp, MII_TG3_CTRL, 0);
1607
1608                         new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
1609                         if (tp->link_config.speed == SPEED_100) {
1610                                 if (tp->link_config.duplex == DUPLEX_FULL)
1611                                         new_adv |= ADVERTISE_100FULL;
1612                                 else
1613                                         new_adv |= ADVERTISE_100HALF;
1614                         } else {
1615                                 if (tp->link_config.duplex == DUPLEX_FULL)
1616                                         new_adv |= ADVERTISE_10FULL;
1617                                 else
1618                                         new_adv |= ADVERTISE_10HALF;
1619                         }
1620                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
1621                 }
1622         }
1623
1624         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
1625             tp->link_config.speed != SPEED_INVALID) {
1626                 u32 bmcr, orig_bmcr;
1627
1628                 tp->link_config.active_speed = tp->link_config.speed;
1629                 tp->link_config.active_duplex = tp->link_config.duplex;
1630
1631                 bmcr = 0;
1632                 switch (tp->link_config.speed) {
1633                 default:
1634                 case SPEED_10:
1635                         break;
1636
1637                 case SPEED_100:
1638                         bmcr |= BMCR_SPEED100;
1639                         break;
1640
1641                 case SPEED_1000:
1642                         bmcr |= TG3_BMCR_SPEED1000;
1643                         break;
1644                 };
1645
1646                 if (tp->link_config.duplex == DUPLEX_FULL)
1647                         bmcr |= BMCR_FULLDPLX;
1648
1649                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
1650                     (bmcr != orig_bmcr)) {
1651                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
1652                         for (i = 0; i < 1500; i++) {
1653                                 u32 tmp;
1654
1655                                 udelay(10);
1656                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
1657                                     tg3_readphy(tp, MII_BMSR, &tmp))
1658                                         continue;
1659                                 if (!(tmp & BMSR_LSTATUS)) {
1660                                         udelay(40);
1661                                         break;
1662                                 }
1663                         }
1664                         tg3_writephy(tp, MII_BMCR, bmcr);
1665                         udelay(40);
1666                 }
1667         } else {
1668                 tg3_writephy(tp, MII_BMCR,
1669                              BMCR_ANENABLE | BMCR_ANRESTART);
1670         }
1671 }
1672
1673 static int tg3_init_5401phy_dsp(struct tg3 *tp)
1674 {
1675         int err;
1676
1677         /* Turn off tap power management. */
1678         /* Set Extended packet length bit */
1679         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1680
1681         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
1682         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
1683
1684         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
1685         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
1686
1687         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1688         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
1689
1690         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
1691         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
1692
1693         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1694         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
1695
1696         udelay(40);
1697
1698         return err;
1699 }
1700
1701 static int tg3_copper_is_advertising_all(struct tg3 *tp)
1702 {
1703         u32 adv_reg, all_mask;
1704
1705         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
1706                 return 0;
1707
1708         all_mask = (ADVERTISE_10HALF | ADVERTISE_10FULL |
1709                     ADVERTISE_100HALF | ADVERTISE_100FULL);
1710         if ((adv_reg & all_mask) != all_mask)
1711                 return 0;
1712         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1713                 u32 tg3_ctrl;
1714
1715                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
1716                         return 0;
1717
1718                 all_mask = (MII_TG3_CTRL_ADV_1000_HALF |
1719                             MII_TG3_CTRL_ADV_1000_FULL);
1720                 if ((tg3_ctrl & all_mask) != all_mask)
1721                         return 0;
1722         }
1723         return 1;
1724 }
1725
1726 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
1727 {
1728         int current_link_up;
1729         u32 bmsr, dummy;
1730         u16 current_speed;
1731         u8 current_duplex;
1732         int i, err;
1733
1734         tw32(MAC_EVENT, 0);
1735
1736         tw32_f(MAC_STATUS,
1737              (MAC_STATUS_SYNC_CHANGED |
1738               MAC_STATUS_CFG_CHANGED |
1739               MAC_STATUS_MI_COMPLETION |
1740               MAC_STATUS_LNKSTATE_CHANGED));
1741         udelay(40);
1742
1743         tp->mi_mode = MAC_MI_MODE_BASE;
1744         tw32_f(MAC_MI_MODE, tp->mi_mode);
1745         udelay(80);
1746
1747         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
1748
1749         /* Some third-party PHYs need to be reset on link going
1750          * down.
1751          */
1752         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1753              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1754              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
1755             netif_carrier_ok(tp->dev)) {
1756                 tg3_readphy(tp, MII_BMSR, &bmsr);
1757                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1758                     !(bmsr & BMSR_LSTATUS))
1759                         force_reset = 1;
1760         }
1761         if (force_reset)
1762                 tg3_phy_reset(tp);
1763
1764         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1765                 tg3_readphy(tp, MII_BMSR, &bmsr);
1766                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
1767                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
1768                         bmsr = 0;
1769
1770                 if (!(bmsr & BMSR_LSTATUS)) {
1771                         err = tg3_init_5401phy_dsp(tp);
1772                         if (err)
1773                                 return err;
1774
1775                         tg3_readphy(tp, MII_BMSR, &bmsr);
1776                         for (i = 0; i < 1000; i++) {
1777                                 udelay(10);
1778                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1779                                     (bmsr & BMSR_LSTATUS)) {
1780                                         udelay(40);
1781                                         break;
1782                                 }
1783                         }
1784
1785                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
1786                             !(bmsr & BMSR_LSTATUS) &&
1787                             tp->link_config.active_speed == SPEED_1000) {
1788                                 err = tg3_phy_reset(tp);
1789                                 if (!err)
1790                                         err = tg3_init_5401phy_dsp(tp);
1791                                 if (err)
1792                                         return err;
1793                         }
1794                 }
1795         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
1796                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
1797                 /* 5701 {A0,B0} CRC bug workaround */
1798                 tg3_writephy(tp, 0x15, 0x0a75);
1799                 tg3_writephy(tp, 0x1c, 0x8c68);
1800                 tg3_writephy(tp, 0x1c, 0x8d68);
1801                 tg3_writephy(tp, 0x1c, 0x8c68);
1802         }
1803
1804         /* Clear pending interrupts... */
1805         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1806         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
1807
1808         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
1809                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
1810         else
1811                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
1812
1813         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
1814             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
1815                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
1816                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
1817                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
1818                 else
1819                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
1820         }
1821
1822         current_link_up = 0;
1823         current_speed = SPEED_INVALID;
1824         current_duplex = DUPLEX_INVALID;
1825
1826         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
1827                 u32 val;
1828
1829                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
1830                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
1831                 if (!(val & (1 << 10))) {
1832                         val |= (1 << 10);
1833                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
1834                         goto relink;
1835                 }
1836         }
1837
1838         bmsr = 0;
1839         for (i = 0; i < 100; i++) {
1840                 tg3_readphy(tp, MII_BMSR, &bmsr);
1841                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
1842                     (bmsr & BMSR_LSTATUS))
1843                         break;
1844                 udelay(40);
1845         }
1846
1847         if (bmsr & BMSR_LSTATUS) {
1848                 u32 aux_stat, bmcr;
1849
1850                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
1851                 for (i = 0; i < 2000; i++) {
1852                         udelay(10);
1853                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
1854                             aux_stat)
1855                                 break;
1856                 }
1857
1858                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
1859                                              &current_speed,
1860                                              &current_duplex);
1861
1862                 bmcr = 0;
1863                 for (i = 0; i < 200; i++) {
1864                         tg3_readphy(tp, MII_BMCR, &bmcr);
1865                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
1866                                 continue;
1867                         if (bmcr && bmcr != 0x7fff)
1868                                 break;
1869                         udelay(10);
1870                 }
1871
1872                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
1873                         if (bmcr & BMCR_ANENABLE) {
1874                                 current_link_up = 1;
1875
1876                                 /* Force autoneg restart if we are exiting
1877                                  * low power mode.
1878                                  */
1879                                 if (!tg3_copper_is_advertising_all(tp))
1880                                         current_link_up = 0;
1881                         } else {
1882                                 current_link_up = 0;
1883                         }
1884                 } else {
1885                         if (!(bmcr & BMCR_ANENABLE) &&
1886                             tp->link_config.speed == current_speed &&
1887                             tp->link_config.duplex == current_duplex) {
1888                                 current_link_up = 1;
1889                         } else {
1890                                 current_link_up = 0;
1891                         }
1892                 }
1893
1894                 tp->link_config.active_speed = current_speed;
1895                 tp->link_config.active_duplex = current_duplex;
1896         }
1897
1898         if (current_link_up == 1 &&
1899             (tp->link_config.active_duplex == DUPLEX_FULL) &&
1900             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
1901                 u32 local_adv, remote_adv;
1902
1903                 if (tg3_readphy(tp, MII_ADVERTISE, &local_adv))
1904                         local_adv = 0;
1905                 local_adv &= (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
1906
1907                 if (tg3_readphy(tp, MII_LPA, &remote_adv))
1908                         remote_adv = 0;
1909
1910                 remote_adv &= (LPA_PAUSE_CAP | LPA_PAUSE_ASYM);
1911
1912                 /* If we are not advertising full pause capability,
1913                  * something is wrong.  Bring the link down and reconfigure.
1914                  */
1915                 if (local_adv != ADVERTISE_PAUSE_CAP) {
1916                         current_link_up = 0;
1917                 } else {
1918                         tg3_setup_flow_control(tp, local_adv, remote_adv);
1919                 }
1920         }
1921 relink:
1922         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
1923                 u32 tmp;
1924
1925                 tg3_phy_copper_begin(tp);
1926
1927                 tg3_readphy(tp, MII_BMSR, &tmp);
1928                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
1929                     (tmp & BMSR_LSTATUS))
1930                         current_link_up = 1;
1931         }
1932
1933         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
1934         if (current_link_up == 1) {
1935                 if (tp->link_config.active_speed == SPEED_100 ||
1936                     tp->link_config.active_speed == SPEED_10)
1937                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
1938                 else
1939                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1940         } else
1941                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
1942
1943         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
1944         if (tp->link_config.active_duplex == DUPLEX_HALF)
1945                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
1946
1947         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
1948         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
1949                 if ((tp->led_ctrl == LED_CTRL_MODE_PHY_2) ||
1950                     (current_link_up == 1 &&
1951                      tp->link_config.active_speed == SPEED_10))
1952                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1953         } else {
1954                 if (current_link_up == 1)
1955                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
1956         }
1957
1958         /* ??? Without this setting Netgear GA302T PHY does not
1959          * ??? send/receive packets...
1960          */
1961         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
1962             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
1963                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
1964                 tw32_f(MAC_MI_MODE, tp->mi_mode);
1965                 udelay(80);
1966         }
1967
1968         tw32_f(MAC_MODE, tp->mac_mode);
1969         udelay(40);
1970
1971         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
1972                 /* Polled via timer. */
1973                 tw32_f(MAC_EVENT, 0);
1974         } else {
1975                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
1976         }
1977         udelay(40);
1978
1979         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
1980             current_link_up == 1 &&
1981             tp->link_config.active_speed == SPEED_1000 &&
1982             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
1983              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
1984                 udelay(120);
1985                 tw32_f(MAC_STATUS,
1986                      (MAC_STATUS_SYNC_CHANGED |
1987                       MAC_STATUS_CFG_CHANGED));
1988                 udelay(40);
1989                 tg3_write_mem(tp,
1990                               NIC_SRAM_FIRMWARE_MBOX,
1991                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
1992         }
1993
1994         if (current_link_up != netif_carrier_ok(tp->dev)) {
1995                 if (current_link_up)
1996                         netif_carrier_on(tp->dev);
1997                 else
1998                         netif_carrier_off(tp->dev);
1999                 tg3_link_report(tp);
2000         }
2001
2002         return 0;
2003 }
2004
2005 struct tg3_fiber_aneginfo {
2006         int state;
2007 #define ANEG_STATE_UNKNOWN              0
2008 #define ANEG_STATE_AN_ENABLE            1
2009 #define ANEG_STATE_RESTART_INIT         2
2010 #define ANEG_STATE_RESTART              3
2011 #define ANEG_STATE_DISABLE_LINK_OK      4
2012 #define ANEG_STATE_ABILITY_DETECT_INIT  5
2013 #define ANEG_STATE_ABILITY_DETECT       6
2014 #define ANEG_STATE_ACK_DETECT_INIT      7
2015 #define ANEG_STATE_ACK_DETECT           8
2016 #define ANEG_STATE_COMPLETE_ACK_INIT    9
2017 #define ANEG_STATE_COMPLETE_ACK         10
2018 #define ANEG_STATE_IDLE_DETECT_INIT     11
2019 #define ANEG_STATE_IDLE_DETECT          12
2020 #define ANEG_STATE_LINK_OK              13
2021 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
2022 #define ANEG_STATE_NEXT_PAGE_WAIT       15
2023
2024         u32 flags;
2025 #define MR_AN_ENABLE            0x00000001
2026 #define MR_RESTART_AN           0x00000002
2027 #define MR_AN_COMPLETE          0x00000004
2028 #define MR_PAGE_RX              0x00000008
2029 #define MR_NP_LOADED            0x00000010
2030 #define MR_TOGGLE_TX            0x00000020
2031 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
2032 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
2033 #define MR_LP_ADV_SYM_PAUSE     0x00000100
2034 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
2035 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
2036 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
2037 #define MR_LP_ADV_NEXT_PAGE     0x00001000
2038 #define MR_TOGGLE_RX            0x00002000
2039 #define MR_NP_RX                0x00004000
2040
2041 #define MR_LINK_OK              0x80000000
2042
2043         unsigned long link_time, cur_time;
2044
2045         u32 ability_match_cfg;
2046         int ability_match_count;
2047
2048         char ability_match, idle_match, ack_match;
2049
2050         u32 txconfig, rxconfig;
2051 #define ANEG_CFG_NP             0x00000080
2052 #define ANEG_CFG_ACK            0x00000040
2053 #define ANEG_CFG_RF2            0x00000020
2054 #define ANEG_CFG_RF1            0x00000010
2055 #define ANEG_CFG_PS2            0x00000001
2056 #define ANEG_CFG_PS1            0x00008000
2057 #define ANEG_CFG_HD             0x00004000
2058 #define ANEG_CFG_FD             0x00002000
2059 #define ANEG_CFG_INVAL          0x00001f06
2060
2061 };
2062 #define ANEG_OK         0
2063 #define ANEG_DONE       1
2064 #define ANEG_TIMER_ENAB 2
2065 #define ANEG_FAILED     -1
2066
2067 #define ANEG_STATE_SETTLE_TIME  10000
2068
2069 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
2070                                    struct tg3_fiber_aneginfo *ap)
2071 {
2072         unsigned long delta;
2073         u32 rx_cfg_reg;
2074         int ret;
2075
2076         if (ap->state == ANEG_STATE_UNKNOWN) {
2077                 ap->rxconfig = 0;
2078                 ap->link_time = 0;
2079                 ap->cur_time = 0;
2080                 ap->ability_match_cfg = 0;
2081                 ap->ability_match_count = 0;
2082                 ap->ability_match = 0;
2083                 ap->idle_match = 0;
2084                 ap->ack_match = 0;
2085         }
2086         ap->cur_time++;
2087
2088         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
2089                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
2090
2091                 if (rx_cfg_reg != ap->ability_match_cfg) {
2092                         ap->ability_match_cfg = rx_cfg_reg;
2093                         ap->ability_match = 0;
2094                         ap->ability_match_count = 0;
2095                 } else {
2096                         if (++ap->ability_match_count > 1) {
2097                                 ap->ability_match = 1;
2098                                 ap->ability_match_cfg = rx_cfg_reg;
2099                         }
2100                 }
2101                 if (rx_cfg_reg & ANEG_CFG_ACK)
2102                         ap->ack_match = 1;
2103                 else
2104                         ap->ack_match = 0;
2105
2106                 ap->idle_match = 0;
2107         } else {
2108                 ap->idle_match = 1;
2109                 ap->ability_match_cfg = 0;
2110                 ap->ability_match_count = 0;
2111                 ap->ability_match = 0;
2112                 ap->ack_match = 0;
2113
2114                 rx_cfg_reg = 0;
2115         }
2116
2117         ap->rxconfig = rx_cfg_reg;
2118         ret = ANEG_OK;
2119
2120         switch(ap->state) {
2121         case ANEG_STATE_UNKNOWN:
2122                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
2123                         ap->state = ANEG_STATE_AN_ENABLE;
2124
2125                 /* fallthru */
2126         case ANEG_STATE_AN_ENABLE:
2127                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
2128                 if (ap->flags & MR_AN_ENABLE) {
2129                         ap->link_time = 0;
2130                         ap->cur_time = 0;
2131                         ap->ability_match_cfg = 0;
2132                         ap->ability_match_count = 0;
2133                         ap->ability_match = 0;
2134                         ap->idle_match = 0;
2135                         ap->ack_match = 0;
2136
2137                         ap->state = ANEG_STATE_RESTART_INIT;
2138                 } else {
2139                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
2140                 }
2141                 break;
2142
2143         case ANEG_STATE_RESTART_INIT:
2144                 ap->link_time = ap->cur_time;
2145                 ap->flags &= ~(MR_NP_LOADED);
2146                 ap->txconfig = 0;
2147                 tw32(MAC_TX_AUTO_NEG, 0);
2148                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2149                 tw32_f(MAC_MODE, tp->mac_mode);
2150                 udelay(40);
2151
2152                 ret = ANEG_TIMER_ENAB;
2153                 ap->state = ANEG_STATE_RESTART;
2154
2155                 /* fallthru */
2156         case ANEG_STATE_RESTART:
2157                 delta = ap->cur_time - ap->link_time;
2158                 if (delta > ANEG_STATE_SETTLE_TIME) {
2159                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
2160                 } else {
2161                         ret = ANEG_TIMER_ENAB;
2162                 }
2163                 break;
2164
2165         case ANEG_STATE_DISABLE_LINK_OK:
2166                 ret = ANEG_DONE;
2167                 break;
2168
2169         case ANEG_STATE_ABILITY_DETECT_INIT:
2170                 ap->flags &= ~(MR_TOGGLE_TX);
2171                 ap->txconfig = (ANEG_CFG_FD | ANEG_CFG_PS1);
2172                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2173                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2174                 tw32_f(MAC_MODE, tp->mac_mode);
2175                 udelay(40);
2176
2177                 ap->state = ANEG_STATE_ABILITY_DETECT;
2178                 break;
2179
2180         case ANEG_STATE_ABILITY_DETECT:
2181                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
2182                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
2183                 }
2184                 break;
2185
2186         case ANEG_STATE_ACK_DETECT_INIT:
2187                 ap->txconfig |= ANEG_CFG_ACK;
2188                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
2189                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
2190                 tw32_f(MAC_MODE, tp->mac_mode);
2191                 udelay(40);
2192
2193                 ap->state = ANEG_STATE_ACK_DETECT;
2194
2195                 /* fallthru */
2196         case ANEG_STATE_ACK_DETECT:
2197                 if (ap->ack_match != 0) {
2198                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
2199                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
2200                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
2201                         } else {
2202                                 ap->state = ANEG_STATE_AN_ENABLE;
2203                         }
2204                 } else if (ap->ability_match != 0 &&
2205                            ap->rxconfig == 0) {
2206                         ap->state = ANEG_STATE_AN_ENABLE;
2207                 }
2208                 break;
2209
2210         case ANEG_STATE_COMPLETE_ACK_INIT:
2211                 if (ap->rxconfig & ANEG_CFG_INVAL) {
2212                         ret = ANEG_FAILED;
2213                         break;
2214                 }
2215                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
2216                                MR_LP_ADV_HALF_DUPLEX |
2217                                MR_LP_ADV_SYM_PAUSE |
2218                                MR_LP_ADV_ASYM_PAUSE |
2219                                MR_LP_ADV_REMOTE_FAULT1 |
2220                                MR_LP_ADV_REMOTE_FAULT2 |
2221                                MR_LP_ADV_NEXT_PAGE |
2222                                MR_TOGGLE_RX |
2223                                MR_NP_RX);
2224                 if (ap->rxconfig & ANEG_CFG_FD)
2225                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
2226                 if (ap->rxconfig & ANEG_CFG_HD)
2227                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
2228                 if (ap->rxconfig & ANEG_CFG_PS1)
2229                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
2230                 if (ap->rxconfig & ANEG_CFG_PS2)
2231                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
2232                 if (ap->rxconfig & ANEG_CFG_RF1)
2233                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
2234                 if (ap->rxconfig & ANEG_CFG_RF2)
2235                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
2236                 if (ap->rxconfig & ANEG_CFG_NP)
2237                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
2238
2239                 ap->link_time = ap->cur_time;
2240
2241                 ap->flags ^= (MR_TOGGLE_TX);
2242                 if (ap->rxconfig & 0x0008)
2243                         ap->flags |= MR_TOGGLE_RX;
2244                 if (ap->rxconfig & ANEG_CFG_NP)
2245                         ap->flags |= MR_NP_RX;
2246                 ap->flags |= MR_PAGE_RX;
2247
2248                 ap->state = ANEG_STATE_COMPLETE_ACK;
2249                 ret = ANEG_TIMER_ENAB;
2250                 break;
2251
2252         case ANEG_STATE_COMPLETE_ACK:
2253                 if (ap->ability_match != 0 &&
2254                     ap->rxconfig == 0) {
2255                         ap->state = ANEG_STATE_AN_ENABLE;
2256                         break;
2257                 }
2258                 delta = ap->cur_time - ap->link_time;
2259                 if (delta > ANEG_STATE_SETTLE_TIME) {
2260                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
2261                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2262                         } else {
2263                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
2264                                     !(ap->flags & MR_NP_RX)) {
2265                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
2266                                 } else {
2267                                         ret = ANEG_FAILED;
2268                                 }
2269                         }
2270                 }
2271                 break;
2272
2273         case ANEG_STATE_IDLE_DETECT_INIT:
2274                 ap->link_time = ap->cur_time;
2275                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2276                 tw32_f(MAC_MODE, tp->mac_mode);
2277                 udelay(40);
2278
2279                 ap->state = ANEG_STATE_IDLE_DETECT;
2280                 ret = ANEG_TIMER_ENAB;
2281                 break;
2282
2283         case ANEG_STATE_IDLE_DETECT:
2284                 if (ap->ability_match != 0 &&
2285                     ap->rxconfig == 0) {
2286                         ap->state = ANEG_STATE_AN_ENABLE;
2287                         break;
2288                 }
2289                 delta = ap->cur_time - ap->link_time;
2290                 if (delta > ANEG_STATE_SETTLE_TIME) {
2291                         /* XXX another gem from the Broadcom driver :( */
2292                         ap->state = ANEG_STATE_LINK_OK;
2293                 }
2294                 break;
2295
2296         case ANEG_STATE_LINK_OK:
2297                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
2298                 ret = ANEG_DONE;
2299                 break;
2300
2301         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
2302                 /* ??? unimplemented */
2303                 break;
2304
2305         case ANEG_STATE_NEXT_PAGE_WAIT:
2306                 /* ??? unimplemented */
2307                 break;
2308
2309         default:
2310                 ret = ANEG_FAILED;
2311                 break;
2312         };
2313
2314         return ret;
2315 }
2316
2317 static int fiber_autoneg(struct tg3 *tp, u32 *flags)
2318 {
2319         int res = 0;
2320         struct tg3_fiber_aneginfo aninfo;
2321         int status = ANEG_FAILED;
2322         unsigned int tick;
2323         u32 tmp;
2324
2325         tw32_f(MAC_TX_AUTO_NEG, 0);
2326
2327         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
2328         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
2329         udelay(40);
2330
2331         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
2332         udelay(40);
2333
2334         memset(&aninfo, 0, sizeof(aninfo));
2335         aninfo.flags |= MR_AN_ENABLE;
2336         aninfo.state = ANEG_STATE_UNKNOWN;
2337         aninfo.cur_time = 0;
2338         tick = 0;
2339         while (++tick < 195000) {
2340                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
2341                 if (status == ANEG_DONE || status == ANEG_FAILED)
2342                         break;
2343
2344                 udelay(1);
2345         }
2346
2347         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
2348         tw32_f(MAC_MODE, tp->mac_mode);
2349         udelay(40);
2350
2351         *flags = aninfo.flags;
2352
2353         if (status == ANEG_DONE &&
2354             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
2355                              MR_LP_ADV_FULL_DUPLEX)))
2356                 res = 1;
2357
2358         return res;
2359 }
2360
2361 static void tg3_init_bcm8002(struct tg3 *tp)
2362 {
2363         u32 mac_status = tr32(MAC_STATUS);
2364         int i;
2365
2366         /* Reset when initting first time or we have a link. */
2367         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
2368             !(mac_status & MAC_STATUS_PCS_SYNCED))
2369                 return;
2370
2371         /* Set PLL lock range. */
2372         tg3_writephy(tp, 0x16, 0x8007);
2373
2374         /* SW reset */
2375         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
2376
2377         /* Wait for reset to complete. */
2378         /* XXX schedule_timeout() ... */
2379         for (i = 0; i < 500; i++)
2380                 udelay(10);
2381
2382         /* Config mode; select PMA/Ch 1 regs. */
2383         tg3_writephy(tp, 0x10, 0x8411);
2384
2385         /* Enable auto-lock and comdet, select txclk for tx. */
2386         tg3_writephy(tp, 0x11, 0x0a10);
2387
2388         tg3_writephy(tp, 0x18, 0x00a0);
2389         tg3_writephy(tp, 0x16, 0x41ff);
2390
2391         /* Assert and deassert POR. */
2392         tg3_writephy(tp, 0x13, 0x0400);
2393         udelay(40);
2394         tg3_writephy(tp, 0x13, 0x0000);
2395
2396         tg3_writephy(tp, 0x11, 0x0a50);
2397         udelay(40);
2398         tg3_writephy(tp, 0x11, 0x0a10);
2399
2400         /* Wait for signal to stabilize */
2401         /* XXX schedule_timeout() ... */
2402         for (i = 0; i < 15000; i++)
2403                 udelay(10);
2404
2405         /* Deselect the channel register so we can read the PHYID
2406          * later.
2407          */
2408         tg3_writephy(tp, 0x10, 0x8011);
2409 }
2410
2411 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
2412 {
2413         u32 sg_dig_ctrl, sg_dig_status;
2414         u32 serdes_cfg, expected_sg_dig_ctrl;
2415         int workaround, port_a;
2416         int current_link_up;
2417
2418         serdes_cfg = 0;
2419         expected_sg_dig_ctrl = 0;
2420         workaround = 0;
2421         port_a = 1;
2422         current_link_up = 0;
2423
2424         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
2425             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
2426                 workaround = 1;
2427                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
2428                         port_a = 0;
2429
2430                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
2431                 /* preserve bits 20-23 for voltage regulator */
2432                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
2433         }
2434
2435         sg_dig_ctrl = tr32(SG_DIG_CTRL);
2436
2437         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
2438                 if (sg_dig_ctrl & (1 << 31)) {
2439                         if (workaround) {
2440                                 u32 val = serdes_cfg;
2441
2442                                 if (port_a)
2443                                         val |= 0xc010000;
2444                                 else
2445                                         val |= 0x4010000;
2446                                 tw32_f(MAC_SERDES_CFG, val);
2447                         }
2448                         tw32_f(SG_DIG_CTRL, 0x01388400);
2449                 }
2450                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
2451                         tg3_setup_flow_control(tp, 0, 0);
2452                         current_link_up = 1;
2453                 }
2454                 goto out;
2455         }
2456
2457         /* Want auto-negotiation.  */
2458         expected_sg_dig_ctrl = 0x81388400;
2459
2460         /* Pause capability */
2461         expected_sg_dig_ctrl |= (1 << 11);
2462
2463         /* Asymettric pause */
2464         expected_sg_dig_ctrl |= (1 << 12);
2465
2466         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
2467                 if (workaround)
2468                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
2469                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | (1 << 30));
2470                 udelay(5);
2471                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
2472
2473                 tp->tg3_flags2 |= TG3_FLG2_PHY_JUST_INITTED;
2474         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
2475                                  MAC_STATUS_SIGNAL_DET)) {
2476                 int i;
2477
2478                 /* Giver time to negotiate (~200ms) */
2479                 for (i = 0; i < 40000; i++) {
2480                         sg_dig_status = tr32(SG_DIG_STATUS);
2481                         if (sg_dig_status & (0x3))
2482                                 break;
2483                         udelay(5);
2484                 }
2485                 mac_status = tr32(MAC_STATUS);
2486
2487                 if ((sg_dig_status & (1 << 1)) &&
2488                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
2489                         u32 local_adv, remote_adv;
2490
2491                         local_adv = ADVERTISE_PAUSE_CAP;
2492                         remote_adv = 0;
2493                         if (sg_dig_status & (1 << 19))
2494                                 remote_adv |= LPA_PAUSE_CAP;
2495                         if (sg_dig_status & (1 << 20))
2496                                 remote_adv |= LPA_PAUSE_ASYM;
2497
2498                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2499                         current_link_up = 1;
2500                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2501                 } else if (!(sg_dig_status & (1 << 1))) {
2502                         if (tp->tg3_flags2 & TG3_FLG2_PHY_JUST_INITTED)
2503                                 tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2504                         else {
2505                                 if (workaround) {
2506                                         u32 val = serdes_cfg;
2507
2508                                         if (port_a)
2509                                                 val |= 0xc010000;
2510                                         else
2511                                                 val |= 0x4010000;
2512
2513                                         tw32_f(MAC_SERDES_CFG, val);
2514                                 }
2515
2516                                 tw32_f(SG_DIG_CTRL, 0x01388400);
2517                                 udelay(40);
2518
2519                                 /* Link parallel detection - link is up */
2520                                 /* only if we have PCS_SYNC and not */
2521                                 /* receiving config code words */
2522                                 mac_status = tr32(MAC_STATUS);
2523                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
2524                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
2525                                         tg3_setup_flow_control(tp, 0, 0);
2526                                         current_link_up = 1;
2527                                 }
2528                         }
2529                 }
2530         }
2531
2532 out:
2533         return current_link_up;
2534 }
2535
2536 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
2537 {
2538         int current_link_up = 0;
2539
2540         if (!(mac_status & MAC_STATUS_PCS_SYNCED)) {
2541                 tp->tg3_flags &= ~TG3_FLAG_GOT_SERDES_FLOWCTL;
2542                 goto out;
2543         }
2544
2545         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2546                 u32 flags;
2547                 int i;
2548
2549                 if (fiber_autoneg(tp, &flags)) {
2550                         u32 local_adv, remote_adv;
2551
2552                         local_adv = ADVERTISE_PAUSE_CAP;
2553                         remote_adv = 0;
2554                         if (flags & MR_LP_ADV_SYM_PAUSE)
2555                                 remote_adv |= LPA_PAUSE_CAP;
2556                         if (flags & MR_LP_ADV_ASYM_PAUSE)
2557                                 remote_adv |= LPA_PAUSE_ASYM;
2558
2559                         tg3_setup_flow_control(tp, local_adv, remote_adv);
2560
2561                         tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2562                         current_link_up = 1;
2563                 }
2564                 for (i = 0; i < 30; i++) {
2565                         udelay(20);
2566                         tw32_f(MAC_STATUS,
2567                                (MAC_STATUS_SYNC_CHANGED |
2568                                 MAC_STATUS_CFG_CHANGED));
2569                         udelay(40);
2570                         if ((tr32(MAC_STATUS) &
2571                              (MAC_STATUS_SYNC_CHANGED |
2572                               MAC_STATUS_CFG_CHANGED)) == 0)
2573                                 break;
2574                 }
2575
2576                 mac_status = tr32(MAC_STATUS);
2577                 if (current_link_up == 0 &&
2578                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
2579                     !(mac_status & MAC_STATUS_RCVD_CFG))
2580                         current_link_up = 1;
2581         } else {
2582                 /* Forcing 1000FD link up. */
2583                 current_link_up = 1;
2584                 tp->tg3_flags |= TG3_FLAG_GOT_SERDES_FLOWCTL;
2585
2586                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
2587                 udelay(40);
2588         }
2589
2590 out:
2591         return current_link_up;
2592 }
2593
2594 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
2595 {
2596         u32 orig_pause_cfg;
2597         u16 orig_active_speed;
2598         u8 orig_active_duplex;
2599         u32 mac_status;
2600         int current_link_up;
2601         int i;
2602
2603         orig_pause_cfg =
2604                 (tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2605                                   TG3_FLAG_TX_PAUSE));
2606         orig_active_speed = tp->link_config.active_speed;
2607         orig_active_duplex = tp->link_config.active_duplex;
2608
2609         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
2610             netif_carrier_ok(tp->dev) &&
2611             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
2612                 mac_status = tr32(MAC_STATUS);
2613                 mac_status &= (MAC_STATUS_PCS_SYNCED |
2614                                MAC_STATUS_SIGNAL_DET |
2615                                MAC_STATUS_CFG_CHANGED |
2616                                MAC_STATUS_RCVD_CFG);
2617                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
2618                                    MAC_STATUS_SIGNAL_DET)) {
2619                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2620                                             MAC_STATUS_CFG_CHANGED));
2621                         return 0;
2622                 }
2623         }
2624
2625         tw32_f(MAC_TX_AUTO_NEG, 0);
2626
2627         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
2628         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
2629         tw32_f(MAC_MODE, tp->mac_mode);
2630         udelay(40);
2631
2632         if (tp->phy_id == PHY_ID_BCM8002)
2633                 tg3_init_bcm8002(tp);
2634
2635         /* Enable link change event even when serdes polling.  */
2636         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2637         udelay(40);
2638
2639         current_link_up = 0;
2640         mac_status = tr32(MAC_STATUS);
2641
2642         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
2643                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
2644         else
2645                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
2646
2647         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
2648         tw32_f(MAC_MODE, tp->mac_mode);
2649         udelay(40);
2650
2651         tp->hw_status->status =
2652                 (SD_STATUS_UPDATED |
2653                  (tp->hw_status->status & ~SD_STATUS_LINK_CHG));
2654
2655         for (i = 0; i < 100; i++) {
2656                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
2657                                     MAC_STATUS_CFG_CHANGED));
2658                 udelay(5);
2659                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
2660                                          MAC_STATUS_CFG_CHANGED)) == 0)
2661                         break;
2662         }
2663
2664         mac_status = tr32(MAC_STATUS);
2665         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
2666                 current_link_up = 0;
2667                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2668                         tw32_f(MAC_MODE, (tp->mac_mode |
2669                                           MAC_MODE_SEND_CONFIGS));
2670                         udelay(1);
2671                         tw32_f(MAC_MODE, tp->mac_mode);
2672                 }
2673         }
2674
2675         if (current_link_up == 1) {
2676                 tp->link_config.active_speed = SPEED_1000;
2677                 tp->link_config.active_duplex = DUPLEX_FULL;
2678                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2679                                     LED_CTRL_LNKLED_OVERRIDE |
2680                                     LED_CTRL_1000MBPS_ON));
2681         } else {
2682                 tp->link_config.active_speed = SPEED_INVALID;
2683                 tp->link_config.active_duplex = DUPLEX_INVALID;
2684                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
2685                                     LED_CTRL_LNKLED_OVERRIDE |
2686                                     LED_CTRL_TRAFFIC_OVERRIDE));
2687         }
2688
2689         if (current_link_up != netif_carrier_ok(tp->dev)) {
2690                 if (current_link_up)
2691                         netif_carrier_on(tp->dev);
2692                 else
2693                         netif_carrier_off(tp->dev);
2694                 tg3_link_report(tp);
2695         } else {
2696                 u32 now_pause_cfg =
2697                         tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
2698                                          TG3_FLAG_TX_PAUSE);
2699                 if (orig_pause_cfg != now_pause_cfg ||
2700                     orig_active_speed != tp->link_config.active_speed ||
2701                     orig_active_duplex != tp->link_config.active_duplex)
2702                         tg3_link_report(tp);
2703         }
2704
2705         return 0;
2706 }
2707
2708 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
2709 {
2710         int current_link_up, err = 0;
2711         u32 bmsr, bmcr;
2712         u16 current_speed;
2713         u8 current_duplex;
2714
2715         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
2716         tw32_f(MAC_MODE, tp->mac_mode);
2717         udelay(40);
2718
2719         tw32(MAC_EVENT, 0);
2720
2721         tw32_f(MAC_STATUS,
2722              (MAC_STATUS_SYNC_CHANGED |
2723               MAC_STATUS_CFG_CHANGED |
2724               MAC_STATUS_MI_COMPLETION |
2725               MAC_STATUS_LNKSTATE_CHANGED));
2726         udelay(40);
2727
2728         if (force_reset)
2729                 tg3_phy_reset(tp);
2730
2731         current_link_up = 0;
2732         current_speed = SPEED_INVALID;
2733         current_duplex = DUPLEX_INVALID;
2734
2735         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2736         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2737         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
2738                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2739                         bmsr |= BMSR_LSTATUS;
2740                 else
2741                         bmsr &= ~BMSR_LSTATUS;
2742         }
2743
2744         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
2745
2746         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
2747             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2748                 /* do nothing, just check for link up at the end */
2749         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
2750                 u32 adv, new_adv;
2751
2752                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2753                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
2754                                   ADVERTISE_1000XPAUSE |
2755                                   ADVERTISE_1000XPSE_ASYM |
2756                                   ADVERTISE_SLCT);
2757
2758                 /* Always advertise symmetric PAUSE just like copper */
2759                 new_adv |= ADVERTISE_1000XPAUSE;
2760
2761                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2762                         new_adv |= ADVERTISE_1000XHALF;
2763                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2764                         new_adv |= ADVERTISE_1000XFULL;
2765
2766                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
2767                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2768                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
2769                         tg3_writephy(tp, MII_BMCR, bmcr);
2770
2771                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2772                         tp->tg3_flags2 |= TG3_FLG2_PHY_JUST_INITTED;
2773                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2774
2775                         return err;
2776                 }
2777         } else {
2778                 u32 new_bmcr;
2779
2780                 bmcr &= ~BMCR_SPEED1000;
2781                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
2782
2783                 if (tp->link_config.duplex == DUPLEX_FULL)
2784                         new_bmcr |= BMCR_FULLDPLX;
2785
2786                 if (new_bmcr != bmcr) {
2787                         /* BMCR_SPEED1000 is a reserved bit that needs
2788                          * to be set on write.
2789                          */
2790                         new_bmcr |= BMCR_SPEED1000;
2791
2792                         /* Force a linkdown */
2793                         if (netif_carrier_ok(tp->dev)) {
2794                                 u32 adv;
2795
2796                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
2797                                 adv &= ~(ADVERTISE_1000XFULL |
2798                                          ADVERTISE_1000XHALF |
2799                                          ADVERTISE_SLCT);
2800                                 tg3_writephy(tp, MII_ADVERTISE, adv);
2801                                 tg3_writephy(tp, MII_BMCR, bmcr |
2802                                                            BMCR_ANRESTART |
2803                                                            BMCR_ANENABLE);
2804                                 udelay(10);
2805                                 netif_carrier_off(tp->dev);
2806                         }
2807                         tg3_writephy(tp, MII_BMCR, new_bmcr);
2808                         bmcr = new_bmcr;
2809                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2810                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
2811                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2812                             ASIC_REV_5714) {
2813                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
2814                                         bmsr |= BMSR_LSTATUS;
2815                                 else
2816                                         bmsr &= ~BMSR_LSTATUS;
2817                         }
2818                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2819                 }
2820         }
2821
2822         if (bmsr & BMSR_LSTATUS) {
2823                 current_speed = SPEED_1000;
2824                 current_link_up = 1;
2825                 if (bmcr & BMCR_FULLDPLX)
2826                         current_duplex = DUPLEX_FULL;
2827                 else
2828                         current_duplex = DUPLEX_HALF;
2829
2830                 if (bmcr & BMCR_ANENABLE) {
2831                         u32 local_adv, remote_adv, common;
2832
2833                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
2834                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
2835                         common = local_adv & remote_adv;
2836                         if (common & (ADVERTISE_1000XHALF |
2837                                       ADVERTISE_1000XFULL)) {
2838                                 if (common & ADVERTISE_1000XFULL)
2839                                         current_duplex = DUPLEX_FULL;
2840                                 else
2841                                         current_duplex = DUPLEX_HALF;
2842
2843                                 tg3_setup_flow_control(tp, local_adv,
2844                                                        remote_adv);
2845                         }
2846                         else
2847                                 current_link_up = 0;
2848                 }
2849         }
2850
2851         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
2852         if (tp->link_config.active_duplex == DUPLEX_HALF)
2853                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
2854
2855         tw32_f(MAC_MODE, tp->mac_mode);
2856         udelay(40);
2857
2858         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
2859
2860         tp->link_config.active_speed = current_speed;
2861         tp->link_config.active_duplex = current_duplex;
2862
2863         if (current_link_up != netif_carrier_ok(tp->dev)) {
2864                 if (current_link_up)
2865                         netif_carrier_on(tp->dev);
2866                 else {
2867                         netif_carrier_off(tp->dev);
2868                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2869                 }
2870                 tg3_link_report(tp);
2871         }
2872         return err;
2873 }
2874
2875 static void tg3_serdes_parallel_detect(struct tg3 *tp)
2876 {
2877         if (tp->tg3_flags2 & TG3_FLG2_PHY_JUST_INITTED) {
2878                 /* Give autoneg time to complete. */
2879                 tp->tg3_flags2 &= ~TG3_FLG2_PHY_JUST_INITTED;
2880                 return;
2881         }
2882         if (!netif_carrier_ok(tp->dev) &&
2883             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
2884                 u32 bmcr;
2885
2886                 tg3_readphy(tp, MII_BMCR, &bmcr);
2887                 if (bmcr & BMCR_ANENABLE) {
2888                         u32 phy1, phy2;
2889
2890                         /* Select shadow register 0x1f */
2891                         tg3_writephy(tp, 0x1c, 0x7c00);
2892                         tg3_readphy(tp, 0x1c, &phy1);
2893
2894                         /* Select expansion interrupt status register */
2895                         tg3_writephy(tp, 0x17, 0x0f01);
2896                         tg3_readphy(tp, 0x15, &phy2);
2897                         tg3_readphy(tp, 0x15, &phy2);
2898
2899                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
2900                                 /* We have signal detect and not receiving
2901                                  * config code words, link is up by parallel
2902                                  * detection.
2903                                  */
2904
2905                                 bmcr &= ~BMCR_ANENABLE;
2906                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
2907                                 tg3_writephy(tp, MII_BMCR, bmcr);
2908                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
2909                         }
2910                 }
2911         }
2912         else if (netif_carrier_ok(tp->dev) &&
2913                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
2914                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
2915                 u32 phy2;
2916
2917                 /* Select expansion interrupt status register */
2918                 tg3_writephy(tp, 0x17, 0x0f01);
2919                 tg3_readphy(tp, 0x15, &phy2);
2920                 if (phy2 & 0x20) {
2921                         u32 bmcr;
2922
2923                         /* Config code words received, turn on autoneg. */
2924                         tg3_readphy(tp, MII_BMCR, &bmcr);
2925                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
2926
2927                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
2928
2929                 }
2930         }
2931 }
2932
2933 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
2934 {
2935         int err;
2936
2937         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
2938                 err = tg3_setup_fiber_phy(tp, force_reset);
2939         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
2940                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
2941         } else {
2942                 err = tg3_setup_copper_phy(tp, force_reset);
2943         }
2944
2945         if (tp->link_config.active_speed == SPEED_1000 &&
2946             tp->link_config.active_duplex == DUPLEX_HALF)
2947                 tw32(MAC_TX_LENGTHS,
2948                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2949                       (6 << TX_LENGTHS_IPG_SHIFT) |
2950                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
2951         else
2952                 tw32(MAC_TX_LENGTHS,
2953                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
2954                       (6 << TX_LENGTHS_IPG_SHIFT) |
2955                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
2956
2957         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
2958                 if (netif_carrier_ok(tp->dev)) {
2959                         tw32(HOSTCC_STAT_COAL_TICKS,
2960                              tp->coal.stats_block_coalesce_usecs);
2961                 } else {
2962                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
2963                 }
2964         }
2965
2966         return err;
2967 }
2968
2969 /* This is called whenever we suspect that the system chipset is re-
2970  * ordering the sequence of MMIO to the tx send mailbox. The symptom
2971  * is bogus tx completions. We try to recover by setting the
2972  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
2973  * in the workqueue.
2974  */
2975 static void tg3_tx_recover(struct tg3 *tp)
2976 {
2977         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
2978                tp->write32_tx_mbox == tg3_write_indirect_mbox);
2979
2980         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
2981                "mapped I/O cycles to the network device, attempting to "
2982                "recover. Please report the problem to the driver maintainer "
2983                "and include system chipset information.\n", tp->dev->name);
2984
2985         spin_lock(&tp->lock);
2986         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
2987         spin_unlock(&tp->lock);
2988 }
2989
2990 /* Tigon3 never reports partial packet sends.  So we do not
2991  * need special logic to handle SKBs that have not had all
2992  * of their frags sent yet, like SunGEM does.
2993  */
2994 static void tg3_tx(struct tg3 *tp)
2995 {
2996         u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
2997         u32 sw_idx = tp->tx_cons;
2998
2999         while (sw_idx != hw_idx) {
3000                 struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
3001                 struct sk_buff *skb = ri->skb;
3002                 int i, tx_bug = 0;
3003
3004                 if (unlikely(skb == NULL)) {
3005                         tg3_tx_recover(tp);
3006                         return;
3007                 }
3008
3009                 pci_unmap_single(tp->pdev,
3010                                  pci_unmap_addr(ri, mapping),
3011                                  skb_headlen(skb),
3012                                  PCI_DMA_TODEVICE);
3013
3014                 ri->skb = NULL;
3015
3016                 sw_idx = NEXT_TX(sw_idx);
3017
3018                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3019                         ri = &tp->tx_buffers[sw_idx];
3020                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
3021                                 tx_bug = 1;
3022
3023                         pci_unmap_page(tp->pdev,
3024                                        pci_unmap_addr(ri, mapping),
3025                                        skb_shinfo(skb)->frags[i].size,
3026                                        PCI_DMA_TODEVICE);
3027
3028                         sw_idx = NEXT_TX(sw_idx);
3029                 }
3030
3031                 dev_kfree_skb(skb);
3032
3033                 if (unlikely(tx_bug)) {
3034                         tg3_tx_recover(tp);
3035                         return;
3036                 }
3037         }
3038
3039         tp->tx_cons = sw_idx;
3040
3041         if (unlikely(netif_queue_stopped(tp->dev))) {
3042                 spin_lock(&tp->tx_lock);
3043                 if (netif_queue_stopped(tp->dev) &&
3044                     (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH))
3045                         netif_wake_queue(tp->dev);
3046                 spin_unlock(&tp->tx_lock);
3047         }
3048 }
3049
3050 /* Returns size of skb allocated or < 0 on error.
3051  *
3052  * We only need to fill in the address because the other members
3053  * of the RX descriptor are invariant, see tg3_init_rings.
3054  *
3055  * Note the purposeful assymetry of cpu vs. chip accesses.  For
3056  * posting buffers we only dirty the first cache line of the RX
3057  * descriptor (containing the address).  Whereas for the RX status
3058  * buffers the cpu only reads the last cacheline of the RX descriptor
3059  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
3060  */
3061 static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
3062                             int src_idx, u32 dest_idx_unmasked)
3063 {
3064         struct tg3_rx_buffer_desc *desc;
3065         struct ring_info *map, *src_map;
3066         struct sk_buff *skb;
3067         dma_addr_t mapping;
3068         int skb_size, dest_idx;
3069
3070         src_map = NULL;
3071         switch (opaque_key) {
3072         case RXD_OPAQUE_RING_STD:
3073                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3074                 desc = &tp->rx_std[dest_idx];
3075                 map = &tp->rx_std_buffers[dest_idx];
3076                 if (src_idx >= 0)
3077                         src_map = &tp->rx_std_buffers[src_idx];
3078                 skb_size = tp->rx_pkt_buf_sz;
3079                 break;
3080
3081         case RXD_OPAQUE_RING_JUMBO:
3082                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3083                 desc = &tp->rx_jumbo[dest_idx];
3084                 map = &tp->rx_jumbo_buffers[dest_idx];
3085                 if (src_idx >= 0)
3086                         src_map = &tp->rx_jumbo_buffers[src_idx];
3087                 skb_size = RX_JUMBO_PKT_BUF_SZ;
3088                 break;
3089
3090         default:
3091                 return -EINVAL;
3092         };
3093
3094         /* Do not overwrite any of the map or rp information
3095          * until we are sure we can commit to a new buffer.
3096          *
3097          * Callers depend upon this behavior and assume that
3098          * we leave everything unchanged if we fail.
3099          */
3100         skb = dev_alloc_skb(skb_size);
3101         if (skb == NULL)
3102                 return -ENOMEM;
3103
3104         skb->dev = tp->dev;
3105         skb_reserve(skb, tp->rx_offset);
3106
3107         mapping = pci_map_single(tp->pdev, skb->data,
3108                                  skb_size - tp->rx_offset,
3109                                  PCI_DMA_FROMDEVICE);
3110
3111         map->skb = skb;
3112         pci_unmap_addr_set(map, mapping, mapping);
3113
3114         if (src_map != NULL)
3115                 src_map->skb = NULL;
3116
3117         desc->addr_hi = ((u64)mapping >> 32);
3118         desc->addr_lo = ((u64)mapping & 0xffffffff);
3119
3120         return skb_size;
3121 }
3122
3123 /* We only need to move over in the address because the other
3124  * members of the RX descriptor are invariant.  See notes above
3125  * tg3_alloc_rx_skb for full details.
3126  */
3127 static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
3128                            int src_idx, u32 dest_idx_unmasked)
3129 {
3130         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
3131         struct ring_info *src_map, *dest_map;
3132         int dest_idx;
3133
3134         switch (opaque_key) {
3135         case RXD_OPAQUE_RING_STD:
3136                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
3137                 dest_desc = &tp->rx_std[dest_idx];
3138                 dest_map = &tp->rx_std_buffers[dest_idx];
3139                 src_desc = &tp->rx_std[src_idx];
3140                 src_map = &tp->rx_std_buffers[src_idx];
3141                 break;
3142
3143         case RXD_OPAQUE_RING_JUMBO:
3144                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
3145                 dest_desc = &tp->rx_jumbo[dest_idx];
3146                 dest_map = &tp->rx_jumbo_buffers[dest_idx];
3147                 src_desc = &tp->rx_jumbo[src_idx];
3148                 src_map = &tp->rx_jumbo_buffers[src_idx];
3149                 break;
3150
3151         default:
3152                 return;
3153         };
3154
3155         dest_map->skb = src_map->skb;
3156         pci_unmap_addr_set(dest_map, mapping,
3157                            pci_unmap_addr(src_map, mapping));
3158         dest_desc->addr_hi = src_desc->addr_hi;
3159         dest_desc->addr_lo = src_desc->addr_lo;
3160
3161         src_map->skb = NULL;
3162 }
3163
3164 #if TG3_VLAN_TAG_USED
3165 static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
3166 {
3167         return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
3168 }
3169 #endif
3170
3171 /* The RX ring scheme is composed of multiple rings which post fresh
3172  * buffers to the chip, and one special ring the chip uses to report
3173  * status back to the host.
3174  *
3175  * The special ring reports the status of received packets to the
3176  * host.  The chip does not write into the original descriptor the
3177  * RX buffer was obtained from.  The chip simply takes the original
3178  * descriptor as provided by the host, updates the status and length
3179  * field, then writes this into the next status ring entry.
3180  *
3181  * Each ring the host uses to post buffers to the chip is described
3182  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
3183  * it is first placed into the on-chip ram.  When the packet's length
3184  * is known, it walks down the TG3_BDINFO entries to select the ring.
3185  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
3186  * which is within the range of the new packet's length is chosen.
3187  *
3188  * The "separate ring for rx status" scheme may sound queer, but it makes
3189  * sense from a cache coherency perspective.  If only the host writes
3190  * to the buffer post rings, and only the chip writes to the rx status
3191  * rings, then cache lines never move beyond shared-modified state.
3192  * If both the host and chip were to write into the same ring, cache line
3193  * eviction could occur since both entities want it in an exclusive state.
3194  */
3195 static int tg3_rx(struct tg3 *tp, int budget)
3196 {
3197         u32 work_mask, rx_std_posted = 0;
3198         u32 sw_idx = tp->rx_rcb_ptr;
3199         u16 hw_idx;
3200         int received;
3201
3202         hw_idx = tp->hw_status->idx[0].rx_producer;
3203         /*
3204          * We need to order the read of hw_idx and the read of
3205          * the opaque cookie.
3206          */
3207         rmb();
3208         work_mask = 0;
3209         received = 0;
3210         while (sw_idx != hw_idx && budget > 0) {
3211                 struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
3212                 unsigned int len;
3213                 struct sk_buff *skb;
3214                 dma_addr_t dma_addr;
3215                 u32 opaque_key, desc_idx, *post_ptr;
3216
3217                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
3218                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
3219                 if (opaque_key == RXD_OPAQUE_RING_STD) {
3220                         dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
3221                                                   mapping);
3222                         skb = tp->rx_std_buffers[desc_idx].skb;
3223                         post_ptr = &tp->rx_std_ptr;
3224                         rx_std_posted++;
3225                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
3226                         dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
3227                                                   mapping);
3228                         skb = tp->rx_jumbo_buffers[desc_idx].skb;
3229                         post_ptr = &tp->rx_jumbo_ptr;
3230                 }
3231                 else {
3232                         goto next_pkt_nopost;
3233                 }
3234
3235                 work_mask |= opaque_key;
3236
3237                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
3238                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
3239                 drop_it:
3240                         tg3_recycle_rx(tp, opaque_key,
3241                                        desc_idx, *post_ptr);
3242                 drop_it_no_recycle:
3243                         /* Other statistics kept track of by card. */
3244                         tp->net_stats.rx_dropped++;
3245                         goto next_pkt;
3246                 }
3247
3248                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */
3249
3250                 if (len > RX_COPY_THRESHOLD
3251                         && tp->rx_offset == 2
3252                         /* rx_offset != 2 iff this is a 5701 card running
3253                          * in PCI-X mode [see tg3_get_invariants()] */
3254                 ) {
3255                         int skb_size;
3256
3257                         skb_size = tg3_alloc_rx_skb(tp, opaque_key,
3258                                                     desc_idx, *post_ptr);
3259                         if (skb_size < 0)
3260                                 goto drop_it;
3261
3262                         pci_unmap_single(tp->pdev, dma_addr,
3263                                          skb_size - tp->rx_offset,
3264                                          PCI_DMA_FROMDEVICE);
3265
3266                         skb_put(skb, len);
3267                 } else {
3268                         struct sk_buff *copy_skb;
3269
3270                         tg3_recycle_rx(tp, opaque_key,
3271                                        desc_idx, *post_ptr);
3272
3273                         copy_skb = dev_alloc_skb(len + 2);
3274                         if (copy_skb == NULL)
3275                                 goto drop_it_no_recycle;
3276
3277                         copy_skb->dev = tp->dev;
3278                         skb_reserve(copy_skb, 2);
3279                         skb_put(copy_skb, len);
3280                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3281                         memcpy(copy_skb->data, skb->data, len);
3282                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
3283
3284                         /* We'll reuse the original ring buffer. */
3285                         skb = copy_skb;
3286                 }
3287
3288                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
3289                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
3290                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
3291                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
3292                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3293                 else
3294                         skb->ip_summed = CHECKSUM_NONE;
3295
3296                 skb->protocol = eth_type_trans(skb, tp->dev);
3297 #if TG3_VLAN_TAG_USED
3298                 if (tp->vlgrp != NULL &&
3299                     desc->type_flags & RXD_FLAG_VLAN) {
3300                         tg3_vlan_rx(tp, skb,
3301                                     desc->err_vlan & RXD_VLAN_MASK);
3302                 } else
3303 #endif
3304                         netif_receive_skb(skb);
3305
3306                 tp->dev->last_rx = jiffies;
3307                 received++;
3308                 budget--;
3309
3310 next_pkt:
3311                 (*post_ptr)++;
3312
3313                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
3314                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
3315
3316                         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX +
3317                                      TG3_64BIT_REG_LOW, idx);
3318                         work_mask &= ~RXD_OPAQUE_RING_STD;
3319                         rx_std_posted = 0;
3320                 }
3321 next_pkt_nopost:
3322                 sw_idx++;
3323                 sw_idx %= TG3_RX_RCB_RING_SIZE(tp);
3324
3325                 /* Refresh hw_idx to see if there is new work */
3326                 if (sw_idx == hw_idx) {
3327                         hw_idx = tp->hw_status->idx[0].rx_producer;
3328                         rmb();
3329                 }
3330         }
3331
3332         /* ACK the status ring. */
3333         tp->rx_rcb_ptr = sw_idx;
3334         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, sw_idx);
3335
3336         /* Refill RX ring(s). */
3337         if (work_mask & RXD_OPAQUE_RING_STD) {
3338                 sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
3339                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
3340                              sw_idx);
3341         }
3342         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
3343                 sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
3344                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
3345                              sw_idx);
3346         }
3347         mmiowb();
3348
3349         return received;
3350 }
3351
3352 static int tg3_poll(struct net_device *netdev, int *budget)
3353 {
3354         struct tg3 *tp = netdev_priv(netdev);
3355         struct tg3_hw_status *sblk = tp->hw_status;
3356         int done;
3357
3358         /* handle link change and other phy events */
3359         if (!(tp->tg3_flags &
3360               (TG3_FLAG_USE_LINKCHG_REG |
3361                TG3_FLAG_POLL_SERDES))) {
3362                 if (sblk->status & SD_STATUS_LINK_CHG) {
3363                         sblk->status = SD_STATUS_UPDATED |
3364                                 (sblk->status & ~SD_STATUS_LINK_CHG);
3365                         spin_lock(&tp->lock);
3366                         tg3_setup_phy(tp, 0);
3367                         spin_unlock(&tp->lock);
3368                 }
3369         }
3370
3371         /* run TX completion thread */
3372         if (sblk->idx[0].tx_consumer != tp->tx_cons) {
3373                 tg3_tx(tp);
3374                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING)) {
3375                         netif_rx_complete(netdev);
3376                         schedule_work(&tp->reset_task);
3377                         return 0;
3378                 }
3379         }
3380
3381         /* run RX thread, within the bounds set by NAPI.
3382          * All RX "locking" is done by ensuring outside
3383          * code synchronizes with dev->poll()
3384          */
3385         if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr) {
3386                 int orig_budget = *budget;
3387                 int work_done;
3388
3389                 if (orig_budget > netdev->quota)
3390                         orig_budget = netdev->quota;
3391
3392                 work_done = tg3_rx(tp, orig_budget);
3393
3394                 *budget -= work_done;
3395                 netdev->quota -= work_done;
3396         }
3397
3398         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
3399                 tp->last_tag = sblk->status_tag;
3400                 rmb();
3401         } else
3402                 sblk->status &= ~SD_STATUS_UPDATED;
3403
3404         /* if no more work, tell net stack and NIC we're done */
3405         done = !tg3_has_work(tp);
3406         if (done) {
3407                 netif_rx_complete(netdev);
3408                 tg3_restart_ints(tp);
3409         }
3410
3411         return (done ? 0 : 1);
3412 }
3413
3414 static void tg3_irq_quiesce(struct tg3 *tp)
3415 {
3416         BUG_ON(tp->irq_sync);
3417
3418         tp->irq_sync = 1;
3419         smp_mb();
3420
3421         synchronize_irq(tp->pdev->irq);
3422 }
3423
3424 static inline int tg3_irq_sync(struct tg3 *tp)
3425 {
3426         return tp->irq_sync;
3427 }
3428
3429 /* Fully shutdown all tg3 driver activity elsewhere in the system.
3430  * If irq_sync is non-zero, then the IRQ handler must be synchronized
3431  * with as well.  Most of the time, this is not necessary except when
3432  * shutting down the device.
3433  */
3434 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
3435 {
3436         if (irq_sync)
3437                 tg3_irq_quiesce(tp);
3438         spin_lock_bh(&tp->lock);
3439 }
3440
3441 static inline void tg3_full_unlock(struct tg3 *tp)
3442 {
3443         spin_unlock_bh(&tp->lock);
3444 }
3445
3446 /* One-shot MSI handler - Chip automatically disables interrupt
3447  * after sending MSI so driver doesn't have to do it.
3448  */
3449 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id, struct pt_regs *regs)
3450 {
3451         struct net_device *dev = dev_id;
3452         struct tg3 *tp = netdev_priv(dev);
3453
3454         prefetch(tp->hw_status);
3455         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3456
3457         if (likely(!tg3_irq_sync(tp)))
3458                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3459
3460         return IRQ_HANDLED;
3461 }
3462
3463 /* MSI ISR - No need to check for interrupt sharing and no need to
3464  * flush status block and interrupt mailbox. PCI ordering rules
3465  * guarantee that MSI will arrive after the status block.
3466  */
3467 static irqreturn_t tg3_msi(int irq, void *dev_id, struct pt_regs *regs)
3468 {
3469         struct net_device *dev = dev_id;
3470         struct tg3 *tp = netdev_priv(dev);
3471
3472         prefetch(tp->hw_status);
3473         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3474         /*
3475          * Writing any value to intr-mbox-0 clears PCI INTA# and
3476          * chip-internal interrupt pending events.
3477          * Writing non-zero to intr-mbox-0 additional tells the
3478          * NIC to stop sending us irqs, engaging "in-intr-handler"
3479          * event coalescing.
3480          */
3481         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
3482         if (likely(!tg3_irq_sync(tp)))
3483                 netif_rx_schedule(dev);         /* schedule NAPI poll */
3484
3485         return IRQ_RETVAL(1);
3486 }
3487
3488 static irqreturn_t tg3_interrupt(int irq, void *dev_id, struct pt_regs *regs)
3489 {
3490         struct net_device *dev = dev_id;
3491         struct tg3 *tp = netdev_priv(dev);
3492         struct tg3_hw_status *sblk = tp->hw_status;
3493         unsigned int handled = 1;
3494
3495         /* In INTx mode, it is possible for the interrupt to arrive at
3496          * the CPU before the status block posted prior to the interrupt.
3497          * Reading the PCI State register will confirm whether the
3498          * interrupt is ours and will flush the status block.
3499          */
3500         if ((sblk->status & SD_STATUS_UPDATED) ||
3501             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3502                 /*
3503                  * Writing any value to intr-mbox-0 clears PCI INTA# and
3504                  * chip-internal interrupt pending events.
3505                  * Writing non-zero to intr-mbox-0 additional tells the
3506                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3507                  * event coalescing.
3508                  */
3509                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3510                              0x00000001);
3511                 if (tg3_irq_sync(tp))
3512                         goto out;
3513                 sblk->status &= ~SD_STATUS_UPDATED;
3514                 if (likely(tg3_has_work(tp))) {
3515                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3516                         netif_rx_schedule(dev);         /* schedule NAPI poll */
3517                 } else {
3518                         /* No work, shared interrupt perhaps?  re-enable
3519                          * interrupts, and flush that PCI write
3520                          */
3521                         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3522                                 0x00000000);
3523                 }
3524         } else {        /* shared interrupt */
3525                 handled = 0;
3526         }
3527 out:
3528         return IRQ_RETVAL(handled);
3529 }
3530
3531 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id, struct pt_regs *regs)
3532 {
3533         struct net_device *dev = dev_id;
3534         struct tg3 *tp = netdev_priv(dev);
3535         struct tg3_hw_status *sblk = tp->hw_status;
3536         unsigned int handled = 1;
3537
3538         /* In INTx mode, it is possible for the interrupt to arrive at
3539          * the CPU before the status block posted prior to the interrupt.
3540          * Reading the PCI State register will confirm whether the
3541          * interrupt is ours and will flush the status block.
3542          */
3543         if ((sblk->status_tag != tp->last_tag) ||
3544             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3545                 /*
3546                  * writing any value to intr-mbox-0 clears PCI INTA# and
3547                  * chip-internal interrupt pending events.
3548                  * writing non-zero to intr-mbox-0 additional tells the
3549                  * NIC to stop sending us irqs, engaging "in-intr-handler"
3550                  * event coalescing.
3551                  */
3552                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3553                              0x00000001);
3554                 if (tg3_irq_sync(tp))
3555                         goto out;
3556                 if (netif_rx_schedule_prep(dev)) {
3557                         prefetch(&tp->rx_rcb[tp->rx_rcb_ptr]);
3558                         /* Update last_tag to mark that this status has been
3559                          * seen. Because interrupt may be shared, we may be
3560                          * racing with tg3_poll(), so only update last_tag
3561                          * if tg3_poll() is not scheduled.
3562                          */
3563                         tp->last_tag = sblk->status_tag;
3564                         __netif_rx_schedule(dev);
3565                 }
3566         } else {        /* shared interrupt */
3567                 handled = 0;
3568         }
3569 out:
3570         return IRQ_RETVAL(handled);
3571 }
3572
3573 /* ISR for interrupt test */
3574 static irqreturn_t tg3_test_isr(int irq, void *dev_id,
3575                 struct pt_regs *regs)
3576 {
3577         struct net_device *dev = dev_id;
3578         struct tg3 *tp = netdev_priv(dev);
3579         struct tg3_hw_status *sblk = tp->hw_status;
3580
3581         if ((sblk->status & SD_STATUS_UPDATED) ||
3582             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
3583                 tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
3584                              0x00000001);
3585                 return IRQ_RETVAL(1);
3586         }
3587         return IRQ_RETVAL(0);
3588 }
3589
3590 static int tg3_init_hw(struct tg3 *, int);
3591 static int tg3_halt(struct tg3 *, int, int);
3592
3593 #ifdef CONFIG_NET_POLL_CONTROLLER
3594 static void tg3_poll_controller(struct net_device *dev)
3595 {
3596         struct tg3 *tp = netdev_priv(dev);
3597
3598         tg3_interrupt(tp->pdev->irq, dev, NULL);
3599 }
3600 #endif
3601
3602 static void tg3_reset_task(void *_data)
3603 {
3604         struct tg3 *tp = _data;
3605         unsigned int restart_timer;
3606
3607         tg3_full_lock(tp, 0);
3608         tp->tg3_flags |= TG3_FLAG_IN_RESET_TASK;
3609
3610         if (!netif_running(tp->dev)) {
3611                 tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3612                 tg3_full_unlock(tp);
3613                 return;
3614         }
3615
3616         tg3_full_unlock(tp);
3617
3618         tg3_netif_stop(tp);
3619
3620         tg3_full_lock(tp, 1);
3621
3622         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
3623         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
3624
3625         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
3626                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
3627                 tp->write32_rx_mbox = tg3_write_flush_reg32;
3628                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
3629                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
3630         }
3631
3632         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
3633         tg3_init_hw(tp, 1);
3634
3635         tg3_netif_start(tp);
3636
3637         if (restart_timer)
3638                 mod_timer(&tp->timer, jiffies + 1);
3639
3640         tp->tg3_flags &= ~TG3_FLAG_IN_RESET_TASK;
3641
3642         tg3_full_unlock(tp);
3643 }
3644
3645 static void tg3_tx_timeout(struct net_device *dev)
3646 {
3647         struct tg3 *tp = netdev_priv(dev);
3648
3649         printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
3650                dev->name);
3651
3652         schedule_work(&tp->reset_task);
3653 }
3654
3655 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
3656 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
3657 {
3658         u32 base = (u32) mapping & 0xffffffff;
3659
3660         return ((base > 0xffffdcc0) &&
3661                 (base + len + 8 < base));
3662 }
3663
3664 /* Test for DMA addresses > 40-bit */
3665 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
3666                                           int len)
3667 {
3668 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
3669         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
3670                 return (((u64) mapping + len) > DMA_40BIT_MASK);
3671         return 0;
3672 #else
3673         return 0;
3674 #endif
3675 }
3676
3677 static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);
3678
3679 /* Workaround 4GB and 40-bit hardware DMA bugs. */
3680 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
3681                                        u32 last_plus_one, u32 *start,
3682                                        u32 base_flags, u32 mss)
3683 {
3684         struct sk_buff *new_skb = skb_copy(skb, GFP_ATOMIC);
3685         dma_addr_t new_addr = 0;
3686         u32 entry = *start;
3687         int i, ret = 0;
3688
3689         if (!new_skb) {
3690                 ret = -1;
3691         } else {
3692                 /* New SKB is guaranteed to be linear. */
3693                 entry = *start;
3694                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
3695                                           PCI_DMA_TODEVICE);
3696                 /* Make sure new skb does not cross any 4G boundaries.
3697                  * Drop the packet if it does.
3698                  */
3699                 if (tg3_4g_overflow_test(new_addr, new_skb->len)) {
3700                         ret = -1;
3701                         dev_kfree_skb(new_skb);
3702                         new_skb = NULL;
3703                 } else {
3704                         tg3_set_txd(tp, entry, new_addr, new_skb->len,
3705                                     base_flags, 1 | (mss << 1));
3706                         *start = NEXT_TX(entry);
3707                 }
3708         }
3709
3710         /* Now clean up the sw ring entries. */
3711         i = 0;
3712         while (entry != last_plus_one) {
3713                 int len;
3714
3715                 if (i == 0)
3716                         len = skb_headlen(skb);
3717                 else
3718                         len = skb_shinfo(skb)->frags[i-1].size;
3719                 pci_unmap_single(tp->pdev,
3720                                  pci_unmap_addr(&tp->tx_buffers[entry], mapping),
3721                                  len, PCI_DMA_TODEVICE);
3722                 if (i == 0) {
3723                         tp->tx_buffers[entry].skb = new_skb;
3724                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
3725                 } else {
3726                         tp->tx_buffers[entry].skb = NULL;
3727                 }
3728                 entry = NEXT_TX(entry);
3729                 i++;
3730         }
3731
3732         dev_kfree_skb(skb);
3733
3734         return ret;
3735 }
3736
3737 static void tg3_set_txd(struct tg3 *tp, int entry,
3738                         dma_addr_t mapping, int len, u32 flags,
3739                         u32 mss_and_is_end)
3740 {
3741         struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];
3742         int is_end = (mss_and_is_end & 0x1);
3743         u32 mss = (mss_and_is_end >> 1);
3744         u32 vlan_tag = 0;
3745
3746         if (is_end)
3747                 flags |= TXD_FLAG_END;
3748         if (flags & TXD_FLAG_VLAN) {
3749                 vlan_tag = flags >> 16;
3750                 flags &= 0xffff;
3751         }
3752         vlan_tag |= (mss << TXD_MSS_SHIFT);
3753
3754         txd->addr_hi = ((u64) mapping >> 32);
3755         txd->addr_lo = ((u64) mapping & 0xffffffff);
3756         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
3757         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
3758 }
3759
3760 /* hard_start_xmit for devices that don't have any bugs and
3761  * support TG3_FLG2_HW_TSO_2 only.
3762  */
3763 static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
3764 {
3765         struct tg3 *tp = netdev_priv(dev);
3766         dma_addr_t mapping;
3767         u32 len, entry, base_flags, mss;
3768
3769         len = skb_headlen(skb);
3770
3771         /* We are running in BH disabled context with netif_tx_lock
3772          * and TX reclaim runs via tp->poll inside of a software
3773          * interrupt.  Furthermore, IRQ processing runs lockless so we have
3774          * no IRQ context deadlocks to worry about either.  Rejoice!
3775          */
3776         if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
3777                 if (!netif_queue_stopped(dev)) {
3778                         netif_stop_queue(dev);
3779
3780                         /* This is a hard error, log it. */
3781                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
3782                                "queue awake!\n", dev->name);
3783                 }
3784                 return NETDEV_TX_BUSY;
3785         }
3786
3787         entry = tp->tx_prod;
3788         base_flags = 0;
3789 #if TG3_TSO_SUPPORT != 0
3790         mss = 0;
3791         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
3792             (mss = skb_shinfo(skb)->gso_size) != 0) {
3793                 int tcp_opt_len, ip_tcp_len;
3794
3795                 if (skb_header_cloned(skb) &&
3796                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
3797                         dev_kfree_skb(skb);
3798                         goto out_unlock;
3799                 }
3800
3801                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
3802                         mss |= (skb_headlen(skb) - ETH_HLEN) << 9;
3803                 else {
3804                         tcp_opt_len = ((skb->h.th->doff - 5) * 4);
3805                         ip_tcp_len = (skb->nh.iph->ihl * 4) +
3806                                      sizeof(struct tcphdr);
3807
3808                         skb->nh.iph->check = 0;
3809                         skb->nh.iph->tot_len = htons(mss + ip_tcp_len +
3810                                                      tcp_opt_len);
3811                         mss |= (ip_tcp_len + tcp_opt_len) << 9;
3812                 }
3813
3814                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
3815                                TXD_FLAG_CPU_POST_DMA);
3816
3817                 skb->h.th->check = 0;
3818
3819         }
3820         else if (skb->ip_summed == CHECKSUM_HW)
3821                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3822 #else
3823         mss = 0;
3824         if (skb->ip_summed == CHECKSUM_HW)
3825                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3826 #endif
3827 #if TG3_VLAN_TAG_USED
3828         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
3829                 base_flags |= (TXD_FLAG_VLAN |
3830                                (vlan_tx_tag_get(skb) << 16));
3831 #endif
3832
3833         /* Queue skb data, a.k.a. the main skb fragment. */
3834         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
3835
3836         tp->tx_buffers[entry].skb = skb;
3837         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3838
3839         tg3_set_txd(tp, entry, mapping, len, base_flags,
3840                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
3841
3842         entry = NEXT_TX(entry);
3843
3844         /* Now loop through additional data fragments, and queue them. */
3845         if (skb_shinfo(skb)->nr_frags > 0) {
3846                 unsigned int i, last;
3847
3848                 last = skb_shinfo(skb)->nr_frags - 1;
3849                 for (i = 0; i <= last; i++) {
3850                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3851
3852                         len = frag->size;
3853                         mapping = pci_map_page(tp->pdev,
3854                                                frag->page,
3855                                                frag->page_offset,
3856                                                len, PCI_DMA_TODEVICE);
3857
3858                         tp->tx_buffers[entry].skb = NULL;
3859                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
3860
3861                         tg3_set_txd(tp, entry, mapping, len,
3862                                     base_flags, (i == last) | (mss << 1));
3863
3864                         entry = NEXT_TX(entry);
3865                 }
3866         }
3867
3868         /* Packets are ready, update Tx producer idx local and on card. */
3869         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
3870
3871         tp->tx_prod = entry;
3872         if (unlikely(TX_BUFFS_AVAIL(tp) <= (MAX_SKB_FRAGS + 1))) {
3873                 spin_lock(&tp->tx_lock);
3874                 netif_stop_queue(dev);
3875                 if (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH)
3876                         netif_wake_queue(tp->dev);
3877                 spin_unlock(&tp->tx_lock);
3878         }
3879
3880 out_unlock:
3881         mmiowb();
3882
3883         dev->trans_start = jiffies;
3884
3885         return NETDEV_TX_OK;
3886 }
3887
3888 #if TG3_TSO_SUPPORT != 0
3889 static int tg3_start_xmit_dma_bug(struct sk_buff *, struct net_device *);
3890
3891 /* Use GSO to workaround a rare TSO bug that may be triggered when the
3892  * TSO header is greater than 80 bytes.
3893  */
3894 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
3895 {
3896         struct sk_buff *segs, *nskb;
3897
3898         /* Estimate the number of fragments in the worst case */
3899         if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->gso_segs * 3))) {
3900                 netif_stop_queue(tp->dev);
3901                 return NETDEV_TX_BUSY;
3902         }
3903
3904         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
3905         if (unlikely(IS_ERR(segs)))
3906                 goto tg3_tso_bug_end;
3907
3908         do {
3909                 nskb = segs;
3910                 segs = segs->next;
3911                 nskb->next = NULL;
3912                 tg3_start_xmit_dma_bug(nskb, tp->dev);
3913         } while (segs);
3914
3915 tg3_tso_bug_end:
3916         dev_kfree_skb(skb);
3917
3918         return NETDEV_TX_OK;
3919 }
3920 #endif
3921
3922 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
3923  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
3924  */
3925 static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
3926 {
3927         struct tg3 *tp = netdev_priv(dev);
3928         dma_addr_t mapping;
3929         u32 len, entry, base_flags, mss;
3930         int would_hit_hwbug;
3931
3932         len = skb_headlen(skb);
3933
3934         /* We are running in BH disabled context with netif_tx_lock
3935          * and TX reclaim runs via tp->poll inside of a software
3936          * interrupt.  Furthermore, IRQ processing runs lockless so we have
3937          * no IRQ context deadlocks to worry about either.  Rejoice!
3938          */
3939         if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
3940                 if (!netif_queue_stopped(dev)) {
3941                         netif_stop_queue(dev);
3942
3943                         /* This is a hard error, log it. */
3944                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
3945                                "queue awake!\n", dev->name);
3946                 }
3947                 return NETDEV_TX_BUSY;
3948         }
3949
3950         entry = tp->tx_prod;
3951         base_flags = 0;
3952         if (skb->ip_summed == CHECKSUM_HW)
3953                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
3954 #if TG3_TSO_SUPPORT != 0
3955         mss = 0;
3956         if (skb->len > (tp->dev->mtu + ETH_HLEN) &&
3957             (mss = skb_shinfo(skb)->gso_size) != 0) {
3958                 int tcp_opt_len, ip_tcp_len, hdr_len;
3959
3960                 if (skb_header_cloned(skb) &&
3961                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
3962                         dev_kfree_skb(skb);
3963                         goto out_unlock;
3964                 }
3965
3966                 tcp_opt_len = ((skb->h.th->doff - 5) * 4);
3967                 ip_tcp_len = (skb->nh.iph->ihl * 4) + sizeof(struct tcphdr);
3968
3969                 hdr_len = ip_tcp_len + tcp_opt_len;
3970                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
3971                              (tp->tg3_flags2 & TG3_FLG2_HW_TSO_1_BUG))
3972                         return (tg3_tso_bug(tp, skb));
3973
3974                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
3975                                TXD_FLAG_CPU_POST_DMA);
3976
3977                 skb->nh.iph->check = 0;
3978                 skb->nh.iph->tot_len = htons(mss + hdr_len);
3979                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
3980                         skb->h.th->check = 0;
3981                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
3982                 }
3983                 else {
3984                         skb->h.th->check =
3985                                 ~csum_tcpudp_magic(skb->nh.iph->saddr,
3986                                                    skb->nh.iph->daddr,
3987                                                    0, IPPROTO_TCP, 0);
3988                 }
3989
3990                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
3991                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
3992                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
3993                                 int tsflags;
3994
3995                                 tsflags = ((skb->nh.iph->ihl - 5) +
3996                                            (tcp_opt_len >> 2));
3997                                 mss |= (tsflags << 11);
3998                         }
3999                 } else {
4000                         if (tcp_opt_len || skb->nh.iph->ihl > 5) {
4001                                 int tsflags;
4002
4003                                 tsflags = ((skb->nh.iph->ihl - 5) +
4004                                            (tcp_opt_len >> 2));
4005                                 base_flags |= tsflags << 12;
4006                         }
4007                 }
4008         }
4009 #else
4010         mss = 0;
4011 #endif
4012 #if TG3_VLAN_TAG_USED
4013         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
4014                 base_flags |= (TXD_FLAG_VLAN |
4015                                (vlan_tx_tag_get(skb) << 16));
4016 #endif
4017
4018         /* Queue skb data, a.k.a. the main skb fragment. */
4019         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
4020
4021         tp->tx_buffers[entry].skb = skb;
4022         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4023
4024         would_hit_hwbug = 0;
4025
4026         if (tg3_4g_overflow_test(mapping, len))
4027                 would_hit_hwbug = 1;
4028
4029         tg3_set_txd(tp, entry, mapping, len, base_flags,
4030                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
4031
4032         entry = NEXT_TX(entry);
4033
4034         /* Now loop through additional data fragments, and queue them. */
4035         if (skb_shinfo(skb)->nr_frags > 0) {
4036                 unsigned int i, last;
4037
4038                 last = skb_shinfo(skb)->nr_frags - 1;
4039                 for (i = 0; i <= last; i++) {
4040                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4041
4042                         len = frag->size;
4043                         mapping = pci_map_page(tp->pdev,
4044                                                frag->page,
4045                                                frag->page_offset,
4046                                                len, PCI_DMA_TODEVICE);
4047
4048                         tp->tx_buffers[entry].skb = NULL;
4049                         pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);
4050
4051                         if (tg3_4g_overflow_test(mapping, len))
4052                                 would_hit_hwbug = 1;
4053
4054                         if (tg3_40bit_overflow_test(tp, mapping, len))
4055                                 would_hit_hwbug = 1;
4056
4057                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
4058                                 tg3_set_txd(tp, entry, mapping, len,
4059                                             base_flags, (i == last)|(mss << 1));
4060                         else
4061                                 tg3_set_txd(tp, entry, mapping, len,
4062                                             base_flags, (i == last));
4063
4064                         entry = NEXT_TX(entry);
4065                 }
4066         }
4067
4068         if (would_hit_hwbug) {
4069                 u32 last_plus_one = entry;
4070                 u32 start;
4071
4072                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
4073                 start &= (TG3_TX_RING_SIZE - 1);
4074
4075                 /* If the workaround fails due to memory/mapping
4076                  * failure, silently drop this packet.
4077                  */
4078                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
4079                                                 &start, base_flags, mss))
4080                         goto out_unlock;
4081
4082                 entry = start;
4083         }
4084
4085         /* Packets are ready, update Tx producer idx local and on card. */
4086         tw32_tx_mbox((MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW), entry);
4087
4088         tp->tx_prod = entry;
4089         if (unlikely(TX_BUFFS_AVAIL(tp) <= (MAX_SKB_FRAGS + 1))) {
4090                 spin_lock(&tp->tx_lock);
4091                 netif_stop_queue(dev);
4092                 if (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH)
4093                         netif_wake_queue(tp->dev);
4094                 spin_unlock(&tp->tx_lock);
4095         }
4096
4097 out_unlock:
4098         mmiowb();
4099
4100         dev->trans_start = jiffies;
4101
4102         return NETDEV_TX_OK;
4103 }
4104
4105 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
4106                                int new_mtu)
4107 {
4108         dev->mtu = new_mtu;
4109
4110         if (new_mtu > ETH_DATA_LEN) {
4111                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4112                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
4113                         ethtool_op_set_tso(dev, 0);
4114                 }
4115                 else
4116                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
4117         } else {
4118                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
4119                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
4120                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
4121         }
4122 }
4123
4124 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
4125 {
4126         struct tg3 *tp = netdev_priv(dev);
4127
4128         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
4129                 return -EINVAL;
4130
4131         if (!netif_running(dev)) {
4132                 /* We'll just catch it later when the
4133                  * device is up'd.
4134                  */
4135                 tg3_set_mtu(dev, tp, new_mtu);
4136                 return 0;
4137         }
4138
4139         tg3_netif_stop(tp);
4140
4141         tg3_full_lock(tp, 1);
4142
4143         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4144
4145         tg3_set_mtu(dev, tp, new_mtu);
4146
4147         tg3_init_hw(tp, 0);
4148
4149         tg3_netif_start(tp);
4150
4151         tg3_full_unlock(tp);
4152
4153         return 0;
4154 }
4155
4156 /* Free up pending packets in all rx/tx rings.
4157  *
4158  * The chip has been shut down and the driver detached from
4159  * the networking, so no interrupts or new tx packets will
4160  * end up in the driver.  tp->{tx,}lock is not held and we are not
4161  * in an interrupt context and thus may sleep.
4162  */
4163 static void tg3_free_rings(struct tg3 *tp)
4164 {
4165         struct ring_info *rxp;
4166         int i;
4167
4168         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4169                 rxp = &tp->rx_std_buffers[i];
4170
4171                 if (rxp->skb == NULL)
4172                         continue;
4173                 pci_unmap_single(tp->pdev,
4174                                  pci_unmap_addr(rxp, mapping),
4175                                  tp->rx_pkt_buf_sz - tp->rx_offset,
4176                                  PCI_DMA_FROMDEVICE);
4177                 dev_kfree_skb_any(rxp->skb);
4178                 rxp->skb = NULL;
4179         }
4180
4181         for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4182                 rxp = &tp->rx_jumbo_buffers[i];
4183
4184                 if (rxp->skb == NULL)
4185                         continue;
4186                 pci_unmap_single(tp->pdev,
4187                                  pci_unmap_addr(rxp, mapping),
4188                                  RX_JUMBO_PKT_BUF_SZ - tp->rx_offset,
4189                                  PCI_DMA_FROMDEVICE);
4190                 dev_kfree_skb_any(rxp->skb);
4191                 rxp->skb = NULL;
4192         }
4193
4194         for (i = 0; i < TG3_TX_RING_SIZE; ) {
4195                 struct tx_ring_info *txp;
4196                 struct sk_buff *skb;
4197                 int j;
4198
4199                 txp = &tp->tx_buffers[i];
4200                 skb = txp->skb;
4201
4202                 if (skb == NULL) {
4203                         i++;
4204                         continue;
4205                 }
4206
4207                 pci_unmap_single(tp->pdev,
4208                                  pci_unmap_addr(txp, mapping),
4209                                  skb_headlen(skb),
4210                                  PCI_DMA_TODEVICE);
4211                 txp->skb = NULL;
4212
4213                 i++;
4214
4215                 for (j = 0; j < skb_shinfo(skb)->nr_frags; j++) {
4216                         txp = &tp->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
4217                         pci_unmap_page(tp->pdev,
4218                                        pci_unmap_addr(txp, mapping),
4219                                        skb_shinfo(skb)->frags[j].size,
4220                                        PCI_DMA_TODEVICE);
4221                         i++;
4222                 }
4223
4224                 dev_kfree_skb_any(skb);
4225         }
4226 }
4227
4228 /* Initialize tx/rx rings for packet processing.
4229  *
4230  * The chip has been shut down and the driver detached from
4231  * the networking, so no interrupts or new tx packets will
4232  * end up in the driver.  tp->{tx,}lock are held and thus
4233  * we may not sleep.
4234  */
4235 static void tg3_init_rings(struct tg3 *tp)
4236 {
4237         u32 i;
4238
4239         /* Free up all the SKBs. */
4240         tg3_free_rings(tp);
4241
4242         /* Zero out all descriptors. */
4243         memset(tp->rx_std, 0, TG3_RX_RING_BYTES);
4244         memset(tp->rx_jumbo, 0, TG3_RX_JUMBO_RING_BYTES);
4245         memset(tp->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
4246         memset(tp->tx_ring, 0, TG3_TX_RING_BYTES);
4247
4248         tp->rx_pkt_buf_sz = RX_PKT_BUF_SZ;
4249         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
4250             (tp->dev->mtu > ETH_DATA_LEN))
4251                 tp->rx_pkt_buf_sz = RX_JUMBO_PKT_BUF_SZ;
4252
4253         /* Initialize invariants of the rings, we only set this
4254          * stuff once.  This works because the card does not
4255          * write into the rx buffer posting rings.
4256          */
4257         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
4258                 struct tg3_rx_buffer_desc *rxd;
4259
4260                 rxd = &tp->rx_std[i];
4261                 rxd->idx_len = (tp->rx_pkt_buf_sz - tp->rx_offset - 64)
4262                         << RXD_LEN_SHIFT;
4263                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
4264                 rxd->opaque = (RXD_OPAQUE_RING_STD |
4265                                (i << RXD_OPAQUE_INDEX_SHIFT));
4266         }
4267
4268         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4269                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
4270                         struct tg3_rx_buffer_desc *rxd;
4271
4272                         rxd = &tp->rx_jumbo[i];
4273                         rxd->idx_len = (RX_JUMBO_PKT_BUF_SZ - tp->rx_offset - 64)
4274                                 << RXD_LEN_SHIFT;
4275                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
4276                                 RXD_FLAG_JUMBO;
4277                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
4278                                (i << RXD_OPAQUE_INDEX_SHIFT));
4279                 }
4280         }
4281
4282         /* Now allocate fresh SKBs for each rx ring. */
4283         for (i = 0; i < tp->rx_pending; i++) {
4284                 if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_STD,
4285                                      -1, i) < 0)
4286                         break;
4287         }
4288
4289         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
4290                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
4291                         if (tg3_alloc_rx_skb(tp, RXD_OPAQUE_RING_JUMBO,
4292                                              -1, i) < 0)
4293                                 break;
4294                 }
4295         }
4296 }
4297
4298 /*
4299  * Must not be invoked with interrupt sources disabled and
4300  * the hardware shutdown down.
4301  */
4302 static void tg3_free_consistent(struct tg3 *tp)
4303 {
4304         kfree(tp->rx_std_buffers);
4305         tp->rx_std_buffers = NULL;
4306         if (tp->rx_std) {
4307                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
4308                                     tp->rx_std, tp->rx_std_mapping);
4309                 tp->rx_std = NULL;
4310         }
4311         if (tp->rx_jumbo) {
4312                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4313                                     tp->rx_jumbo, tp->rx_jumbo_mapping);
4314                 tp->rx_jumbo = NULL;
4315         }
4316         if (tp->rx_rcb) {
4317                 pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4318                                     tp->rx_rcb, tp->rx_rcb_mapping);
4319                 tp->rx_rcb = NULL;
4320         }
4321         if (tp->tx_ring) {
4322                 pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
4323                         tp->tx_ring, tp->tx_desc_mapping);
4324                 tp->tx_ring = NULL;
4325         }
4326         if (tp->hw_status) {
4327                 pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
4328                                     tp->hw_status, tp->status_mapping);
4329                 tp->hw_status = NULL;
4330         }
4331         if (tp->hw_stats) {
4332                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
4333                                     tp->hw_stats, tp->stats_mapping);
4334                 tp->hw_stats = NULL;
4335         }
4336 }
4337
4338 /*
4339  * Must not be invoked with interrupt sources disabled and
4340  * the hardware shutdown down.  Can sleep.
4341  */
4342 static int tg3_alloc_consistent(struct tg3 *tp)
4343 {
4344         tp->rx_std_buffers = kmalloc((sizeof(struct ring_info) *
4345                                       (TG3_RX_RING_SIZE +
4346                                        TG3_RX_JUMBO_RING_SIZE)) +
4347                                      (sizeof(struct tx_ring_info) *
4348                                       TG3_TX_RING_SIZE),
4349                                      GFP_KERNEL);
4350         if (!tp->rx_std_buffers)
4351                 return -ENOMEM;
4352
4353         memset(tp->rx_std_buffers, 0,
4354                (sizeof(struct ring_info) *
4355                 (TG3_RX_RING_SIZE +
4356                  TG3_RX_JUMBO_RING_SIZE)) +
4357                (sizeof(struct tx_ring_info) *
4358                 TG3_TX_RING_SIZE));
4359
4360         tp->rx_jumbo_buffers = &tp->rx_std_buffers[TG3_RX_RING_SIZE];
4361         tp->tx_buffers = (struct tx_ring_info *)
4362                 &tp->rx_jumbo_buffers[TG3_RX_JUMBO_RING_SIZE];
4363
4364         tp->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
4365                                           &tp->rx_std_mapping);
4366         if (!tp->rx_std)
4367                 goto err_out;
4368
4369         tp->rx_jumbo = pci_alloc_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
4370                                             &tp->rx_jumbo_mapping);
4371
4372         if (!tp->rx_jumbo)
4373                 goto err_out;
4374
4375         tp->rx_rcb = pci_alloc_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
4376                                           &tp->rx_rcb_mapping);
4377         if (!tp->rx_rcb)
4378                 goto err_out;
4379
4380         tp->tx_ring = pci_alloc_consistent(tp->pdev, TG3_TX_RING_BYTES,
4381                                            &tp->tx_desc_mapping);
4382         if (!tp->tx_ring)
4383                 goto err_out;
4384
4385         tp->hw_status = pci_alloc_consistent(tp->pdev,
4386                                              TG3_HW_STATUS_SIZE,
4387                                              &tp->status_mapping);
4388         if (!tp->hw_status)
4389                 goto err_out;
4390
4391         tp->hw_stats = pci_alloc_consistent(tp->pdev,
4392                                             sizeof(struct tg3_hw_stats),
4393                                             &tp->stats_mapping);
4394         if (!tp->hw_stats)
4395                 goto err_out;
4396
4397         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4398         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4399
4400         return 0;
4401
4402 err_out:
4403         tg3_free_consistent(tp);
4404         return -ENOMEM;
4405 }
4406
4407 #define MAX_WAIT_CNT 1000
4408
4409 /* To stop a block, clear the enable bit and poll till it
4410  * clears.  tp->lock is held.
4411  */
4412 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
4413 {
4414         unsigned int i;
4415         u32 val;
4416
4417         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
4418                 switch (ofs) {
4419                 case RCVLSC_MODE:
4420                 case DMAC_MODE:
4421                 case MBFREE_MODE:
4422                 case BUFMGR_MODE:
4423                 case MEMARB_MODE:
4424                         /* We can't enable/disable these bits of the
4425                          * 5705/5750, just say success.
4426                          */
4427                         return 0;
4428
4429                 default:
4430                         break;
4431                 };
4432         }
4433
4434         val = tr32(ofs);
4435         val &= ~enable_bit;
4436         tw32_f(ofs, val);
4437
4438         for (i = 0; i < MAX_WAIT_CNT; i++) {
4439                 udelay(100);
4440                 val = tr32(ofs);
4441                 if ((val & enable_bit) == 0)
4442                         break;
4443         }
4444
4445         if (i == MAX_WAIT_CNT && !silent) {
4446                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
4447                        "ofs=%lx enable_bit=%x\n",
4448                        ofs, enable_bit);
4449                 return -ENODEV;
4450         }
4451
4452         return 0;
4453 }
4454
4455 /* tp->lock is held. */
4456 static int tg3_abort_hw(struct tg3 *tp, int silent)
4457 {
4458         int i, err;
4459
4460         tg3_disable_ints(tp);
4461
4462         tp->rx_mode &= ~RX_MODE_ENABLE;
4463         tw32_f(MAC_RX_MODE, tp->rx_mode);
4464         udelay(10);
4465
4466         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
4467         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
4468         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
4469         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
4470         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
4471         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
4472
4473         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
4474         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
4475         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
4476         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
4477         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
4478         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
4479         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
4480
4481         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
4482         tw32_f(MAC_MODE, tp->mac_mode);
4483         udelay(40);
4484
4485         tp->tx_mode &= ~TX_MODE_ENABLE;
4486         tw32_f(MAC_TX_MODE, tp->tx_mode);
4487
4488         for (i = 0; i < MAX_WAIT_CNT; i++) {
4489                 udelay(100);
4490                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
4491                         break;
4492         }
4493         if (i >= MAX_WAIT_CNT) {
4494                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
4495                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
4496                        tp->dev->name, tr32(MAC_TX_MODE));
4497                 err |= -ENODEV;
4498         }
4499
4500         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
4501         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
4502         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
4503
4504         tw32(FTQ_RESET, 0xffffffff);
4505         tw32(FTQ_RESET, 0x00000000);
4506
4507         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
4508         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
4509
4510         if (tp->hw_status)
4511                 memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
4512         if (tp->hw_stats)
4513                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
4514
4515         return err;
4516 }
4517
4518 /* tp->lock is held. */
4519 static int tg3_nvram_lock(struct tg3 *tp)
4520 {
4521         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4522                 int i;
4523
4524                 if (tp->nvram_lock_cnt == 0) {
4525                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
4526                         for (i = 0; i < 8000; i++) {
4527                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
4528                                         break;
4529                                 udelay(20);
4530                         }
4531                         if (i == 8000) {
4532                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
4533                                 return -ENODEV;
4534                         }
4535                 }
4536                 tp->nvram_lock_cnt++;
4537         }
4538         return 0;
4539 }
4540
4541 /* tp->lock is held. */
4542 static void tg3_nvram_unlock(struct tg3 *tp)
4543 {
4544         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
4545                 if (tp->nvram_lock_cnt > 0)
4546                         tp->nvram_lock_cnt--;
4547                 if (tp->nvram_lock_cnt == 0)
4548                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
4549         }
4550 }
4551
4552 /* tp->lock is held. */
4553 static void tg3_enable_nvram_access(struct tg3 *tp)
4554 {
4555         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4556             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4557                 u32 nvaccess = tr32(NVRAM_ACCESS);
4558
4559                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
4560         }
4561 }
4562
4563 /* tp->lock is held. */
4564 static void tg3_disable_nvram_access(struct tg3 *tp)
4565 {
4566         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
4567             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
4568                 u32 nvaccess = tr32(NVRAM_ACCESS);
4569
4570                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
4571         }
4572 }
4573
4574 /* tp->lock is held. */
4575 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
4576 {
4577         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
4578                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
4579
4580         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4581                 switch (kind) {
4582                 case RESET_KIND_INIT:
4583                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4584                                       DRV_STATE_START);
4585                         break;
4586
4587                 case RESET_KIND_SHUTDOWN:
4588                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4589                                       DRV_STATE_UNLOAD);
4590                         break;
4591
4592                 case RESET_KIND_SUSPEND:
4593                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4594                                       DRV_STATE_SUSPEND);
4595                         break;
4596
4597                 default:
4598                         break;
4599                 };
4600         }
4601 }
4602
4603 /* tp->lock is held. */
4604 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
4605 {
4606         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
4607                 switch (kind) {
4608                 case RESET_KIND_INIT:
4609                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4610                                       DRV_STATE_START_DONE);
4611                         break;
4612
4613                 case RESET_KIND_SHUTDOWN:
4614                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4615                                       DRV_STATE_UNLOAD_DONE);
4616                         break;
4617
4618                 default:
4619                         break;
4620                 };
4621         }
4622 }
4623
4624 /* tp->lock is held. */
4625 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
4626 {
4627         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4628                 switch (kind) {
4629                 case RESET_KIND_INIT:
4630                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4631                                       DRV_STATE_START);
4632                         break;
4633
4634                 case RESET_KIND_SHUTDOWN:
4635                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4636                                       DRV_STATE_UNLOAD);
4637                         break;
4638
4639                 case RESET_KIND_SUSPEND:
4640                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
4641                                       DRV_STATE_SUSPEND);
4642                         break;
4643
4644                 default:
4645                         break;
4646                 };
4647         }
4648 }
4649
4650 static void tg3_stop_fw(struct tg3 *);
4651
4652 /* tp->lock is held. */
4653 static int tg3_chip_reset(struct tg3 *tp)
4654 {
4655         u32 val;
4656         void (*write_op)(struct tg3 *, u32, u32);
4657         int i;
4658
4659         tg3_nvram_lock(tp);
4660
4661         /* No matching tg3_nvram_unlock() after this because
4662          * chip reset below will undo the nvram lock.
4663          */
4664         tp->nvram_lock_cnt = 0;
4665
4666         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
4667             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
4668             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
4669                 tw32(GRC_FASTBOOT_PC, 0);
4670
4671         /*
4672          * We must avoid the readl() that normally takes place.
4673          * It locks machines, causes machine checks, and other
4674          * fun things.  So, temporarily disable the 5701
4675          * hardware workaround, while we do the reset.
4676          */
4677         write_op = tp->write32;
4678         if (write_op == tg3_write_flush_reg32)
4679                 tp->write32 = tg3_write32;
4680
4681         /* do the reset */
4682         val = GRC_MISC_CFG_CORECLK_RESET;
4683
4684         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4685                 if (tr32(0x7e2c) == 0x60) {
4686                         tw32(0x7e2c, 0x20);
4687                 }
4688                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4689                         tw32(GRC_MISC_CFG, (1 << 29));
4690                         val |= (1 << 29);
4691                 }
4692         }
4693
4694         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
4695                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
4696         tw32(GRC_MISC_CFG, val);
4697
4698         /* restore 5701 hardware bug workaround write method */
4699         tp->write32 = write_op;
4700
4701         /* Unfortunately, we have to delay before the PCI read back.
4702          * Some 575X chips even will not respond to a PCI cfg access
4703          * when the reset command is given to the chip.
4704          *
4705          * How do these hardware designers expect things to work
4706          * properly if the PCI write is posted for a long period
4707          * of time?  It is always necessary to have some method by
4708          * which a register read back can occur to push the write
4709          * out which does the reset.
4710          *
4711          * For most tg3 variants the trick below was working.
4712          * Ho hum...
4713          */
4714         udelay(120);
4715
4716         /* Flush PCI posted writes.  The normal MMIO registers
4717          * are inaccessible at this time so this is the only
4718          * way to make this reliably (actually, this is no longer
4719          * the case, see above).  I tried to use indirect
4720          * register read/write but this upset some 5701 variants.
4721          */
4722         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
4723
4724         udelay(120);
4725
4726         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
4727                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
4728                         int i;
4729                         u32 cfg_val;
4730
4731                         /* Wait for link training to complete.  */
4732                         for (i = 0; i < 5000; i++)
4733                                 udelay(100);
4734
4735                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
4736                         pci_write_config_dword(tp->pdev, 0xc4,
4737                                                cfg_val | (1 << 15));
4738                 }
4739                 /* Set PCIE max payload size and clear error status.  */
4740                 pci_write_config_dword(tp->pdev, 0xd8, 0xf5000);
4741         }
4742
4743         /* Re-enable indirect register accesses. */
4744         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
4745                                tp->misc_host_ctrl);
4746
4747         /* Set MAX PCI retry to zero. */
4748         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
4749         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
4750             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
4751                 val |= PCISTATE_RETRY_SAME_DMA;
4752         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
4753
4754         pci_restore_state(tp->pdev);
4755
4756         /* Make sure PCI-X relaxed ordering bit is clear. */
4757         pci_read_config_dword(tp->pdev, TG3PCI_X_CAPS, &val);
4758         val &= ~PCIX_CAPS_RELAXED_ORDERING;
4759         pci_write_config_dword(tp->pdev, TG3PCI_X_CAPS, val);
4760
4761         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
4762                 u32 val;
4763
4764                 /* Chip reset on 5780 will reset MSI enable bit,
4765                  * so need to restore it.
4766                  */
4767                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
4768                         u16 ctrl;
4769
4770                         pci_read_config_word(tp->pdev,
4771                                              tp->msi_cap + PCI_MSI_FLAGS,
4772                                              &ctrl);
4773                         pci_write_config_word(tp->pdev,
4774                                               tp->msi_cap + PCI_MSI_FLAGS,
4775                                               ctrl | PCI_MSI_FLAGS_ENABLE);
4776                         val = tr32(MSGINT_MODE);
4777                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
4778                 }
4779
4780                 val = tr32(MEMARB_MODE);
4781                 tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
4782
4783         } else
4784                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
4785
4786         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
4787                 tg3_stop_fw(tp);
4788                 tw32(0x5000, 0x400);
4789         }
4790
4791         tw32(GRC_MODE, tp->grc_mode);
4792
4793         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
4794                 u32 val = tr32(0xc4);
4795
4796                 tw32(0xc4, val | (1 << 15));
4797         }
4798
4799         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
4800             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
4801                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
4802                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
4803                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
4804                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
4805         }
4806
4807         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
4808                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
4809                 tw32_f(MAC_MODE, tp->mac_mode);
4810         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
4811                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
4812                 tw32_f(MAC_MODE, tp->mac_mode);
4813         } else
4814                 tw32_f(MAC_MODE, 0);
4815         udelay(40);
4816
4817         /* Wait for firmware initialization to complete. */
4818         for (i = 0; i < 100000; i++) {
4819                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
4820                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
4821                         break;
4822                 udelay(10);
4823         }
4824
4825         /* Chip might not be fitted with firmare.  Some Sun onboard
4826          * parts are configured like that.  So don't signal the timeout
4827          * of the above loop as an error, but do report the lack of
4828          * running firmware once.
4829          */
4830         if (i >= 100000 &&
4831             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
4832                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
4833
4834                 printk(KERN_INFO PFX "%s: No firmware running.\n",
4835                        tp->dev->name);
4836         }
4837
4838         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
4839             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
4840                 u32 val = tr32(0x7c00);
4841
4842                 tw32(0x7c00, val | (1 << 25));
4843         }
4844
4845         /* Reprobe ASF enable state.  */
4846         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
4847         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
4848         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
4849         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
4850                 u32 nic_cfg;
4851
4852                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
4853                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
4854                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
4855                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
4856                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
4857                 }
4858         }
4859
4860         return 0;
4861 }
4862
4863 /* tp->lock is held. */
4864 static void tg3_stop_fw(struct tg3 *tp)
4865 {
4866         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
4867                 u32 val;
4868                 int i;
4869
4870                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
4871                 val = tr32(GRC_RX_CPU_EVENT);
4872                 val |= (1 << 14);
4873                 tw32(GRC_RX_CPU_EVENT, val);
4874
4875                 /* Wait for RX cpu to ACK the event.  */
4876                 for (i = 0; i < 100; i++) {
4877                         if (!(tr32(GRC_RX_CPU_EVENT) & (1 << 14)))
4878                                 break;
4879                         udelay(1);
4880                 }
4881         }
4882 }
4883
4884 /* tp->lock is held. */
4885 static int tg3_halt(struct tg3 *tp, int kind, int silent)
4886 {
4887         int err;
4888
4889         tg3_stop_fw(tp);
4890
4891         tg3_write_sig_pre_reset(tp, kind);
4892
4893         tg3_abort_hw(tp, silent);
4894         err = tg3_chip_reset(tp);
4895
4896         tg3_write_sig_legacy(tp, kind);
4897         tg3_write_sig_post_reset(tp, kind);
4898
4899         if (err)
4900                 return err;
4901
4902         return 0;
4903 }
4904
4905 #define TG3_FW_RELEASE_MAJOR    0x0
4906 #define TG3_FW_RELASE_MINOR     0x0
4907 #define TG3_FW_RELEASE_FIX      0x0
4908 #define TG3_FW_START_ADDR       0x08000000
4909 #define TG3_FW_TEXT_ADDR        0x08000000
4910 #define TG3_FW_TEXT_LEN         0x9c0
4911 #define TG3_FW_RODATA_ADDR      0x080009c0
4912 #define TG3_FW_RODATA_LEN       0x60
4913 #define TG3_FW_DATA_ADDR        0x08000a40
4914 #define TG3_FW_DATA_LEN         0x20
4915 #define TG3_FW_SBSS_ADDR        0x08000a60
4916 #define TG3_FW_SBSS_LEN         0xc
4917 #define TG3_FW_BSS_ADDR         0x08000a70
4918 #define TG3_FW_BSS_LEN          0x10
4919
4920 static u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
4921         0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
4922         0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
4923         0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
4924         0x0e00021c, 0x00000000, 0x0000000d, 0x00000000, 0x00000000, 0x00000000,
4925         0x27bdffe0, 0x3c1cc000, 0xafbf0018, 0xaf80680c, 0x0e00004c, 0x241b2105,
4926         0x97850000, 0x97870002, 0x9782002c, 0x9783002e, 0x3c040800, 0x248409c0,
4927         0xafa00014, 0x00021400, 0x00621825, 0x00052c00, 0xafa30010, 0x8f860010,
4928         0x00e52825, 0x0e000060, 0x24070102, 0x3c02ac00, 0x34420100, 0x3c03ac01,
4929         0x34630100, 0xaf820490, 0x3c02ffff, 0xaf820494, 0xaf830498, 0xaf82049c,
4930         0x24020001, 0xaf825ce0, 0x0e00003f, 0xaf825d00, 0x0e000140, 0x00000000,
4931         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x2402ffff, 0xaf825404, 0x8f835400,
4932         0x34630400, 0xaf835400, 0xaf825404, 0x3c020800, 0x24420034, 0xaf82541c,
4933         0x03e00008, 0xaf805400, 0x00000000, 0x00000000, 0x3c020800, 0x34423000,
4934         0x3c030800, 0x34633000, 0x3c040800, 0x348437ff, 0x3c010800, 0xac220a64,
4935         0x24020040, 0x3c010800, 0xac220a68, 0x3c010800, 0xac200a60, 0xac600000,
4936         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
4937         0x00804821, 0x8faa0010, 0x3c020800, 0x8c420a60, 0x3c040800, 0x8c840a68,
4938         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010800, 0xac230a60, 0x14400003,
4939         0x00004021, 0x3c010800, 0xac200a60, 0x3c020800, 0x8c420a60, 0x3c030800,
4940         0x8c630a64, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
4941         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020800, 0x8c420a60,
4942         0x3c030800, 0x8c630a64, 0x8f84680c, 0x00021140, 0x00431021, 0xac440008,
4943         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
4944         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4945         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4946         0, 0, 0, 0, 0, 0,
4947         0x02000008, 0x00000000, 0x0a0001e3, 0x3c0a0001, 0x0a0001e3, 0x3c0a0002,
4948         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4949         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4950         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4951         0x0a0001e3, 0x3c0a0007, 0x0a0001e3, 0x3c0a0008, 0x0a0001e3, 0x3c0a0009,
4952         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000b,
4953         0x0a0001e3, 0x3c0a000c, 0x0a0001e3, 0x3c0a000d, 0x0a0001e3, 0x00000000,
4954         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a000e, 0x0a0001e3, 0x00000000,
4955         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4956         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000, 0x0a0001e3, 0x00000000,
4957         0x0a0001e3, 0x00000000, 0x0a0001e3, 0x3c0a0013, 0x0a0001e3, 0x3c0a0014,
4958         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4959         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4960         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
4961         0x27bdffe0, 0x00001821, 0x00001021, 0xafbf0018, 0xafb10014, 0xafb00010,
4962         0x3c010800, 0x00220821, 0xac200a70, 0x3c010800, 0x00220821, 0xac200a74,
4963         0x3c010800, 0x00220821, 0xac200a78, 0x24630001, 0x1860fff5, 0x2442000c,
4964         0x24110001, 0x8f906810, 0x32020004, 0x14400005, 0x24040001, 0x3c020800,
4965         0x8c420a78, 0x18400003, 0x00002021, 0x0e000182, 0x00000000, 0x32020001,
4966         0x10400003, 0x00000000, 0x0e000169, 0x00000000, 0x0a000153, 0xaf915028,
4967         0x8fbf0018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020, 0x3c050800,
4968         0x8ca50a70, 0x3c060800, 0x8cc60a80, 0x3c070800, 0x8ce70a78, 0x27bdffe0,
4969         0x3c040800, 0x248409d0, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014,
4970         0x0e00017b, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x24020001,
4971         0x8f836810, 0x00821004, 0x00021027, 0x00621824, 0x03e00008, 0xaf836810,
4972         0x27bdffd8, 0xafbf0024, 0x1080002e, 0xafb00020, 0x8f825cec, 0xafa20018,
4973         0x8f825cec, 0x3c100800, 0x26100a78, 0xafa2001c, 0x34028000, 0xaf825cec,
4974         0x8e020000, 0x18400016, 0x00000000, 0x3c020800, 0x94420a74, 0x8fa3001c,
4975         0x000221c0, 0xac830004, 0x8fa2001c, 0x3c010800, 0x0e000201, 0xac220a74,
4976         0x10400005, 0x00000000, 0x8e020000, 0x24420001, 0x0a0001df, 0xae020000,
4977         0x3c020800, 0x8c420a70, 0x00021c02, 0x000321c0, 0x0a0001c5, 0xafa2001c,
4978         0x0e000201, 0x00000000, 0x1040001f, 0x00000000, 0x8e020000, 0x8fa3001c,
4979         0x24420001, 0x3c010800, 0xac230a70, 0x3c010800, 0xac230a74, 0x0a0001df,
4980         0xae020000, 0x3c100800, 0x26100a78, 0x8e020000, 0x18400028, 0x00000000,
4981         0x0e000201, 0x00000000, 0x14400024, 0x00000000, 0x8e020000, 0x3c030800,
4982         0x8c630a70, 0x2442ffff, 0xafa3001c, 0x18400006, 0xae020000, 0x00031402,
4983         0x000221c0, 0x8c820004, 0x3c010800, 0xac220a70, 0x97a2001e, 0x2442ff00,
4984         0x2c420300, 0x1440000b, 0x24024000, 0x3c040800, 0x248409dc, 0xafa00010,
4985         0xafa00014, 0x8fa6001c, 0x24050008, 0x0e000060, 0x00003821, 0x0a0001df,
4986         0x00000000, 0xaf825cf8, 0x3c020800, 0x8c420a40, 0x8fa3001c, 0x24420001,
4987         0xaf835cf8, 0x3c010800, 0xac220a40, 0x8fbf0024, 0x8fb00020, 0x03e00008,
4988         0x27bd0028, 0x27bdffe0, 0x3c040800, 0x248409e8, 0x00002821, 0x00003021,
4989         0x00003821, 0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x8fbf0018,
4990         0x03e00008, 0x27bd0020, 0x8f82680c, 0x8f85680c, 0x00021827, 0x0003182b,
4991         0x00031823, 0x00431024, 0x00441021, 0x00a2282b, 0x10a00006, 0x00000000,
4992         0x00401821, 0x8f82680c, 0x0043102b, 0x1440fffd, 0x00000000, 0x03e00008,
4993         0x00000000, 0x3c040800, 0x8c840000, 0x3c030800, 0x8c630a40, 0x0064102b,
4994         0x54400002, 0x00831023, 0x00641023, 0x2c420008, 0x03e00008, 0x38420001,
4995         0x27bdffe0, 0x00802821, 0x3c040800, 0x24840a00, 0x00003021, 0x00003821,
4996         0xafbf0018, 0xafa00010, 0x0e000060, 0xafa00014, 0x0a000216, 0x00000000,
4997         0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000, 0x27bdffe0, 0x3c1cc000,
4998         0xafbf0018, 0x0e00004c, 0xaf80680c, 0x3c040800, 0x24840a10, 0x03802821,
4999         0x00003021, 0x00003821, 0xafa00010, 0x0e000060, 0xafa00014, 0x2402ffff,
5000         0xaf825404, 0x3c0200aa, 0x0e000234, 0xaf825434, 0x8fbf0018, 0x03e00008,
5001         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe8, 0xafb00010,
5002         0x24100001, 0xafbf0014, 0x3c01c003, 0xac200000, 0x8f826810, 0x30422000,
5003         0x10400003, 0x00000000, 0x0e000246, 0x00000000, 0x0a00023a, 0xaf905428,
5004         0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x27bdfff8, 0x8f845d0c,
5005         0x3c0200ff, 0x3c030800, 0x8c630a50, 0x3442fff8, 0x00821024, 0x1043001e,
5006         0x3c0500ff, 0x34a5fff8, 0x3c06c003, 0x3c074000, 0x00851824, 0x8c620010,
5007         0x3c010800, 0xac230a50, 0x30420008, 0x10400005, 0x00871025, 0x8cc20000,
5008         0x24420001, 0xacc20000, 0x00871025, 0xaf825d0c, 0x8fa20000, 0x24420001,
5009         0xafa20000, 0x8fa20000, 0x8fa20000, 0x24420001, 0xafa20000, 0x8fa20000,
5010         0x8f845d0c, 0x3c030800, 0x8c630a50, 0x00851024, 0x1443ffe8, 0x00851824,
5011         0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
5012 };
5013
5014 static u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
5015         0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
5016         0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
5017         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
5018         0x00000000, 0x00000000, 0x4d61696e, 0x43707542, 0x00000000, 0x00000000,
5019         0x00000000
5020 };
5021
5022 #if 0 /* All zeros, don't eat up space with it. */
5023 u32 tg3FwData[(TG3_FW_DATA_LEN / sizeof(u32)) + 1] = {
5024         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5025         0x00000000, 0x00000000, 0x00000000, 0x00000000
5026 };
5027 #endif
5028
5029 #define RX_CPU_SCRATCH_BASE     0x30000
5030 #define RX_CPU_SCRATCH_SIZE     0x04000
5031 #define TX_CPU_SCRATCH_BASE     0x34000
5032 #define TX_CPU_SCRATCH_SIZE     0x04000
5033
5034 /* tp->lock is held. */
5035 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
5036 {
5037         int i;
5038
5039         BUG_ON(offset == TX_CPU_BASE &&
5040             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
5041
5042         if (offset == RX_CPU_BASE) {
5043                 for (i = 0; i < 10000; i++) {
5044                         tw32(offset + CPU_STATE, 0xffffffff);
5045                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5046                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5047                                 break;
5048                 }
5049
5050                 tw32(offset + CPU_STATE, 0xffffffff);
5051                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
5052                 udelay(10);
5053         } else {
5054                 for (i = 0; i < 10000; i++) {
5055                         tw32(offset + CPU_STATE, 0xffffffff);
5056                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
5057                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
5058                                 break;
5059                 }
5060         }
5061
5062         if (i >= 10000) {
5063                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
5064                        "and %s CPU\n",
5065                        tp->dev->name,
5066                        (offset == RX_CPU_BASE ? "RX" : "TX"));
5067                 return -ENODEV;
5068         }
5069
5070         /* Clear firmware's nvram arbitration. */
5071         if (tp->tg3_flags & TG3_FLAG_NVRAM)
5072                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
5073         return 0;
5074 }
5075
5076 struct fw_info {
5077         unsigned int text_base;
5078         unsigned int text_len;
5079         u32 *text_data;
5080         unsigned int rodata_base;
5081         unsigned int rodata_len;
5082         u32 *rodata_data;
5083         unsigned int data_base;
5084         unsigned int data_len;
5085         u32 *data_data;
5086 };
5087
5088 /* tp->lock is held. */
5089 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
5090                                  int cpu_scratch_size, struct fw_info *info)
5091 {
5092         int err, lock_err, i;
5093         void (*write_op)(struct tg3 *, u32, u32);
5094
5095         if (cpu_base == TX_CPU_BASE &&
5096             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5097                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
5098                        "TX cpu firmware on %s which is 5705.\n",
5099                        tp->dev->name);
5100                 return -EINVAL;
5101         }
5102
5103         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
5104                 write_op = tg3_write_mem;
5105         else
5106                 write_op = tg3_write_indirect_reg32;
5107
5108         /* It is possible that bootcode is still loading at this point.
5109          * Get the nvram lock first before halting the cpu.
5110          */
5111         lock_err = tg3_nvram_lock(tp);
5112         err = tg3_halt_cpu(tp, cpu_base);
5113         if (!lock_err)
5114                 tg3_nvram_unlock(tp);
5115         if (err)
5116                 goto out;
5117
5118         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
5119                 write_op(tp, cpu_scratch_base + i, 0);
5120         tw32(cpu_base + CPU_STATE, 0xffffffff);
5121         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
5122         for (i = 0; i < (info->text_len / sizeof(u32)); i++)
5123                 write_op(tp, (cpu_scratch_base +
5124                               (info->text_base & 0xffff) +
5125                               (i * sizeof(u32))),
5126                          (info->text_data ?
5127                           info->text_data[i] : 0));
5128         for (i = 0; i < (info->rodata_len / sizeof(u32)); i++)
5129                 write_op(tp, (cpu_scratch_base +
5130                               (info->rodata_base & 0xffff) +
5131                               (i * sizeof(u32))),
5132                          (info->rodata_data ?
5133                           info->rodata_data[i] : 0));
5134         for (i = 0; i < (info->data_len / sizeof(u32)); i++)
5135                 write_op(tp, (cpu_scratch_base +
5136                               (info->data_base & 0xffff) +
5137                               (i * sizeof(u32))),
5138                          (info->data_data ?
5139                           info->data_data[i] : 0));
5140
5141         err = 0;
5142
5143 out:
5144         return err;
5145 }
5146
5147 /* tp->lock is held. */
5148 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
5149 {
5150         struct fw_info info;
5151         int err, i;
5152
5153         info.text_base = TG3_FW_TEXT_ADDR;
5154         info.text_len = TG3_FW_TEXT_LEN;
5155         info.text_data = &tg3FwText[0];
5156         info.rodata_base = TG3_FW_RODATA_ADDR;
5157         info.rodata_len = TG3_FW_RODATA_LEN;
5158         info.rodata_data = &tg3FwRodata[0];
5159         info.data_base = TG3_FW_DATA_ADDR;
5160         info.data_len = TG3_FW_DATA_LEN;
5161         info.data_data = NULL;
5162
5163         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
5164                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
5165                                     &info);
5166         if (err)
5167                 return err;
5168
5169         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
5170                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
5171                                     &info);
5172         if (err)
5173                 return err;
5174
5175         /* Now startup only the RX cpu. */
5176         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5177         tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5178
5179         for (i = 0; i < 5; i++) {
5180                 if (tr32(RX_CPU_BASE + CPU_PC) == TG3_FW_TEXT_ADDR)
5181                         break;
5182                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5183                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
5184                 tw32_f(RX_CPU_BASE + CPU_PC,    TG3_FW_TEXT_ADDR);
5185                 udelay(1000);
5186         }
5187         if (i >= 5) {
5188                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
5189                        "to set RX CPU PC, is %08x should be %08x\n",
5190                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
5191                        TG3_FW_TEXT_ADDR);
5192                 return -ENODEV;
5193         }
5194         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
5195         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
5196
5197         return 0;
5198 }
5199
5200 #if TG3_TSO_SUPPORT != 0
5201
5202 #define TG3_TSO_FW_RELEASE_MAJOR        0x1
5203 #define TG3_TSO_FW_RELASE_MINOR         0x6
5204 #define TG3_TSO_FW_RELEASE_FIX          0x0
5205 #define TG3_TSO_FW_START_ADDR           0x08000000
5206 #define TG3_TSO_FW_TEXT_ADDR            0x08000000
5207 #define TG3_TSO_FW_TEXT_LEN             0x1aa0
5208 #define TG3_TSO_FW_RODATA_ADDR          0x08001aa0
5209 #define TG3_TSO_FW_RODATA_LEN           0x60
5210 #define TG3_TSO_FW_DATA_ADDR            0x08001b20
5211 #define TG3_TSO_FW_DATA_LEN             0x30
5212 #define TG3_TSO_FW_SBSS_ADDR            0x08001b50
5213 #define TG3_TSO_FW_SBSS_LEN             0x2c
5214 #define TG3_TSO_FW_BSS_ADDR             0x08001b80
5215 #define TG3_TSO_FW_BSS_LEN              0x894
5216
5217 static u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
5218         0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
5219         0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
5220         0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5221         0xafbf0018, 0x0e0005d8, 0x34840002, 0x0e000668, 0x00000000, 0x3c030800,
5222         0x90631b68, 0x24020002, 0x3c040800, 0x24841aac, 0x14620003, 0x24050001,
5223         0x3c040800, 0x24841aa0, 0x24060006, 0x00003821, 0xafa00010, 0x0e00067c,
5224         0xafa00014, 0x8f625c50, 0x34420001, 0xaf625c50, 0x8f625c90, 0x34420001,
5225         0xaf625c90, 0x2402ffff, 0x0e000034, 0xaf625404, 0x8fbf0018, 0x03e00008,
5226         0x27bd0020, 0x00000000, 0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c,
5227         0xafb20018, 0xafb10014, 0x0e00005b, 0xafb00010, 0x24120002, 0x24110001,
5228         0x8f706820, 0x32020100, 0x10400003, 0x00000000, 0x0e0000bb, 0x00000000,
5229         0x8f706820, 0x32022000, 0x10400004, 0x32020001, 0x0e0001f0, 0x24040001,
5230         0x32020001, 0x10400003, 0x00000000, 0x0e0000a3, 0x00000000, 0x3c020800,
5231         0x90421b98, 0x14520003, 0x00000000, 0x0e0004c0, 0x00000000, 0x0a00003c,
5232         0xaf715028, 0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008,
5233         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ac0, 0x00002821, 0x00003021,
5234         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x3c040800,
5235         0x248423d8, 0xa4800000, 0x3c010800, 0xa0201b98, 0x3c010800, 0xac201b9c,
5236         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5237         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bbc, 0x8f624434, 0x3c010800,
5238         0xac221b88, 0x8f624438, 0x3c010800, 0xac221b8c, 0x8f624410, 0xac80f7a8,
5239         0x3c010800, 0xac201b84, 0x3c010800, 0xac2023e0, 0x3c010800, 0xac2023c8,
5240         0x3c010800, 0xac2023cc, 0x3c010800, 0xac202400, 0x3c010800, 0xac221b90,
5241         0x8f620068, 0x24030007, 0x00021702, 0x10430005, 0x00000000, 0x8f620068,
5242         0x00021702, 0x14400004, 0x24020001, 0x3c010800, 0x0a000097, 0xac20240c,
5243         0xac820034, 0x3c040800, 0x24841acc, 0x3c050800, 0x8ca5240c, 0x00003021,
5244         0x00003821, 0xafa00010, 0x0e00067c, 0xafa00014, 0x8fbf0018, 0x03e00008,
5245         0x27bd0020, 0x27bdffe0, 0x3c040800, 0x24841ad8, 0x00002821, 0x00003021,
5246         0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014, 0x0e00005b,
5247         0x00000000, 0x0e0000b4, 0x00002021, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5248         0x24020001, 0x8f636820, 0x00821004, 0x00021027, 0x00621824, 0x03e00008,
5249         0xaf636820, 0x27bdffd0, 0xafbf002c, 0xafb60028, 0xafb50024, 0xafb40020,
5250         0xafb3001c, 0xafb20018, 0xafb10014, 0xafb00010, 0x8f675c5c, 0x3c030800,
5251         0x24631bbc, 0x8c620000, 0x14470005, 0x3c0200ff, 0x3c020800, 0x90421b98,
5252         0x14400119, 0x3c0200ff, 0x3442fff8, 0x00e28824, 0xac670000, 0x00111902,
5253         0x306300ff, 0x30e20003, 0x000211c0, 0x00622825, 0x00a04021, 0x00071602,
5254         0x3c030800, 0x90631b98, 0x3044000f, 0x14600036, 0x00804821, 0x24020001,
5255         0x3c010800, 0xa0221b98, 0x00051100, 0x00821025, 0x3c010800, 0xac201b9c,
5256         0x3c010800, 0xac201ba0, 0x3c010800, 0xac201ba4, 0x3c010800, 0xac201bac,
5257         0x3c010800, 0xac201bb8, 0x3c010800, 0xac201bb0, 0x3c010800, 0xac201bb4,
5258         0x3c010800, 0xa42223d8, 0x9622000c, 0x30437fff, 0x3c010800, 0xa4222410,
5259         0x30428000, 0x3c010800, 0xa4231bc6, 0x10400005, 0x24020001, 0x3c010800,
5260         0xac2223f4, 0x0a000102, 0x2406003e, 0x24060036, 0x3c010800, 0xac2023f4,
5261         0x9622000a, 0x3c030800, 0x94631bc6, 0x3c010800, 0xac2023f0, 0x3c010800,
5262         0xac2023f8, 0x00021302, 0x00021080, 0x00c21021, 0x00621821, 0x3c010800,
5263         0xa42223d0, 0x3c010800, 0x0a000115, 0xa4231b96, 0x9622000c, 0x3c010800,
5264         0xa42223ec, 0x3c040800, 0x24841b9c, 0x8c820000, 0x00021100, 0x3c010800,
5265         0x00220821, 0xac311bc8, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5266         0xac271bcc, 0x8c820000, 0x25030001, 0x306601ff, 0x00021100, 0x3c010800,
5267         0x00220821, 0xac261bd0, 0x8c820000, 0x00021100, 0x3c010800, 0x00220821,
5268         0xac291bd4, 0x96230008, 0x3c020800, 0x8c421bac, 0x00432821, 0x3c010800,
5269         0xac251bac, 0x9622000a, 0x30420004, 0x14400018, 0x00061100, 0x8f630c14,
5270         0x3063000f, 0x2c620002, 0x1440000b, 0x3c02c000, 0x8f630c14, 0x3c020800,
5271         0x8c421b40, 0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002,
5272         0x1040fff7, 0x3c02c000, 0x00e21825, 0xaf635c5c, 0x8f625c50, 0x30420002,
5273         0x10400014, 0x00000000, 0x0a000147, 0x00000000, 0x3c030800, 0x8c631b80,
5274         0x3c040800, 0x94841b94, 0x01221025, 0x3c010800, 0xa42223da, 0x24020001,
5275         0x3c010800, 0xac221bb8, 0x24630001, 0x0085202a, 0x3c010800, 0x10800003,
5276         0xac231b80, 0x3c010800, 0xa4251b94, 0x3c060800, 0x24c61b9c, 0x8cc20000,
5277         0x24420001, 0xacc20000, 0x28420080, 0x14400005, 0x00000000, 0x0e000656,
5278         0x24040002, 0x0a0001e6, 0x00000000, 0x3c020800, 0x8c421bb8, 0x10400078,
5279         0x24020001, 0x3c050800, 0x90a51b98, 0x14a20072, 0x00000000, 0x3c150800,
5280         0x96b51b96, 0x3c040800, 0x8c841bac, 0x32a3ffff, 0x0083102a, 0x1440006c,
5281         0x00000000, 0x14830003, 0x00000000, 0x3c010800, 0xac2523f0, 0x1060005c,
5282         0x00009021, 0x24d60004, 0x0060a021, 0x24d30014, 0x8ec20000, 0x00028100,
5283         0x3c110800, 0x02308821, 0x0e000625, 0x8e311bc8, 0x00402821, 0x10a00054,
5284         0x00000000, 0x9628000a, 0x31020040, 0x10400005, 0x2407180c, 0x8e22000c,
5285         0x2407188c, 0x00021400, 0xaca20018, 0x3c030800, 0x00701821, 0x8c631bd0,
5286         0x3c020800, 0x00501021, 0x8c421bd4, 0x00031d00, 0x00021400, 0x00621825,
5287         0xaca30014, 0x8ec30004, 0x96220008, 0x00432023, 0x3242ffff, 0x3083ffff,
5288         0x00431021, 0x0282102a, 0x14400002, 0x02b23023, 0x00803021, 0x8e620000,
5289         0x30c4ffff, 0x00441021, 0xae620000, 0x8e220000, 0xaca20000, 0x8e220004,
5290         0x8e63fff4, 0x00431021, 0xaca20004, 0xa4a6000e, 0x8e62fff4, 0x00441021,
5291         0xae62fff4, 0x96230008, 0x0043102a, 0x14400005, 0x02469021, 0x8e62fff0,
5292         0xae60fff4, 0x24420001, 0xae62fff0, 0xaca00008, 0x3242ffff, 0x14540008,
5293         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x24020905, 0xa4a2000c,
5294         0x0a0001cb, 0x34e70020, 0xa4a2000c, 0x3c020800, 0x8c4223f0, 0x10400003,
5295         0x3c024b65, 0x0a0001d3, 0x34427654, 0x3c02b49a, 0x344289ab, 0xaca2001c,
5296         0x30e2ffff, 0xaca20010, 0x0e0005a2, 0x00a02021, 0x3242ffff, 0x0054102b,
5297         0x1440ffa9, 0x00000000, 0x24020002, 0x3c010800, 0x0a0001e6, 0xa0221b98,
5298         0x8ec2083c, 0x24420001, 0x0a0001e6, 0xaec2083c, 0x0e0004c0, 0x00000000,
5299         0x8fbf002c, 0x8fb60028, 0x8fb50024, 0x8fb40020, 0x8fb3001c, 0x8fb20018,
5300         0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0030, 0x27bdffd0, 0xafbf0028,
5301         0xafb30024, 0xafb20020, 0xafb1001c, 0xafb00018, 0x8f725c9c, 0x3c0200ff,
5302         0x3442fff8, 0x3c070800, 0x24e71bb4, 0x02428824, 0x9623000e, 0x8ce20000,
5303         0x00431021, 0xace20000, 0x8e220010, 0x30420020, 0x14400011, 0x00809821,
5304         0x0e00063b, 0x02202021, 0x3c02c000, 0x02421825, 0xaf635c9c, 0x8f625c90,
5305         0x30420002, 0x1040011e, 0x00000000, 0xaf635c9c, 0x8f625c90, 0x30420002,
5306         0x10400119, 0x00000000, 0x0a00020d, 0x00000000, 0x8e240008, 0x8e230014,
5307         0x00041402, 0x000231c0, 0x00031502, 0x304201ff, 0x2442ffff, 0x3042007f,
5308         0x00031942, 0x30637800, 0x00021100, 0x24424000, 0x00624821, 0x9522000a,
5309         0x3084ffff, 0x30420008, 0x104000b0, 0x000429c0, 0x3c020800, 0x8c422400,
5310         0x14400024, 0x24c50008, 0x94c20014, 0x3c010800, 0xa42223d0, 0x8cc40010,
5311         0x00041402, 0x3c010800, 0xa42223d2, 0x3c010800, 0xa42423d4, 0x94c2000e,
5312         0x3083ffff, 0x00431023, 0x3c010800, 0xac222408, 0x94c2001a, 0x3c010800,
5313         0xac262400, 0x3c010800, 0xac322404, 0x3c010800, 0xac2223fc, 0x3c02c000,
5314         0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e5, 0x00000000,
5315         0xaf635c9c, 0x8f625c90, 0x30420002, 0x104000e0, 0x00000000, 0x0a000246,
5316         0x00000000, 0x94c2000e, 0x3c030800, 0x946323d4, 0x00434023, 0x3103ffff,
5317         0x2c620008, 0x1040001c, 0x00000000, 0x94c20014, 0x24420028, 0x00a22821,
5318         0x00031042, 0x1840000b, 0x00002021, 0x24e60848, 0x00403821, 0x94a30000,
5319         0x8cc20000, 0x24840001, 0x00431021, 0xacc20000, 0x0087102a, 0x1440fff9,
5320         0x24a50002, 0x31020001, 0x1040001f, 0x3c024000, 0x3c040800, 0x248423fc,
5321         0xa0a00001, 0x94a30000, 0x8c820000, 0x00431021, 0x0a000285, 0xac820000,
5322         0x8f626800, 0x3c030010, 0x00431024, 0x10400009, 0x00000000, 0x94c2001a,
5323         0x3c030800, 0x8c6323fc, 0x00431021, 0x3c010800, 0xac2223fc, 0x0a000286,
5324         0x3c024000, 0x94c2001a, 0x94c4001c, 0x3c030800, 0x8c6323fc, 0x00441023,
5325         0x00621821, 0x3c010800, 0xac2323fc, 0x3c024000, 0x02421825, 0xaf635c9c,
5326         0x8f625c90, 0x30420002, 0x1440fffc, 0x00000000, 0x9522000a, 0x30420010,
5327         0x1040009b, 0x00000000, 0x3c030800, 0x946323d4, 0x3c070800, 0x24e72400,
5328         0x8ce40000, 0x8f626800, 0x24630030, 0x00832821, 0x3c030010, 0x00431024,
5329         0x1440000a, 0x00000000, 0x94a20004, 0x3c040800, 0x8c842408, 0x3c030800,
5330         0x8c6323fc, 0x00441023, 0x00621821, 0x3c010800, 0xac2323fc, 0x3c040800,
5331         0x8c8423fc, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402, 0x00822021,
5332         0x00041027, 0xa4a20006, 0x3c030800, 0x8c632404, 0x3c0200ff, 0x3442fff8,
5333         0x00628824, 0x96220008, 0x24050001, 0x24034000, 0x000231c0, 0x00801021,
5334         0xa4c2001a, 0xa4c0001c, 0xace00000, 0x3c010800, 0xac251b60, 0xaf635cb8,
5335         0x8f625cb0, 0x30420002, 0x10400003, 0x00000000, 0x3c010800, 0xac201b60,
5336         0x8e220008, 0xaf625cb8, 0x8f625cb0, 0x30420002, 0x10400003, 0x00000000,
5337         0x3c010800, 0xac201b60, 0x3c020800, 0x8c421b60, 0x1040ffec, 0x00000000,
5338         0x3c040800, 0x0e00063b, 0x8c842404, 0x0a00032a, 0x00000000, 0x3c030800,
5339         0x90631b98, 0x24020002, 0x14620003, 0x3c034b65, 0x0a0002e1, 0x00008021,
5340         0x8e22001c, 0x34637654, 0x10430002, 0x24100002, 0x24100001, 0x00c02021,
5341         0x0e000350, 0x02003021, 0x24020003, 0x3c010800, 0xa0221b98, 0x24020002,
5342         0x1202000a, 0x24020001, 0x3c030800, 0x8c6323f0, 0x10620006, 0x00000000,
5343         0x3c020800, 0x944223d8, 0x00021400, 0x0a00031f, 0xae220014, 0x3c040800,
5344         0x248423da, 0x94820000, 0x00021400, 0xae220014, 0x3c020800, 0x8c421bbc,
5345         0x3c03c000, 0x3c010800, 0xa0201b98, 0x00431025, 0xaf625c5c, 0x8f625c50,
5346         0x30420002, 0x10400009, 0x00000000, 0x2484f7e2, 0x8c820000, 0x00431025,
5347         0xaf625c5c, 0x8f625c50, 0x30420002, 0x1440fffa, 0x00000000, 0x3c020800,
5348         0x24421b84, 0x8c430000, 0x24630001, 0xac430000, 0x8f630c14, 0x3063000f,
5349         0x2c620002, 0x1440000c, 0x3c024000, 0x8f630c14, 0x3c020800, 0x8c421b40,
5350         0x3063000f, 0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7,
5351         0x00000000, 0x3c024000, 0x02421825, 0xaf635c9c, 0x8f625c90, 0x30420002,
5352         0x1440fffc, 0x00000000, 0x12600003, 0x00000000, 0x0e0004c0, 0x00000000,
5353         0x8fbf0028, 0x8fb30024, 0x8fb20020, 0x8fb1001c, 0x8fb00018, 0x03e00008,
5354         0x27bd0030, 0x8f634450, 0x3c040800, 0x24841b88, 0x8c820000, 0x00031c02,
5355         0x0043102b, 0x14400007, 0x3c038000, 0x8c840004, 0x8f624450, 0x00021c02,
5356         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5357         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3c024000,
5358         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00000000,
5359         0x03e00008, 0x00000000, 0x27bdffe0, 0x00805821, 0x14c00011, 0x256e0008,
5360         0x3c020800, 0x8c4223f4, 0x10400007, 0x24020016, 0x3c010800, 0xa42223d2,
5361         0x2402002a, 0x3c010800, 0x0a000364, 0xa42223d4, 0x8d670010, 0x00071402,
5362         0x3c010800, 0xa42223d2, 0x3c010800, 0xa42723d4, 0x3c040800, 0x948423d4,
5363         0x3c030800, 0x946323d2, 0x95cf0006, 0x3c020800, 0x944223d0, 0x00832023,
5364         0x01e2c023, 0x3065ffff, 0x24a20028, 0x01c24821, 0x3082ffff, 0x14c0001a,
5365         0x01226021, 0x9582000c, 0x3042003f, 0x3c010800, 0xa42223d6, 0x95820004,
5366         0x95830006, 0x3c010800, 0xac2023e4, 0x3c010800, 0xac2023e8, 0x00021400,
5367         0x00431025, 0x3c010800, 0xac221bc0, 0x95220004, 0x3c010800, 0xa4221bc4,
5368         0x95230002, 0x01e51023, 0x0043102a, 0x10400010, 0x24020001, 0x3c010800,
5369         0x0a000398, 0xac2223f8, 0x3c030800, 0x8c6323e8, 0x3c020800, 0x94421bc4,
5370         0x00431021, 0xa5220004, 0x3c020800, 0x94421bc0, 0xa5820004, 0x3c020800,
5371         0x8c421bc0, 0xa5820006, 0x3c020800, 0x8c4223f0, 0x3c0d0800, 0x8dad23e4,
5372         0x3c0a0800, 0x144000e5, 0x8d4a23e8, 0x3c020800, 0x94421bc4, 0x004a1821,
5373         0x3063ffff, 0x0062182b, 0x24020002, 0x10c2000d, 0x01435023, 0x3c020800,
5374         0x944223d6, 0x30420009, 0x10400008, 0x00000000, 0x9582000c, 0x3042fff6,
5375         0xa582000c, 0x3c020800, 0x944223d6, 0x30420009, 0x01a26823, 0x3c020800,
5376         0x8c4223f8, 0x1040004a, 0x01203821, 0x3c020800, 0x944223d2, 0x00004021,
5377         0xa520000a, 0x01e21023, 0xa5220002, 0x3082ffff, 0x00021042, 0x18400008,
5378         0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021, 0x0103102a,
5379         0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061402,
5380         0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021, 0x2527000c,
5381         0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004, 0x1440fffb,
5382         0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023, 0x01803821,
5383         0x3082ffff, 0xa4e00010, 0x00621821, 0x00021042, 0x18400010, 0x00c33021,
5384         0x00404821, 0x94e20000, 0x24e70002, 0x00c23021, 0x30e2007f, 0x14400006,
5385         0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80, 0x00625824, 0x25670008,
5386         0x0109102a, 0x1440fff3, 0x00000000, 0x30820001, 0x10400005, 0x00061c02,
5387         0xa0e00001, 0x94e20000, 0x00c23021, 0x00061c02, 0x30c2ffff, 0x00623021,
5388         0x00061402, 0x00c23021, 0x0a00047d, 0x30c6ffff, 0x24020002, 0x14c20081,
5389         0x00000000, 0x3c020800, 0x8c42240c, 0x14400007, 0x00000000, 0x3c020800,
5390         0x944223d2, 0x95230002, 0x01e21023, 0x10620077, 0x00000000, 0x3c020800,
5391         0x944223d2, 0x01e21023, 0xa5220002, 0x3c020800, 0x8c42240c, 0x1040001a,
5392         0x31e3ffff, 0x8dc70010, 0x3c020800, 0x94421b96, 0x00e04021, 0x00072c02,
5393         0x00aa2021, 0x00431023, 0x00823823, 0x00072402, 0x30e2ffff, 0x00823821,
5394         0x00071027, 0xa522000a, 0x3102ffff, 0x3c040800, 0x948423d4, 0x00453023,
5395         0x00e02821, 0x00641823, 0x006d1821, 0x00c33021, 0x00061c02, 0x30c2ffff,
5396         0x0a00047d, 0x00623021, 0x01203821, 0x00004021, 0x3082ffff, 0x00021042,
5397         0x18400008, 0x00003021, 0x00401821, 0x94e20000, 0x25080001, 0x00c23021,
5398         0x0103102a, 0x1440fffb, 0x24e70002, 0x00061c02, 0x30c2ffff, 0x00623021,
5399         0x00061402, 0x00c23021, 0x00c02821, 0x00061027, 0xa522000a, 0x00003021,
5400         0x2527000c, 0x00004021, 0x94e20000, 0x25080001, 0x00c23021, 0x2d020004,
5401         0x1440fffb, 0x24e70002, 0x95220002, 0x00004021, 0x91230009, 0x00442023,
5402         0x01803821, 0x3082ffff, 0xa4e00010, 0x3c040800, 0x948423d4, 0x00621821,
5403         0x00c33021, 0x00061c02, 0x30c2ffff, 0x00623021, 0x00061c02, 0x3c020800,
5404         0x944223d0, 0x00c34821, 0x00441023, 0x00021fc2, 0x00431021, 0x00021043,
5405         0x18400010, 0x00003021, 0x00402021, 0x94e20000, 0x24e70002, 0x00c23021,
5406         0x30e2007f, 0x14400006, 0x25080001, 0x8d630000, 0x3c02007f, 0x3442ff80,
5407         0x00625824, 0x25670008, 0x0104102a, 0x1440fff3, 0x00000000, 0x3c020800,
5408         0x944223ec, 0x00c23021, 0x3122ffff, 0x00c23021, 0x00061c02, 0x30c2ffff,
5409         0x00623021, 0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010,
5410         0xadc00014, 0x0a00049d, 0xadc00000, 0x8dc70010, 0x00e04021, 0x11400007,
5411         0x00072c02, 0x00aa3021, 0x00061402, 0x30c3ffff, 0x00433021, 0x00061402,
5412         0x00c22821, 0x00051027, 0xa522000a, 0x3c030800, 0x946323d4, 0x3102ffff,
5413         0x01e21021, 0x00433023, 0x00cd3021, 0x00061c02, 0x30c2ffff, 0x00623021,
5414         0x00061402, 0x00c23021, 0x00c04021, 0x00061027, 0xa5820010, 0x3102ffff,
5415         0x00051c00, 0x00431025, 0xadc20010, 0x3c020800, 0x8c4223f4, 0x10400005,
5416         0x2de205eb, 0x14400002, 0x25e2fff2, 0x34028870, 0xa5c20034, 0x3c030800,
5417         0x246323e8, 0x8c620000, 0x24420001, 0xac620000, 0x3c040800, 0x8c8423e4,
5418         0x3c020800, 0x8c421bc0, 0x3303ffff, 0x00832021, 0x00431821, 0x0062102b,
5419         0x3c010800, 0xac2423e4, 0x10400003, 0x2482ffff, 0x3c010800, 0xac2223e4,
5420         0x3c010800, 0xac231bc0, 0x03e00008, 0x27bd0020, 0x27bdffb8, 0x3c050800,
5421         0x24a51b96, 0xafbf0044, 0xafbe0040, 0xafb7003c, 0xafb60038, 0xafb50034,
5422         0xafb40030, 0xafb3002c, 0xafb20028, 0xafb10024, 0xafb00020, 0x94a90000,
5423         0x3c020800, 0x944223d0, 0x3c030800, 0x8c631bb0, 0x3c040800, 0x8c841bac,
5424         0x01221023, 0x0064182a, 0xa7a9001e, 0x106000be, 0xa7a20016, 0x24be0022,
5425         0x97b6001e, 0x24b3001a, 0x24b70016, 0x8fc20000, 0x14400008, 0x00000000,
5426         0x8fc2fff8, 0x97a30016, 0x8fc4fff4, 0x00431021, 0x0082202a, 0x148000b0,
5427         0x00000000, 0x97d50818, 0x32a2ffff, 0x104000a3, 0x00009021, 0x0040a021,
5428         0x00008821, 0x0e000625, 0x00000000, 0x00403021, 0x14c00007, 0x00000000,
5429         0x3c020800, 0x8c4223dc, 0x24420001, 0x3c010800, 0x0a000596, 0xac2223dc,
5430         0x3c100800, 0x02118021, 0x8e101bc8, 0x9608000a, 0x31020040, 0x10400005,
5431         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x31020080,
5432         0x54400001, 0x34e70010, 0x3c020800, 0x00511021, 0x8c421bd0, 0x3c030800,
5433         0x00711821, 0x8c631bd4, 0x00021500, 0x00031c00, 0x00431025, 0xacc20014,
5434         0x96040008, 0x3242ffff, 0x00821021, 0x0282102a, 0x14400002, 0x02b22823,
5435         0x00802821, 0x8e020000, 0x02459021, 0xacc20000, 0x8e020004, 0x00c02021,
5436         0x26310010, 0xac820004, 0x30e2ffff, 0xac800008, 0xa485000e, 0xac820010,
5437         0x24020305, 0x0e0005a2, 0xa482000c, 0x3242ffff, 0x0054102b, 0x1440ffc5,
5438         0x3242ffff, 0x0a00058e, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5439         0x10400067, 0x00000000, 0x8e62fff0, 0x00028900, 0x3c100800, 0x02118021,
5440         0x0e000625, 0x8e101bc8, 0x00403021, 0x14c00005, 0x00000000, 0x8e62082c,
5441         0x24420001, 0x0a000596, 0xae62082c, 0x9608000a, 0x31020040, 0x10400005,
5442         0x2407180c, 0x8e02000c, 0x2407188c, 0x00021400, 0xacc20018, 0x3c020800,
5443         0x00511021, 0x8c421bd0, 0x3c030800, 0x00711821, 0x8c631bd4, 0x00021500,
5444         0x00031c00, 0x00431025, 0xacc20014, 0x8e63fff4, 0x96020008, 0x00432023,
5445         0x3242ffff, 0x3083ffff, 0x00431021, 0x02c2102a, 0x10400003, 0x00802821,
5446         0x97a9001e, 0x01322823, 0x8e620000, 0x30a4ffff, 0x00441021, 0xae620000,
5447         0xa4c5000e, 0x8e020000, 0xacc20000, 0x8e020004, 0x8e63fff4, 0x00431021,
5448         0xacc20004, 0x8e63fff4, 0x96020008, 0x00641821, 0x0062102a, 0x14400006,
5449         0x02459021, 0x8e62fff0, 0xae60fff4, 0x24420001, 0x0a000571, 0xae62fff0,
5450         0xae63fff4, 0xacc00008, 0x3242ffff, 0x10560003, 0x31020004, 0x10400006,
5451         0x24020305, 0x31020080, 0x54400001, 0x34e70010, 0x34e70020, 0x24020905,
5452         0xa4c2000c, 0x8ee30000, 0x8ee20004, 0x14620007, 0x3c02b49a, 0x8ee20860,
5453         0x54400001, 0x34e70400, 0x3c024b65, 0x0a000588, 0x34427654, 0x344289ab,
5454         0xacc2001c, 0x30e2ffff, 0xacc20010, 0x0e0005a2, 0x00c02021, 0x3242ffff,
5455         0x0056102b, 0x1440ff9b, 0x00000000, 0x8e620000, 0x8e63fffc, 0x0043102a,
5456         0x1440ff48, 0x00000000, 0x8fbf0044, 0x8fbe0040, 0x8fb7003c, 0x8fb60038,
5457         0x8fb50034, 0x8fb40030, 0x8fb3002c, 0x8fb20028, 0x8fb10024, 0x8fb00020,
5458         0x03e00008, 0x27bd0048, 0x27bdffe8, 0xafbf0014, 0xafb00010, 0x8f624450,
5459         0x8f634410, 0x0a0005b1, 0x00808021, 0x8f626820, 0x30422000, 0x10400003,
5460         0x00000000, 0x0e0001f0, 0x00002021, 0x8f624450, 0x8f634410, 0x3042ffff,
5461         0x0043102b, 0x1440fff5, 0x00000000, 0x8f630c14, 0x3063000f, 0x2c620002,
5462         0x1440000b, 0x00000000, 0x8f630c14, 0x3c020800, 0x8c421b40, 0x3063000f,
5463         0x24420001, 0x3c010800, 0xac221b40, 0x2c620002, 0x1040fff7, 0x00000000,
5464         0xaf705c18, 0x8f625c10, 0x30420002, 0x10400009, 0x00000000, 0x8f626820,
5465         0x30422000, 0x1040fff8, 0x00000000, 0x0e0001f0, 0x00002021, 0x0a0005c4,
5466         0x00000000, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000,
5467         0x00000000, 0x00000000, 0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010,
5468         0xaf60680c, 0x8f626804, 0x34420082, 0xaf626804, 0x8f634000, 0x24020b50,
5469         0x3c010800, 0xac221b54, 0x24020b78, 0x3c010800, 0xac221b64, 0x34630002,
5470         0xaf634000, 0x0e000605, 0x00808021, 0x3c010800, 0xa0221b68, 0x304200ff,
5471         0x24030002, 0x14430005, 0x00000000, 0x3c020800, 0x8c421b54, 0x0a0005f8,
5472         0xac5000c0, 0x3c020800, 0x8c421b54, 0xac5000bc, 0x8f624434, 0x8f634438,
5473         0x8f644410, 0x3c010800, 0xac221b5c, 0x3c010800, 0xac231b6c, 0x3c010800,
5474         0xac241b58, 0x8fbf0014, 0x8fb00010, 0x03e00008, 0x27bd0018, 0x3c040800,
5475         0x8c870000, 0x3c03aa55, 0x3463aa55, 0x3c06c003, 0xac830000, 0x8cc20000,
5476         0x14430007, 0x24050002, 0x3c0355aa, 0x346355aa, 0xac830000, 0x8cc20000,
5477         0x50430001, 0x24050001, 0x3c020800, 0xac470000, 0x03e00008, 0x00a01021,
5478         0x27bdfff8, 0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe,
5479         0x00000000, 0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008,
5480         0x27bd0008, 0x8f634450, 0x3c020800, 0x8c421b5c, 0x00031c02, 0x0043102b,
5481         0x14400008, 0x3c038000, 0x3c040800, 0x8c841b6c, 0x8f624450, 0x00021c02,
5482         0x0083102b, 0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024,
5483         0x1440fffd, 0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff,
5484         0x2442e000, 0x2c422001, 0x14400003, 0x3c024000, 0x0a000648, 0x2402ffff,
5485         0x00822025, 0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021,
5486         0x03e00008, 0x00000000, 0x8f624450, 0x3c030800, 0x8c631b58, 0x0a000651,
5487         0x3042ffff, 0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000,
5488         0x03e00008, 0x00000000, 0x27bdffe0, 0x00802821, 0x3c040800, 0x24841af0,
5489         0x00003021, 0x00003821, 0xafbf0018, 0xafa00010, 0x0e00067c, 0xafa00014,
5490         0x0a000660, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x00000000,
5491         0x00000000, 0x00000000, 0x3c020800, 0x34423000, 0x3c030800, 0x34633000,
5492         0x3c040800, 0x348437ff, 0x3c010800, 0xac221b74, 0x24020040, 0x3c010800,
5493         0xac221b78, 0x3c010800, 0xac201b70, 0xac600000, 0x24630004, 0x0083102b,
5494         0x5040fffd, 0xac600000, 0x03e00008, 0x00000000, 0x00804821, 0x8faa0010,
5495         0x3c020800, 0x8c421b70, 0x3c040800, 0x8c841b78, 0x8fab0014, 0x24430001,
5496         0x0044102b, 0x3c010800, 0xac231b70, 0x14400003, 0x00004021, 0x3c010800,
5497         0xac201b70, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74, 0x91240000,
5498         0x00021140, 0x00431021, 0x00481021, 0x25080001, 0xa0440000, 0x29020008,
5499         0x1440fff4, 0x25290001, 0x3c020800, 0x8c421b70, 0x3c030800, 0x8c631b74,
5500         0x8f64680c, 0x00021140, 0x00431021, 0xac440008, 0xac45000c, 0xac460010,
5501         0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
5502 };
5503
5504 static u32 tg3TsoFwRodata[] = {
5505         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
5506         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
5507         0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
5508         0x00000000, 0x00000000, 0x66617461, 0x6c457272, 0x00000000, 0x00000000,
5509         0x00000000,
5510 };
5511
5512 static u32 tg3TsoFwData[] = {
5513         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
5514         0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
5515         0x00000000,
5516 };
5517
5518 /* 5705 needs a special version of the TSO firmware.  */
5519 #define TG3_TSO5_FW_RELEASE_MAJOR       0x1
5520 #define TG3_TSO5_FW_RELASE_MINOR        0x2
5521 #define TG3_TSO5_FW_RELEASE_FIX         0x0
5522 #define TG3_TSO5_FW_START_ADDR          0x00010000
5523 #define TG3_TSO5_FW_TEXT_ADDR           0x00010000
5524 #define TG3_TSO5_FW_TEXT_LEN            0xe90
5525 #define TG3_TSO5_FW_RODATA_ADDR         0x00010e90
5526 #define TG3_TSO5_FW_RODATA_LEN          0x50
5527 #define TG3_TSO5_FW_DATA_ADDR           0x00010f00
5528 #define TG3_TSO5_FW_DATA_LEN            0x20
5529 #define TG3_TSO5_FW_SBSS_ADDR           0x00010f20
5530 #define TG3_TSO5_FW_SBSS_LEN            0x28
5531 #define TG3_TSO5_FW_BSS_ADDR            0x00010f50
5532 #define TG3_TSO5_FW_BSS_LEN             0x88
5533
5534 static u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
5535         0x0c004003, 0x00000000, 0x00010f04, 0x00000000, 0x10000003, 0x00000000,
5536         0x0000000d, 0x0000000d, 0x3c1d0001, 0x37bde000, 0x03a0f021, 0x3c100001,
5537         0x26100000, 0x0c004010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
5538         0xafbf0018, 0x0c0042e8, 0x34840002, 0x0c004364, 0x00000000, 0x3c030001,
5539         0x90630f34, 0x24020002, 0x3c040001, 0x24840e9c, 0x14620003, 0x24050001,
5540         0x3c040001, 0x24840e90, 0x24060002, 0x00003821, 0xafa00010, 0x0c004378,
5541         0xafa00014, 0x0c00402c, 0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5542         0x00000000, 0x00000000, 0x27bdffe0, 0xafbf001c, 0xafb20018, 0xafb10014,
5543         0x0c0042d4, 0xafb00010, 0x3c128000, 0x24110001, 0x8f706810, 0x32020400,
5544         0x10400007, 0x00000000, 0x8f641008, 0x00921024, 0x14400003, 0x00000000,
5545         0x0c004064, 0x00000000, 0x3c020001, 0x90420f56, 0x10510003, 0x32020200,
5546         0x1040fff1, 0x00000000, 0x0c0041b4, 0x00000000, 0x08004034, 0x00000000,
5547         0x8fbf001c, 0x8fb20018, 0x8fb10014, 0x8fb00010, 0x03e00008, 0x27bd0020,
5548         0x27bdffe0, 0x3c040001, 0x24840eb0, 0x00002821, 0x00003021, 0x00003821,
5549         0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0000d021, 0x24020130,
5550         0xaf625000, 0x3c010001, 0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018,
5551         0x03e00008, 0x27bd0020, 0x00000000, 0x00000000, 0x3c030001, 0x24630f60,
5552         0x90620000, 0x27bdfff0, 0x14400003, 0x0080c021, 0x08004073, 0x00004821,
5553         0x3c022000, 0x03021024, 0x10400003, 0x24090002, 0x08004073, 0xa0600000,
5554         0x24090001, 0x00181040, 0x30431f80, 0x346f8008, 0x1520004b, 0x25eb0028,
5555         0x3c040001, 0x00832021, 0x8c848010, 0x3c050001, 0x24a50f7a, 0x00041402,
5556         0xa0a20000, 0x3c010001, 0xa0240f7b, 0x3c020001, 0x00431021, 0x94428014,
5557         0x3c010001, 0xa0220f7c, 0x3c0c0001, 0x01836021, 0x8d8c8018, 0x304200ff,
5558         0x24420008, 0x000220c3, 0x24020001, 0x3c010001, 0xa0220f60, 0x0124102b,
5559         0x1040000c, 0x00003821, 0x24a6000e, 0x01602821, 0x8ca20000, 0x8ca30004,
5560         0x24a50008, 0x24e70001, 0xacc20000, 0xacc30004, 0x00e4102b, 0x1440fff8,
5561         0x24c60008, 0x00003821, 0x3c080001, 0x25080f7b, 0x91060000, 0x3c020001,
5562         0x90420f7c, 0x2503000d, 0x00c32821, 0x00461023, 0x00021fc2, 0x00431021,
5563         0x00021043, 0x1840000c, 0x00002021, 0x91020001, 0x00461023, 0x00021fc2,
5564         0x00431021, 0x00021843, 0x94a20000, 0x24e70001, 0x00822021, 0x00e3102a,
5565         0x1440fffb, 0x24a50002, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
5566         0x00822021, 0x3c02ffff, 0x01821024, 0x3083ffff, 0x00431025, 0x3c010001,
5567         0x080040fa, 0xac220f80, 0x3c050001, 0x24a50f7c, 0x90a20000, 0x3c0c0001,
5568         0x01836021, 0x8d8c8018, 0x000220c2, 0x1080000e, 0x00003821, 0x01603021,
5569         0x24a5000c, 0x8ca20000, 0x8ca30004, 0x24a50008, 0x24e70001, 0xacc20000,
5570         0xacc30004, 0x00e4102b, 0x1440fff8, 0x24c60008, 0x3c050001, 0x24a50f7c,
5571         0x90a20000, 0x30430007, 0x24020004, 0x10620011, 0x28620005, 0x10400005,
5572         0x24020002, 0x10620008, 0x000710c0, 0x080040fa, 0x00000000, 0x24020006,
5573         0x1062000e, 0x000710c0, 0x080040fa, 0x00000000, 0x00a21821, 0x9463000c,
5574         0x004b1021, 0x080040fa, 0xa4430000, 0x000710c0, 0x00a21821, 0x8c63000c,
5575         0x004b1021, 0x080040fa, 0xac430000, 0x00a21821, 0x8c63000c, 0x004b2021,
5576         0x00a21021, 0xac830000, 0x94420010, 0xa4820004, 0x95e70006, 0x3c020001,
5577         0x90420f7c, 0x3c030001, 0x90630f7a, 0x00e2c823, 0x3c020001, 0x90420f7b,
5578         0x24630028, 0x01e34021, 0x24420028, 0x15200012, 0x01e23021, 0x94c2000c,
5579         0x3c010001, 0xa4220f78, 0x94c20004, 0x94c30006, 0x3c010001, 0xa4200f76,
5580         0x3c010001, 0xa4200f72, 0x00021400, 0x00431025, 0x3c010001, 0xac220f6c,
5581         0x95020004, 0x3c010001, 0x08004124, 0xa4220f70, 0x3c020001, 0x94420f70,
5582         0x3c030001, 0x94630f72, 0x00431021, 0xa5020004, 0x3c020001, 0x94420f6c,
5583         0xa4c20004, 0x3c020001, 0x8c420f6c, 0xa4c20006, 0x3c040001, 0x94840f72,
5584         0x3c020001, 0x94420f70, 0x3c0a0001, 0x954a0f76, 0x00441821, 0x3063ffff,
5585         0x0062182a, 0x24020002, 0x1122000b, 0x00832023, 0x3c030001, 0x94630f78,
5586         0x30620009, 0x10400006, 0x3062fff6, 0xa4c2000c, 0x3c020001, 0x94420f78,
5587         0x30420009, 0x01425023, 0x24020001, 0x1122001b, 0x29220002, 0x50400005,
5588         0x24020002, 0x11200007, 0x31a2ffff, 0x08004197, 0x00000000, 0x1122001d,
5589         0x24020016, 0x08004197, 0x31a2ffff, 0x3c0e0001, 0x95ce0f80, 0x10800005,
5590         0x01806821, 0x01c42021, 0x00041c02, 0x3082ffff, 0x00627021, 0x000e1027,
5591         0xa502000a, 0x3c030001, 0x90630f7b, 0x31a2ffff, 0x00e21021, 0x0800418d,
5592         0x00432023, 0x3c020001, 0x94420f80, 0x00442021, 0x00041c02, 0x3082ffff,
5593         0x00622021, 0x00807021, 0x00041027, 0x08004185, 0xa502000a, 0x3c050001,
5594         0x24a50f7a, 0x90a30000, 0x14620002, 0x24e2fff2, 0xa5e20034, 0x90a20000,
5595         0x00e21023, 0xa5020002, 0x3c030001, 0x94630f80, 0x3c020001, 0x94420f5a,
5596         0x30e5ffff, 0x00641821, 0x00451023, 0x00622023, 0x00041c02, 0x3082ffff,
5597         0x00622021, 0x00041027, 0xa502000a, 0x3c030001, 0x90630f7c, 0x24620001,
5598         0x14a20005, 0x00807021, 0x01631021, 0x90420000, 0x08004185, 0x00026200,
5599         0x24620002, 0x14a20003, 0x306200fe, 0x004b1021, 0x944c0000, 0x3c020001,
5600         0x94420f82, 0x3183ffff, 0x3c040001, 0x90840f7b, 0x00431021, 0x00e21021,
5601         0x00442023, 0x008a2021, 0x00041c02, 0x3082ffff, 0x00622021, 0x00041402,
5602         0x00822021, 0x00806821, 0x00041027, 0xa4c20010, 0x31a2ffff, 0x000e1c00,
5603         0x00431025, 0x3c040001, 0x24840f72, 0xade20010, 0x94820000, 0x3c050001,
5604         0x94a50f76, 0x3c030001, 0x8c630f6c, 0x24420001, 0x00b92821, 0xa4820000,
5605         0x3322ffff, 0x00622021, 0x0083182b, 0x3c010001, 0xa4250f76, 0x10600003,
5606         0x24a2ffff, 0x3c010001, 0xa4220f76, 0x3c024000, 0x03021025, 0x3c010001,
5607         0xac240f6c, 0xaf621008, 0x03e00008, 0x27bd0010, 0x3c030001, 0x90630f56,
5608         0x27bdffe8, 0x24020001, 0xafbf0014, 0x10620026, 0xafb00010, 0x8f620cf4,
5609         0x2442ffff, 0x3042007f, 0x00021100, 0x8c434000, 0x3c010001, 0xac230f64,
5610         0x8c434008, 0x24444000, 0x8c5c4004, 0x30620040, 0x14400002, 0x24020088,
5611         0x24020008, 0x3c010001, 0xa4220f68, 0x30620004, 0x10400005, 0x24020001,
5612         0x3c010001, 0xa0220f57, 0x080041d5, 0x00031402, 0x3c010001, 0xa0200f57,
5613         0x00031402, 0x3c010001, 0xa4220f54, 0x9483000c, 0x24020001, 0x3c010001,
5614         0xa4200f50, 0x3c010001, 0xa0220f56, 0x3c010001, 0xa4230f62, 0x24020001,
5615         0x1342001e, 0x00000000, 0x13400005, 0x24020003, 0x13420067, 0x00000000,
5616         0x080042cf, 0x00000000, 0x3c020001, 0x94420f62, 0x241a0001, 0x3c010001,
5617         0xa4200f5e, 0x3c010001, 0xa4200f52, 0x304407ff, 0x00021bc2, 0x00031823,
5618         0x3063003e, 0x34630036, 0x00021242, 0x3042003c, 0x00621821, 0x3c010001,
5619         0xa4240f58, 0x00832021, 0x24630030, 0x3c010001, 0xa4240f5a, 0x3c010001,
5620         0xa4230f5c, 0x3c060001, 0x24c60f52, 0x94c50000, 0x94c30002, 0x3c040001,
5621         0x94840f5a, 0x00651021, 0x0044102a, 0x10400013, 0x3c108000, 0x00a31021,
5622         0xa4c20000, 0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008,
5623         0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4,
5624         0x00501024, 0x104000b7, 0x00000000, 0x0800420f, 0x00000000, 0x3c030001,
5625         0x94630f50, 0x00851023, 0xa4c40000, 0x00621821, 0x3042ffff, 0x3c010001,
5626         0xa4230f50, 0xaf620ce8, 0x3c020001, 0x94420f68, 0x34420024, 0xaf620cec,
5627         0x94c30002, 0x3c020001, 0x94420f50, 0x14620012, 0x3c028000, 0x3c108000,
5628         0x3c02a000, 0xaf620cf4, 0x3c010001, 0xa0200f56, 0x8f641008, 0x00901024,
5629         0x14400003, 0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024,
5630         0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003, 0xaf620cf4, 0x3c108000,
5631         0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064, 0x00000000,
5632         0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x080042cf, 0x241a0003,
5633         0x3c070001, 0x24e70f50, 0x94e20000, 0x03821021, 0xaf620ce0, 0x3c020001,
5634         0x8c420f64, 0xaf620ce4, 0x3c050001, 0x94a50f54, 0x94e30000, 0x3c040001,
5635         0x94840f58, 0x3c020001, 0x94420f5e, 0x00a32823, 0x00822023, 0x30a6ffff,
5636         0x3083ffff, 0x00c3102b, 0x14400043, 0x00000000, 0x3c020001, 0x94420f5c,
5637         0x00021400, 0x00621025, 0xaf620ce8, 0x94e20000, 0x3c030001, 0x94630f54,
5638         0x00441021, 0xa4e20000, 0x3042ffff, 0x14430021, 0x3c020008, 0x3c020001,
5639         0x90420f57, 0x10400006, 0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624,
5640         0x0800427c, 0x0000d021, 0x3c020001, 0x94420f68, 0x3c030008, 0x34630624,
5641         0x00431025, 0xaf620cec, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
5642         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
5643         0x00000000, 0x8f620cf4, 0x00501024, 0x10400015, 0x00000000, 0x08004283,
5644         0x00000000, 0x3c030001, 0x94630f68, 0x34420624, 0x3c108000, 0x00621825,
5645         0x3c028000, 0xaf630cec, 0xaf620cf4, 0x8f641008, 0x00901024, 0x14400003,
5646         0x00000000, 0x0c004064, 0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7,
5647         0x00000000, 0x3c010001, 0x080042cf, 0xa4200f5e, 0x3c020001, 0x94420f5c,
5648         0x00021400, 0x00c21025, 0xaf620ce8, 0x3c020001, 0x90420f57, 0x10400009,
5649         0x3c03000c, 0x3c020001, 0x94420f68, 0x34630624, 0x0000d021, 0x00431025,
5650         0xaf620cec, 0x080042c1, 0x3c108000, 0x3c020001, 0x94420f68, 0x3c030008,
5651         0x34630604, 0x00431025, 0xaf620cec, 0x3c020001, 0x94420f5e, 0x00451021,
5652         0x3c010001, 0xa4220f5e, 0x3c108000, 0x3c02a000, 0xaf620cf4, 0x3c010001,
5653         0xa0200f56, 0x8f641008, 0x00901024, 0x14400003, 0x00000000, 0x0c004064,
5654         0x00000000, 0x8f620cf4, 0x00501024, 0x1440fff7, 0x00000000, 0x8fbf0014,
5655         0x8fb00010, 0x03e00008, 0x27bd0018, 0x00000000, 0x27bdffe0, 0x3c040001,
5656         0x24840ec0, 0x00002821, 0x00003021, 0x00003821, 0xafbf0018, 0xafa00010,
5657         0x0c004378, 0xafa00014, 0x0000d021, 0x24020130, 0xaf625000, 0x3c010001,
5658         0xa4200f50, 0x3c010001, 0xa0200f57, 0x8fbf0018, 0x03e00008, 0x27bd0020,
5659         0x27bdffe8, 0x3c1bc000, 0xafbf0014, 0xafb00010, 0xaf60680c, 0x8f626804,
5660         0x34420082, 0xaf626804, 0x8f634000, 0x24020b50, 0x3c010001, 0xac220f20,
5661         0x24020b78, 0x3c010001, 0xac220f30, 0x34630002, 0xaf634000, 0x0c004315,
5662         0x00808021, 0x3c010001, 0xa0220f34, 0x304200ff, 0x24030002, 0x14430005,
5663         0x00000000, 0x3c020001, 0x8c420f20, 0x08004308, 0xac5000c0, 0x3c020001,
5664         0x8c420f20, 0xac5000bc, 0x8f624434, 0x8f634438, 0x8f644410, 0x3c010001,
5665         0xac220f28, 0x3c010001, 0xac230f38, 0x3c010001, 0xac240f24, 0x8fbf0014,
5666         0x8fb00010, 0x03e00008, 0x27bd0018, 0x03e00008, 0x24020001, 0x27bdfff8,
5667         0x18800009, 0x00002821, 0x8f63680c, 0x8f62680c, 0x1043fffe, 0x00000000,
5668         0x24a50001, 0x00a4102a, 0x1440fff9, 0x00000000, 0x03e00008, 0x27bd0008,
5669         0x8f634450, 0x3c020001, 0x8c420f28, 0x00031c02, 0x0043102b, 0x14400008,
5670         0x3c038000, 0x3c040001, 0x8c840f38, 0x8f624450, 0x00021c02, 0x0083102b,
5671         0x1040fffc, 0x3c038000, 0xaf634444, 0x8f624444, 0x00431024, 0x1440fffd,
5672         0x00000000, 0x8f624448, 0x03e00008, 0x3042ffff, 0x3082ffff, 0x2442e000,
5673         0x2c422001, 0x14400003, 0x3c024000, 0x08004347, 0x2402ffff, 0x00822025,
5674         0xaf645c38, 0x8f625c30, 0x30420002, 0x1440fffc, 0x00001021, 0x03e00008,
5675         0x00000000, 0x8f624450, 0x3c030001, 0x8c630f24, 0x08004350, 0x3042ffff,
5676         0x8f624450, 0x3042ffff, 0x0043102b, 0x1440fffc, 0x00000000, 0x03e00008,
5677         0x00000000, 0x27bdffe0, 0x00802821, 0x3c040001, 0x24840ed0, 0x00003021,
5678         0x00003821, 0xafbf0018, 0xafa00010, 0x0c004378, 0xafa00014, 0x0800435f,
5679         0x00000000, 0x8fbf0018, 0x03e00008, 0x27bd0020, 0x3c020001, 0x3442d600,
5680         0x3c030001, 0x3463d600, 0x3c040001, 0x3484ddff, 0x3c010001, 0xac220f40,
5681         0x24020040, 0x3c010001, 0xac220f44, 0x3c010001, 0xac200f3c, 0xac600000,
5682         0x24630004, 0x0083102b, 0x5040fffd, 0xac600000, 0x03e00008, 0x00000000,
5683         0x00804821, 0x8faa0010, 0x3c020001, 0x8c420f3c, 0x3c040001, 0x8c840f44,
5684         0x8fab0014, 0x24430001, 0x0044102b, 0x3c010001, 0xac230f3c, 0x14400003,
5685         0x00004021, 0x3c010001, 0xac200f3c, 0x3c020001, 0x8c420f3c, 0x3c030001,
5686         0x8c630f40, 0x91240000, 0x00021140, 0x00431021, 0x00481021, 0x25080001,
5687         0xa0440000, 0x29020008, 0x1440fff4, 0x25290001, 0x3c020001, 0x8c420f3c,
5688         0x3c030001, 0x8c630f40, 0x8f64680c, 0x00021140, 0x00431021, 0xac440008,
5689         0xac45000c, 0xac460010, 0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c,
5690         0x00000000, 0x00000000, 0x00000000,
5691 };
5692
5693 static u32 tg3Tso5FwRodata[(TG3_TSO5_FW_RODATA_LEN / 4) + 1] = {
5694         0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
5695         0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x00000000, 0x00000000,
5696         0x73746b6f, 0x66666c64, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
5697         0x00000000, 0x00000000, 0x00000000,
5698 };
5699
5700 static u32 tg3Tso5FwData[(TG3_TSO5_FW_DATA_LEN / 4) + 1] = {
5701         0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x322e3000, 0x00000000,
5702         0x00000000, 0x00000000, 0x00000000,
5703 };
5704
5705 /* tp->lock is held. */
5706 static int tg3_load_tso_firmware(struct tg3 *tp)
5707 {
5708         struct fw_info info;
5709         unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
5710         int err, i;
5711
5712         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
5713                 return 0;
5714
5715         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
5716                 info.text_base = TG3_TSO5_FW_TEXT_ADDR;
5717                 info.text_len = TG3_TSO5_FW_TEXT_LEN;
5718                 info.text_data = &tg3Tso5FwText[0];
5719                 info.rodata_base = TG3_TSO5_FW_RODATA_ADDR;
5720                 info.rodata_len = TG3_TSO5_FW_RODATA_LEN;
5721                 info.rodata_data = &tg3Tso5FwRodata[0];
5722                 info.data_base = TG3_TSO5_FW_DATA_ADDR;
5723                 info.data_len = TG3_TSO5_FW_DATA_LEN;
5724                 info.data_data = &tg3Tso5FwData[0];
5725                 cpu_base = RX_CPU_BASE;
5726                 cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
5727                 cpu_scratch_size = (info.text_len +
5728                                     info.rodata_len +
5729                                     info.data_len +
5730                                     TG3_TSO5_FW_SBSS_LEN +
5731                                     TG3_TSO5_FW_BSS_LEN);
5732         } else {
5733                 info.text_base = TG3_TSO_FW_TEXT_ADDR;
5734                 info.text_len = TG3_TSO_FW_TEXT_LEN;
5735                 info.text_data = &tg3TsoFwText[0];
5736                 info.rodata_base = TG3_TSO_FW_RODATA_ADDR;
5737                 info.rodata_len = TG3_TSO_FW_RODATA_LEN;
5738                 info.rodata_data = &tg3TsoFwRodata[0];
5739                 info.data_base = TG3_TSO_FW_DATA_ADDR;
5740                 info.data_len = TG3_TSO_FW_DATA_LEN;
5741                 info.data_data = &tg3TsoFwData[0];
5742                 cpu_base = TX_CPU_BASE;
5743                 cpu_scratch_base = TX_CPU_SCRATCH_BASE;
5744                 cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
5745         }
5746
5747         err = tg3_load_firmware_cpu(tp, cpu_base,
5748                                     cpu_scratch_base, cpu_scratch_size,
5749                                     &info);
5750         if (err)
5751                 return err;
5752
5753         /* Now startup the cpu. */
5754         tw32(cpu_base + CPU_STATE, 0xffffffff);
5755         tw32_f(cpu_base + CPU_PC,    info.text_base);
5756
5757         for (i = 0; i < 5; i++) {
5758                 if (tr32(cpu_base + CPU_PC) == info.text_base)
5759                         break;
5760                 tw32(cpu_base + CPU_STATE, 0xffffffff);
5761                 tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
5762                 tw32_f(cpu_base + CPU_PC,    info.text_base);
5763                 udelay(1000);
5764         }
5765         if (i >= 5) {
5766                 printk(KERN_ERR PFX "tg3_load_tso_firmware fails for %s "
5767                        "to set CPU PC, is %08x should be %08x\n",
5768                        tp->dev->name, tr32(cpu_base + CPU_PC),
5769                        info.text_base);
5770                 return -ENODEV;
5771         }
5772         tw32(cpu_base + CPU_STATE, 0xffffffff);
5773         tw32_f(cpu_base + CPU_MODE,  0x00000000);
5774         return 0;
5775 }
5776
5777 #endif /* TG3_TSO_SUPPORT != 0 */
5778
5779 /* tp->lock is held. */
5780 static void __tg3_set_mac_addr(struct tg3 *tp)
5781 {
5782         u32 addr_high, addr_low;
5783         int i;
5784
5785         addr_high = ((tp->dev->dev_addr[0] << 8) |
5786                      tp->dev->dev_addr[1]);
5787         addr_low = ((tp->dev->dev_addr[2] << 24) |
5788                     (tp->dev->dev_addr[3] << 16) |
5789                     (tp->dev->dev_addr[4] <<  8) |
5790                     (tp->dev->dev_addr[5] <<  0));
5791         for (i = 0; i < 4; i++) {
5792                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
5793                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
5794         }
5795
5796         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
5797             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
5798                 for (i = 0; i < 12; i++) {
5799                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
5800                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
5801                 }
5802         }
5803
5804         addr_high = (tp->dev->dev_addr[0] +
5805                      tp->dev->dev_addr[1] +
5806                      tp->dev->dev_addr[2] +
5807                      tp->dev->dev_addr[3] +
5808                      tp->dev->dev_addr[4] +
5809                      tp->dev->dev_addr[5]) &
5810                 TX_BACKOFF_SEED_MASK;
5811         tw32(MAC_TX_BACKOFF_SEED, addr_high);
5812 }
5813
5814 static int tg3_set_mac_addr(struct net_device *dev, void *p)
5815 {
5816         struct tg3 *tp = netdev_priv(dev);
5817         struct sockaddr *addr = p;
5818
5819         if (!is_valid_ether_addr(addr->sa_data))
5820                 return -EINVAL;
5821
5822         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
5823
5824         if (!netif_running(dev))
5825                 return 0;
5826
5827         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
5828                 /* Reset chip so that ASF can re-init any MAC addresses it
5829                  * needs.
5830                  */
5831                 tg3_netif_stop(tp);
5832                 tg3_full_lock(tp, 1);
5833
5834                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
5835                 tg3_init_hw(tp, 0);
5836
5837                 tg3_netif_start(tp);
5838                 tg3_full_unlock(tp);
5839         } else {
5840                 spin_lock_bh(&tp->lock);
5841                 __tg3_set_mac_addr(tp);
5842                 spin_unlock_bh(&tp->lock);
5843         }
5844
5845         return 0;
5846 }
5847
5848 /* tp->lock is held. */
5849 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
5850                            dma_addr_t mapping, u32 maxlen_flags,
5851                            u32 nic_addr)
5852 {
5853         tg3_write_mem(tp,
5854                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
5855                       ((u64) mapping >> 32));
5856         tg3_write_mem(tp,
5857                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
5858                       ((u64) mapping & 0xffffffff));
5859         tg3_write_mem(tp,
5860                       (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
5861                        maxlen_flags);
5862
5863         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
5864                 tg3_write_mem(tp,
5865                               (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
5866                               nic_addr);
5867 }
5868
5869 static void __tg3_set_rx_mode(struct net_device *);
5870 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
5871 {
5872         tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
5873         tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
5874         tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
5875         tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
5876         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5877                 tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
5878                 tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
5879         }
5880         tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
5881         tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
5882         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
5883                 u32 val = ec->stats_block_coalesce_usecs;
5884
5885                 if (!netif_carrier_ok(tp->dev))
5886                         val = 0;
5887
5888                 tw32(HOSTCC_STAT_COAL_TICKS, val);
5889         }
5890 }
5891
5892 /* tp->lock is held. */
5893 static int tg3_reset_hw(struct tg3 *tp, int reset_phy)
5894 {
5895         u32 val, rdmac_mode;
5896         int i, err, limit;
5897
5898         tg3_disable_ints(tp);
5899
5900         tg3_stop_fw(tp);
5901
5902         tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
5903
5904         if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
5905                 tg3_abort_hw(tp, 1);
5906         }
5907
5908         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) && reset_phy)
5909                 tg3_phy_reset(tp);
5910
5911         err = tg3_chip_reset(tp);
5912         if (err)
5913                 return err;
5914
5915         tg3_write_sig_legacy(tp, RESET_KIND_INIT);
5916
5917         /* This works around an issue with Athlon chipsets on
5918          * B3 tigon3 silicon.  This bit has no effect on any
5919          * other revision.  But do not set this on PCI Express
5920          * chips.
5921          */
5922         if (!(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
5923                 tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
5924         tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
5925
5926         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
5927             (tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
5928                 val = tr32(TG3PCI_PCISTATE);
5929                 val |= PCISTATE_RETRY_SAME_DMA;
5930                 tw32(TG3PCI_PCISTATE, val);
5931         }
5932
5933         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_BX) {
5934                 /* Enable some hw fixes.  */
5935                 val = tr32(TG3PCI_MSI_DATA);
5936                 val |= (1 << 26) | (1 << 28) | (1 << 29);
5937                 tw32(TG3PCI_MSI_DATA, val);
5938         }
5939
5940         /* Descriptor ring init may make accesses to the
5941          * NIC SRAM area to setup the TX descriptors, so we
5942          * can only do this after the hardware has been
5943          * successfully reset.
5944          */
5945         tg3_init_rings(tp);
5946
5947         /* This value is determined during the probe time DMA
5948          * engine test, tg3_test_dma.
5949          */
5950         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
5951
5952         tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
5953                           GRC_MODE_4X_NIC_SEND_RINGS |
5954                           GRC_MODE_NO_TX_PHDR_CSUM |
5955                           GRC_MODE_NO_RX_PHDR_CSUM);
5956         tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
5957
5958         /* Pseudo-header checksum is done by hardware logic and not
5959          * the offload processers, so make the chip do the pseudo-
5960          * header checksums on receive.  For transmit it is more
5961          * convenient to do the pseudo-header checksum in software
5962          * as Linux does that on transmit for us in all cases.
5963          */
5964         tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
5965
5966         tw32(GRC_MODE,
5967              tp->grc_mode |
5968              (GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP));
5969
5970         /* Setup the timer prescalar register.  Clock is always 66Mhz. */
5971         val = tr32(GRC_MISC_CFG);
5972         val &= ~0xff;
5973         val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
5974         tw32(GRC_MISC_CFG, val);
5975
5976         /* Initialize MBUF/DESC pool. */
5977         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
5978                 /* Do nothing.  */
5979         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705) {
5980                 tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
5981                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
5982                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
5983                 else
5984                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
5985                 tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
5986                 tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
5987         }
5988 #if TG3_TSO_SUPPORT != 0
5989         else if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
5990                 int fw_len;
5991
5992                 fw_len = (TG3_TSO5_FW_TEXT_LEN +
5993                           TG3_TSO5_FW_RODATA_LEN +
5994                           TG3_TSO5_FW_DATA_LEN +
5995                           TG3_TSO5_FW_SBSS_LEN +
5996                           TG3_TSO5_FW_BSS_LEN);
5997                 fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
5998                 tw32(BUFMGR_MB_POOL_ADDR,
5999                      NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
6000                 tw32(BUFMGR_MB_POOL_SIZE,
6001                      NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
6002         }
6003 #endif
6004
6005         if (tp->dev->mtu <= ETH_DATA_LEN) {
6006                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6007                      tp->bufmgr_config.mbuf_read_dma_low_water);
6008                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6009                      tp->bufmgr_config.mbuf_mac_rx_low_water);
6010                 tw32(BUFMGR_MB_HIGH_WATER,
6011                      tp->bufmgr_config.mbuf_high_water);
6012         } else {
6013                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
6014                      tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
6015                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
6016                      tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
6017                 tw32(BUFMGR_MB_HIGH_WATER,
6018                      tp->bufmgr_config.mbuf_high_water_jumbo);
6019         }
6020         tw32(BUFMGR_DMA_LOW_WATER,
6021              tp->bufmgr_config.dma_low_water);
6022         tw32(BUFMGR_DMA_HIGH_WATER,
6023              tp->bufmgr_config.dma_high_water);
6024
6025         tw32(BUFMGR_MODE, BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE);
6026         for (i = 0; i < 2000; i++) {
6027                 if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
6028                         break;
6029                 udelay(10);
6030         }
6031         if (i >= 2000) {
6032                 printk(KERN_ERR PFX "tg3_reset_hw cannot enable BUFMGR for %s.\n",
6033                        tp->dev->name);
6034                 return -ENODEV;
6035         }
6036
6037         /* Setup replenish threshold. */
6038         val = tp->rx_pending / 8;
6039         if (val == 0)
6040                 val = 1;
6041         else if (val > tp->rx_std_max_post)
6042                 val = tp->rx_std_max_post;
6043
6044         tw32(RCVBDI_STD_THRESH, val);
6045
6046         /* Initialize TG3_BDINFO's at:
6047          *  RCVDBDI_STD_BD:     standard eth size rx ring
6048          *  RCVDBDI_JUMBO_BD:   jumbo frame rx ring
6049          *  RCVDBDI_MINI_BD:    small frame rx ring (??? does not work)
6050          *
6051          * like so:
6052          *  TG3_BDINFO_HOST_ADDR:       high/low parts of DMA address of ring
6053          *  TG3_BDINFO_MAXLEN_FLAGS:    (rx max buffer size << 16) |
6054          *                              ring attribute flags
6055          *  TG3_BDINFO_NIC_ADDR:        location of descriptors in nic SRAM
6056          *
6057          * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
6058          * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
6059          *
6060          * The size of each ring is fixed in the firmware, but the location is
6061          * configurable.
6062          */
6063         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6064              ((u64) tp->rx_std_mapping >> 32));
6065         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6066              ((u64) tp->rx_std_mapping & 0xffffffff));
6067         tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
6068              NIC_SRAM_RX_BUFFER_DESC);
6069
6070         /* Don't even try to program the JUMBO/MINI buffer descriptor
6071          * configs on 5705.
6072          */
6073         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
6074                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6075                      RX_STD_MAX_SIZE_5705 << BDINFO_FLAGS_MAXLEN_SHIFT);
6076         } else {
6077                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS,
6078                      RX_STD_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6079
6080                 tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
6081                      BDINFO_FLAGS_DISABLED);
6082
6083                 /* Setup replenish threshold. */
6084                 tw32(RCVBDI_JUMBO_THRESH, tp->rx_jumbo_pending / 8);
6085
6086                 if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
6087                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
6088                              ((u64) tp->rx_jumbo_mapping >> 32));
6089                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
6090                              ((u64) tp->rx_jumbo_mapping & 0xffffffff));
6091                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6092                              RX_JUMBO_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT);
6093                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
6094                              NIC_SRAM_RX_JUMBO_BUFFER_DESC);
6095                 } else {
6096                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
6097                              BDINFO_FLAGS_DISABLED);
6098                 }
6099
6100         }
6101
6102         /* There is only one send ring on 5705/5750, no need to explicitly
6103          * disable the others.
6104          */
6105         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6106                 /* Clear out send RCB ring in SRAM. */
6107                 for (i = NIC_SRAM_SEND_RCB; i < NIC_SRAM_RCV_RET_RCB; i += TG3_BDINFO_SIZE)
6108                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6109                                       BDINFO_FLAGS_DISABLED);
6110         }
6111
6112         tp->tx_prod = 0;
6113         tp->tx_cons = 0;
6114         tw32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6115         tw32_tx_mbox(MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW, 0);
6116
6117         tg3_set_bdinfo(tp, NIC_SRAM_SEND_RCB,
6118                        tp->tx_desc_mapping,
6119                        (TG3_TX_RING_SIZE <<
6120                         BDINFO_FLAGS_MAXLEN_SHIFT),
6121                        NIC_SRAM_TX_BUFFER_DESC);
6122
6123         /* There is only one receive return ring on 5705/5750, no need
6124          * to explicitly disable the others.
6125          */
6126         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6127                 for (i = NIC_SRAM_RCV_RET_RCB; i < NIC_SRAM_STATS_BLK;
6128                      i += TG3_BDINFO_SIZE) {
6129                         tg3_write_mem(tp, i + TG3_BDINFO_MAXLEN_FLAGS,
6130                                       BDINFO_FLAGS_DISABLED);
6131                 }
6132         }
6133
6134         tp->rx_rcb_ptr = 0;
6135         tw32_rx_mbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW, 0);
6136
6137         tg3_set_bdinfo(tp, NIC_SRAM_RCV_RET_RCB,
6138                        tp->rx_rcb_mapping,
6139                        (TG3_RX_RCB_RING_SIZE(tp) <<
6140                         BDINFO_FLAGS_MAXLEN_SHIFT),
6141                        0);
6142
6143         tp->rx_std_ptr = tp->rx_pending;
6144         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
6145                      tp->rx_std_ptr);
6146
6147         tp->rx_jumbo_ptr = (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) ?
6148                                                 tp->rx_jumbo_pending : 0;
6149         tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
6150                      tp->rx_jumbo_ptr);
6151
6152         /* Initialize MAC address and backoff seed. */
6153         __tg3_set_mac_addr(tp);
6154
6155         /* MTU + ethernet header + FCS + optional VLAN tag */
6156         tw32(MAC_RX_MTU_SIZE, tp->dev->mtu + ETH_HLEN + 8);
6157
6158         /* The slot time is changed by tg3_setup_phy if we
6159          * run at gigabit with half duplex.
6160          */
6161         tw32(MAC_TX_LENGTHS,
6162              (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
6163              (6 << TX_LENGTHS_IPG_SHIFT) |
6164              (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
6165
6166         /* Receive rules. */
6167         tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
6168         tw32(RCVLPC_CONFIG, 0x0181);
6169
6170         /* Calculate RDMAC_MODE setting early, we need it to determine
6171          * the RCVLPC_STATE_ENABLE mask.
6172          */
6173         rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
6174                       RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
6175                       RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
6176                       RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
6177                       RDMAC_MODE_LNGREAD_ENAB);
6178         if (tp->tg3_flags & TG3_FLAG_SPLIT_MODE)
6179                 rdmac_mode |= RDMAC_MODE_SPLIT_ENABLE;
6180
6181         /* If statement applies to 5705 and 5750 PCI devices only */
6182         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6183              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6184             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)) {
6185                 if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE &&
6186                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
6187                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
6188                         rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
6189                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6190                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788)) {
6191                         rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6192                 }
6193         }
6194
6195         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
6196                 rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
6197
6198 #if TG3_TSO_SUPPORT != 0
6199         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6200                 rdmac_mode |= (1 << 27);
6201 #endif
6202
6203         /* Receive/send statistics. */
6204         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
6205                 val = tr32(RCVLPC_STATS_ENABLE);
6206                 val &= ~RCVLPC_STATSENAB_DACK_FIX;
6207                 tw32(RCVLPC_STATS_ENABLE, val);
6208         } else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
6209                    (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
6210                 val = tr32(RCVLPC_STATS_ENABLE);
6211                 val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
6212                 tw32(RCVLPC_STATS_ENABLE, val);
6213         } else {
6214                 tw32(RCVLPC_STATS_ENABLE, 0xffffff);
6215         }
6216         tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
6217         tw32(SNDDATAI_STATSENAB, 0xffffff);
6218         tw32(SNDDATAI_STATSCTRL,
6219              (SNDDATAI_SCTRL_ENABLE |
6220               SNDDATAI_SCTRL_FASTUPD));
6221
6222         /* Setup host coalescing engine. */
6223         tw32(HOSTCC_MODE, 0);
6224         for (i = 0; i < 2000; i++) {
6225                 if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
6226                         break;
6227                 udelay(10);
6228         }
6229
6230         __tg3_set_coalesce(tp, &tp->coal);
6231
6232         /* set status block DMA address */
6233         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6234              ((u64) tp->status_mapping >> 32));
6235         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6236              ((u64) tp->status_mapping & 0xffffffff));
6237
6238         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6239                 /* Status/statistics block address.  See tg3_timer,
6240                  * the tg3_periodic_fetch_stats call there, and
6241                  * tg3_get_stats to see how this works for 5705/5750 chips.
6242                  */
6243                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
6244                      ((u64) tp->stats_mapping >> 32));
6245                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
6246                      ((u64) tp->stats_mapping & 0xffffffff));
6247                 tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
6248                 tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
6249         }
6250
6251         tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
6252
6253         tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
6254         tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
6255         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6256                 tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
6257
6258         /* Clear statistics/status block in chip, and status block in ram. */
6259         for (i = NIC_SRAM_STATS_BLK;
6260              i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
6261              i += sizeof(u32)) {
6262                 tg3_write_mem(tp, i, 0);
6263                 udelay(40);
6264         }
6265         memset(tp->hw_status, 0, TG3_HW_STATUS_SIZE);
6266
6267         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
6268                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
6269                 /* reset to prevent losing 1st rx packet intermittently */
6270                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6271                 udelay(10);
6272         }
6273
6274         tp->mac_mode = MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
6275                 MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE | MAC_MODE_FHDE_ENABLE;
6276         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
6277         udelay(40);
6278
6279         /* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
6280          * If TG3_FLAG_EEPROM_WRITE_PROT is set, we should read the
6281          * register to preserve the GPIO settings for LOMs. The GPIOs,
6282          * whether used as inputs or outputs, are set by boot code after
6283          * reset.
6284          */
6285         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
6286                 u32 gpio_mask;
6287
6288                 gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE2 |
6289                             GRC_LCLCTRL_GPIO_OUTPUT0 | GRC_LCLCTRL_GPIO_OUTPUT2;
6290
6291                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
6292                         gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
6293                                      GRC_LCLCTRL_GPIO_OUTPUT3;
6294
6295                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
6296                         gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
6297
6298                 tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
6299
6300                 /* GPIO1 must be driven high for eeprom write protect */
6301                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
6302                                        GRC_LCLCTRL_GPIO_OUTPUT1);
6303         }
6304         tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6305         udelay(100);
6306
6307         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0);
6308         tp->last_tag = 0;
6309
6310         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6311                 tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
6312                 udelay(40);
6313         }
6314
6315         val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
6316                WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
6317                WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
6318                WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
6319                WDMAC_MODE_LNGREAD_ENAB);
6320
6321         /* If statement applies to 5705 and 5750 PCI devices only */
6322         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
6323              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
6324             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
6325                 if ((tp->tg3_flags & TG3_FLG2_TSO_CAPABLE) &&
6326                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
6327                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
6328                         /* nothing */
6329                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
6330                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
6331                            !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
6332                         val |= WDMAC_MODE_RX_ACCEL;
6333                 }
6334         }
6335
6336         /* Enable host coalescing bug fix */
6337         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755) ||
6338             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787))
6339                 val |= (1 << 29);
6340
6341         tw32_f(WDMAC_MODE, val);
6342         udelay(40);
6343
6344         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0) {
6345                 val = tr32(TG3PCI_X_CAPS);
6346                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703) {
6347                         val &= ~PCIX_CAPS_BURST_MASK;
6348                         val |= (PCIX_CAPS_MAX_BURST_CPIOB << PCIX_CAPS_BURST_SHIFT);
6349                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
6350                         val &= ~(PCIX_CAPS_SPLIT_MASK | PCIX_CAPS_BURST_MASK);
6351                         val |= (PCIX_CAPS_MAX_BURST_CPIOB << PCIX_CAPS_BURST_SHIFT);
6352                         if (tp->tg3_flags & TG3_FLAG_SPLIT_MODE)
6353                                 val |= (tp->split_mode_max_reqs <<
6354                                         PCIX_CAPS_SPLIT_SHIFT);
6355                 }
6356                 tw32(TG3PCI_X_CAPS, val);
6357         }
6358
6359         tw32_f(RDMAC_MODE, rdmac_mode);
6360         udelay(40);
6361
6362         tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
6363         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6364                 tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
6365         tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
6366         tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
6367         tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
6368         tw32(RCVDBDI_MODE, RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ);
6369         tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
6370 #if TG3_TSO_SUPPORT != 0
6371         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6372                 tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
6373 #endif
6374         tw32(SNDBDI_MODE, SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE);
6375         tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
6376
6377         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
6378                 err = tg3_load_5701_a0_firmware_fix(tp);
6379                 if (err)
6380                         return err;
6381         }
6382
6383 #if TG3_TSO_SUPPORT != 0
6384         if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
6385                 err = tg3_load_tso_firmware(tp);
6386                 if (err)
6387                         return err;
6388         }
6389 #endif
6390
6391         tp->tx_mode = TX_MODE_ENABLE;
6392         tw32_f(MAC_TX_MODE, tp->tx_mode);
6393         udelay(100);
6394
6395         tp->rx_mode = RX_MODE_ENABLE;
6396         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
6397                 tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
6398
6399         tw32_f(MAC_RX_MODE, tp->rx_mode);
6400         udelay(10);
6401
6402         if (tp->link_config.phy_is_low_power) {
6403                 tp->link_config.phy_is_low_power = 0;
6404                 tp->link_config.speed = tp->link_config.orig_speed;
6405                 tp->link_config.duplex = tp->link_config.orig_duplex;
6406                 tp->link_config.autoneg = tp->link_config.orig_autoneg;
6407         }
6408
6409         tp->mi_mode = MAC_MI_MODE_BASE;
6410         tw32_f(MAC_MI_MODE, tp->mi_mode);
6411         udelay(80);
6412
6413         tw32(MAC_LED_CTRL, tp->led_ctrl);
6414
6415         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
6416         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
6417                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
6418                 udelay(10);
6419         }
6420         tw32_f(MAC_RX_MODE, tp->rx_mode);
6421         udelay(10);
6422
6423         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
6424                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
6425                         !(tp->tg3_flags2 & TG3_FLG2_SERDES_PREEMPHASIS)) {
6426                         /* Set drive transmission level to 1.2V  */
6427                         /* only if the signal pre-emphasis bit is not set  */
6428                         val = tr32(MAC_SERDES_CFG);
6429                         val &= 0xfffff000;
6430                         val |= 0x880;
6431                         tw32(MAC_SERDES_CFG, val);
6432                 }
6433                 if (tp->pci_chip_rev_id == CHIPREV_ID_5703_A1)
6434                         tw32(MAC_SERDES_CFG, 0x616000);
6435         }
6436
6437         /* Prevent chip from dropping frames when flow control
6438          * is enabled.
6439          */
6440         tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, 2);
6441
6442         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
6443             (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
6444                 /* Use hardware link auto-negotiation */
6445                 tp->tg3_flags2 |= TG3_FLG2_HW_AUTONEG;
6446         }
6447
6448         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) &&
6449             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
6450                 u32 tmp;
6451
6452                 tmp = tr32(SERDES_RX_CTRL);
6453                 tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
6454                 tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
6455                 tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
6456                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
6457         }
6458
6459         err = tg3_setup_phy(tp, reset_phy);
6460         if (err)
6461                 return err;
6462
6463         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
6464                 u32 tmp;
6465
6466                 /* Clear CRC stats. */
6467                 if (!tg3_readphy(tp, 0x1e, &tmp)) {
6468                         tg3_writephy(tp, 0x1e, tmp | 0x8000);
6469                         tg3_readphy(tp, 0x14, &tmp);
6470                 }
6471         }
6472
6473         __tg3_set_rx_mode(tp->dev);
6474
6475         /* Initialize receive rules. */
6476         tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
6477         tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
6478         tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
6479         tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
6480
6481         if ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
6482             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
6483                 limit = 8;
6484         else
6485                 limit = 16;
6486         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF)
6487                 limit -= 4;
6488         switch (limit) {
6489         case 16:
6490                 tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
6491         case 15:
6492                 tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
6493         case 14:
6494                 tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
6495         case 13:
6496                 tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
6497         case 12:
6498                 tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
6499         case 11:
6500                 tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
6501         case 10:
6502                 tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
6503         case 9:
6504                 tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
6505         case 8:
6506                 tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
6507         case 7:
6508                 tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
6509         case 6:
6510                 tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
6511         case 5:
6512                 tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
6513         case 4:
6514                 /* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
6515         case 3:
6516                 /* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
6517         case 2:
6518         case 1:
6519
6520         default:
6521                 break;
6522         };
6523
6524         tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
6525
6526         return 0;
6527 }
6528
6529 /* Called at device open time to get the chip ready for
6530  * packet processing.  Invoked with tp->lock held.
6531  */
6532 static int tg3_init_hw(struct tg3 *tp, int reset_phy)
6533 {
6534         int err;
6535
6536         /* Force the chip into D0. */
6537         err = tg3_set_power_state(tp, PCI_D0);
6538         if (err)
6539                 goto out;
6540
6541         tg3_switch_clocks(tp);
6542
6543         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
6544
6545         err = tg3_reset_hw(tp, reset_phy);
6546
6547 out:
6548         return err;
6549 }
6550
6551 #define TG3_STAT_ADD32(PSTAT, REG) \
6552 do {    u32 __val = tr32(REG); \
6553         (PSTAT)->low += __val; \
6554         if ((PSTAT)->low < __val) \
6555                 (PSTAT)->high += 1; \
6556 } while (0)
6557
6558 static void tg3_periodic_fetch_stats(struct tg3 *tp)
6559 {
6560         struct tg3_hw_stats *sp = tp->hw_stats;
6561
6562         if (!netif_carrier_ok(tp->dev))
6563                 return;
6564
6565         TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
6566         TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
6567         TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
6568         TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
6569         TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
6570         TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
6571         TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
6572         TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
6573         TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
6574         TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
6575         TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
6576         TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
6577         TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
6578
6579         TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
6580         TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
6581         TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
6582         TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
6583         TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
6584         TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
6585         TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
6586         TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
6587         TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
6588         TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
6589         TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
6590         TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
6591         TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
6592         TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
6593
6594         TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
6595         TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
6596         TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
6597 }
6598
6599 static void tg3_timer(unsigned long __opaque)
6600 {
6601         struct tg3 *tp = (struct tg3 *) __opaque;
6602
6603         if (tp->irq_sync)
6604                 goto restart_timer;
6605
6606         spin_lock(&tp->lock);
6607
6608         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
6609                 /* All of this garbage is because when using non-tagged
6610                  * IRQ status the mailbox/status_block protocol the chip
6611                  * uses with the cpu is race prone.
6612                  */
6613                 if (tp->hw_status->status & SD_STATUS_UPDATED) {
6614                         tw32(GRC_LOCAL_CTRL,
6615                              tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
6616                 } else {
6617                         tw32(HOSTCC_MODE, tp->coalesce_mode |
6618                              (HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW));
6619                 }
6620
6621                 if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
6622                         tp->tg3_flags2 |= TG3_FLG2_RESTART_TIMER;
6623                         spin_unlock(&tp->lock);
6624                         schedule_work(&tp->reset_task);
6625                         return;
6626                 }
6627         }
6628
6629         /* This part only runs once per second. */
6630         if (!--tp->timer_counter) {
6631                 if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
6632                         tg3_periodic_fetch_stats(tp);
6633
6634                 if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
6635                         u32 mac_stat;
6636                         int phy_event;
6637
6638                         mac_stat = tr32(MAC_STATUS);
6639
6640                         phy_event = 0;
6641                         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) {
6642                                 if (mac_stat & MAC_STATUS_MI_INTERRUPT)
6643                                         phy_event = 1;
6644                         } else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
6645                                 phy_event = 1;
6646
6647                         if (phy_event)
6648                                 tg3_setup_phy(tp, 0);
6649                 } else if (tp->tg3_flags & TG3_FLAG_POLL_SERDES) {
6650                         u32 mac_stat = tr32(MAC_STATUS);
6651                         int need_setup = 0;
6652
6653                         if (netif_carrier_ok(tp->dev) &&
6654                             (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
6655                                 need_setup = 1;
6656                         }
6657                         if (! netif_carrier_ok(tp->dev) &&
6658                             (mac_stat & (MAC_STATUS_PCS_SYNCED |
6659                                          MAC_STATUS_SIGNAL_DET))) {
6660                                 need_setup = 1;
6661                         }
6662                         if (need_setup) {
6663                                 tw32_f(MAC_MODE,
6664                                      (tp->mac_mode &
6665                                       ~MAC_MODE_PORT_MODE_MASK));
6666                                 udelay(40);
6667                                 tw32_f(MAC_MODE, tp->mac_mode);
6668                                 udelay(40);
6669                                 tg3_setup_phy(tp, 0);
6670                         }
6671                 } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
6672                         tg3_serdes_parallel_detect(tp);
6673
6674                 tp->timer_counter = tp->timer_multiplier;
6675         }
6676
6677         /* Heartbeat is only sent once every 2 seconds.  */
6678         if (!--tp->asf_counter) {
6679                 if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
6680                         u32 val;
6681
6682                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
6683                                       FWCMD_NICDRV_ALIVE2);
6684                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
6685                         /* 5 seconds timeout */
6686                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, 5);
6687                         val = tr32(GRC_RX_CPU_EVENT);
6688                         val |= (1 << 14);
6689                         tw32(GRC_RX_CPU_EVENT, val);
6690                 }
6691                 tp->asf_counter = tp->asf_multiplier;
6692         }
6693
6694         spin_unlock(&tp->lock);
6695
6696 restart_timer:
6697         tp->timer.expires = jiffies + tp->timer_offset;
6698         add_timer(&tp->timer);
6699 }
6700
6701 static int tg3_request_irq(struct tg3 *tp)
6702 {
6703         irqreturn_t (*fn)(int, void *, struct pt_regs *);
6704         unsigned long flags;
6705         struct net_device *dev = tp->dev;
6706
6707         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6708                 fn = tg3_msi;
6709                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
6710                         fn = tg3_msi_1shot;
6711                 flags = IRQF_SAMPLE_RANDOM;
6712         } else {
6713                 fn = tg3_interrupt;
6714                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
6715                         fn = tg3_interrupt_tagged;
6716                 flags = IRQF_SHARED | IRQF_SAMPLE_RANDOM;
6717         }
6718         return (request_irq(tp->pdev->irq, fn, flags, dev->name, dev));
6719 }
6720
6721 static int tg3_test_interrupt(struct tg3 *tp)
6722 {
6723         struct net_device *dev = tp->dev;
6724         int err, i;
6725         u32 int_mbox = 0;
6726
6727         if (!netif_running(dev))
6728                 return -ENODEV;
6729
6730         tg3_disable_ints(tp);
6731
6732         free_irq(tp->pdev->irq, dev);
6733
6734         err = request_irq(tp->pdev->irq, tg3_test_isr,
6735                           IRQF_SHARED | IRQF_SAMPLE_RANDOM, dev->name, dev);
6736         if (err)
6737                 return err;
6738
6739         tp->hw_status->status &= ~SD_STATUS_UPDATED;
6740         tg3_enable_ints(tp);
6741
6742         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
6743                HOSTCC_MODE_NOW);
6744
6745         for (i = 0; i < 5; i++) {
6746                 int_mbox = tr32_mailbox(MAILBOX_INTERRUPT_0 +
6747                                         TG3_64BIT_REG_LOW);
6748                 if (int_mbox != 0)
6749                         break;
6750                 msleep(10);
6751         }
6752
6753         tg3_disable_ints(tp);
6754
6755         free_irq(tp->pdev->irq, dev);
6756
6757         err = tg3_request_irq(tp);
6758
6759         if (err)
6760                 return err;
6761
6762         if (int_mbox != 0)
6763                 return 0;
6764
6765         return -EIO;
6766 }
6767
6768 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
6769  * successfully restored
6770  */
6771 static int tg3_test_msi(struct tg3 *tp)
6772 {
6773         struct net_device *dev = tp->dev;
6774         int err;
6775         u16 pci_cmd;
6776
6777         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSI))
6778                 return 0;
6779
6780         /* Turn off SERR reporting in case MSI terminates with Master
6781          * Abort.
6782          */
6783         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
6784         pci_write_config_word(tp->pdev, PCI_COMMAND,
6785                               pci_cmd & ~PCI_COMMAND_SERR);
6786
6787         err = tg3_test_interrupt(tp);
6788
6789         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
6790
6791         if (!err)
6792                 return 0;
6793
6794         /* other failures */
6795         if (err != -EIO)
6796                 return err;
6797
6798         /* MSI test failed, go back to INTx mode */
6799         printk(KERN_WARNING PFX "%s: No interrupt was generated using MSI, "
6800                "switching to INTx mode. Please report this failure to "
6801                "the PCI maintainer and include system chipset information.\n",
6802                        tp->dev->name);
6803
6804         free_irq(tp->pdev->irq, dev);
6805         pci_disable_msi(tp->pdev);
6806
6807         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6808
6809         err = tg3_request_irq(tp);
6810         if (err)
6811                 return err;
6812
6813         /* Need to reset the chip because the MSI cycle may have terminated
6814          * with Master Abort.
6815          */
6816         tg3_full_lock(tp, 1);
6817
6818         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
6819         err = tg3_init_hw(tp, 1);
6820
6821         tg3_full_unlock(tp);
6822
6823         if (err)
6824                 free_irq(tp->pdev->irq, dev);
6825
6826         return err;
6827 }
6828
6829 static int tg3_open(struct net_device *dev)
6830 {
6831         struct tg3 *tp = netdev_priv(dev);
6832         int err;
6833
6834         tg3_full_lock(tp, 0);
6835
6836         err = tg3_set_power_state(tp, PCI_D0);
6837         if (err)
6838                 return err;
6839
6840         tg3_disable_ints(tp);
6841         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
6842
6843         tg3_full_unlock(tp);
6844
6845         /* The placement of this call is tied
6846          * to the setup and use of Host TX descriptors.
6847          */
6848         err = tg3_alloc_consistent(tp);
6849         if (err)
6850                 return err;
6851
6852         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
6853             (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5750_AX) &&
6854             (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5750_BX) &&
6855             !((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) &&
6856               (tp->pdev_peer == tp->pdev))) {
6857                 /* All MSI supporting chips should support tagged
6858                  * status.  Assert that this is the case.
6859                  */
6860                 if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
6861                         printk(KERN_WARNING PFX "%s: MSI without TAGGED? "
6862                                "Not using MSI.\n", tp->dev->name);
6863                 } else if (pci_enable_msi(tp->pdev) == 0) {
6864                         u32 msi_mode;
6865
6866                         msi_mode = tr32(MSGINT_MODE);
6867                         tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
6868                         tp->tg3_flags2 |= TG3_FLG2_USING_MSI;
6869                 }
6870         }
6871         err = tg3_request_irq(tp);
6872
6873         if (err) {
6874                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6875                         pci_disable_msi(tp->pdev);
6876                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6877                 }
6878                 tg3_free_consistent(tp);
6879                 return err;
6880         }
6881
6882         tg3_full_lock(tp, 0);
6883
6884         err = tg3_init_hw(tp, 1);
6885         if (err) {
6886                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
6887                 tg3_free_rings(tp);
6888         } else {
6889                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
6890                         tp->timer_offset = HZ;
6891                 else
6892                         tp->timer_offset = HZ / 10;
6893
6894                 BUG_ON(tp->timer_offset > HZ);
6895                 tp->timer_counter = tp->timer_multiplier =
6896                         (HZ / tp->timer_offset);
6897                 tp->asf_counter = tp->asf_multiplier =
6898                         ((HZ / tp->timer_offset) * 2);
6899
6900                 init_timer(&tp->timer);
6901                 tp->timer.expires = jiffies + tp->timer_offset;
6902                 tp->timer.data = (unsigned long) tp;
6903                 tp->timer.function = tg3_timer;
6904         }
6905
6906         tg3_full_unlock(tp);
6907
6908         if (err) {
6909                 free_irq(tp->pdev->irq, dev);
6910                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6911                         pci_disable_msi(tp->pdev);
6912                         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6913                 }
6914                 tg3_free_consistent(tp);
6915                 return err;
6916         }
6917
6918         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6919                 err = tg3_test_msi(tp);
6920
6921                 if (err) {
6922                         tg3_full_lock(tp, 0);
6923
6924                         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6925                                 pci_disable_msi(tp->pdev);
6926                                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
6927                         }
6928                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
6929                         tg3_free_rings(tp);
6930                         tg3_free_consistent(tp);
6931
6932                         tg3_full_unlock(tp);
6933
6934                         return err;
6935                 }
6936
6937                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6938                         if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI) {
6939                                 u32 val = tr32(0x7c04);
6940
6941                                 tw32(0x7c04, val | (1 << 29));
6942                         }
6943                 }
6944         }
6945
6946         tg3_full_lock(tp, 0);
6947
6948         add_timer(&tp->timer);
6949         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
6950         tg3_enable_ints(tp);
6951
6952         tg3_full_unlock(tp);
6953
6954         netif_start_queue(dev);
6955
6956         return 0;
6957 }
6958
6959 #if 0
6960 /*static*/ void tg3_dump_state(struct tg3 *tp)
6961 {
6962         u32 val32, val32_2, val32_3, val32_4, val32_5;
6963         u16 val16;
6964         int i;
6965
6966         pci_read_config_word(tp->pdev, PCI_STATUS, &val16);
6967         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE, &val32);
6968         printk("DEBUG: PCI status [%04x] TG3PCI state[%08x]\n",
6969                val16, val32);
6970
6971         /* MAC block */
6972         printk("DEBUG: MAC_MODE[%08x] MAC_STATUS[%08x]\n",
6973                tr32(MAC_MODE), tr32(MAC_STATUS));
6974         printk("       MAC_EVENT[%08x] MAC_LED_CTRL[%08x]\n",
6975                tr32(MAC_EVENT), tr32(MAC_LED_CTRL));
6976         printk("DEBUG: MAC_TX_MODE[%08x] MAC_TX_STATUS[%08x]\n",
6977                tr32(MAC_TX_MODE), tr32(MAC_TX_STATUS));
6978         printk("       MAC_RX_MODE[%08x] MAC_RX_STATUS[%08x]\n",
6979                tr32(MAC_RX_MODE), tr32(MAC_RX_STATUS));
6980
6981         /* Send data initiator control block */
6982         printk("DEBUG: SNDDATAI_MODE[%08x] SNDDATAI_STATUS[%08x]\n",
6983                tr32(SNDDATAI_MODE), tr32(SNDDATAI_STATUS));
6984         printk("       SNDDATAI_STATSCTRL[%08x]\n",
6985                tr32(SNDDATAI_STATSCTRL));
6986
6987         /* Send data completion control block */
6988         printk("DEBUG: SNDDATAC_MODE[%08x]\n", tr32(SNDDATAC_MODE));
6989
6990         /* Send BD ring selector block */
6991         printk("DEBUG: SNDBDS_MODE[%08x] SNDBDS_STATUS[%08x]\n",
6992                tr32(SNDBDS_MODE), tr32(SNDBDS_STATUS));
6993
6994         /* Send BD initiator control block */
6995         printk("DEBUG: SNDBDI_MODE[%08x] SNDBDI_STATUS[%08x]\n",
6996                tr32(SNDBDI_MODE), tr32(SNDBDI_STATUS));
6997
6998         /* Send BD completion control block */
6999         printk("DEBUG: SNDBDC_MODE[%08x]\n", tr32(SNDBDC_MODE));
7000
7001         /* Receive list placement control block */
7002         printk("DEBUG: RCVLPC_MODE[%08x] RCVLPC_STATUS[%08x]\n",
7003                tr32(RCVLPC_MODE), tr32(RCVLPC_STATUS));
7004         printk("       RCVLPC_STATSCTRL[%08x]\n",
7005                tr32(RCVLPC_STATSCTRL));
7006
7007         /* Receive data and receive BD initiator control block */
7008         printk("DEBUG: RCVDBDI_MODE[%08x] RCVDBDI_STATUS[%08x]\n",
7009                tr32(RCVDBDI_MODE), tr32(RCVDBDI_STATUS));
7010
7011         /* Receive data completion control block */
7012         printk("DEBUG: RCVDCC_MODE[%08x]\n",
7013                tr32(RCVDCC_MODE));
7014
7015         /* Receive BD initiator control block */
7016         printk("DEBUG: RCVBDI_MODE[%08x] RCVBDI_STATUS[%08x]\n",
7017                tr32(RCVBDI_MODE), tr32(RCVBDI_STATUS));
7018
7019         /* Receive BD completion control block */
7020         printk("DEBUG: RCVCC_MODE[%08x] RCVCC_STATUS[%08x]\n",
7021                tr32(RCVCC_MODE), tr32(RCVCC_STATUS));
7022
7023         /* Receive list selector control block */
7024         printk("DEBUG: RCVLSC_MODE[%08x] RCVLSC_STATUS[%08x]\n",
7025                tr32(RCVLSC_MODE), tr32(RCVLSC_STATUS));
7026
7027         /* Mbuf cluster free block */
7028         printk("DEBUG: MBFREE_MODE[%08x] MBFREE_STATUS[%08x]\n",
7029                tr32(MBFREE_MODE), tr32(MBFREE_STATUS));
7030
7031         /* Host coalescing control block */
7032         printk("DEBUG: HOSTCC_MODE[%08x] HOSTCC_STATUS[%08x]\n",
7033                tr32(HOSTCC_MODE), tr32(HOSTCC_STATUS));
7034         printk("DEBUG: HOSTCC_STATS_BLK_HOST_ADDR[%08x%08x]\n",
7035                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7036                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7037         printk("DEBUG: HOSTCC_STATUS_BLK_HOST_ADDR[%08x%08x]\n",
7038                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
7039                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
7040         printk("DEBUG: HOSTCC_STATS_BLK_NIC_ADDR[%08x]\n",
7041                tr32(HOSTCC_STATS_BLK_NIC_ADDR));
7042         printk("DEBUG: HOSTCC_STATUS_BLK_NIC_ADDR[%08x]\n",
7043                tr32(HOSTCC_STATUS_BLK_NIC_ADDR));
7044
7045         /* Memory arbiter control block */
7046         printk("DEBUG: MEMARB_MODE[%08x] MEMARB_STATUS[%08x]\n",
7047                tr32(MEMARB_MODE), tr32(MEMARB_STATUS));
7048
7049         /* Buffer manager control block */
7050         printk("DEBUG: BUFMGR_MODE[%08x] BUFMGR_STATUS[%08x]\n",
7051                tr32(BUFMGR_MODE), tr32(BUFMGR_STATUS));
7052         printk("DEBUG: BUFMGR_MB_POOL_ADDR[%08x] BUFMGR_MB_POOL_SIZE[%08x]\n",
7053                tr32(BUFMGR_MB_POOL_ADDR), tr32(BUFMGR_MB_POOL_SIZE));
7054         printk("DEBUG: BUFMGR_DMA_DESC_POOL_ADDR[%08x] "
7055                "BUFMGR_DMA_DESC_POOL_SIZE[%08x]\n",
7056                tr32(BUFMGR_DMA_DESC_POOL_ADDR),
7057                tr32(BUFMGR_DMA_DESC_POOL_SIZE));
7058
7059         /* Read DMA control block */
7060         printk("DEBUG: RDMAC_MODE[%08x] RDMAC_STATUS[%08x]\n",
7061                tr32(RDMAC_MODE), tr32(RDMAC_STATUS));
7062
7063         /* Write DMA control block */
7064         printk("DEBUG: WDMAC_MODE[%08x] WDMAC_STATUS[%08x]\n",
7065                tr32(WDMAC_MODE), tr32(WDMAC_STATUS));
7066
7067         /* DMA completion block */
7068         printk("DEBUG: DMAC_MODE[%08x]\n",
7069                tr32(DMAC_MODE));
7070
7071         /* GRC block */
7072         printk("DEBUG: GRC_MODE[%08x] GRC_MISC_CFG[%08x]\n",
7073                tr32(GRC_MODE), tr32(GRC_MISC_CFG));
7074         printk("DEBUG: GRC_LOCAL_CTRL[%08x]\n",
7075                tr32(GRC_LOCAL_CTRL));
7076
7077         /* TG3_BDINFOs */
7078         printk("DEBUG: RCVDBDI_JUMBO_BD[%08x%08x:%08x:%08x]\n",
7079                tr32(RCVDBDI_JUMBO_BD + 0x0),
7080                tr32(RCVDBDI_JUMBO_BD + 0x4),
7081                tr32(RCVDBDI_JUMBO_BD + 0x8),
7082                tr32(RCVDBDI_JUMBO_BD + 0xc));
7083         printk("DEBUG: RCVDBDI_STD_BD[%08x%08x:%08x:%08x]\n",
7084                tr32(RCVDBDI_STD_BD + 0x0),
7085                tr32(RCVDBDI_STD_BD + 0x4),
7086                tr32(RCVDBDI_STD_BD + 0x8),
7087                tr32(RCVDBDI_STD_BD + 0xc));
7088         printk("DEBUG: RCVDBDI_MINI_BD[%08x%08x:%08x:%08x]\n",
7089                tr32(RCVDBDI_MINI_BD + 0x0),
7090                tr32(RCVDBDI_MINI_BD + 0x4),
7091                tr32(RCVDBDI_MINI_BD + 0x8),
7092                tr32(RCVDBDI_MINI_BD + 0xc));
7093
7094         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x0, &val32);
7095         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x4, &val32_2);
7096         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x8, &val32_3);
7097         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0xc, &val32_4);
7098         printk("DEBUG: SRAM_SEND_RCB_0[%08x%08x:%08x:%08x]\n",
7099                val32, val32_2, val32_3, val32_4);
7100
7101         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x0, &val32);
7102         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x4, &val32_2);
7103         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x8, &val32_3);
7104         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0xc, &val32_4);
7105         printk("DEBUG: SRAM_RCV_RET_RCB_0[%08x%08x:%08x:%08x]\n",
7106                val32, val32_2, val32_3, val32_4);
7107
7108         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x0, &val32);
7109         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x4, &val32_2);
7110         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x8, &val32_3);
7111         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0xc, &val32_4);
7112         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x10, &val32_5);
7113         printk("DEBUG: SRAM_STATUS_BLK[%08x:%08x:%08x:%08x:%08x]\n",
7114                val32, val32_2, val32_3, val32_4, val32_5);
7115
7116         /* SW status block */
7117         printk("DEBUG: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
7118                tp->hw_status->status,
7119                tp->hw_status->status_tag,
7120                tp->hw_status->rx_jumbo_consumer,
7121                tp->hw_status->rx_consumer,
7122                tp->hw_status->rx_mini_consumer,
7123                tp->hw_status->idx[0].rx_producer,
7124                tp->hw_status->idx[0].tx_consumer);
7125
7126         /* SW statistics block */
7127         printk("DEBUG: Host statistics block [%08x:%08x:%08x:%08x]\n",
7128                ((u32 *)tp->hw_stats)[0],
7129                ((u32 *)tp->hw_stats)[1],
7130                ((u32 *)tp->hw_stats)[2],
7131                ((u32 *)tp->hw_stats)[3]);
7132
7133         /* Mailboxes */
7134         printk("DEBUG: SNDHOST_PROD[%08x%08x] SNDNIC_PROD[%08x%08x]\n",
7135                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x0),
7136                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x4),
7137                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x0),
7138                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x4));
7139
7140         /* NIC side send descriptors. */
7141         for (i = 0; i < 6; i++) {
7142                 unsigned long txd;
7143
7144                 txd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_TX_BUFFER_DESC
7145                         + (i * sizeof(struct tg3_tx_buffer_desc));
7146                 printk("DEBUG: NIC TXD(%d)[%08x:%08x:%08x:%08x]\n",
7147                        i,
7148                        readl(txd + 0x0), readl(txd + 0x4),
7149                        readl(txd + 0x8), readl(txd + 0xc));
7150         }
7151
7152         /* NIC side RX descriptors. */
7153         for (i = 0; i < 6; i++) {
7154                 unsigned long rxd;
7155
7156                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_BUFFER_DESC
7157                         + (i * sizeof(struct tg3_rx_buffer_desc));
7158                 printk("DEBUG: NIC RXD_STD(%d)[0][%08x:%08x:%08x:%08x]\n",
7159                        i,
7160                        readl(rxd + 0x0), readl(rxd + 0x4),
7161                        readl(rxd + 0x8), readl(rxd + 0xc));
7162                 rxd += (4 * sizeof(u32));
7163                 printk("DEBUG: NIC RXD_STD(%d)[1][%08x:%08x:%08x:%08x]\n",
7164                        i,
7165                        readl(rxd + 0x0), readl(rxd + 0x4),
7166                        readl(rxd + 0x8), readl(rxd + 0xc));
7167         }
7168
7169         for (i = 0; i < 6; i++) {
7170                 unsigned long rxd;
7171
7172                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_JUMBO_BUFFER_DESC
7173                         + (i * sizeof(struct tg3_rx_buffer_desc));
7174                 printk("DEBUG: NIC RXD_JUMBO(%d)[0][%08x:%08x:%08x:%08x]\n",
7175                        i,
7176                        readl(rxd + 0x0), readl(rxd + 0x4),
7177                        readl(rxd + 0x8), readl(rxd + 0xc));
7178                 rxd += (4 * sizeof(u32));
7179                 printk("DEBUG: NIC RXD_JUMBO(%d)[1][%08x:%08x:%08x:%08x]\n",
7180                        i,
7181                        readl(rxd + 0x0), readl(rxd + 0x4),
7182                        readl(rxd + 0x8), readl(rxd + 0xc));
7183         }
7184 }
7185 #endif
7186
7187 static struct net_device_stats *tg3_get_stats(struct net_device *);
7188 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *);
7189
7190 static int tg3_close(struct net_device *dev)
7191 {
7192         struct tg3 *tp = netdev_priv(dev);
7193
7194         /* Calling flush_scheduled_work() may deadlock because
7195          * linkwatch_event() may be on the workqueue and it will try to get
7196          * the rtnl_lock which we are holding.
7197          */
7198         while (tp->tg3_flags & TG3_FLAG_IN_RESET_TASK)
7199                 msleep(1);
7200
7201         netif_stop_queue(dev);
7202
7203         del_timer_sync(&tp->timer);
7204
7205         tg3_full_lock(tp, 1);
7206 #if 0
7207         tg3_dump_state(tp);
7208 #endif
7209
7210         tg3_disable_ints(tp);
7211
7212         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7213         tg3_free_rings(tp);
7214         tp->tg3_flags &=
7215                 ~(TG3_FLAG_INIT_COMPLETE |
7216                   TG3_FLAG_GOT_SERDES_FLOWCTL);
7217
7218         tg3_full_unlock(tp);
7219
7220         free_irq(tp->pdev->irq, dev);
7221         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
7222                 pci_disable_msi(tp->pdev);
7223                 tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
7224         }
7225
7226         memcpy(&tp->net_stats_prev, tg3_get_stats(tp->dev),
7227                sizeof(tp->net_stats_prev));
7228         memcpy(&tp->estats_prev, tg3_get_estats(tp),
7229                sizeof(tp->estats_prev));
7230
7231         tg3_free_consistent(tp);
7232
7233         tg3_set_power_state(tp, PCI_D3hot);
7234
7235         netif_carrier_off(tp->dev);
7236
7237         return 0;
7238 }
7239
7240 static inline unsigned long get_stat64(tg3_stat64_t *val)
7241 {
7242         unsigned long ret;
7243
7244 #if (BITS_PER_LONG == 32)
7245         ret = val->low;
7246 #else
7247         ret = ((u64)val->high << 32) | ((u64)val->low);
7248 #endif
7249         return ret;
7250 }
7251
7252 static unsigned long calc_crc_errors(struct tg3 *tp)
7253 {
7254         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7255
7256         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7257             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
7258              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
7259                 u32 val;
7260
7261                 spin_lock_bh(&tp->lock);
7262                 if (!tg3_readphy(tp, 0x1e, &val)) {
7263                         tg3_writephy(tp, 0x1e, val | 0x8000);
7264                         tg3_readphy(tp, 0x14, &val);
7265                 } else
7266                         val = 0;
7267                 spin_unlock_bh(&tp->lock);
7268
7269                 tp->phy_crc_errors += val;
7270
7271                 return tp->phy_crc_errors;
7272         }
7273
7274         return get_stat64(&hw_stats->rx_fcs_errors);
7275 }
7276
7277 #define ESTAT_ADD(member) \
7278         estats->member =        old_estats->member + \
7279                                 get_stat64(&hw_stats->member)
7280
7281 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *tp)
7282 {
7283         struct tg3_ethtool_stats *estats = &tp->estats;
7284         struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
7285         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7286
7287         if (!hw_stats)
7288                 return old_estats;
7289
7290         ESTAT_ADD(rx_octets);
7291         ESTAT_ADD(rx_fragments);
7292         ESTAT_ADD(rx_ucast_packets);
7293         ESTAT_ADD(rx_mcast_packets);
7294         ESTAT_ADD(rx_bcast_packets);
7295         ESTAT_ADD(rx_fcs_errors);
7296         ESTAT_ADD(rx_align_errors);
7297         ESTAT_ADD(rx_xon_pause_rcvd);
7298         ESTAT_ADD(rx_xoff_pause_rcvd);
7299         ESTAT_ADD(rx_mac_ctrl_rcvd);
7300         ESTAT_ADD(rx_xoff_entered);
7301         ESTAT_ADD(rx_frame_too_long_errors);
7302         ESTAT_ADD(rx_jabbers);
7303         ESTAT_ADD(rx_undersize_packets);
7304         ESTAT_ADD(rx_in_length_errors);
7305         ESTAT_ADD(rx_out_length_errors);
7306         ESTAT_ADD(rx_64_or_less_octet_packets);
7307         ESTAT_ADD(rx_65_to_127_octet_packets);
7308         ESTAT_ADD(rx_128_to_255_octet_packets);
7309         ESTAT_ADD(rx_256_to_511_octet_packets);
7310         ESTAT_ADD(rx_512_to_1023_octet_packets);
7311         ESTAT_ADD(rx_1024_to_1522_octet_packets);
7312         ESTAT_ADD(rx_1523_to_2047_octet_packets);
7313         ESTAT_ADD(rx_2048_to_4095_octet_packets);
7314         ESTAT_ADD(rx_4096_to_8191_octet_packets);
7315         ESTAT_ADD(rx_8192_to_9022_octet_packets);
7316
7317         ESTAT_ADD(tx_octets);
7318         ESTAT_ADD(tx_collisions);
7319         ESTAT_ADD(tx_xon_sent);
7320         ESTAT_ADD(tx_xoff_sent);
7321         ESTAT_ADD(tx_flow_control);
7322         ESTAT_ADD(tx_mac_errors);
7323         ESTAT_ADD(tx_single_collisions);
7324         ESTAT_ADD(tx_mult_collisions);
7325         ESTAT_ADD(tx_deferred);
7326         ESTAT_ADD(tx_excessive_collisions);
7327         ESTAT_ADD(tx_late_collisions);
7328         ESTAT_ADD(tx_collide_2times);
7329         ESTAT_ADD(tx_collide_3times);
7330         ESTAT_ADD(tx_collide_4times);
7331         ESTAT_ADD(tx_collide_5times);
7332         ESTAT_ADD(tx_collide_6times);
7333         ESTAT_ADD(tx_collide_7times);
7334         ESTAT_ADD(tx_collide_8times);
7335         ESTAT_ADD(tx_collide_9times);
7336         ESTAT_ADD(tx_collide_10times);
7337         ESTAT_ADD(tx_collide_11times);
7338         ESTAT_ADD(tx_collide_12times);
7339         ESTAT_ADD(tx_collide_13times);
7340         ESTAT_ADD(tx_collide_14times);
7341         ESTAT_ADD(tx_collide_15times);
7342         ESTAT_ADD(tx_ucast_packets);
7343         ESTAT_ADD(tx_mcast_packets);
7344         ESTAT_ADD(tx_bcast_packets);
7345         ESTAT_ADD(tx_carrier_sense_errors);
7346         ESTAT_ADD(tx_discards);
7347         ESTAT_ADD(tx_errors);
7348
7349         ESTAT_ADD(dma_writeq_full);
7350         ESTAT_ADD(dma_write_prioq_full);
7351         ESTAT_ADD(rxbds_empty);
7352         ESTAT_ADD(rx_discards);
7353         ESTAT_ADD(rx_errors);
7354         ESTAT_ADD(rx_threshold_hit);
7355
7356         ESTAT_ADD(dma_readq_full);
7357         ESTAT_ADD(dma_read_prioq_full);
7358         ESTAT_ADD(tx_comp_queue_full);
7359
7360         ESTAT_ADD(ring_set_send_prod_index);
7361         ESTAT_ADD(ring_status_update);
7362         ESTAT_ADD(nic_irqs);
7363         ESTAT_ADD(nic_avoided_irqs);
7364         ESTAT_ADD(nic_tx_threshold_hit);
7365
7366         return estats;
7367 }
7368
7369 static struct net_device_stats *tg3_get_stats(struct net_device *dev)
7370 {
7371         struct tg3 *tp = netdev_priv(dev);
7372         struct net_device_stats *stats = &tp->net_stats;
7373         struct net_device_stats *old_stats = &tp->net_stats_prev;
7374         struct tg3_hw_stats *hw_stats = tp->hw_stats;
7375
7376         if (!hw_stats)
7377                 return old_stats;
7378
7379         stats->rx_packets = old_stats->rx_packets +
7380                 get_stat64(&hw_stats->rx_ucast_packets) +
7381                 get_stat64(&hw_stats->rx_mcast_packets) +
7382                 get_stat64(&hw_stats->rx_bcast_packets);
7383
7384         stats->tx_packets = old_stats->tx_packets +
7385                 get_stat64(&hw_stats->tx_ucast_packets) +
7386                 get_stat64(&hw_stats->tx_mcast_packets) +
7387                 get_stat64(&hw_stats->tx_bcast_packets);
7388
7389         stats->rx_bytes = old_stats->rx_bytes +
7390                 get_stat64(&hw_stats->rx_octets);
7391         stats->tx_bytes = old_stats->tx_bytes +
7392                 get_stat64(&hw_stats->tx_octets);
7393
7394         stats->rx_errors = old_stats->rx_errors +
7395                 get_stat64(&hw_stats->rx_errors);
7396         stats->tx_errors = old_stats->tx_errors +
7397                 get_stat64(&hw_stats->tx_errors) +
7398                 get_stat64(&hw_stats->tx_mac_errors) +
7399                 get_stat64(&hw_stats->tx_carrier_sense_errors) +
7400                 get_stat64(&hw_stats->tx_discards);
7401
7402         stats->multicast = old_stats->multicast +
7403                 get_stat64(&hw_stats->rx_mcast_packets);
7404         stats->collisions = old_stats->collisions +
7405                 get_stat64(&hw_stats->tx_collisions);
7406
7407         stats->rx_length_errors = old_stats->rx_length_errors +
7408                 get_stat64(&hw_stats->rx_frame_too_long_errors) +
7409                 get_stat64(&hw_stats->rx_undersize_packets);
7410
7411         stats->rx_over_errors = old_stats->rx_over_errors +
7412                 get_stat64(&hw_stats->rxbds_empty);
7413         stats->rx_frame_errors = old_stats->rx_frame_errors +
7414                 get_stat64(&hw_stats->rx_align_errors);
7415         stats->tx_aborted_errors = old_stats->tx_aborted_errors +
7416                 get_stat64(&hw_stats->tx_discards);
7417         stats->tx_carrier_errors = old_stats->tx_carrier_errors +
7418                 get_stat64(&hw_stats->tx_carrier_sense_errors);
7419
7420         stats->rx_crc_errors = old_stats->rx_crc_errors +
7421                 calc_crc_errors(tp);
7422
7423         stats->rx_missed_errors = old_stats->rx_missed_errors +
7424                 get_stat64(&hw_stats->rx_discards);
7425
7426         return stats;
7427 }
7428
7429 static inline u32 calc_crc(unsigned char *buf, int len)
7430 {
7431         u32 reg;
7432         u32 tmp;
7433         int j, k;
7434
7435         reg = 0xffffffff;
7436
7437         for (j = 0; j < len; j++) {
7438                 reg ^= buf[j];
7439
7440                 for (k = 0; k < 8; k++) {
7441                         tmp = reg & 0x01;
7442
7443                         reg >>= 1;
7444
7445                         if (tmp) {
7446                                 reg ^= 0xedb88320;
7447                         }
7448                 }
7449         }
7450
7451         return ~reg;
7452 }
7453
7454 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
7455 {
7456         /* accept or reject all multicast frames */
7457         tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
7458         tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
7459         tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
7460         tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
7461 }
7462
7463 static void __tg3_set_rx_mode(struct net_device *dev)
7464 {
7465         struct tg3 *tp = netdev_priv(dev);
7466         u32 rx_mode;
7467
7468         rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
7469                                   RX_MODE_KEEP_VLAN_TAG);
7470
7471         /* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
7472          * flag clear.
7473          */
7474 #if TG3_VLAN_TAG_USED
7475         if (!tp->vlgrp &&
7476             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
7477                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
7478 #else
7479         /* By definition, VLAN is disabled always in this
7480          * case.
7481          */
7482         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
7483                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
7484 #endif
7485
7486         if (dev->flags & IFF_PROMISC) {
7487                 /* Promiscuous mode. */
7488                 rx_mode |= RX_MODE_PROMISC;
7489         } else if (dev->flags & IFF_ALLMULTI) {
7490                 /* Accept all multicast. */
7491                 tg3_set_multi (tp, 1);
7492         } else if (dev->mc_count < 1) {
7493                 /* Reject all multicast. */
7494                 tg3_set_multi (tp, 0);
7495         } else {
7496                 /* Accept one or more multicast(s). */
7497                 struct dev_mc_list *mclist;
7498                 unsigned int i;
7499                 u32 mc_filter[4] = { 0, };
7500                 u32 regidx;
7501                 u32 bit;
7502                 u32 crc;
7503
7504                 for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count;
7505                      i++, mclist = mclist->next) {
7506
7507                         crc = calc_crc (mclist->dmi_addr, ETH_ALEN);
7508                         bit = ~crc & 0x7f;
7509                         regidx = (bit & 0x60) >> 5;
7510                         bit &= 0x1f;
7511                         mc_filter[regidx] |= (1 << bit);
7512                 }
7513
7514                 tw32(MAC_HASH_REG_0, mc_filter[0]);
7515                 tw32(MAC_HASH_REG_1, mc_filter[1]);
7516                 tw32(MAC_HASH_REG_2, mc_filter[2]);
7517                 tw32(MAC_HASH_REG_3, mc_filter[3]);
7518         }
7519
7520         if (rx_mode != tp->rx_mode) {
7521                 tp->rx_mode = rx_mode;
7522                 tw32_f(MAC_RX_MODE, rx_mode);
7523                 udelay(10);
7524         }
7525 }
7526
7527 static void tg3_set_rx_mode(struct net_device *dev)
7528 {
7529         struct tg3 *tp = netdev_priv(dev);
7530
7531         if (!netif_running(dev))
7532                 return;
7533
7534         tg3_full_lock(tp, 0);
7535         __tg3_set_rx_mode(dev);
7536         tg3_full_unlock(tp);
7537 }
7538
7539 #define TG3_REGDUMP_LEN         (32 * 1024)
7540
7541 static int tg3_get_regs_len(struct net_device *dev)
7542 {
7543         return TG3_REGDUMP_LEN;
7544 }
7545
7546 static void tg3_get_regs(struct net_device *dev,
7547                 struct ethtool_regs *regs, void *_p)
7548 {
7549         u32 *p = _p;
7550         struct tg3 *tp = netdev_priv(dev);
7551         u8 *orig_p = _p;
7552         int i;
7553
7554         regs->version = 0;
7555
7556         memset(p, 0, TG3_REGDUMP_LEN);
7557
7558         if (tp->link_config.phy_is_low_power)
7559                 return;
7560
7561         tg3_full_lock(tp, 0);
7562
7563 #define __GET_REG32(reg)        (*(p)++ = tr32(reg))
7564 #define GET_REG32_LOOP(base,len)                \
7565 do {    p = (u32 *)(orig_p + (base));           \
7566         for (i = 0; i < len; i += 4)            \
7567                 __GET_REG32((base) + i);        \
7568 } while (0)
7569 #define GET_REG32_1(reg)                        \
7570 do {    p = (u32 *)(orig_p + (reg));            \
7571         __GET_REG32((reg));                     \
7572 } while (0)
7573
7574         GET_REG32_LOOP(TG3PCI_VENDOR, 0xb0);
7575         GET_REG32_LOOP(MAILBOX_INTERRUPT_0, 0x200);
7576         GET_REG32_LOOP(MAC_MODE, 0x4f0);
7577         GET_REG32_LOOP(SNDDATAI_MODE, 0xe0);
7578         GET_REG32_1(SNDDATAC_MODE);
7579         GET_REG32_LOOP(SNDBDS_MODE, 0x80);
7580         GET_REG32_LOOP(SNDBDI_MODE, 0x48);
7581         GET_REG32_1(SNDBDC_MODE);
7582         GET_REG32_LOOP(RCVLPC_MODE, 0x20);
7583         GET_REG32_LOOP(RCVLPC_SELLST_BASE, 0x15c);
7584         GET_REG32_LOOP(RCVDBDI_MODE, 0x0c);
7585         GET_REG32_LOOP(RCVDBDI_JUMBO_BD, 0x3c);
7586         GET_REG32_LOOP(RCVDBDI_BD_PROD_IDX_0, 0x44);
7587         GET_REG32_1(RCVDCC_MODE);
7588         GET_REG32_LOOP(RCVBDI_MODE, 0x20);
7589         GET_REG32_LOOP(RCVCC_MODE, 0x14);
7590         GET_REG32_LOOP(RCVLSC_MODE, 0x08);
7591         GET_REG32_1(MBFREE_MODE);
7592         GET_REG32_LOOP(HOSTCC_MODE, 0x100);
7593         GET_REG32_LOOP(MEMARB_MODE, 0x10);
7594         GET_REG32_LOOP(BUFMGR_MODE, 0x58);
7595         GET_REG32_LOOP(RDMAC_MODE, 0x08);
7596         GET_REG32_LOOP(WDMAC_MODE, 0x08);
7597         GET_REG32_1(RX_CPU_MODE);
7598         GET_REG32_1(RX_CPU_STATE);
7599         GET_REG32_1(RX_CPU_PGMCTR);
7600         GET_REG32_1(RX_CPU_HWBKPT);
7601         GET_REG32_1(TX_CPU_MODE);
7602         GET_REG32_1(TX_CPU_STATE);
7603         GET_REG32_1(TX_CPU_PGMCTR);
7604         GET_REG32_LOOP(GRCMBOX_INTERRUPT_0, 0x110);
7605         GET_REG32_LOOP(FTQ_RESET, 0x120);
7606         GET_REG32_LOOP(MSGINT_MODE, 0x0c);
7607         GET_REG32_1(DMAC_MODE);
7608         GET_REG32_LOOP(GRC_MODE, 0x4c);
7609         if (tp->tg3_flags & TG3_FLAG_NVRAM)
7610                 GET_REG32_LOOP(NVRAM_CMD, 0x24);
7611
7612 #undef __GET_REG32
7613 #undef GET_REG32_LOOP
7614 #undef GET_REG32_1
7615
7616         tg3_full_unlock(tp);
7617 }
7618
7619 static int tg3_get_eeprom_len(struct net_device *dev)
7620 {
7621         struct tg3 *tp = netdev_priv(dev);
7622
7623         return tp->nvram_size;
7624 }
7625
7626 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val);
7627 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val);
7628
7629 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
7630 {
7631         struct tg3 *tp = netdev_priv(dev);
7632         int ret;
7633         u8  *pd;
7634         u32 i, offset, len, val, b_offset, b_count;
7635
7636         if (tp->link_config.phy_is_low_power)
7637                 return -EAGAIN;
7638
7639         offset = eeprom->offset;
7640         len = eeprom->len;
7641         eeprom->len = 0;
7642
7643         eeprom->magic = TG3_EEPROM_MAGIC;
7644
7645         if (offset & 3) {
7646                 /* adjustments to start on required 4 byte boundary */
7647                 b_offset = offset & 3;
7648                 b_count = 4 - b_offset;
7649                 if (b_count > len) {
7650                         /* i.e. offset=1 len=2 */
7651                         b_count = len;
7652                 }
7653                 ret = tg3_nvram_read(tp, offset-b_offset, &val);
7654                 if (ret)
7655                         return ret;
7656                 val = cpu_to_le32(val);
7657                 memcpy(data, ((char*)&val) + b_offset, b_count);
7658                 len -= b_count;
7659                 offset += b_count;
7660                 eeprom->len += b_count;
7661         }
7662
7663         /* read bytes upto the last 4 byte boundary */
7664         pd = &data[eeprom->len];
7665         for (i = 0; i < (len - (len & 3)); i += 4) {
7666                 ret = tg3_nvram_read(tp, offset + i, &val);
7667                 if (ret) {
7668                         eeprom->len += i;
7669                         return ret;
7670                 }
7671                 val = cpu_to_le32(val);
7672                 memcpy(pd + i, &val, 4);
7673         }
7674         eeprom->len += i;
7675
7676         if (len & 3) {
7677                 /* read last bytes not ending on 4 byte boundary */
7678                 pd = &data[eeprom->len];
7679                 b_count = len & 3;
7680                 b_offset = offset + len - b_count;
7681                 ret = tg3_nvram_read(tp, b_offset, &val);
7682                 if (ret)
7683                         return ret;
7684                 val = cpu_to_le32(val);
7685                 memcpy(pd, ((char*)&val), b_count);
7686                 eeprom->len += b_count;
7687         }
7688         return 0;
7689 }
7690
7691 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf);
7692
7693 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
7694 {
7695         struct tg3 *tp = netdev_priv(dev);
7696         int ret;
7697         u32 offset, len, b_offset, odd_len, start, end;
7698         u8 *buf;
7699
7700         if (tp->link_config.phy_is_low_power)
7701                 return -EAGAIN;
7702
7703         if (eeprom->magic != TG3_EEPROM_MAGIC)
7704                 return -EINVAL;
7705
7706         offset = eeprom->offset;
7707         len = eeprom->len;
7708
7709         if ((b_offset = (offset & 3))) {
7710                 /* adjustments to start on required 4 byte boundary */
7711                 ret = tg3_nvram_read(tp, offset-b_offset, &start);
7712                 if (ret)
7713                         return ret;
7714                 start = cpu_to_le32(start);
7715                 len += b_offset;
7716                 offset &= ~3;
7717                 if (len < 4)
7718                         len = 4;
7719         }
7720
7721         odd_len = 0;
7722         if (len & 3) {
7723                 /* adjustments to end on required 4 byte boundary */
7724                 odd_len = 1;
7725                 len = (len + 3) & ~3;
7726                 ret = tg3_nvram_read(tp, offset+len-4, &end);
7727                 if (ret)
7728                         return ret;
7729                 end = cpu_to_le32(end);
7730         }
7731
7732         buf = data;
7733         if (b_offset || odd_len) {
7734                 buf = kmalloc(len, GFP_KERNEL);
7735                 if (buf == 0)
7736                         return -ENOMEM;
7737                 if (b_offset)
7738                         memcpy(buf, &start, 4);
7739                 if (odd_len)
7740                         memcpy(buf+len-4, &end, 4);
7741                 memcpy(buf + b_offset, data, eeprom->len);
7742         }
7743
7744         ret = tg3_nvram_write_block(tp, offset, len, buf);
7745
7746         if (buf != data)
7747                 kfree(buf);
7748
7749         return ret;
7750 }
7751
7752 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
7753 {
7754         struct tg3 *tp = netdev_priv(dev);
7755
7756         cmd->supported = (SUPPORTED_Autoneg);
7757
7758         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
7759                 cmd->supported |= (SUPPORTED_1000baseT_Half |
7760                                    SUPPORTED_1000baseT_Full);
7761
7762         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
7763                 cmd->supported |= (SUPPORTED_100baseT_Half |
7764                                   SUPPORTED_100baseT_Full |
7765                                   SUPPORTED_10baseT_Half |
7766                                   SUPPORTED_10baseT_Full |
7767                                   SUPPORTED_MII);
7768                 cmd->port = PORT_TP;
7769         } else {
7770                 cmd->supported |= SUPPORTED_FIBRE;
7771                 cmd->port = PORT_FIBRE;
7772         }
7773
7774         cmd->advertising = tp->link_config.advertising;
7775         if (netif_running(dev)) {
7776                 cmd->speed = tp->link_config.active_speed;
7777                 cmd->duplex = tp->link_config.active_duplex;
7778         }
7779         cmd->phy_address = PHY_ADDR;
7780         cmd->transceiver = 0;
7781         cmd->autoneg = tp->link_config.autoneg;
7782         cmd->maxtxpkt = 0;
7783         cmd->maxrxpkt = 0;
7784         return 0;
7785 }
7786
7787 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
7788 {
7789         struct tg3 *tp = netdev_priv(dev);
7790
7791         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) {
7792                 /* These are the only valid advertisement bits allowed.  */
7793                 if (cmd->autoneg == AUTONEG_ENABLE &&
7794                     (cmd->advertising & ~(ADVERTISED_1000baseT_Half |
7795                                           ADVERTISED_1000baseT_Full |
7796                                           ADVERTISED_Autoneg |
7797                                           ADVERTISED_FIBRE)))
7798                         return -EINVAL;
7799                 /* Fiber can only do SPEED_1000.  */
7800                 else if ((cmd->autoneg != AUTONEG_ENABLE) &&
7801                          (cmd->speed != SPEED_1000))
7802                         return -EINVAL;
7803         /* Copper cannot force SPEED_1000.  */
7804         } else if ((cmd->autoneg != AUTONEG_ENABLE) &&
7805                    (cmd->speed == SPEED_1000))
7806                 return -EINVAL;
7807         else if ((cmd->speed == SPEED_1000) &&
7808                  (tp->tg3_flags2 & TG3_FLAG_10_100_ONLY))
7809                 return -EINVAL;
7810
7811         tg3_full_lock(tp, 0);
7812
7813         tp->link_config.autoneg = cmd->autoneg;
7814         if (cmd->autoneg == AUTONEG_ENABLE) {
7815                 tp->link_config.advertising = cmd->advertising;
7816                 tp->link_config.speed = SPEED_INVALID;
7817                 tp->link_config.duplex = DUPLEX_INVALID;
7818         } else {
7819                 tp->link_config.advertising = 0;
7820                 tp->link_config.speed = cmd->speed;
7821                 tp->link_config.duplex = cmd->duplex;
7822         }
7823
7824         if (netif_running(dev))
7825                 tg3_setup_phy(tp, 1);
7826
7827         tg3_full_unlock(tp);
7828
7829         return 0;
7830 }
7831
7832 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
7833 {
7834         struct tg3 *tp = netdev_priv(dev);
7835
7836         strcpy(info->driver, DRV_MODULE_NAME);
7837         strcpy(info->version, DRV_MODULE_VERSION);
7838         strcpy(info->fw_version, tp->fw_ver);
7839         strcpy(info->bus_info, pci_name(tp->pdev));
7840 }
7841
7842 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
7843 {
7844         struct tg3 *tp = netdev_priv(dev);
7845
7846         wol->supported = WAKE_MAGIC;
7847         wol->wolopts = 0;
7848         if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)
7849                 wol->wolopts = WAKE_MAGIC;
7850         memset(&wol->sopass, 0, sizeof(wol->sopass));
7851 }
7852
7853 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
7854 {
7855         struct tg3 *tp = netdev_priv(dev);
7856
7857         if (wol->wolopts & ~WAKE_MAGIC)
7858                 return -EINVAL;
7859         if ((wol->wolopts & WAKE_MAGIC) &&
7860             tp->tg3_flags2 & TG3_FLG2_PHY_SERDES &&
7861             !(tp->tg3_flags & TG3_FLAG_SERDES_WOL_CAP))
7862                 return -EINVAL;
7863
7864         spin_lock_bh(&tp->lock);
7865         if (wol->wolopts & WAKE_MAGIC)
7866                 tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
7867         else
7868                 tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
7869         spin_unlock_bh(&tp->lock);
7870
7871         return 0;
7872 }
7873
7874 static u32 tg3_get_msglevel(struct net_device *dev)
7875 {
7876         struct tg3 *tp = netdev_priv(dev);
7877         return tp->msg_enable;
7878 }
7879
7880 static void tg3_set_msglevel(struct net_device *dev, u32 value)
7881 {
7882         struct tg3 *tp = netdev_priv(dev);
7883         tp->msg_enable = value;
7884 }
7885
7886 #if TG3_TSO_SUPPORT != 0
7887 static int tg3_set_tso(struct net_device *dev, u32 value)
7888 {
7889         struct tg3 *tp = netdev_priv(dev);
7890
7891         if (!(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
7892                 if (value)
7893                         return -EINVAL;
7894                 return 0;
7895         }
7896         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) {
7897                 if (value)
7898                         dev->features |= NETIF_F_TSO6;
7899                 else
7900                         dev->features &= ~NETIF_F_TSO6;
7901         }
7902         return ethtool_op_set_tso(dev, value);
7903 }
7904 #endif
7905
7906 static int tg3_nway_reset(struct net_device *dev)
7907 {
7908         struct tg3 *tp = netdev_priv(dev);
7909         u32 bmcr;
7910         int r;
7911
7912         if (!netif_running(dev))
7913                 return -EAGAIN;
7914
7915         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
7916                 return -EINVAL;
7917
7918         spin_lock_bh(&tp->lock);
7919         r = -EINVAL;
7920         tg3_readphy(tp, MII_BMCR, &bmcr);
7921         if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
7922             ((bmcr & BMCR_ANENABLE) ||
7923              (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT))) {
7924                 tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
7925                                            BMCR_ANENABLE);
7926                 r = 0;
7927         }
7928         spin_unlock_bh(&tp->lock);
7929
7930         return r;
7931 }
7932
7933 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
7934 {
7935         struct tg3 *tp = netdev_priv(dev);
7936
7937         ering->rx_max_pending = TG3_RX_RING_SIZE - 1;
7938         ering->rx_mini_max_pending = 0;
7939         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
7940                 ering->rx_jumbo_max_pending = TG3_RX_JUMBO_RING_SIZE - 1;
7941         else
7942                 ering->rx_jumbo_max_pending = 0;
7943
7944         ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
7945
7946         ering->rx_pending = tp->rx_pending;
7947         ering->rx_mini_pending = 0;
7948         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
7949                 ering->rx_jumbo_pending = tp->rx_jumbo_pending;
7950         else
7951                 ering->rx_jumbo_pending = 0;
7952
7953         ering->tx_pending = tp->tx_pending;
7954 }
7955
7956 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
7957 {
7958         struct tg3 *tp = netdev_priv(dev);
7959         int irq_sync = 0;
7960
7961         if ((ering->rx_pending > TG3_RX_RING_SIZE - 1) ||
7962             (ering->rx_jumbo_pending > TG3_RX_JUMBO_RING_SIZE - 1) ||
7963             (ering->tx_pending > TG3_TX_RING_SIZE - 1))
7964                 return -EINVAL;
7965
7966         if (netif_running(dev)) {
7967                 tg3_netif_stop(tp);
7968                 irq_sync = 1;
7969         }
7970
7971         tg3_full_lock(tp, irq_sync);
7972
7973         tp->rx_pending = ering->rx_pending;
7974
7975         if ((tp->tg3_flags2 & TG3_FLG2_MAX_RXPEND_64) &&
7976             tp->rx_pending > 63)
7977                 tp->rx_pending = 63;
7978         tp->rx_jumbo_pending = ering->rx_jumbo_pending;
7979         tp->tx_pending = ering->tx_pending;
7980
7981         if (netif_running(dev)) {
7982                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
7983                 tg3_init_hw(tp, 1);
7984                 tg3_netif_start(tp);
7985         }
7986
7987         tg3_full_unlock(tp);
7988
7989         return 0;
7990 }
7991
7992 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
7993 {
7994         struct tg3 *tp = netdev_priv(dev);
7995
7996         epause->autoneg = (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) != 0;
7997         epause->rx_pause = (tp->tg3_flags & TG3_FLAG_RX_PAUSE) != 0;
7998         epause->tx_pause = (tp->tg3_flags & TG3_FLAG_TX_PAUSE) != 0;
7999 }
8000
8001 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
8002 {
8003         struct tg3 *tp = netdev_priv(dev);
8004         int irq_sync = 0;
8005
8006         if (netif_running(dev)) {
8007                 tg3_netif_stop(tp);
8008                 irq_sync = 1;
8009         }
8010
8011         tg3_full_lock(tp, irq_sync);
8012
8013         if (epause->autoneg)
8014                 tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
8015         else
8016                 tp->tg3_flags &= ~TG3_FLAG_PAUSE_AUTONEG;
8017         if (epause->rx_pause)
8018                 tp->tg3_flags |= TG3_FLAG_RX_PAUSE;
8019         else
8020                 tp->tg3_flags &= ~TG3_FLAG_RX_PAUSE;
8021         if (epause->tx_pause)
8022                 tp->tg3_flags |= TG3_FLAG_TX_PAUSE;
8023         else
8024                 tp->tg3_flags &= ~TG3_FLAG_TX_PAUSE;
8025
8026         if (netif_running(dev)) {
8027                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8028                 tg3_init_hw(tp, 1);
8029                 tg3_netif_start(tp);
8030         }
8031
8032         tg3_full_unlock(tp);
8033
8034         return 0;
8035 }
8036
8037 static u32 tg3_get_rx_csum(struct net_device *dev)
8038 {
8039         struct tg3 *tp = netdev_priv(dev);
8040         return (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0;
8041 }
8042
8043 static int tg3_set_rx_csum(struct net_device *dev, u32 data)
8044 {
8045         struct tg3 *tp = netdev_priv(dev);
8046
8047         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8048                 if (data != 0)
8049                         return -EINVAL;
8050                 return 0;
8051         }
8052
8053         spin_lock_bh(&tp->lock);
8054         if (data)
8055                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
8056         else
8057                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
8058         spin_unlock_bh(&tp->lock);
8059
8060         return 0;
8061 }
8062
8063 static int tg3_set_tx_csum(struct net_device *dev, u32 data)
8064 {
8065         struct tg3 *tp = netdev_priv(dev);
8066
8067         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
8068                 if (data != 0)
8069                         return -EINVAL;
8070                 return 0;
8071         }
8072
8073         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
8074             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
8075                 ethtool_op_set_tx_hw_csum(dev, data);
8076         else
8077                 ethtool_op_set_tx_csum(dev, data);
8078
8079         return 0;
8080 }
8081
8082 static int tg3_get_stats_count (struct net_device *dev)
8083 {
8084         return TG3_NUM_STATS;
8085 }
8086
8087 static int tg3_get_test_count (struct net_device *dev)
8088 {
8089         return TG3_NUM_TEST;
8090 }
8091
8092 static void tg3_get_strings (struct net_device *dev, u32 stringset, u8 *buf)
8093 {
8094         switch (stringset) {
8095         case ETH_SS_STATS:
8096                 memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
8097                 break;
8098         case ETH_SS_TEST:
8099                 memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
8100                 break;
8101         default:
8102                 WARN_ON(1);     /* we need a WARN() */
8103                 break;
8104         }
8105 }
8106
8107 static int tg3_phys_id(struct net_device *dev, u32 data)
8108 {
8109         struct tg3 *tp = netdev_priv(dev);
8110         int i;
8111
8112         if (!netif_running(tp->dev))
8113                 return -EAGAIN;
8114
8115         if (data == 0)
8116                 data = 2;
8117
8118         for (i = 0; i < (data * 2); i++) {
8119                 if ((i % 2) == 0)
8120                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8121                                            LED_CTRL_1000MBPS_ON |
8122                                            LED_CTRL_100MBPS_ON |
8123                                            LED_CTRL_10MBPS_ON |
8124                                            LED_CTRL_TRAFFIC_OVERRIDE |
8125                                            LED_CTRL_TRAFFIC_BLINK |
8126                                            LED_CTRL_TRAFFIC_LED);
8127
8128                 else
8129                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
8130                                            LED_CTRL_TRAFFIC_OVERRIDE);
8131
8132                 if (msleep_interruptible(500))
8133                         break;
8134         }
8135         tw32(MAC_LED_CTRL, tp->led_ctrl);
8136         return 0;
8137 }
8138
8139 static void tg3_get_ethtool_stats (struct net_device *dev,
8140                                    struct ethtool_stats *estats, u64 *tmp_stats)
8141 {
8142         struct tg3 *tp = netdev_priv(dev);
8143         memcpy(tmp_stats, tg3_get_estats(tp), sizeof(tp->estats));
8144 }
8145
8146 #define NVRAM_TEST_SIZE 0x100
8147 #define NVRAM_SELFBOOT_FORMAT1_SIZE 0x14
8148
8149 static int tg3_test_nvram(struct tg3 *tp)
8150 {
8151         u32 *buf, csum, magic;
8152         int i, j, err = 0, size;
8153
8154         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
8155                 return -EIO;
8156
8157         if (magic == TG3_EEPROM_MAGIC)
8158                 size = NVRAM_TEST_SIZE;
8159         else if ((magic & 0xff000000) == 0xa5000000) {
8160                 if ((magic & 0xe00000) == 0x200000)
8161                         size = NVRAM_SELFBOOT_FORMAT1_SIZE;
8162                 else
8163                         return 0;
8164         } else
8165                 return -EIO;
8166
8167         buf = kmalloc(size, GFP_KERNEL);
8168         if (buf == NULL)
8169                 return -ENOMEM;
8170
8171         err = -EIO;
8172         for (i = 0, j = 0; i < size; i += 4, j++) {
8173                 u32 val;
8174
8175                 if ((err = tg3_nvram_read(tp, i, &val)) != 0)
8176                         break;
8177                 buf[j] = cpu_to_le32(val);
8178         }
8179         if (i < size)
8180                 goto out;
8181
8182         /* Selfboot format */
8183         if (cpu_to_be32(buf[0]) != TG3_EEPROM_MAGIC) {
8184                 u8 *buf8 = (u8 *) buf, csum8 = 0;
8185
8186                 for (i = 0; i < size; i++)
8187                         csum8 += buf8[i];
8188
8189                 if (csum8 == 0) {
8190                         err = 0;
8191                         goto out;
8192                 }
8193
8194                 err = -EIO;
8195                 goto out;
8196         }
8197
8198         /* Bootstrap checksum at offset 0x10 */
8199         csum = calc_crc((unsigned char *) buf, 0x10);
8200         if(csum != cpu_to_le32(buf[0x10/4]))
8201                 goto out;
8202
8203         /* Manufacturing block starts at offset 0x74, checksum at 0xfc */
8204         csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
8205         if (csum != cpu_to_le32(buf[0xfc/4]))
8206                  goto out;
8207
8208         err = 0;
8209
8210 out:
8211         kfree(buf);
8212         return err;
8213 }
8214
8215 #define TG3_SERDES_TIMEOUT_SEC  2
8216 #define TG3_COPPER_TIMEOUT_SEC  6
8217
8218 static int tg3_test_link(struct tg3 *tp)
8219 {
8220         int i, max;
8221
8222         if (!netif_running(tp->dev))
8223                 return -ENODEV;
8224
8225         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
8226                 max = TG3_SERDES_TIMEOUT_SEC;
8227         else
8228                 max = TG3_COPPER_TIMEOUT_SEC;
8229
8230         for (i = 0; i < max; i++) {
8231                 if (netif_carrier_ok(tp->dev))
8232                         return 0;
8233
8234                 if (msleep_interruptible(1000))
8235                         break;
8236         }
8237
8238         return -EIO;
8239 }
8240
8241 /* Only test the commonly used registers */
8242 static int tg3_test_registers(struct tg3 *tp)
8243 {
8244         int i, is_5705;
8245         u32 offset, read_mask, write_mask, val, save_val, read_val;
8246         static struct {
8247                 u16 offset;
8248                 u16 flags;
8249 #define TG3_FL_5705     0x1
8250 #define TG3_FL_NOT_5705 0x2
8251 #define TG3_FL_NOT_5788 0x4
8252                 u32 read_mask;
8253                 u32 write_mask;
8254         } reg_tbl[] = {
8255                 /* MAC Control Registers */
8256                 { MAC_MODE, TG3_FL_NOT_5705,
8257                         0x00000000, 0x00ef6f8c },
8258                 { MAC_MODE, TG3_FL_5705,
8259                         0x00000000, 0x01ef6b8c },
8260                 { MAC_STATUS, TG3_FL_NOT_5705,
8261                         0x03800107, 0x00000000 },
8262                 { MAC_STATUS, TG3_FL_5705,
8263                         0x03800100, 0x00000000 },
8264                 { MAC_ADDR_0_HIGH, 0x0000,
8265                         0x00000000, 0x0000ffff },
8266                 { MAC_ADDR_0_LOW, 0x0000,
8267                         0x00000000, 0xffffffff },
8268                 { MAC_RX_MTU_SIZE, 0x0000,
8269                         0x00000000, 0x0000ffff },
8270                 { MAC_TX_MODE, 0x0000,
8271                         0x00000000, 0x00000070 },
8272                 { MAC_TX_LENGTHS, 0x0000,
8273                         0x00000000, 0x00003fff },
8274                 { MAC_RX_MODE, TG3_FL_NOT_5705,
8275                         0x00000000, 0x000007fc },
8276                 { MAC_RX_MODE, TG3_FL_5705,
8277                         0x00000000, 0x000007dc },
8278                 { MAC_HASH_REG_0, 0x0000,
8279                         0x00000000, 0xffffffff },
8280                 { MAC_HASH_REG_1, 0x0000,
8281                         0x00000000, 0xffffffff },
8282                 { MAC_HASH_REG_2, 0x0000,
8283                         0x00000000, 0xffffffff },
8284                 { MAC_HASH_REG_3, 0x0000,
8285                         0x00000000, 0xffffffff },
8286
8287                 /* Receive Data and Receive BD Initiator Control Registers. */
8288                 { RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
8289                         0x00000000, 0xffffffff },
8290                 { RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
8291                         0x00000000, 0xffffffff },
8292                 { RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
8293                         0x00000000, 0x00000003 },
8294                 { RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
8295                         0x00000000, 0xffffffff },
8296                 { RCVDBDI_STD_BD+0, 0x0000,
8297                         0x00000000, 0xffffffff },
8298                 { RCVDBDI_STD_BD+4, 0x0000,
8299                         0x00000000, 0xffffffff },
8300                 { RCVDBDI_STD_BD+8, 0x0000,
8301                         0x00000000, 0xffff0002 },
8302                 { RCVDBDI_STD_BD+0xc, 0x0000,
8303                         0x00000000, 0xffffffff },
8304
8305                 /* Receive BD Initiator Control Registers. */
8306                 { RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
8307                         0x00000000, 0xffffffff },
8308                 { RCVBDI_STD_THRESH, TG3_FL_5705,
8309                         0x00000000, 0x000003ff },
8310                 { RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
8311                         0x00000000, 0xffffffff },
8312
8313                 /* Host Coalescing Control Registers. */
8314                 { HOSTCC_MODE, TG3_FL_NOT_5705,
8315                         0x00000000, 0x00000004 },
8316                 { HOSTCC_MODE, TG3_FL_5705,
8317                         0x00000000, 0x000000f6 },
8318                 { HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
8319                         0x00000000, 0xffffffff },
8320                 { HOSTCC_RXCOL_TICKS, TG3_FL_5705,
8321                         0x00000000, 0x000003ff },
8322                 { HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
8323                         0x00000000, 0xffffffff },
8324                 { HOSTCC_TXCOL_TICKS, TG3_FL_5705,
8325                         0x00000000, 0x000003ff },
8326                 { HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
8327                         0x00000000, 0xffffffff },
8328                 { HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
8329                         0x00000000, 0x000000ff },
8330                 { HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
8331                         0x00000000, 0xffffffff },
8332                 { HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
8333                         0x00000000, 0x000000ff },
8334                 { HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
8335                         0x00000000, 0xffffffff },
8336                 { HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
8337                         0x00000000, 0xffffffff },
8338                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
8339                         0x00000000, 0xffffffff },
8340                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
8341                         0x00000000, 0x000000ff },
8342                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
8343                         0x00000000, 0xffffffff },
8344                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
8345                         0x00000000, 0x000000ff },
8346                 { HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
8347                         0x00000000, 0xffffffff },
8348                 { HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
8349                         0x00000000, 0xffffffff },
8350                 { HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
8351                         0x00000000, 0xffffffff },
8352                 { HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
8353                         0x00000000, 0xffffffff },
8354                 { HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
8355                         0x00000000, 0xffffffff },
8356                 { HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
8357                         0xffffffff, 0x00000000 },
8358                 { HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
8359                         0xffffffff, 0x00000000 },
8360
8361                 /* Buffer Manager Control Registers. */
8362                 { BUFMGR_MB_POOL_ADDR, 0x0000,
8363                         0x00000000, 0x007fff80 },
8364                 { BUFMGR_MB_POOL_SIZE, 0x0000,
8365                         0x00000000, 0x007fffff },
8366                 { BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
8367                         0x00000000, 0x0000003f },
8368                 { BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
8369                         0x00000000, 0x000001ff },
8370                 { BUFMGR_MB_HIGH_WATER, 0x0000,
8371                         0x00000000, 0x000001ff },
8372                 { BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
8373                         0xffffffff, 0x00000000 },
8374                 { BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
8375                         0xffffffff, 0x00000000 },
8376
8377                 /* Mailbox Registers */
8378                 { GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
8379                         0x00000000, 0x000001ff },
8380                 { GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
8381                         0x00000000, 0x000001ff },
8382                 { GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
8383                         0x00000000, 0x000007ff },
8384                 { GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
8385                         0x00000000, 0x000001ff },
8386
8387                 { 0xffff, 0x0000, 0x00000000, 0x00000000 },
8388         };
8389
8390         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
8391                 is_5705 = 1;
8392         else
8393                 is_5705 = 0;
8394
8395         for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
8396                 if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
8397                         continue;
8398
8399                 if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
8400                         continue;
8401
8402                 if ((tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
8403                     (reg_tbl[i].flags & TG3_FL_NOT_5788))
8404                         continue;
8405
8406                 offset = (u32) reg_tbl[i].offset;
8407                 read_mask = reg_tbl[i].read_mask;
8408                 write_mask = reg_tbl[i].write_mask;
8409
8410                 /* Save the original register content */
8411                 save_val = tr32(offset);
8412
8413                 /* Determine the read-only value. */
8414                 read_val = save_val & read_mask;
8415
8416                 /* Write zero to the register, then make sure the read-only bits
8417                  * are not changed and the read/write bits are all zeros.
8418                  */
8419                 tw32(offset, 0);
8420
8421                 val = tr32(offset);
8422
8423                 /* Test the read-only and read/write bits. */
8424                 if (((val & read_mask) != read_val) || (val & write_mask))
8425                         goto out;
8426
8427                 /* Write ones to all the bits defined by RdMask and WrMask, then
8428                  * make sure the read-only bits are not changed and the
8429                  * read/write bits are all ones.
8430                  */
8431                 tw32(offset, read_mask | write_mask);
8432
8433                 val = tr32(offset);
8434
8435                 /* Test the read-only bits. */
8436                 if ((val & read_mask) != read_val)
8437                         goto out;
8438
8439                 /* Test the read/write bits. */
8440                 if ((val & write_mask) != write_mask)
8441                         goto out;
8442
8443                 tw32(offset, save_val);
8444         }
8445
8446         return 0;
8447
8448 out:
8449         printk(KERN_ERR PFX "Register test failed at offset %x\n", offset);
8450         tw32(offset, save_val);
8451         return -EIO;
8452 }
8453
8454 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
8455 {
8456         static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
8457         int i;
8458         u32 j;
8459
8460         for (i = 0; i < sizeof(test_pattern)/sizeof(u32); i++) {
8461                 for (j = 0; j < len; j += 4) {
8462                         u32 val;
8463
8464                         tg3_write_mem(tp, offset + j, test_pattern[i]);
8465                         tg3_read_mem(tp, offset + j, &val);
8466                         if (val != test_pattern[i])
8467                                 return -EIO;
8468                 }
8469         }
8470         return 0;
8471 }
8472
8473 static int tg3_test_memory(struct tg3 *tp)
8474 {
8475         static struct mem_entry {
8476                 u32 offset;
8477                 u32 len;
8478         } mem_tbl_570x[] = {
8479                 { 0x00000000, 0x00b50},
8480                 { 0x00002000, 0x1c000},
8481                 { 0xffffffff, 0x00000}
8482         }, mem_tbl_5705[] = {
8483                 { 0x00000100, 0x0000c},
8484                 { 0x00000200, 0x00008},
8485                 { 0x00004000, 0x00800},
8486                 { 0x00006000, 0x01000},
8487                 { 0x00008000, 0x02000},
8488                 { 0x00010000, 0x0e000},
8489                 { 0xffffffff, 0x00000}
8490         }, mem_tbl_5755[] = {
8491                 { 0x00000200, 0x00008},
8492                 { 0x00004000, 0x00800},
8493                 { 0x00006000, 0x00800},
8494                 { 0x00008000, 0x02000},
8495                 { 0x00010000, 0x0c000},
8496                 { 0xffffffff, 0x00000}
8497         };
8498         struct mem_entry *mem_tbl;
8499         int err = 0;
8500         int i;
8501
8502         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
8503                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
8504                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
8505                         mem_tbl = mem_tbl_5755;
8506                 else
8507                         mem_tbl = mem_tbl_5705;
8508         } else
8509                 mem_tbl = mem_tbl_570x;
8510
8511         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
8512                 if ((err = tg3_do_mem_test(tp, mem_tbl[i].offset,
8513                     mem_tbl[i].len)) != 0)
8514                         break;
8515         }
8516
8517         return err;
8518 }
8519
8520 #define TG3_MAC_LOOPBACK        0
8521 #define TG3_PHY_LOOPBACK        1
8522
8523 static int tg3_run_loopback(struct tg3 *tp, int loopback_mode)
8524 {
8525         u32 mac_mode, rx_start_idx, rx_idx, tx_idx, opaque_key;
8526         u32 desc_idx;
8527         struct sk_buff *skb, *rx_skb;
8528         u8 *tx_data;
8529         dma_addr_t map;
8530         int num_pkts, tx_len, rx_len, i, err;
8531         struct tg3_rx_buffer_desc *desc;
8532
8533         if (loopback_mode == TG3_MAC_LOOPBACK) {
8534                 /* HW errata - mac loopback fails in some cases on 5780.
8535                  * Normal traffic and PHY loopback are not affected by
8536                  * errata.
8537                  */
8538                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780)
8539                         return 0;
8540
8541                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
8542                            MAC_MODE_PORT_INT_LPBACK | MAC_MODE_LINK_POLARITY |
8543                            MAC_MODE_PORT_MODE_GMII;
8544                 tw32(MAC_MODE, mac_mode);
8545         } else if (loopback_mode == TG3_PHY_LOOPBACK) {
8546                 tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK | BMCR_FULLDPLX |
8547                                            BMCR_SPEED1000);
8548                 udelay(40);
8549                 /* reset to prevent losing 1st rx packet intermittently */
8550                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
8551                         tw32_f(MAC_RX_MODE, RX_MODE_RESET);
8552                         udelay(10);
8553                         tw32_f(MAC_RX_MODE, tp->rx_mode);
8554                 }
8555                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
8556                            MAC_MODE_LINK_POLARITY | MAC_MODE_PORT_MODE_GMII;
8557                 if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
8558                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
8559                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
8560                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
8561                 }
8562                 tw32(MAC_MODE, mac_mode);
8563         }
8564         else
8565                 return -EINVAL;
8566
8567         err = -EIO;
8568
8569         tx_len = 1514;
8570         skb = dev_alloc_skb(tx_len);
8571         if (!skb)
8572                 return -ENOMEM;
8573
8574         tx_data = skb_put(skb, tx_len);
8575         memcpy(tx_data, tp->dev->dev_addr, 6);
8576         memset(tx_data + 6, 0x0, 8);
8577
8578         tw32(MAC_RX_MTU_SIZE, tx_len + 4);
8579
8580         for (i = 14; i < tx_len; i++)
8581                 tx_data[i] = (u8) (i & 0xff);
8582
8583         map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
8584
8585         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
8586              HOSTCC_MODE_NOW);
8587
8588         udelay(10);
8589
8590         rx_start_idx = tp->hw_status->idx[0].rx_producer;
8591
8592         num_pkts = 0;
8593
8594         tg3_set_txd(tp, tp->tx_prod, map, tx_len, 0, 1);
8595
8596         tp->tx_prod++;
8597         num_pkts++;
8598
8599         tw32_tx_mbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW,
8600                      tp->tx_prod);
8601         tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW);
8602
8603         udelay(10);
8604
8605         for (i = 0; i < 10; i++) {
8606                 tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
8607                        HOSTCC_MODE_NOW);
8608
8609                 udelay(10);
8610
8611                 tx_idx = tp->hw_status->idx[0].tx_consumer;
8612                 rx_idx = tp->hw_status->idx[0].rx_producer;
8613                 if ((tx_idx == tp->tx_prod) &&
8614                     (rx_idx == (rx_start_idx + num_pkts)))
8615                         break;
8616         }
8617
8618         pci_unmap_single(tp->pdev, map, tx_len, PCI_DMA_TODEVICE);
8619         dev_kfree_skb(skb);
8620
8621         if (tx_idx != tp->tx_prod)
8622                 goto out;
8623
8624         if (rx_idx != rx_start_idx + num_pkts)
8625                 goto out;
8626
8627         desc = &tp->rx_rcb[rx_start_idx];
8628         desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
8629         opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
8630         if (opaque_key != RXD_OPAQUE_RING_STD)
8631                 goto out;
8632
8633         if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
8634             (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
8635                 goto out;
8636
8637         rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4;
8638         if (rx_len != tx_len)
8639                 goto out;
8640
8641         rx_skb = tp->rx_std_buffers[desc_idx].skb;
8642
8643         map = pci_unmap_addr(&tp->rx_std_buffers[desc_idx], mapping);
8644         pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len, PCI_DMA_FROMDEVICE);
8645
8646         for (i = 14; i < tx_len; i++) {
8647                 if (*(rx_skb->data + i) != (u8) (i & 0xff))
8648                         goto out;
8649         }
8650         err = 0;
8651
8652         /* tg3_free_rings will unmap and free the rx_skb */
8653 out:
8654         return err;
8655 }
8656
8657 #define TG3_MAC_LOOPBACK_FAILED         1
8658 #define TG3_PHY_LOOPBACK_FAILED         2
8659 #define TG3_LOOPBACK_FAILED             (TG3_MAC_LOOPBACK_FAILED |      \
8660                                          TG3_PHY_LOOPBACK_FAILED)
8661
8662 static int tg3_test_loopback(struct tg3 *tp)
8663 {
8664         int err = 0;
8665
8666         if (!netif_running(tp->dev))
8667                 return TG3_LOOPBACK_FAILED;
8668
8669         tg3_reset_hw(tp, 1);
8670
8671         if (tg3_run_loopback(tp, TG3_MAC_LOOPBACK))
8672                 err |= TG3_MAC_LOOPBACK_FAILED;
8673         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
8674                 if (tg3_run_loopback(tp, TG3_PHY_LOOPBACK))
8675                         err |= TG3_PHY_LOOPBACK_FAILED;
8676         }
8677
8678         return err;
8679 }
8680
8681 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
8682                           u64 *data)
8683 {
8684         struct tg3 *tp = netdev_priv(dev);
8685
8686         if (tp->link_config.phy_is_low_power)
8687                 tg3_set_power_state(tp, PCI_D0);
8688
8689         memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
8690
8691         if (tg3_test_nvram(tp) != 0) {
8692                 etest->flags |= ETH_TEST_FL_FAILED;
8693                 data[0] = 1;
8694         }
8695         if (tg3_test_link(tp) != 0) {
8696                 etest->flags |= ETH_TEST_FL_FAILED;
8697                 data[1] = 1;
8698         }
8699         if (etest->flags & ETH_TEST_FL_OFFLINE) {
8700                 int err, irq_sync = 0;
8701
8702                 if (netif_running(dev)) {
8703                         tg3_netif_stop(tp);
8704                         irq_sync = 1;
8705                 }
8706
8707                 tg3_full_lock(tp, irq_sync);
8708
8709                 tg3_halt(tp, RESET_KIND_SUSPEND, 1);
8710                 err = tg3_nvram_lock(tp);
8711                 tg3_halt_cpu(tp, RX_CPU_BASE);
8712                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
8713                         tg3_halt_cpu(tp, TX_CPU_BASE);
8714                 if (!err)
8715                         tg3_nvram_unlock(tp);
8716
8717                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
8718                         tg3_phy_reset(tp);
8719
8720                 if (tg3_test_registers(tp) != 0) {
8721                         etest->flags |= ETH_TEST_FL_FAILED;
8722                         data[2] = 1;
8723                 }
8724                 if (tg3_test_memory(tp) != 0) {
8725                         etest->flags |= ETH_TEST_FL_FAILED;
8726                         data[3] = 1;
8727                 }
8728                 if ((data[4] = tg3_test_loopback(tp)) != 0)
8729                         etest->flags |= ETH_TEST_FL_FAILED;
8730
8731                 tg3_full_unlock(tp);
8732
8733                 if (tg3_test_interrupt(tp) != 0) {
8734                         etest->flags |= ETH_TEST_FL_FAILED;
8735                         data[5] = 1;
8736                 }
8737
8738                 tg3_full_lock(tp, 0);
8739
8740                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8741                 if (netif_running(dev)) {
8742                         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
8743                         tg3_init_hw(tp, 1);
8744                         tg3_netif_start(tp);
8745                 }
8746
8747                 tg3_full_unlock(tp);
8748         }
8749         if (tp->link_config.phy_is_low_power)
8750                 tg3_set_power_state(tp, PCI_D3hot);
8751
8752 }
8753
8754 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
8755 {
8756         struct mii_ioctl_data *data = if_mii(ifr);
8757         struct tg3 *tp = netdev_priv(dev);
8758         int err;
8759
8760         switch(cmd) {
8761         case SIOCGMIIPHY:
8762                 data->phy_id = PHY_ADDR;
8763
8764                 /* fallthru */
8765         case SIOCGMIIREG: {
8766                 u32 mii_regval;
8767
8768                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
8769                         break;                  /* We have no PHY */
8770
8771                 if (tp->link_config.phy_is_low_power)
8772                         return -EAGAIN;
8773
8774                 spin_lock_bh(&tp->lock);
8775                 err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval);
8776                 spin_unlock_bh(&tp->lock);
8777
8778                 data->val_out = mii_regval;
8779
8780                 return err;
8781         }
8782
8783         case SIOCSMIIREG:
8784                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
8785                         break;                  /* We have no PHY */
8786
8787                 if (!capable(CAP_NET_ADMIN))
8788                         return -EPERM;
8789
8790                 if (tp->link_config.phy_is_low_power)
8791                         return -EAGAIN;
8792
8793                 spin_lock_bh(&tp->lock);
8794                 err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in);
8795                 spin_unlock_bh(&tp->lock);
8796
8797                 return err;
8798
8799         default:
8800                 /* do nothing */
8801                 break;
8802         }
8803         return -EOPNOTSUPP;
8804 }
8805
8806 #if TG3_VLAN_TAG_USED
8807 static void tg3_vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
8808 {
8809         struct tg3 *tp = netdev_priv(dev);
8810
8811         if (netif_running(dev))
8812                 tg3_netif_stop(tp);
8813
8814         tg3_full_lock(tp, 0);
8815
8816         tp->vlgrp = grp;
8817
8818         /* Update RX_MODE_KEEP_VLAN_TAG bit in RX_MODE register. */
8819         __tg3_set_rx_mode(dev);
8820
8821         tg3_full_unlock(tp);
8822
8823         if (netif_running(dev))
8824                 tg3_netif_start(tp);
8825 }
8826
8827 static void tg3_vlan_rx_kill_vid(struct net_device *dev, unsigned short vid)
8828 {
8829         struct tg3 *tp = netdev_priv(dev);
8830
8831         if (netif_running(dev))
8832                 tg3_netif_stop(tp);
8833
8834         tg3_full_lock(tp, 0);
8835         if (tp->vlgrp)
8836                 tp->vlgrp->vlan_devices[vid] = NULL;
8837         tg3_full_unlock(tp);
8838
8839         if (netif_running(dev))
8840                 tg3_netif_start(tp);
8841 }
8842 #endif
8843
8844 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
8845 {
8846         struct tg3 *tp = netdev_priv(dev);
8847
8848         memcpy(ec, &tp->coal, sizeof(*ec));
8849         return 0;
8850 }
8851
8852 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
8853 {
8854         struct tg3 *tp = netdev_priv(dev);
8855         u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
8856         u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
8857
8858         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
8859                 max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
8860                 max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
8861                 max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
8862                 min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
8863         }
8864
8865         if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
8866             (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
8867             (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
8868             (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
8869             (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
8870             (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
8871             (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
8872             (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
8873             (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
8874             (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
8875                 return -EINVAL;
8876
8877         /* No rx interrupts will be generated if both are zero */
8878         if ((ec->rx_coalesce_usecs == 0) &&
8879             (ec->rx_max_coalesced_frames == 0))
8880                 return -EINVAL;
8881
8882         /* No tx interrupts will be generated if both are zero */
8883         if ((ec->tx_coalesce_usecs == 0) &&
8884             (ec->tx_max_coalesced_frames == 0))
8885                 return -EINVAL;
8886
8887         /* Only copy relevant parameters, ignore all others. */
8888         tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
8889         tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
8890         tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
8891         tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
8892         tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
8893         tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
8894         tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
8895         tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
8896         tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
8897
8898         if (netif_running(dev)) {
8899                 tg3_full_lock(tp, 0);
8900                 __tg3_set_coalesce(tp, &tp->coal);
8901                 tg3_full_unlock(tp);
8902         }
8903         return 0;
8904 }
8905
8906 static struct ethtool_ops tg3_ethtool_ops = {
8907         .get_settings           = tg3_get_settings,
8908         .set_settings           = tg3_set_settings,
8909         .get_drvinfo            = tg3_get_drvinfo,
8910         .get_regs_len           = tg3_get_regs_len,
8911         .get_regs               = tg3_get_regs,
8912         .get_wol                = tg3_get_wol,
8913         .set_wol                = tg3_set_wol,
8914         .get_msglevel           = tg3_get_msglevel,
8915         .set_msglevel           = tg3_set_msglevel,
8916         .nway_reset             = tg3_nway_reset,
8917         .get_link               = ethtool_op_get_link,
8918         .get_eeprom_len         = tg3_get_eeprom_len,
8919         .get_eeprom             = tg3_get_eeprom,
8920         .set_eeprom             = tg3_set_eeprom,
8921         .get_ringparam          = tg3_get_ringparam,
8922         .set_ringparam          = tg3_set_ringparam,
8923         .get_pauseparam         = tg3_get_pauseparam,
8924         .set_pauseparam         = tg3_set_pauseparam,
8925         .get_rx_csum            = tg3_get_rx_csum,
8926         .set_rx_csum            = tg3_set_rx_csum,
8927         .get_tx_csum            = ethtool_op_get_tx_csum,
8928         .set_tx_csum            = tg3_set_tx_csum,
8929         .get_sg                 = ethtool_op_get_sg,
8930         .set_sg                 = ethtool_op_set_sg,
8931 #if TG3_TSO_SUPPORT != 0
8932         .get_tso                = ethtool_op_get_tso,
8933         .set_tso                = tg3_set_tso,
8934 #endif
8935         .self_test_count        = tg3_get_test_count,
8936         .self_test              = tg3_self_test,
8937         .get_strings            = tg3_get_strings,
8938         .phys_id                = tg3_phys_id,
8939         .get_stats_count        = tg3_get_stats_count,
8940         .get_ethtool_stats      = tg3_get_ethtool_stats,
8941         .get_coalesce           = tg3_get_coalesce,
8942         .set_coalesce           = tg3_set_coalesce,
8943         .get_perm_addr          = ethtool_op_get_perm_addr,
8944 };
8945
8946 static void __devinit tg3_get_eeprom_size(struct tg3 *tp)
8947 {
8948         u32 cursize, val, magic;
8949
8950         tp->nvram_size = EEPROM_CHIP_SIZE;
8951
8952         if (tg3_nvram_read_swab(tp, 0, &magic) != 0)
8953                 return;
8954
8955         if ((magic != TG3_EEPROM_MAGIC) && ((magic & 0xff000000) != 0xa5000000))
8956                 return;
8957
8958         /*
8959          * Size the chip by reading offsets at increasing powers of two.
8960          * When we encounter our validation signature, we know the addressing
8961          * has wrapped around, and thus have our chip size.
8962          */
8963         cursize = 0x10;
8964
8965         while (cursize < tp->nvram_size) {
8966                 if (tg3_nvram_read_swab(tp, cursize, &val) != 0)
8967                         return;
8968
8969                 if (val == magic)
8970                         break;
8971
8972                 cursize <<= 1;
8973         }
8974
8975         tp->nvram_size = cursize;
8976 }
8977
8978 static void __devinit tg3_get_nvram_size(struct tg3 *tp)
8979 {
8980         u32 val;
8981
8982         if (tg3_nvram_read_swab(tp, 0, &val) != 0)
8983                 return;
8984
8985         /* Selfboot format */
8986         if (val != TG3_EEPROM_MAGIC) {
8987                 tg3_get_eeprom_size(tp);
8988                 return;
8989         }
8990
8991         if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
8992                 if (val != 0) {
8993                         tp->nvram_size = (val >> 16) * 1024;
8994                         return;
8995                 }
8996         }
8997         tp->nvram_size = 0x20000;
8998 }
8999
9000 static void __devinit tg3_get_nvram_info(struct tg3 *tp)
9001 {
9002         u32 nvcfg1;
9003
9004         nvcfg1 = tr32(NVRAM_CFG1);
9005         if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
9006                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
9007         }
9008         else {
9009                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9010                 tw32(NVRAM_CFG1, nvcfg1);
9011         }
9012
9013         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) ||
9014             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
9015                 switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
9016                         case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
9017                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9018                                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9019                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9020                                 break;
9021                         case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
9022                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9023                                 tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
9024                                 break;
9025                         case FLASH_VENDOR_ATMEL_EEPROM:
9026                                 tp->nvram_jedecnum = JEDEC_ATMEL;
9027                                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9028                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9029                                 break;
9030                         case FLASH_VENDOR_ST:
9031                                 tp->nvram_jedecnum = JEDEC_ST;
9032                                 tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
9033                                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9034                                 break;
9035                         case FLASH_VENDOR_SAIFUN:
9036                                 tp->nvram_jedecnum = JEDEC_SAIFUN;
9037                                 tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
9038                                 break;
9039                         case FLASH_VENDOR_SST_SMALL:
9040                         case FLASH_VENDOR_SST_LARGE:
9041                                 tp->nvram_jedecnum = JEDEC_SST;
9042                                 tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
9043                                 break;
9044                 }
9045         }
9046         else {
9047                 tp->nvram_jedecnum = JEDEC_ATMEL;
9048                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
9049                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9050         }
9051 }
9052
9053 static void __devinit tg3_get_5752_nvram_info(struct tg3 *tp)
9054 {
9055         u32 nvcfg1;
9056
9057         nvcfg1 = tr32(NVRAM_CFG1);
9058
9059         /* NVRAM protection for TPM */
9060         if (nvcfg1 & (1 << 27))
9061                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9062
9063         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9064                 case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
9065                 case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
9066                         tp->nvram_jedecnum = JEDEC_ATMEL;
9067                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9068                         break;
9069                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9070                         tp->nvram_jedecnum = JEDEC_ATMEL;
9071                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9072                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9073                         break;
9074                 case FLASH_5752VENDOR_ST_M45PE10:
9075                 case FLASH_5752VENDOR_ST_M45PE20:
9076                 case FLASH_5752VENDOR_ST_M45PE40:
9077                         tp->nvram_jedecnum = JEDEC_ST;
9078                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9079                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9080                         break;
9081         }
9082
9083         if (tp->tg3_flags2 & TG3_FLG2_FLASH) {
9084                 switch (nvcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
9085                         case FLASH_5752PAGE_SIZE_256:
9086                                 tp->nvram_pagesize = 256;
9087                                 break;
9088                         case FLASH_5752PAGE_SIZE_512:
9089                                 tp->nvram_pagesize = 512;
9090                                 break;
9091                         case FLASH_5752PAGE_SIZE_1K:
9092                                 tp->nvram_pagesize = 1024;
9093                                 break;
9094                         case FLASH_5752PAGE_SIZE_2K:
9095                                 tp->nvram_pagesize = 2048;
9096                                 break;
9097                         case FLASH_5752PAGE_SIZE_4K:
9098                                 tp->nvram_pagesize = 4096;
9099                                 break;
9100                         case FLASH_5752PAGE_SIZE_264:
9101                                 tp->nvram_pagesize = 264;
9102                                 break;
9103                 }
9104         }
9105         else {
9106                 /* For eeprom, set pagesize to maximum eeprom size */
9107                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9108
9109                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9110                 tw32(NVRAM_CFG1, nvcfg1);
9111         }
9112 }
9113
9114 static void __devinit tg3_get_5755_nvram_info(struct tg3 *tp)
9115 {
9116         u32 nvcfg1;
9117
9118         nvcfg1 = tr32(NVRAM_CFG1);
9119
9120         /* NVRAM protection for TPM */
9121         if (nvcfg1 & (1 << 27))
9122                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
9123
9124         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9125                 case FLASH_5755VENDOR_ATMEL_EEPROM_64KHZ:
9126                 case FLASH_5755VENDOR_ATMEL_EEPROM_376KHZ:
9127                         tp->nvram_jedecnum = JEDEC_ATMEL;
9128                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9129                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9130
9131                         nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9132                         tw32(NVRAM_CFG1, nvcfg1);
9133                         break;
9134                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9135                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9136                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9137                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9138                 case FLASH_5755VENDOR_ATMEL_FLASH_4:
9139                         tp->nvram_jedecnum = JEDEC_ATMEL;
9140                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9141                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9142                         tp->nvram_pagesize = 264;
9143                         break;
9144                 case FLASH_5752VENDOR_ST_M45PE10:
9145                 case FLASH_5752VENDOR_ST_M45PE20:
9146                 case FLASH_5752VENDOR_ST_M45PE40:
9147                         tp->nvram_jedecnum = JEDEC_ST;
9148                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9149                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9150                         tp->nvram_pagesize = 256;
9151                         break;
9152         }
9153 }
9154
9155 static void __devinit tg3_get_5787_nvram_info(struct tg3 *tp)
9156 {
9157         u32 nvcfg1;
9158
9159         nvcfg1 = tr32(NVRAM_CFG1);
9160
9161         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
9162                 case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
9163                 case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
9164                 case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
9165                 case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
9166                         tp->nvram_jedecnum = JEDEC_ATMEL;
9167                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9168                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
9169
9170                         nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
9171                         tw32(NVRAM_CFG1, nvcfg1);
9172                         break;
9173                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
9174                 case FLASH_5755VENDOR_ATMEL_FLASH_1:
9175                 case FLASH_5755VENDOR_ATMEL_FLASH_2:
9176                 case FLASH_5755VENDOR_ATMEL_FLASH_3:
9177                         tp->nvram_jedecnum = JEDEC_ATMEL;
9178                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9179                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9180                         tp->nvram_pagesize = 264;
9181                         break;
9182                 case FLASH_5752VENDOR_ST_M45PE10:
9183                 case FLASH_5752VENDOR_ST_M45PE20:
9184                 case FLASH_5752VENDOR_ST_M45PE40:
9185                         tp->nvram_jedecnum = JEDEC_ST;
9186                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
9187                         tp->tg3_flags2 |= TG3_FLG2_FLASH;
9188                         tp->nvram_pagesize = 256;
9189                         break;
9190         }
9191 }
9192
9193 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
9194 static void __devinit tg3_nvram_init(struct tg3 *tp)
9195 {
9196         int j;
9197
9198         tw32_f(GRC_EEPROM_ADDR,
9199              (EEPROM_ADDR_FSM_RESET |
9200               (EEPROM_DEFAULT_CLOCK_PERIOD <<
9201                EEPROM_ADDR_CLKPERD_SHIFT)));
9202
9203         /* XXX schedule_timeout() ... */
9204         for (j = 0; j < 100; j++)
9205                 udelay(10);
9206
9207         /* Enable seeprom accesses. */
9208         tw32_f(GRC_LOCAL_CTRL,
9209              tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
9210         udelay(100);
9211
9212         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
9213             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
9214                 tp->tg3_flags |= TG3_FLAG_NVRAM;
9215
9216                 if (tg3_nvram_lock(tp)) {
9217                         printk(KERN_WARNING PFX "%s: Cannot get nvarm lock, "
9218                                "tg3_nvram_init failed.\n", tp->dev->name);
9219                         return;
9220                 }
9221                 tg3_enable_nvram_access(tp);
9222
9223                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
9224                         tg3_get_5752_nvram_info(tp);
9225                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
9226                         tg3_get_5755_nvram_info(tp);
9227                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
9228                         tg3_get_5787_nvram_info(tp);
9229                 else
9230                         tg3_get_nvram_info(tp);
9231
9232                 tg3_get_nvram_size(tp);
9233
9234                 tg3_disable_nvram_access(tp);
9235                 tg3_nvram_unlock(tp);
9236
9237         } else {
9238                 tp->tg3_flags &= ~(TG3_FLAG_NVRAM | TG3_FLAG_NVRAM_BUFFERED);
9239
9240                 tg3_get_eeprom_size(tp);
9241         }
9242 }
9243
9244 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
9245                                         u32 offset, u32 *val)
9246 {
9247         u32 tmp;
9248         int i;
9249
9250         if (offset > EEPROM_ADDR_ADDR_MASK ||
9251             (offset % 4) != 0)
9252                 return -EINVAL;
9253
9254         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
9255                                         EEPROM_ADDR_DEVID_MASK |
9256                                         EEPROM_ADDR_READ);
9257         tw32(GRC_EEPROM_ADDR,
9258              tmp |
9259              (0 << EEPROM_ADDR_DEVID_SHIFT) |
9260              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
9261               EEPROM_ADDR_ADDR_MASK) |
9262              EEPROM_ADDR_READ | EEPROM_ADDR_START);
9263
9264         for (i = 0; i < 10000; i++) {
9265                 tmp = tr32(GRC_EEPROM_ADDR);
9266
9267                 if (tmp & EEPROM_ADDR_COMPLETE)
9268                         break;
9269                 udelay(100);
9270         }
9271         if (!(tmp & EEPROM_ADDR_COMPLETE))
9272                 return -EBUSY;
9273
9274         *val = tr32(GRC_EEPROM_DATA);
9275         return 0;
9276 }
9277
9278 #define NVRAM_CMD_TIMEOUT 10000
9279
9280 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
9281 {
9282         int i;
9283
9284         tw32(NVRAM_CMD, nvram_cmd);
9285         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
9286                 udelay(10);
9287                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
9288                         udelay(10);
9289                         break;
9290                 }
9291         }
9292         if (i == NVRAM_CMD_TIMEOUT) {
9293                 return -EBUSY;
9294         }
9295         return 0;
9296 }
9297
9298 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
9299 {
9300         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
9301             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
9302             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
9303             (tp->nvram_jedecnum == JEDEC_ATMEL))
9304
9305                 addr = ((addr / tp->nvram_pagesize) <<
9306                         ATMEL_AT45DB0X1B_PAGE_POS) +
9307                        (addr % tp->nvram_pagesize);
9308
9309         return addr;
9310 }
9311
9312 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
9313 {
9314         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
9315             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
9316             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
9317             (tp->nvram_jedecnum == JEDEC_ATMEL))
9318
9319                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
9320                         tp->nvram_pagesize) +
9321                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
9322
9323         return addr;
9324 }
9325
9326 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
9327 {
9328         int ret;
9329
9330         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
9331                 return tg3_nvram_read_using_eeprom(tp, offset, val);
9332
9333         offset = tg3_nvram_phys_addr(tp, offset);
9334
9335         if (offset > NVRAM_ADDR_MSK)
9336                 return -EINVAL;
9337
9338         ret = tg3_nvram_lock(tp);
9339         if (ret)
9340                 return ret;
9341
9342         tg3_enable_nvram_access(tp);
9343
9344         tw32(NVRAM_ADDR, offset);
9345         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
9346                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
9347
9348         if (ret == 0)
9349                 *val = swab32(tr32(NVRAM_RDDATA));
9350
9351         tg3_disable_nvram_access(tp);
9352
9353         tg3_nvram_unlock(tp);
9354
9355         return ret;
9356 }
9357
9358 static int tg3_nvram_read_swab(struct tg3 *tp, u32 offset, u32 *val)
9359 {
9360         int err;
9361         u32 tmp;
9362
9363         err = tg3_nvram_read(tp, offset, &tmp);
9364         *val = swab32(tmp);
9365         return err;
9366 }
9367
9368 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
9369                                     u32 offset, u32 len, u8 *buf)
9370 {
9371         int i, j, rc = 0;
9372         u32 val;
9373
9374         for (i = 0; i < len; i += 4) {
9375                 u32 addr, data;
9376
9377                 addr = offset + i;
9378
9379                 memcpy(&data, buf + i, 4);
9380
9381                 tw32(GRC_EEPROM_DATA, cpu_to_le32(data));
9382
9383                 val = tr32(GRC_EEPROM_ADDR);
9384                 tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
9385
9386                 val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
9387                         EEPROM_ADDR_READ);
9388                 tw32(GRC_EEPROM_ADDR, val |
9389                         (0 << EEPROM_ADDR_DEVID_SHIFT) |
9390                         (addr & EEPROM_ADDR_ADDR_MASK) |
9391                         EEPROM_ADDR_START |
9392                         EEPROM_ADDR_WRITE);
9393
9394                 for (j = 0; j < 10000; j++) {
9395                         val = tr32(GRC_EEPROM_ADDR);
9396
9397                         if (val & EEPROM_ADDR_COMPLETE)
9398                                 break;
9399                         udelay(100);
9400                 }
9401                 if (!(val & EEPROM_ADDR_COMPLETE)) {
9402                         rc = -EBUSY;
9403                         break;
9404                 }
9405         }
9406
9407         return rc;
9408 }
9409
9410 /* offset and length are dword aligned */
9411 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
9412                 u8 *buf)
9413 {
9414         int ret = 0;
9415         u32 pagesize = tp->nvram_pagesize;
9416         u32 pagemask = pagesize - 1;
9417         u32 nvram_cmd;
9418         u8 *tmp;
9419
9420         tmp = kmalloc(pagesize, GFP_KERNEL);
9421         if (tmp == NULL)
9422                 return -ENOMEM;
9423
9424         while (len) {
9425                 int j;
9426                 u32 phy_addr, page_off, size;
9427
9428                 phy_addr = offset & ~pagemask;
9429
9430                 for (j = 0; j < pagesize; j += 4) {
9431                         if ((ret = tg3_nvram_read(tp, phy_addr + j,
9432                                                 (u32 *) (tmp + j))))
9433                                 break;
9434                 }
9435                 if (ret)
9436                         break;
9437
9438                 page_off = offset & pagemask;
9439                 size = pagesize;
9440                 if (len < size)
9441                         size = len;
9442
9443                 len -= size;
9444
9445                 memcpy(tmp + page_off, buf, size);
9446
9447                 offset = offset + (pagesize - page_off);
9448
9449                 tg3_enable_nvram_access(tp);
9450
9451                 /*
9452                  * Before we can erase the flash page, we need
9453                  * to issue a special "write enable" command.
9454                  */
9455                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
9456
9457                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
9458                         break;
9459
9460                 /* Erase the target page */
9461                 tw32(NVRAM_ADDR, phy_addr);
9462
9463                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
9464                         NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
9465
9466                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
9467                         break;
9468
9469                 /* Issue another write enable to start the write. */
9470                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
9471
9472                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
9473                         break;
9474
9475                 for (j = 0; j < pagesize; j += 4) {
9476                         u32 data;
9477
9478                         data = *((u32 *) (tmp + j));
9479                         tw32(NVRAM_WRDATA, cpu_to_be32(data));
9480
9481                         tw32(NVRAM_ADDR, phy_addr + j);
9482
9483                         nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
9484                                 NVRAM_CMD_WR;
9485
9486                         if (j == 0)
9487                                 nvram_cmd |= NVRAM_CMD_FIRST;
9488                         else if (j == (pagesize - 4))
9489                                 nvram_cmd |= NVRAM_CMD_LAST;
9490
9491                         if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
9492                                 break;
9493                 }
9494                 if (ret)
9495                         break;
9496         }
9497
9498         nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
9499         tg3_nvram_exec_cmd(tp, nvram_cmd);
9500
9501         kfree(tmp);
9502
9503         return ret;
9504 }
9505
9506 /* offset and length are dword aligned */
9507 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
9508                 u8 *buf)
9509 {
9510         int i, ret = 0;
9511
9512         for (i = 0; i < len; i += 4, offset += 4) {
9513                 u32 data, page_off, phy_addr, nvram_cmd;
9514
9515                 memcpy(&data, buf + i, 4);
9516                 tw32(NVRAM_WRDATA, cpu_to_be32(data));
9517
9518                 page_off = offset % tp->nvram_pagesize;
9519
9520                 phy_addr = tg3_nvram_phys_addr(tp, offset);
9521
9522                 tw32(NVRAM_ADDR, phy_addr);
9523
9524                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
9525
9526                 if ((page_off == 0) || (i == 0))
9527                         nvram_cmd |= NVRAM_CMD_FIRST;
9528                 if (page_off == (tp->nvram_pagesize - 4))
9529                         nvram_cmd |= NVRAM_CMD_LAST;
9530
9531                 if (i == (len - 4))
9532                         nvram_cmd |= NVRAM_CMD_LAST;
9533
9534                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752) &&
9535                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755) &&
9536                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787) &&
9537                     (tp->nvram_jedecnum == JEDEC_ST) &&
9538                     (nvram_cmd & NVRAM_CMD_FIRST)) {
9539
9540                         if ((ret = tg3_nvram_exec_cmd(tp,
9541                                 NVRAM_CMD_WREN | NVRAM_CMD_GO |
9542                                 NVRAM_CMD_DONE)))
9543
9544                                 break;
9545                 }
9546                 if (!(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
9547                         /* We always do complete word writes to eeprom. */
9548                         nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
9549                 }
9550
9551                 if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
9552                         break;
9553         }
9554         return ret;
9555 }
9556
9557 /* offset and length are dword aligned */
9558 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
9559 {
9560         int ret;
9561
9562         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
9563                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
9564                        ~GRC_LCLCTRL_GPIO_OUTPUT1);
9565                 udelay(40);
9566         }
9567
9568         if (!(tp->tg3_flags & TG3_FLAG_NVRAM)) {
9569                 ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
9570         }
9571         else {
9572                 u32 grc_mode;
9573
9574                 ret = tg3_nvram_lock(tp);
9575                 if (ret)
9576                         return ret;
9577
9578                 tg3_enable_nvram_access(tp);
9579                 if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
9580                     !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM))
9581                         tw32(NVRAM_WRITE1, 0x406);
9582
9583                 grc_mode = tr32(GRC_MODE);
9584                 tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
9585
9586                 if ((tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) ||
9587                         !(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
9588
9589                         ret = tg3_nvram_write_block_buffered(tp, offset, len,
9590                                 buf);
9591                 }
9592                 else {
9593                         ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
9594                                 buf);
9595                 }
9596
9597                 grc_mode = tr32(GRC_MODE);
9598                 tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
9599
9600                 tg3_disable_nvram_access(tp);
9601                 tg3_nvram_unlock(tp);
9602         }
9603
9604         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
9605                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
9606                 udelay(40);
9607         }
9608
9609         return ret;
9610 }
9611
9612 struct subsys_tbl_ent {
9613         u16 subsys_vendor, subsys_devid;
9614         u32 phy_id;
9615 };
9616
9617 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
9618         /* Broadcom boards. */
9619         { PCI_VENDOR_ID_BROADCOM, 0x1644, PHY_ID_BCM5401 }, /* BCM95700A6 */
9620         { PCI_VENDOR_ID_BROADCOM, 0x0001, PHY_ID_BCM5701 }, /* BCM95701A5 */
9621         { PCI_VENDOR_ID_BROADCOM, 0x0002, PHY_ID_BCM8002 }, /* BCM95700T6 */
9622         { PCI_VENDOR_ID_BROADCOM, 0x0003, 0 },              /* BCM95700A9 */
9623         { PCI_VENDOR_ID_BROADCOM, 0x0005, PHY_ID_BCM5701 }, /* BCM95701T1 */
9624         { PCI_VENDOR_ID_BROADCOM, 0x0006, PHY_ID_BCM5701 }, /* BCM95701T8 */
9625         { PCI_VENDOR_ID_BROADCOM, 0x0007, 0 },              /* BCM95701A7 */
9626         { PCI_VENDOR_ID_BROADCOM, 0x0008, PHY_ID_BCM5701 }, /* BCM95701A10 */
9627         { PCI_VENDOR_ID_BROADCOM, 0x8008, PHY_ID_BCM5701 }, /* BCM95701A12 */
9628         { PCI_VENDOR_ID_BROADCOM, 0x0009, PHY_ID_BCM5703 }, /* BCM95703Ax1 */
9629         { PCI_VENDOR_ID_BROADCOM, 0x8009, PHY_ID_BCM5703 }, /* BCM95703Ax2 */
9630
9631         /* 3com boards. */
9632         { PCI_VENDOR_ID_3COM, 0x1000, PHY_ID_BCM5401 }, /* 3C996T */
9633         { PCI_VENDOR_ID_3COM, 0x1006, PHY_ID_BCM5701 }, /* 3C996BT */
9634         { PCI_VENDOR_ID_3COM, 0x1004, 0 },              /* 3C996SX */
9635         { PCI_VENDOR_ID_3COM, 0x1007, PHY_ID_BCM5701 }, /* 3C1000T */
9636         { PCI_VENDOR_ID_3COM, 0x1008, PHY_ID_BCM5701 }, /* 3C940BR01 */
9637
9638         /* DELL boards. */
9639         { PCI_VENDOR_ID_DELL, 0x00d1, PHY_ID_BCM5401 }, /* VIPER */
9640         { PCI_VENDOR_ID_DELL, 0x0106, PHY_ID_BCM5401 }, /* JAGUAR */
9641         { PCI_VENDOR_ID_DELL, 0x0109, PHY_ID_BCM5411 }, /* MERLOT */
9642         { PCI_VENDOR_ID_DELL, 0x010a, PHY_ID_BCM5411 }, /* SLIM_MERLOT */
9643
9644         /* Compaq boards. */
9645         { PCI_VENDOR_ID_COMPAQ, 0x007c, PHY_ID_BCM5701 }, /* BANSHEE */
9646         { PCI_VENDOR_ID_COMPAQ, 0x009a, PHY_ID_BCM5701 }, /* BANSHEE_2 */
9647         { PCI_VENDOR_ID_COMPAQ, 0x007d, 0 },              /* CHANGELING */
9648         { PCI_VENDOR_ID_COMPAQ, 0x0085, PHY_ID_BCM5701 }, /* NC7780 */
9649         { PCI_VENDOR_ID_COMPAQ, 0x0099, PHY_ID_BCM5701 }, /* NC7780_2 */
9650
9651         /* IBM boards. */
9652         { PCI_VENDOR_ID_IBM, 0x0281, 0 } /* IBM??? */
9653 };
9654
9655 static inline struct subsys_tbl_ent *lookup_by_subsys(struct tg3 *tp)
9656 {
9657         int i;
9658
9659         for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
9660                 if ((subsys_id_to_phy_id[i].subsys_vendor ==
9661                      tp->pdev->subsystem_vendor) &&
9662                     (subsys_id_to_phy_id[i].subsys_devid ==
9663                      tp->pdev->subsystem_device))
9664                         return &subsys_id_to_phy_id[i];
9665         }
9666         return NULL;
9667 }
9668
9669 static void __devinit tg3_get_eeprom_hw_cfg(struct tg3 *tp)
9670 {
9671         u32 val;
9672         u16 pmcsr;
9673
9674         /* On some early chips the SRAM cannot be accessed in D3hot state,
9675          * so need make sure we're in D0.
9676          */
9677         pci_read_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, &pmcsr);
9678         pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
9679         pci_write_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, pmcsr);
9680         msleep(1);
9681
9682         /* Make sure register accesses (indirect or otherwise)
9683          * will function correctly.
9684          */
9685         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
9686                                tp->misc_host_ctrl);
9687
9688         /* The memory arbiter has to be enabled in order for SRAM accesses
9689          * to succeed.  Normally on powerup the tg3 chip firmware will make
9690          * sure it is enabled, but other entities such as system netboot
9691          * code might disable it.
9692          */
9693         val = tr32(MEMARB_MODE);
9694         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
9695
9696         tp->phy_id = PHY_ID_INVALID;
9697         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
9698
9699         /* Assume an onboard device by default.  */
9700         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
9701
9702         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
9703         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
9704                 u32 nic_cfg, led_cfg;
9705                 u32 nic_phy_id, ver, cfg2 = 0, eeprom_phy_id;
9706                 int eeprom_phy_serdes = 0;
9707
9708                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
9709                 tp->nic_sram_data_cfg = nic_cfg;
9710
9711                 tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
9712                 ver >>= NIC_SRAM_DATA_VER_SHIFT;
9713                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700) &&
9714                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) &&
9715                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5703) &&
9716                     (ver > 0) && (ver < 0x100))
9717                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
9718
9719                 if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
9720                     NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
9721                         eeprom_phy_serdes = 1;
9722
9723                 tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
9724                 if (nic_phy_id != 0) {
9725                         u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
9726                         u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
9727
9728                         eeprom_phy_id  = (id1 >> 16) << 10;
9729                         eeprom_phy_id |= (id2 & 0xfc00) << 16;
9730                         eeprom_phy_id |= (id2 & 0x03ff) <<  0;
9731                 } else
9732                         eeprom_phy_id = 0;
9733
9734                 tp->phy_id = eeprom_phy_id;
9735                 if (eeprom_phy_serdes) {
9736                         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
9737                                 tp->tg3_flags2 |= TG3_FLG2_MII_SERDES;
9738                         else
9739                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
9740                 }
9741
9742                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
9743                         led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
9744                                     SHASTA_EXT_LED_MODE_MASK);
9745                 else
9746                         led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
9747
9748                 switch (led_cfg) {
9749                 default:
9750                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
9751                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
9752                         break;
9753
9754                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
9755                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
9756                         break;
9757
9758                 case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
9759                         tp->led_ctrl = LED_CTRL_MODE_MAC;
9760
9761                         /* Default to PHY_1_MODE if 0 (MAC_MODE) is
9762                          * read on some older 5700/5701 bootcode.
9763                          */
9764                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
9765                             ASIC_REV_5700 ||
9766                             GET_ASIC_REV(tp->pci_chip_rev_id) ==
9767                             ASIC_REV_5701)
9768                                 tp->led_ctrl = LED_CTRL_MODE_PHY_1;
9769
9770                         break;
9771
9772                 case SHASTA_EXT_LED_SHARED:
9773                         tp->led_ctrl = LED_CTRL_MODE_SHARED;
9774                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
9775                             tp->pci_chip_rev_id != CHIPREV_ID_5750_A1)
9776                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
9777                                                  LED_CTRL_MODE_PHY_2);
9778                         break;
9779
9780                 case SHASTA_EXT_LED_MAC:
9781                         tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
9782                         break;
9783
9784                 case SHASTA_EXT_LED_COMBO:
9785                         tp->led_ctrl = LED_CTRL_MODE_COMBO;
9786                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0)
9787                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
9788                                                  LED_CTRL_MODE_PHY_2);
9789                         break;
9790
9791                 };
9792
9793                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
9794                      GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) &&
9795                     tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
9796                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
9797
9798                 if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP)
9799                         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
9800                 else
9801                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
9802
9803                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
9804                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
9805                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
9806                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
9807                 }
9808                 if (nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL)
9809                         tp->tg3_flags |= TG3_FLAG_SERDES_WOL_CAP;
9810
9811                 if (cfg2 & (1 << 17))
9812                         tp->tg3_flags2 |= TG3_FLG2_CAPACITIVE_COUPLING;
9813
9814                 /* serdes signal pre-emphasis in register 0x590 set by */
9815                 /* bootcode if bit 18 is set */
9816                 if (cfg2 & (1 << 18))
9817                         tp->tg3_flags2 |= TG3_FLG2_SERDES_PREEMPHASIS;
9818         }
9819 }
9820
9821 static int __devinit tg3_phy_probe(struct tg3 *tp)
9822 {
9823         u32 hw_phy_id_1, hw_phy_id_2;
9824         u32 hw_phy_id, hw_phy_id_masked;
9825         int err;
9826
9827         /* Reading the PHY ID register can conflict with ASF
9828          * firwmare access to the PHY hardware.
9829          */
9830         err = 0;
9831         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
9832                 hw_phy_id = hw_phy_id_masked = PHY_ID_INVALID;
9833         } else {
9834                 /* Now read the physical PHY_ID from the chip and verify
9835                  * that it is sane.  If it doesn't look good, we fall back
9836                  * to either the hard-coded table based PHY_ID and failing
9837                  * that the value found in the eeprom area.
9838                  */
9839                 err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
9840                 err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
9841
9842                 hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
9843                 hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
9844                 hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
9845
9846                 hw_phy_id_masked = hw_phy_id & PHY_ID_MASK;
9847         }
9848
9849         if (!err && KNOWN_PHY_ID(hw_phy_id_masked)) {
9850                 tp->phy_id = hw_phy_id;
9851                 if (hw_phy_id_masked == PHY_ID_BCM8002)
9852                         tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
9853                 else
9854                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_SERDES;
9855         } else {
9856                 if (tp->phy_id != PHY_ID_INVALID) {
9857                         /* Do nothing, phy ID already set up in
9858                          * tg3_get_eeprom_hw_cfg().
9859                          */
9860                 } else {
9861                         struct subsys_tbl_ent *p;
9862
9863                         /* No eeprom signature?  Try the hardcoded
9864                          * subsys device table.
9865                          */
9866                         p = lookup_by_subsys(tp);
9867                         if (!p)
9868                                 return -ENODEV;
9869
9870                         tp->phy_id = p->phy_id;
9871                         if (!tp->phy_id ||
9872                             tp->phy_id == PHY_ID_BCM8002)
9873                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
9874                 }
9875         }
9876
9877         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) &&
9878             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
9879                 u32 bmsr, adv_reg, tg3_ctrl;
9880
9881                 tg3_readphy(tp, MII_BMSR, &bmsr);
9882                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
9883                     (bmsr & BMSR_LSTATUS))
9884                         goto skip_phy_reset;
9885
9886                 err = tg3_phy_reset(tp);
9887                 if (err)
9888                         return err;
9889
9890                 adv_reg = (ADVERTISE_10HALF | ADVERTISE_10FULL |
9891                            ADVERTISE_100HALF | ADVERTISE_100FULL |
9892                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
9893                 tg3_ctrl = 0;
9894                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
9895                         tg3_ctrl = (MII_TG3_CTRL_ADV_1000_HALF |
9896                                     MII_TG3_CTRL_ADV_1000_FULL);
9897                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
9898                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
9899                                 tg3_ctrl |= (MII_TG3_CTRL_AS_MASTER |
9900                                              MII_TG3_CTRL_ENABLE_AS_MASTER);
9901                 }
9902
9903                 if (!tg3_copper_is_advertising_all(tp)) {
9904                         tg3_writephy(tp, MII_ADVERTISE, adv_reg);
9905
9906                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
9907                                 tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
9908
9909                         tg3_writephy(tp, MII_BMCR,
9910                                      BMCR_ANENABLE | BMCR_ANRESTART);
9911                 }
9912                 tg3_phy_set_wirespeed(tp);
9913
9914                 tg3_writephy(tp, MII_ADVERTISE, adv_reg);
9915                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
9916                         tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
9917         }
9918
9919 skip_phy_reset:
9920         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
9921                 err = tg3_init_5401phy_dsp(tp);
9922                 if (err)
9923                         return err;
9924         }
9925
9926         if (!err && ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)) {
9927                 err = tg3_init_5401phy_dsp(tp);
9928         }
9929
9930         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
9931                 tp->link_config.advertising =
9932                         (ADVERTISED_1000baseT_Half |
9933                          ADVERTISED_1000baseT_Full |
9934                          ADVERTISED_Autoneg |
9935                          ADVERTISED_FIBRE);
9936         if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
9937                 tp->link_config.advertising &=
9938                         ~(ADVERTISED_1000baseT_Half |
9939                           ADVERTISED_1000baseT_Full);
9940
9941         return err;
9942 }
9943
9944 static void __devinit tg3_read_partno(struct tg3 *tp)
9945 {
9946         unsigned char vpd_data[256];
9947         int i;
9948         u32 magic;
9949
9950         if (tg3_nvram_read_swab(tp, 0x0, &magic))
9951                 goto out_not_found;
9952
9953         if (magic == TG3_EEPROM_MAGIC) {
9954                 for (i = 0; i < 256; i += 4) {
9955                         u32 tmp;
9956
9957                         if (tg3_nvram_read(tp, 0x100 + i, &tmp))
9958                                 goto out_not_found;
9959
9960                         vpd_data[i + 0] = ((tmp >>  0) & 0xff);
9961                         vpd_data[i + 1] = ((tmp >>  8) & 0xff);
9962                         vpd_data[i + 2] = ((tmp >> 16) & 0xff);
9963                         vpd_data[i + 3] = ((tmp >> 24) & 0xff);
9964                 }
9965         } else {
9966                 int vpd_cap;
9967
9968                 vpd_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_VPD);
9969                 for (i = 0; i < 256; i += 4) {
9970                         u32 tmp, j = 0;
9971                         u16 tmp16;
9972
9973                         pci_write_config_word(tp->pdev, vpd_cap + PCI_VPD_ADDR,
9974                                               i);
9975                         while (j++ < 100) {
9976                                 pci_read_config_word(tp->pdev, vpd_cap +
9977                                                      PCI_VPD_ADDR, &tmp16);
9978                                 if (tmp16 & 0x8000)
9979                                         break;
9980                                 msleep(1);
9981                         }
9982                         if (!(tmp16 & 0x8000))
9983                                 goto out_not_found;
9984
9985                         pci_read_config_dword(tp->pdev, vpd_cap + PCI_VPD_DATA,
9986                                               &tmp);
9987                         tmp = cpu_to_le32(tmp);
9988                         memcpy(&vpd_data[i], &tmp, 4);
9989                 }
9990         }
9991
9992         /* Now parse and find the part number. */
9993         for (i = 0; i < 256; ) {
9994                 unsigned char val = vpd_data[i];
9995                 int block_end;
9996
9997                 if (val == 0x82 || val == 0x91) {
9998                         i = (i + 3 +
9999                              (vpd_data[i + 1] +
10000                               (vpd_data[i + 2] << 8)));
10001                         continue;
10002                 }
10003
10004                 if (val != 0x90)
10005                         goto out_not_found;
10006
10007                 block_end = (i + 3 +
10008                              (vpd_data[i + 1] +
10009                               (vpd_data[i + 2] << 8)));
10010                 i += 3;
10011                 while (i < block_end) {
10012                         if (vpd_data[i + 0] == 'P' &&
10013                             vpd_data[i + 1] == 'N') {
10014                                 int partno_len = vpd_data[i + 2];
10015
10016                                 if (partno_len > 24)
10017                                         goto out_not_found;
10018
10019                                 memcpy(tp->board_part_number,
10020                                        &vpd_data[i + 3],
10021                                        partno_len);
10022
10023                                 /* Success. */
10024                                 return;
10025                         }
10026                 }
10027
10028                 /* Part number not found. */
10029                 goto out_not_found;
10030         }
10031
10032 out_not_found:
10033         strcpy(tp->board_part_number, "none");
10034 }
10035
10036 static void __devinit tg3_read_fw_ver(struct tg3 *tp)
10037 {
10038         u32 val, offset, start;
10039
10040         if (tg3_nvram_read_swab(tp, 0, &val))
10041                 return;
10042
10043         if (val != TG3_EEPROM_MAGIC)
10044                 return;
10045
10046         if (tg3_nvram_read_swab(tp, 0xc, &offset) ||
10047             tg3_nvram_read_swab(tp, 0x4, &start))
10048                 return;
10049
10050         offset = tg3_nvram_logical_addr(tp, offset);
10051         if (tg3_nvram_read_swab(tp, offset, &val))
10052                 return;
10053
10054         if ((val & 0xfc000000) == 0x0c000000) {
10055                 u32 ver_offset, addr;
10056                 int i;
10057
10058                 if (tg3_nvram_read_swab(tp, offset + 4, &val) ||
10059                     tg3_nvram_read_swab(tp, offset + 8, &ver_offset))
10060                         return;
10061
10062                 if (val != 0)
10063                         return;
10064
10065                 addr = offset + ver_offset - start;
10066                 for (i = 0; i < 16; i += 4) {
10067                         if (tg3_nvram_read(tp, addr + i, &val))
10068                                 return;
10069
10070                         val = cpu_to_le32(val);
10071                         memcpy(tp->fw_ver + i, &val, 4);
10072                 }
10073         }
10074 }
10075
10076 static int __devinit tg3_get_invariants(struct tg3 *tp)
10077 {
10078         static struct pci_device_id write_reorder_chipsets[] = {
10079                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
10080                              PCI_DEVICE_ID_AMD_FE_GATE_700C) },
10081                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
10082                              PCI_DEVICE_ID_AMD_8131_BRIDGE) },
10083                 { PCI_DEVICE(PCI_VENDOR_ID_VIA,
10084                              PCI_DEVICE_ID_VIA_8385_0) },
10085                 { },
10086         };
10087         u32 misc_ctrl_reg;
10088         u32 cacheline_sz_reg;
10089         u32 pci_state_reg, grc_misc_cfg;
10090         u32 val;
10091         u16 pci_cmd;
10092         int err;
10093
10094         /* Force memory write invalidate off.  If we leave it on,
10095          * then on 5700_BX chips we have to enable a workaround.
10096          * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
10097          * to match the cacheline size.  The Broadcom driver have this
10098          * workaround but turns MWI off all the times so never uses
10099          * it.  This seems to suggest that the workaround is insufficient.
10100          */
10101         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
10102         pci_cmd &= ~PCI_COMMAND_INVALIDATE;
10103         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
10104
10105         /* It is absolutely critical that TG3PCI_MISC_HOST_CTRL
10106          * has the register indirect write enable bit set before
10107          * we try to access any of the MMIO registers.  It is also
10108          * critical that the PCI-X hw workaround situation is decided
10109          * before that as well.
10110          */
10111         pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10112                               &misc_ctrl_reg);
10113
10114         tp->pci_chip_rev_id = (misc_ctrl_reg >>
10115                                MISC_HOST_CTRL_CHIPREV_SHIFT);
10116
10117         /* Wrong chip ID in 5752 A0. This code can be removed later
10118          * as A0 is not in production.
10119          */
10120         if (tp->pci_chip_rev_id == CHIPREV_ID_5752_A0_HW)
10121                 tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
10122
10123         /* If we have 5702/03 A1 or A2 on certain ICH chipsets,
10124          * we need to disable memory and use config. cycles
10125          * only to access all registers. The 5702/03 chips
10126          * can mistakenly decode the special cycles from the
10127          * ICH chipsets as memory write cycles, causing corruption
10128          * of register and memory space. Only certain ICH bridges
10129          * will drive special cycles with non-zero data during the
10130          * address phase which can fall within the 5703's address
10131          * range. This is not an ICH bug as the PCI spec allows
10132          * non-zero address during special cycles. However, only
10133          * these ICH bridges are known to drive non-zero addresses
10134          * during special cycles.
10135          *
10136          * Since special cycles do not cross PCI bridges, we only
10137          * enable this workaround if the 5703 is on the secondary
10138          * bus of these ICH bridges.
10139          */
10140         if ((tp->pci_chip_rev_id == CHIPREV_ID_5703_A1) ||
10141             (tp->pci_chip_rev_id == CHIPREV_ID_5703_A2)) {
10142                 static struct tg3_dev_id {
10143                         u32     vendor;
10144                         u32     device;
10145                         u32     rev;
10146                 } ich_chipsets[] = {
10147                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
10148                           PCI_ANY_ID },
10149                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
10150                           PCI_ANY_ID },
10151                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
10152                           0xa },
10153                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
10154                           PCI_ANY_ID },
10155                         { },
10156                 };
10157                 struct tg3_dev_id *pci_id = &ich_chipsets[0];
10158                 struct pci_dev *bridge = NULL;
10159
10160                 while (pci_id->vendor != 0) {
10161                         bridge = pci_get_device(pci_id->vendor, pci_id->device,
10162                                                 bridge);
10163                         if (!bridge) {
10164                                 pci_id++;
10165                                 continue;
10166                         }
10167                         if (pci_id->rev != PCI_ANY_ID) {
10168                                 u8 rev;
10169
10170                                 pci_read_config_byte(bridge, PCI_REVISION_ID,
10171                                                      &rev);
10172                                 if (rev > pci_id->rev)
10173                                         continue;
10174                         }
10175                         if (bridge->subordinate &&
10176                             (bridge->subordinate->number ==
10177                              tp->pdev->bus->number)) {
10178
10179                                 tp->tg3_flags2 |= TG3_FLG2_ICH_WORKAROUND;
10180                                 pci_dev_put(bridge);
10181                                 break;
10182                         }
10183                 }
10184         }
10185
10186         /* The EPB bridge inside 5714, 5715, and 5780 cannot support
10187          * DMA addresses > 40-bit. This bridge may have other additional
10188          * 57xx devices behind it in some 4-port NIC designs for example.
10189          * Any tg3 device found behind the bridge will also need the 40-bit
10190          * DMA workaround.
10191          */
10192         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
10193             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
10194                 tp->tg3_flags2 |= TG3_FLG2_5780_CLASS;
10195                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
10196                 tp->msi_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_MSI);
10197         }
10198         else {
10199                 struct pci_dev *bridge = NULL;
10200
10201                 do {
10202                         bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
10203                                                 PCI_DEVICE_ID_SERVERWORKS_EPB,
10204                                                 bridge);
10205                         if (bridge && bridge->subordinate &&
10206                             (bridge->subordinate->number <=
10207                              tp->pdev->bus->number) &&
10208                             (bridge->subordinate->subordinate >=
10209                              tp->pdev->bus->number)) {
10210                                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
10211                                 pci_dev_put(bridge);
10212                                 break;
10213                         }
10214                 } while (bridge);
10215         }
10216
10217         /* Initialize misc host control in PCI block. */
10218         tp->misc_host_ctrl |= (misc_ctrl_reg &
10219                                MISC_HOST_CTRL_CHIPREV);
10220         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10221                                tp->misc_host_ctrl);
10222
10223         pci_read_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
10224                               &cacheline_sz_reg);
10225
10226         tp->pci_cacheline_sz = (cacheline_sz_reg >>  0) & 0xff;
10227         tp->pci_lat_timer    = (cacheline_sz_reg >>  8) & 0xff;
10228         tp->pci_hdr_type     = (cacheline_sz_reg >> 16) & 0xff;
10229         tp->pci_bist         = (cacheline_sz_reg >> 24) & 0xff;
10230
10231         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
10232             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
10233             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10234             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
10235             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
10236                 tp->tg3_flags2 |= TG3_FLG2_5750_PLUS;
10237
10238         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) ||
10239             (tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
10240                 tp->tg3_flags2 |= TG3_FLG2_5705_PLUS;
10241
10242         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
10243                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10244                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787) {
10245                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_2;
10246                         tp->tg3_flags2 |= TG3_FLG2_1SHOT_MSI;
10247                 } else {
10248                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_1 |
10249                                           TG3_FLG2_HW_TSO_1_BUG;
10250                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
10251                                 ASIC_REV_5750 &&
10252                             tp->pci_chip_rev_id >= CHIPREV_ID_5750_C2)
10253                                 tp->tg3_flags2 &= ~TG3_FLG2_HW_TSO_1_BUG;
10254                 }
10255         }
10256
10257         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705 &&
10258             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5750 &&
10259             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752 &&
10260             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5755 &&
10261             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5787)
10262                 tp->tg3_flags2 |= TG3_FLG2_JUMBO_CAPABLE;
10263
10264         if (pci_find_capability(tp->pdev, PCI_CAP_ID_EXP) != 0)
10265                 tp->tg3_flags2 |= TG3_FLG2_PCI_EXPRESS;
10266
10267         /* If we have an AMD 762 or VIA K8T800 chipset, write
10268          * reordering to the mailbox registers done by the host
10269          * controller can cause major troubles.  We read back from
10270          * every mailbox register write to force the writes to be
10271          * posted to the chip in order.
10272          */
10273         if (pci_dev_present(write_reorder_chipsets) &&
10274             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
10275                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
10276
10277         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
10278             tp->pci_lat_timer < 64) {
10279                 tp->pci_lat_timer = 64;
10280
10281                 cacheline_sz_reg  = ((tp->pci_cacheline_sz & 0xff) <<  0);
10282                 cacheline_sz_reg |= ((tp->pci_lat_timer    & 0xff) <<  8);
10283                 cacheline_sz_reg |= ((tp->pci_hdr_type     & 0xff) << 16);
10284                 cacheline_sz_reg |= ((tp->pci_bist         & 0xff) << 24);
10285
10286                 pci_write_config_dword(tp->pdev, TG3PCI_CACHELINESZ,
10287                                        cacheline_sz_reg);
10288         }
10289
10290         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
10291                               &pci_state_reg);
10292
10293         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0) {
10294                 tp->tg3_flags |= TG3_FLAG_PCIX_MODE;
10295
10296                 /* If this is a 5700 BX chipset, and we are in PCI-X
10297                  * mode, enable register write workaround.
10298                  *
10299                  * The workaround is to use indirect register accesses
10300                  * for all chip writes not to mailbox registers.
10301                  */
10302                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX) {
10303                         u32 pm_reg;
10304                         u16 pci_cmd;
10305
10306                         tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
10307
10308                         /* The chip can have it's power management PCI config
10309                          * space registers clobbered due to this bug.
10310                          * So explicitly force the chip into D0 here.
10311                          */
10312                         pci_read_config_dword(tp->pdev, TG3PCI_PM_CTRL_STAT,
10313                                               &pm_reg);
10314                         pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
10315                         pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
10316                         pci_write_config_dword(tp->pdev, TG3PCI_PM_CTRL_STAT,
10317                                                pm_reg);
10318
10319                         /* Also, force SERR#/PERR# in PCI command. */
10320                         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
10321                         pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
10322                         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
10323                 }
10324         }
10325
10326         /* 5700 BX chips need to have their TX producer index mailboxes
10327          * written twice to workaround a bug.
10328          */
10329         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX)
10330                 tp->tg3_flags |= TG3_FLAG_TXD_MBOX_HWBUG;
10331
10332         /* Back to back register writes can cause problems on this chip,
10333          * the workaround is to read back all reg writes except those to
10334          * mailbox regs.  See tg3_write_indirect_reg32().
10335          *
10336          * PCI Express 5750_A0 rev chips need this workaround too.
10337          */
10338         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
10339             ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
10340              tp->pci_chip_rev_id == CHIPREV_ID_5750_A0))
10341                 tp->tg3_flags |= TG3_FLAG_5701_REG_WRITE_BUG;
10342
10343         if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
10344                 tp->tg3_flags |= TG3_FLAG_PCI_HIGH_SPEED;
10345         if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
10346                 tp->tg3_flags |= TG3_FLAG_PCI_32BIT;
10347
10348         /* Chip-specific fixup from Broadcom driver */
10349         if ((tp->pci_chip_rev_id == CHIPREV_ID_5704_A0) &&
10350             (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
10351                 pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
10352                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
10353         }
10354
10355         /* Default fast path register access methods */
10356         tp->read32 = tg3_read32;
10357         tp->write32 = tg3_write32;
10358         tp->read32_mbox = tg3_read32;
10359         tp->write32_mbox = tg3_write32;
10360         tp->write32_tx_mbox = tg3_write32;
10361         tp->write32_rx_mbox = tg3_write32;
10362
10363         /* Various workaround register access methods */
10364         if (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG)
10365                 tp->write32 = tg3_write_indirect_reg32;
10366         else if (tp->tg3_flags & TG3_FLAG_5701_REG_WRITE_BUG)
10367                 tp->write32 = tg3_write_flush_reg32;
10368
10369         if ((tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG) ||
10370             (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)) {
10371                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
10372                 if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
10373                         tp->write32_rx_mbox = tg3_write_flush_reg32;
10374         }
10375
10376         if (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND) {
10377                 tp->read32 = tg3_read_indirect_reg32;
10378                 tp->write32 = tg3_write_indirect_reg32;
10379                 tp->read32_mbox = tg3_read_indirect_mbox;
10380                 tp->write32_mbox = tg3_write_indirect_mbox;
10381                 tp->write32_tx_mbox = tg3_write_indirect_mbox;
10382                 tp->write32_rx_mbox = tg3_write_indirect_mbox;
10383
10384                 iounmap(tp->regs);
10385                 tp->regs = NULL;
10386
10387                 pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
10388                 pci_cmd &= ~PCI_COMMAND_MEMORY;
10389                 pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
10390         }
10391
10392         if (tp->write32 == tg3_write_indirect_reg32 ||
10393             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
10394              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10395               GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)))
10396                 tp->tg3_flags |= TG3_FLAG_SRAM_USE_CONFIG;
10397
10398         /* Get eeprom hw config before calling tg3_set_power_state().
10399          * In particular, the TG3_FLAG_EEPROM_WRITE_PROT flag must be
10400          * determined before calling tg3_set_power_state() so that
10401          * we know whether or not to switch out of Vaux power.
10402          * When the flag is set, it means that GPIO1 is used for eeprom
10403          * write protect and also implies that it is a LOM where GPIOs
10404          * are not used to switch power.
10405          */
10406         tg3_get_eeprom_hw_cfg(tp);
10407
10408         /* Set up tp->grc_local_ctrl before calling tg3_set_power_state().
10409          * GPIO1 driven high will bring 5700's external PHY out of reset.
10410          * It is also used as eeprom write protect on LOMs.
10411          */
10412         tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
10413         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
10414             (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
10415                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
10416                                        GRC_LCLCTRL_GPIO_OUTPUT1);
10417         /* Unused GPIO3 must be driven as output on 5752 because there
10418          * are no pull-up resistors on unused GPIO pins.
10419          */
10420         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
10421                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
10422
10423         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
10424                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
10425
10426         /* Force the chip into D0. */
10427         err = tg3_set_power_state(tp, PCI_D0);
10428         if (err) {
10429                 printk(KERN_ERR PFX "(%s) transition to D0 failed\n",
10430                        pci_name(tp->pdev));
10431                 return err;
10432         }
10433
10434         /* 5700 B0 chips do not support checksumming correctly due
10435          * to hardware bugs.
10436          */
10437         if (tp->pci_chip_rev_id == CHIPREV_ID_5700_B0)
10438                 tp->tg3_flags |= TG3_FLAG_BROKEN_CHECKSUMS;
10439
10440         /* Derive initial jumbo mode from MTU assigned in
10441          * ether_setup() via the alloc_etherdev() call
10442          */
10443         if (tp->dev->mtu > ETH_DATA_LEN &&
10444             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
10445                 tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
10446
10447         /* Determine WakeOnLan speed to use. */
10448         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
10449             tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
10450             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0 ||
10451             tp->pci_chip_rev_id == CHIPREV_ID_5701_B2) {
10452                 tp->tg3_flags &= ~(TG3_FLAG_WOL_SPEED_100MB);
10453         } else {
10454                 tp->tg3_flags |= TG3_FLAG_WOL_SPEED_100MB;
10455         }
10456
10457         /* A few boards don't want Ethernet@WireSpeed phy feature */
10458         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
10459             ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
10460              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) &&
10461              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A1)) ||
10462             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
10463                 tp->tg3_flags2 |= TG3_FLG2_NO_ETH_WIRE_SPEED;
10464
10465         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5703_AX ||
10466             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_AX)
10467                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADC_BUG;
10468         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0)
10469                 tp->tg3_flags2 |= TG3_FLG2_PHY_5704_A0_BUG;
10470
10471         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
10472                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10473                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
10474                         tp->tg3_flags2 |= TG3_FLG2_PHY_JITTER_BUG;
10475                 else
10476                         tp->tg3_flags2 |= TG3_FLG2_PHY_BER_BUG;
10477         }
10478
10479         tp->coalesce_mode = 0;
10480         if (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_AX &&
10481             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_BX)
10482                 tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
10483
10484         /* Initialize MAC MI mode, polling disabled. */
10485         tw32_f(MAC_MI_MODE, tp->mi_mode);
10486         udelay(80);
10487
10488         /* Initialize data/descriptor byte/word swapping. */
10489         val = tr32(GRC_MODE);
10490         val &= GRC_MODE_HOST_STACKUP;
10491         tw32(GRC_MODE, val | tp->grc_mode);
10492
10493         tg3_switch_clocks(tp);
10494
10495         /* Clear this out for sanity. */
10496         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
10497
10498         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
10499                               &pci_state_reg);
10500         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
10501             (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) == 0) {
10502                 u32 chiprevid = GET_CHIP_REV_ID(tp->misc_host_ctrl);
10503
10504                 if (chiprevid == CHIPREV_ID_5701_A0 ||
10505                     chiprevid == CHIPREV_ID_5701_B0 ||
10506                     chiprevid == CHIPREV_ID_5701_B2 ||
10507                     chiprevid == CHIPREV_ID_5701_B5) {
10508                         void __iomem *sram_base;
10509
10510                         /* Write some dummy words into the SRAM status block
10511                          * area, see if it reads back correctly.  If the return
10512                          * value is bad, force enable the PCIX workaround.
10513                          */
10514                         sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
10515
10516                         writel(0x00000000, sram_base);
10517                         writel(0x00000000, sram_base + 4);
10518                         writel(0xffffffff, sram_base + 4);
10519                         if (readl(sram_base) != 0x00000000)
10520                                 tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
10521                 }
10522         }
10523
10524         udelay(50);
10525         tg3_nvram_init(tp);
10526
10527         grc_misc_cfg = tr32(GRC_MISC_CFG);
10528         grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
10529
10530         /* Broadcom's driver says that CIOBE multisplit has a bug */
10531 #if 0
10532         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
10533             grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5704CIOBE) {
10534                 tp->tg3_flags |= TG3_FLAG_SPLIT_MODE;
10535                 tp->split_mode_max_reqs = SPLIT_MODE_5704_MAX_REQ;
10536         }
10537 #endif
10538         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
10539             (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
10540              grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
10541                 tp->tg3_flags2 |= TG3_FLG2_IS_5788;
10542
10543         if (!(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
10544             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700))
10545                 tp->tg3_flags |= TG3_FLAG_TAGGED_STATUS;
10546         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
10547                 tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
10548                                       HOSTCC_MODE_CLRTICK_TXBD);
10549
10550                 tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
10551                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
10552                                        tp->misc_host_ctrl);
10553         }
10554
10555         /* these are limited to 10/100 only */
10556         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
10557              (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
10558             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
10559              tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
10560              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901 ||
10561               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901_2 ||
10562               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5705F)) ||
10563             (tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
10564              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5751F ||
10565               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5753F)))
10566                 tp->tg3_flags |= TG3_FLAG_10_100_ONLY;
10567
10568         err = tg3_phy_probe(tp);
10569         if (err) {
10570                 printk(KERN_ERR PFX "(%s) phy probe failed, err %d\n",
10571                        pci_name(tp->pdev), err);
10572                 /* ... but do not return immediately ... */
10573         }
10574
10575         tg3_read_partno(tp);
10576         tg3_read_fw_ver(tp);
10577
10578         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
10579                 tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
10580         } else {
10581                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
10582                         tp->tg3_flags |= TG3_FLAG_USE_MI_INTERRUPT;
10583                 else
10584                         tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
10585         }
10586
10587         /* 5700 {AX,BX} chips have a broken status block link
10588          * change bit implementation, so we must use the
10589          * status register in those cases.
10590          */
10591         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
10592                 tp->tg3_flags |= TG3_FLAG_USE_LINKCHG_REG;
10593         else
10594                 tp->tg3_flags &= ~TG3_FLAG_USE_LINKCHG_REG;
10595
10596         /* The led_ctrl is set during tg3_phy_probe, here we might
10597          * have to force the link status polling mechanism based
10598          * upon subsystem IDs.
10599          */
10600         if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
10601             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
10602                 tp->tg3_flags |= (TG3_FLAG_USE_MI_INTERRUPT |
10603                                   TG3_FLAG_USE_LINKCHG_REG);
10604         }
10605
10606         /* For all SERDES we poll the MAC status register. */
10607         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
10608                 tp->tg3_flags |= TG3_FLAG_POLL_SERDES;
10609         else
10610                 tp->tg3_flags &= ~TG3_FLAG_POLL_SERDES;
10611
10612         /* All chips before 5787 can get confused if TX buffers
10613          * straddle the 4GB address boundary in some cases.
10614          */
10615         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
10616             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
10617                 tp->dev->hard_start_xmit = tg3_start_xmit;
10618         else
10619                 tp->dev->hard_start_xmit = tg3_start_xmit_dma_bug;
10620
10621         tp->rx_offset = 2;
10622         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
10623             (tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0)
10624                 tp->rx_offset = 0;
10625
10626         tp->rx_std_max_post = TG3_RX_RING_SIZE;
10627
10628         /* Increment the rx prod index on the rx std ring by at most
10629          * 8 for these chips to workaround hw errata.
10630          */
10631         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
10632             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
10633             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
10634                 tp->rx_std_max_post = 8;
10635
10636         /* By default, disable wake-on-lan.  User can change this
10637          * using ETHTOOL_SWOL.
10638          */
10639         tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
10640
10641         return err;
10642 }
10643
10644 #ifdef CONFIG_SPARC64
10645 static int __devinit tg3_get_macaddr_sparc(struct tg3 *tp)
10646 {
10647         struct net_device *dev = tp->dev;
10648         struct pci_dev *pdev = tp->pdev;
10649         struct pcidev_cookie *pcp = pdev->sysdata;
10650
10651         if (pcp != NULL) {
10652                 unsigned char *addr;
10653                 int len;
10654
10655                 addr = of_get_property(pcp->prom_node, "local-mac-address",
10656                                         &len);
10657                 if (addr && len == 6) {
10658                         memcpy(dev->dev_addr, addr, 6);
10659                         memcpy(dev->perm_addr, dev->dev_addr, 6);
10660                         return 0;
10661                 }
10662         }
10663         return -ENODEV;
10664 }
10665
10666 static int __devinit tg3_get_default_macaddr_sparc(struct tg3 *tp)
10667 {
10668         struct net_device *dev = tp->dev;
10669
10670         memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
10671         memcpy(dev->perm_addr, idprom->id_ethaddr, 6);
10672         return 0;
10673 }
10674 #endif
10675
10676 static int __devinit tg3_get_device_address(struct tg3 *tp)
10677 {
10678         struct net_device *dev = tp->dev;
10679         u32 hi, lo, mac_offset;
10680         int addr_ok = 0;
10681
10682 #ifdef CONFIG_SPARC64
10683         if (!tg3_get_macaddr_sparc(tp))
10684                 return 0;
10685 #endif
10686
10687         mac_offset = 0x7c;
10688         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
10689             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
10690                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
10691                         mac_offset = 0xcc;
10692                 if (tg3_nvram_lock(tp))
10693                         tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
10694                 else
10695                         tg3_nvram_unlock(tp);
10696         }
10697
10698         /* First try to get it from MAC address mailbox. */
10699         tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
10700         if ((hi >> 16) == 0x484b) {
10701                 dev->dev_addr[0] = (hi >>  8) & 0xff;
10702                 dev->dev_addr[1] = (hi >>  0) & 0xff;
10703
10704                 tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
10705                 dev->dev_addr[2] = (lo >> 24) & 0xff;
10706                 dev->dev_addr[3] = (lo >> 16) & 0xff;
10707                 dev->dev_addr[4] = (lo >>  8) & 0xff;
10708                 dev->dev_addr[5] = (lo >>  0) & 0xff;
10709
10710                 /* Some old bootcode may report a 0 MAC address in SRAM */
10711                 addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
10712         }
10713         if (!addr_ok) {
10714                 /* Next, try NVRAM. */
10715                 if (!tg3_nvram_read(tp, mac_offset + 0, &hi) &&
10716                     !tg3_nvram_read(tp, mac_offset + 4, &lo)) {
10717                         dev->dev_addr[0] = ((hi >> 16) & 0xff);
10718                         dev->dev_addr[1] = ((hi >> 24) & 0xff);
10719                         dev->dev_addr[2] = ((lo >>  0) & 0xff);
10720                         dev->dev_addr[3] = ((lo >>  8) & 0xff);
10721                         dev->dev_addr[4] = ((lo >> 16) & 0xff);
10722                         dev->dev_addr[5] = ((lo >> 24) & 0xff);
10723                 }
10724                 /* Finally just fetch it out of the MAC control regs. */
10725                 else {
10726                         hi = tr32(MAC_ADDR_0_HIGH);
10727                         lo = tr32(MAC_ADDR_0_LOW);
10728
10729                         dev->dev_addr[5] = lo & 0xff;
10730                         dev->dev_addr[4] = (lo >> 8) & 0xff;
10731                         dev->dev_addr[3] = (lo >> 16) & 0xff;
10732                         dev->dev_addr[2] = (lo >> 24) & 0xff;
10733                         dev->dev_addr[1] = hi & 0xff;
10734                         dev->dev_addr[0] = (hi >> 8) & 0xff;
10735                 }
10736         }
10737
10738         if (!is_valid_ether_addr(&dev->dev_addr[0])) {
10739 #ifdef CONFIG_SPARC64
10740                 if (!tg3_get_default_macaddr_sparc(tp))
10741                         return 0;
10742 #endif
10743                 return -EINVAL;
10744         }
10745         memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10746         return 0;
10747 }
10748
10749 #define BOUNDARY_SINGLE_CACHELINE       1
10750 #define BOUNDARY_MULTI_CACHELINE        2
10751
10752 static u32 __devinit tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
10753 {
10754         int cacheline_size;
10755         u8 byte;
10756         int goal;
10757
10758         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
10759         if (byte == 0)
10760                 cacheline_size = 1024;
10761         else
10762                 cacheline_size = (int) byte * 4;
10763
10764         /* On 5703 and later chips, the boundary bits have no
10765          * effect.
10766          */
10767         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
10768             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
10769             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
10770                 goto out;
10771
10772 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
10773         goal = BOUNDARY_MULTI_CACHELINE;
10774 #else
10775 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
10776         goal = BOUNDARY_SINGLE_CACHELINE;
10777 #else
10778         goal = 0;
10779 #endif
10780 #endif
10781
10782         if (!goal)
10783                 goto out;
10784
10785         /* PCI controllers on most RISC systems tend to disconnect
10786          * when a device tries to burst across a cache-line boundary.
10787          * Therefore, letting tg3 do so just wastes PCI bandwidth.
10788          *
10789          * Unfortunately, for PCI-E there are only limited
10790          * write-side controls for this, and thus for reads
10791          * we will still get the disconnects.  We'll also waste
10792          * these PCI cycles for both read and write for chips
10793          * other than 5700 and 5701 which do not implement the
10794          * boundary bits.
10795          */
10796         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
10797             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
10798                 switch (cacheline_size) {
10799                 case 16:
10800                 case 32:
10801                 case 64:
10802                 case 128:
10803                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10804                                 val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
10805                                         DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
10806                         } else {
10807                                 val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
10808                                         DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
10809                         }
10810                         break;
10811
10812                 case 256:
10813                         val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
10814                                 DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
10815                         break;
10816
10817                 default:
10818                         val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
10819                                 DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
10820                         break;
10821                 };
10822         } else if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
10823                 switch (cacheline_size) {
10824                 case 16:
10825                 case 32:
10826                 case 64:
10827                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10828                                 val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
10829                                 val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
10830                                 break;
10831                         }
10832                         /* fallthrough */
10833                 case 128:
10834                 default:
10835                         val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
10836                         val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
10837                         break;
10838                 };
10839         } else {
10840                 switch (cacheline_size) {
10841                 case 16:
10842                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10843                                 val |= (DMA_RWCTRL_READ_BNDRY_16 |
10844                                         DMA_RWCTRL_WRITE_BNDRY_16);
10845                                 break;
10846                         }
10847                         /* fallthrough */
10848                 case 32:
10849                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10850                                 val |= (DMA_RWCTRL_READ_BNDRY_32 |
10851                                         DMA_RWCTRL_WRITE_BNDRY_32);
10852                                 break;
10853                         }
10854                         /* fallthrough */
10855                 case 64:
10856                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10857                                 val |= (DMA_RWCTRL_READ_BNDRY_64 |
10858                                         DMA_RWCTRL_WRITE_BNDRY_64);
10859                                 break;
10860                         }
10861                         /* fallthrough */
10862                 case 128:
10863                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
10864                                 val |= (DMA_RWCTRL_READ_BNDRY_128 |
10865                                         DMA_RWCTRL_WRITE_BNDRY_128);
10866                                 break;
10867                         }
10868                         /* fallthrough */
10869                 case 256:
10870                         val |= (DMA_RWCTRL_READ_BNDRY_256 |
10871                                 DMA_RWCTRL_WRITE_BNDRY_256);
10872                         break;
10873                 case 512:
10874                         val |= (DMA_RWCTRL_READ_BNDRY_512 |
10875                                 DMA_RWCTRL_WRITE_BNDRY_512);
10876                         break;
10877                 case 1024:
10878                 default:
10879                         val |= (DMA_RWCTRL_READ_BNDRY_1024 |
10880                                 DMA_RWCTRL_WRITE_BNDRY_1024);
10881                         break;
10882                 };
10883         }
10884
10885 out:
10886         return val;
10887 }
10888
10889 static int __devinit tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma, int size, int to_device)
10890 {
10891         struct tg3_internal_buffer_desc test_desc;
10892         u32 sram_dma_descs;
10893         int i, ret;
10894
10895         sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
10896
10897         tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
10898         tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
10899         tw32(RDMAC_STATUS, 0);
10900         tw32(WDMAC_STATUS, 0);
10901
10902         tw32(BUFMGR_MODE, 0);
10903         tw32(FTQ_RESET, 0);
10904
10905         test_desc.addr_hi = ((u64) buf_dma) >> 32;
10906         test_desc.addr_lo = buf_dma & 0xffffffff;
10907         test_desc.nic_mbuf = 0x00002100;
10908         test_desc.len = size;
10909
10910         /*
10911          * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
10912          * the *second* time the tg3 driver was getting loaded after an
10913          * initial scan.
10914          *
10915          * Broadcom tells me:
10916          *   ...the DMA engine is connected to the GRC block and a DMA
10917          *   reset may affect the GRC block in some unpredictable way...
10918          *   The behavior of resets to individual blocks has not been tested.
10919          *
10920          * Broadcom noted the GRC reset will also reset all sub-components.
10921          */
10922         if (to_device) {
10923                 test_desc.cqid_sqid = (13 << 8) | 2;
10924
10925                 tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
10926                 udelay(40);
10927         } else {
10928                 test_desc.cqid_sqid = (16 << 8) | 7;
10929
10930                 tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
10931                 udelay(40);
10932         }
10933         test_desc.flags = 0x00000005;
10934
10935         for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
10936                 u32 val;
10937
10938                 val = *(((u32 *)&test_desc) + i);
10939                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
10940                                        sram_dma_descs + (i * sizeof(u32)));
10941                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
10942         }
10943         pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
10944
10945         if (to_device) {
10946                 tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
10947         } else {
10948                 tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
10949         }
10950
10951         ret = -ENODEV;
10952         for (i = 0; i < 40; i++) {
10953                 u32 val;
10954
10955                 if (to_device)
10956                         val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
10957                 else
10958                         val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
10959                 if ((val & 0xffff) == sram_dma_descs) {
10960                         ret = 0;
10961                         break;
10962                 }
10963
10964                 udelay(100);
10965         }
10966
10967         return ret;
10968 }
10969
10970 #define TEST_BUFFER_SIZE        0x2000
10971
10972 static int __devinit tg3_test_dma(struct tg3 *tp)
10973 {
10974         dma_addr_t buf_dma;
10975         u32 *buf, saved_dma_rwctrl;
10976         int ret;
10977
10978         buf = pci_alloc_consistent(tp->pdev, TEST_BUFFER_SIZE, &buf_dma);
10979         if (!buf) {
10980                 ret = -ENOMEM;
10981                 goto out_nofree;
10982         }
10983
10984         tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
10985                           (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
10986
10987         tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
10988
10989         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
10990                 /* DMA read watermark not used on PCIE */
10991                 tp->dma_rwctrl |= 0x00180000;
10992         } else if (!(tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
10993                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
10994                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)
10995                         tp->dma_rwctrl |= 0x003f0000;
10996                 else
10997                         tp->dma_rwctrl |= 0x003f000f;
10998         } else {
10999                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
11000                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
11001                         u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
11002
11003                         /* If the 5704 is behind the EPB bridge, we can
11004                          * do the less restrictive ONE_DMA workaround for
11005                          * better performance.
11006                          */
11007                         if ((tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) &&
11008                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
11009                                 tp->dma_rwctrl |= 0x8000;
11010                         else if (ccval == 0x6 || ccval == 0x7)
11011                                 tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
11012
11013                         /* Set bit 23 to enable PCIX hw bug fix */
11014                         tp->dma_rwctrl |= 0x009f0000;
11015                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780) {
11016                         /* 5780 always in PCIX mode */
11017                         tp->dma_rwctrl |= 0x00144000;
11018                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
11019                         /* 5714 always in PCIX mode */
11020                         tp->dma_rwctrl |= 0x00148000;
11021                 } else {
11022                         tp->dma_rwctrl |= 0x001b000f;
11023                 }
11024         }
11025
11026         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
11027             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
11028                 tp->dma_rwctrl &= 0xfffffff0;
11029
11030         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11031             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
11032                 /* Remove this if it causes problems for some boards. */
11033                 tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
11034
11035                 /* On 5700/5701 chips, we need to set this bit.
11036                  * Otherwise the chip will issue cacheline transactions
11037                  * to streamable DMA memory with not all the byte
11038                  * enables turned on.  This is an error on several
11039                  * RISC PCI controllers, in particular sparc64.
11040                  *
11041                  * On 5703/5704 chips, this bit has been reassigned
11042                  * a different meaning.  In particular, it is used
11043                  * on those chips to enable a PCI-X workaround.
11044                  */
11045                 tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
11046         }
11047
11048         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
11049
11050 #if 0
11051         /* Unneeded, already done by tg3_get_invariants.  */
11052         tg3_switch_clocks(tp);
11053 #endif
11054
11055         ret = 0;
11056         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
11057             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
11058                 goto out;
11059
11060         /* It is best to perform DMA test with maximum write burst size
11061          * to expose the 5700/5701 write DMA bug.
11062          */
11063         saved_dma_rwctrl = tp->dma_rwctrl;
11064         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
11065         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
11066
11067         while (1) {
11068                 u32 *p = buf, i;
11069
11070                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
11071                         p[i] = i;
11072
11073                 /* Send the buffer to the chip. */
11074                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 1);
11075                 if (ret) {
11076                         printk(KERN_ERR "tg3_test_dma() Write the buffer failed %d\n", ret);
11077                         break;
11078                 }
11079
11080 #if 0
11081                 /* validate data reached card RAM correctly. */
11082                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
11083                         u32 val;
11084                         tg3_read_mem(tp, 0x2100 + (i*4), &val);
11085                         if (le32_to_cpu(val) != p[i]) {
11086                                 printk(KERN_ERR "  tg3_test_dma()  Card buffer corrupted on write! (%d != %d)\n", val, i);
11087                                 /* ret = -ENODEV here? */
11088                         }
11089                         p[i] = 0;
11090                 }
11091 #endif
11092                 /* Now read it back. */
11093                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 0);
11094                 if (ret) {
11095                         printk(KERN_ERR "tg3_test_dma() Read the buffer failed %d\n", ret);
11096
11097                         break;
11098                 }
11099
11100                 /* Verify it. */
11101                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
11102                         if (p[i] == i)
11103                                 continue;
11104
11105                         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
11106                             DMA_RWCTRL_WRITE_BNDRY_16) {
11107                                 tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
11108                                 tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
11109                                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
11110                                 break;
11111                         } else {
11112                                 printk(KERN_ERR "tg3_test_dma() buffer corrupted on read back! (%d != %d)\n", p[i], i);
11113                                 ret = -ENODEV;
11114                                 goto out;
11115                         }
11116                 }
11117
11118                 if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
11119                         /* Success. */
11120                         ret = 0;
11121                         break;
11122                 }
11123         }
11124         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
11125             DMA_RWCTRL_WRITE_BNDRY_16) {
11126                 static struct pci_device_id dma_wait_state_chipsets[] = {
11127                         { PCI_DEVICE(PCI_VENDOR_ID_APPLE,
11128                                      PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
11129                         { },
11130                 };
11131
11132                 /* DMA test passed without adjusting DMA boundary,
11133                  * now look for chipsets that are known to expose the
11134                  * DMA bug without failing the test.
11135                  */
11136                 if (pci_dev_present(dma_wait_state_chipsets)) {
11137                         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
11138                         tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
11139                 }
11140                 else
11141                         /* Safe to use the calculated DMA boundary. */
11142                         tp->dma_rwctrl = saved_dma_rwctrl;
11143
11144                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
11145         }
11146
11147 out:
11148         pci_free_consistent(tp->pdev, TEST_BUFFER_SIZE, buf, buf_dma);
11149 out_nofree:
11150         return ret;
11151 }
11152
11153 static void __devinit tg3_init_link_config(struct tg3 *tp)
11154 {
11155         tp->link_config.advertising =
11156                 (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
11157                  ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
11158                  ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
11159                  ADVERTISED_Autoneg | ADVERTISED_MII);
11160         tp->link_config.speed = SPEED_INVALID;
11161         tp->link_config.duplex = DUPLEX_INVALID;
11162         tp->link_config.autoneg = AUTONEG_ENABLE;
11163         tp->link_config.active_speed = SPEED_INVALID;
11164         tp->link_config.active_duplex = DUPLEX_INVALID;
11165         tp->link_config.phy_is_low_power = 0;
11166         tp->link_config.orig_speed = SPEED_INVALID;
11167         tp->link_config.orig_duplex = DUPLEX_INVALID;
11168         tp->link_config.orig_autoneg = AUTONEG_INVALID;
11169 }
11170
11171 static void __devinit tg3_init_bufmgr_config(struct tg3 *tp)
11172 {
11173         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
11174                 tp->bufmgr_config.mbuf_read_dma_low_water =
11175                         DEFAULT_MB_RDMA_LOW_WATER_5705;
11176                 tp->bufmgr_config.mbuf_mac_rx_low_water =
11177                         DEFAULT_MB_MACRX_LOW_WATER_5705;
11178                 tp->bufmgr_config.mbuf_high_water =
11179                         DEFAULT_MB_HIGH_WATER_5705;
11180
11181                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
11182                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
11183                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
11184                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
11185                 tp->bufmgr_config.mbuf_high_water_jumbo =
11186                         DEFAULT_MB_HIGH_WATER_JUMBO_5780;
11187         } else {
11188                 tp->bufmgr_config.mbuf_read_dma_low_water =
11189                         DEFAULT_MB_RDMA_LOW_WATER;
11190                 tp->bufmgr_config.mbuf_mac_rx_low_water =
11191                         DEFAULT_MB_MACRX_LOW_WATER;
11192                 tp->bufmgr_config.mbuf_high_water =
11193                         DEFAULT_MB_HIGH_WATER;
11194
11195                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
11196                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
11197                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
11198                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
11199                 tp->bufmgr_config.mbuf_high_water_jumbo =
11200                         DEFAULT_MB_HIGH_WATER_JUMBO;
11201         }
11202
11203         tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
11204         tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
11205 }
11206
11207 static char * __devinit tg3_phy_string(struct tg3 *tp)
11208 {
11209         switch (tp->phy_id & PHY_ID_MASK) {
11210         case PHY_ID_BCM5400:    return "5400";
11211         case PHY_ID_BCM5401:    return "5401";
11212         case PHY_ID_BCM5411:    return "5411";
11213         case PHY_ID_BCM5701:    return "5701";
11214         case PHY_ID_BCM5703:    return "5703";
11215         case PHY_ID_BCM5704:    return "5704";
11216         case PHY_ID_BCM5705:    return "5705";
11217         case PHY_ID_BCM5750:    return "5750";
11218         case PHY_ID_BCM5752:    return "5752";
11219         case PHY_ID_BCM5714:    return "5714";
11220         case PHY_ID_BCM5780:    return "5780";
11221         case PHY_ID_BCM5755:    return "5755";
11222         case PHY_ID_BCM5787:    return "5787";
11223         case PHY_ID_BCM8002:    return "8002/serdes";
11224         case 0:                 return "serdes";
11225         default:                return "unknown";
11226         };
11227 }
11228
11229 static char * __devinit tg3_bus_string(struct tg3 *tp, char *str)
11230 {
11231         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
11232                 strcpy(str, "PCI Express");
11233                 return str;
11234         } else if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
11235                 u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
11236
11237                 strcpy(str, "PCIX:");
11238
11239                 if ((clock_ctrl == 7) ||
11240                     ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
11241                      GRC_MISC_CFG_BOARD_ID_5704CIOBE))
11242                         strcat(str, "133MHz");
11243                 else if (clock_ctrl == 0)
11244                         strcat(str, "33MHz");
11245                 else if (clock_ctrl == 2)
11246                         strcat(str, "50MHz");
11247                 else if (clock_ctrl == 4)
11248                         strcat(str, "66MHz");
11249                 else if (clock_ctrl == 6)
11250                         strcat(str, "100MHz");
11251         } else {
11252                 strcpy(str, "PCI:");
11253                 if (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED)
11254                         strcat(str, "66MHz");
11255                 else
11256                         strcat(str, "33MHz");
11257         }
11258         if (tp->tg3_flags & TG3_FLAG_PCI_32BIT)
11259                 strcat(str, ":32-bit");
11260         else
11261                 strcat(str, ":64-bit");
11262         return str;
11263 }
11264
11265 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *tp)
11266 {
11267         struct pci_dev *peer;
11268         unsigned int func, devnr = tp->pdev->devfn & ~7;
11269
11270         for (func = 0; func < 8; func++) {
11271                 peer = pci_get_slot(tp->pdev->bus, devnr | func);
11272                 if (peer && peer != tp->pdev)
11273                         break;
11274                 pci_dev_put(peer);
11275         }
11276         /* 5704 can be configured in single-port mode, set peer to
11277          * tp->pdev in that case.
11278          */
11279         if (!peer) {
11280                 peer = tp->pdev;
11281                 return peer;
11282         }
11283
11284         /*
11285          * We don't need to keep the refcount elevated; there's no way
11286          * to remove one half of this device without removing the other
11287          */
11288         pci_dev_put(peer);
11289
11290         return peer;
11291 }
11292
11293 static void __devinit tg3_init_coal(struct tg3 *tp)
11294 {
11295         struct ethtool_coalesce *ec = &tp->coal;
11296
11297         memset(ec, 0, sizeof(*ec));
11298         ec->cmd = ETHTOOL_GCOALESCE;
11299         ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
11300         ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
11301         ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
11302         ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
11303         ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
11304         ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
11305         ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
11306         ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
11307         ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
11308
11309         if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
11310                                  HOSTCC_MODE_CLRTICK_TXBD)) {
11311                 ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
11312                 ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
11313                 ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
11314                 ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
11315         }
11316
11317         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
11318                 ec->rx_coalesce_usecs_irq = 0;
11319                 ec->tx_coalesce_usecs_irq = 0;
11320                 ec->stats_block_coalesce_usecs = 0;
11321         }
11322 }
11323
11324 static int __devinit tg3_init_one(struct pci_dev *pdev,
11325                                   const struct pci_device_id *ent)
11326 {
11327         static int tg3_version_printed = 0;
11328         unsigned long tg3reg_base, tg3reg_len;
11329         struct net_device *dev;
11330         struct tg3 *tp;
11331         int i, err, pm_cap;
11332         char str[40];
11333         u64 dma_mask, persist_dma_mask;
11334
11335         if (tg3_version_printed++ == 0)
11336                 printk(KERN_INFO "%s", version);
11337
11338         err = pci_enable_device(pdev);
11339         if (err) {
11340                 printk(KERN_ERR PFX "Cannot enable PCI device, "
11341                        "aborting.\n");
11342                 return err;
11343         }
11344
11345         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11346                 printk(KERN_ERR PFX "Cannot find proper PCI device "
11347                        "base address, aborting.\n");
11348                 err = -ENODEV;
11349                 goto err_out_disable_pdev;
11350         }
11351
11352         err = pci_request_regions(pdev, DRV_MODULE_NAME);
11353         if (err) {
11354                 printk(KERN_ERR PFX "Cannot obtain PCI resources, "
11355                        "aborting.\n");
11356                 goto err_out_disable_pdev;
11357         }
11358
11359         pci_set_master(pdev);
11360
11361         /* Find power-management capability. */
11362         pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
11363         if (pm_cap == 0) {
11364                 printk(KERN_ERR PFX "Cannot find PowerManagement capability, "
11365                        "aborting.\n");
11366                 err = -EIO;
11367                 goto err_out_free_res;
11368         }
11369
11370         tg3reg_base = pci_resource_start(pdev, 0);
11371         tg3reg_len = pci_resource_len(pdev, 0);
11372
11373         dev = alloc_etherdev(sizeof(*tp));
11374         if (!dev) {
11375                 printk(KERN_ERR PFX "Etherdev alloc failed, aborting.\n");
11376                 err = -ENOMEM;
11377                 goto err_out_free_res;
11378         }
11379
11380         SET_MODULE_OWNER(dev);
11381         SET_NETDEV_DEV(dev, &pdev->dev);
11382
11383 #if TG3_VLAN_TAG_USED
11384         dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
11385         dev->vlan_rx_register = tg3_vlan_rx_register;
11386         dev->vlan_rx_kill_vid = tg3_vlan_rx_kill_vid;
11387 #endif
11388
11389         tp = netdev_priv(dev);
11390         tp->pdev = pdev;
11391         tp->dev = dev;
11392         tp->pm_cap = pm_cap;
11393         tp->mac_mode = TG3_DEF_MAC_MODE;
11394         tp->rx_mode = TG3_DEF_RX_MODE;
11395         tp->tx_mode = TG3_DEF_TX_MODE;
11396         tp->mi_mode = MAC_MI_MODE_BASE;
11397         if (tg3_debug > 0)
11398                 tp->msg_enable = tg3_debug;
11399         else
11400                 tp->msg_enable = TG3_DEF_MSG_ENABLE;
11401
11402         /* The word/byte swap controls here control register access byte
11403          * swapping.  DMA data byte swapping is controlled in the GRC_MODE
11404          * setting below.
11405          */
11406         tp->misc_host_ctrl =
11407                 MISC_HOST_CTRL_MASK_PCI_INT |
11408                 MISC_HOST_CTRL_WORD_SWAP |
11409                 MISC_HOST_CTRL_INDIR_ACCESS |
11410                 MISC_HOST_CTRL_PCISTATE_RW;
11411
11412         /* The NONFRM (non-frame) byte/word swap controls take effect
11413          * on descriptor entries, anything which isn't packet data.
11414          *
11415          * The StrongARM chips on the board (one for tx, one for rx)
11416          * are running in big-endian mode.
11417          */
11418         tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
11419                         GRC_MODE_WSWAP_NONFRM_DATA);
11420 #ifdef __BIG_ENDIAN
11421         tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
11422 #endif
11423         spin_lock_init(&tp->lock);
11424         spin_lock_init(&tp->tx_lock);
11425         spin_lock_init(&tp->indirect_lock);
11426         INIT_WORK(&tp->reset_task, tg3_reset_task, tp);
11427
11428         tp->regs = ioremap_nocache(tg3reg_base, tg3reg_len);
11429         if (tp->regs == 0UL) {
11430                 printk(KERN_ERR PFX "Cannot map device registers, "
11431                        "aborting.\n");
11432                 err = -ENOMEM;
11433                 goto err_out_free_dev;
11434         }
11435
11436         tg3_init_link_config(tp);
11437
11438         tp->rx_pending = TG3_DEF_RX_RING_PENDING;
11439         tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
11440         tp->tx_pending = TG3_DEF_TX_RING_PENDING;
11441
11442         dev->open = tg3_open;
11443         dev->stop = tg3_close;
11444         dev->get_stats = tg3_get_stats;
11445         dev->set_multicast_list = tg3_set_rx_mode;
11446         dev->set_mac_address = tg3_set_mac_addr;
11447         dev->do_ioctl = tg3_ioctl;
11448         dev->tx_timeout = tg3_tx_timeout;
11449         dev->poll = tg3_poll;
11450         dev->ethtool_ops = &tg3_ethtool_ops;
11451         dev->weight = 64;
11452         dev->watchdog_timeo = TG3_TX_TIMEOUT;
11453         dev->change_mtu = tg3_change_mtu;
11454         dev->irq = pdev->irq;
11455 #ifdef CONFIG_NET_POLL_CONTROLLER
11456         dev->poll_controller = tg3_poll_controller;
11457 #endif
11458
11459         err = tg3_get_invariants(tp);
11460         if (err) {
11461                 printk(KERN_ERR PFX "Problem fetching invariants of chip, "
11462                        "aborting.\n");
11463                 goto err_out_iounmap;
11464         }
11465
11466         /* The EPB bridge inside 5714, 5715, and 5780 and any
11467          * device behind the EPB cannot support DMA addresses > 40-bit.
11468          * On 64-bit systems with IOMMU, use 40-bit dma_mask.
11469          * On 64-bit systems without IOMMU, use 64-bit dma_mask and
11470          * do DMA address check in tg3_start_xmit().
11471          */
11472         if (tp->tg3_flags2 & TG3_FLG2_IS_5788)
11473                 persist_dma_mask = dma_mask = DMA_32BIT_MASK;
11474         else if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) {
11475                 persist_dma_mask = dma_mask = DMA_40BIT_MASK;
11476 #ifdef CONFIG_HIGHMEM
11477                 dma_mask = DMA_64BIT_MASK;
11478 #endif
11479         } else
11480                 persist_dma_mask = dma_mask = DMA_64BIT_MASK;
11481
11482         /* Configure DMA attributes. */
11483         if (dma_mask > DMA_32BIT_MASK) {
11484                 err = pci_set_dma_mask(pdev, dma_mask);
11485                 if (!err) {
11486                         dev->features |= NETIF_F_HIGHDMA;
11487                         err = pci_set_consistent_dma_mask(pdev,
11488                                                           persist_dma_mask);
11489                         if (err < 0) {
11490                                 printk(KERN_ERR PFX "Unable to obtain 64 bit "
11491                                        "DMA for consistent allocations\n");
11492                                 goto err_out_iounmap;
11493                         }
11494                 }
11495         }
11496         if (err || dma_mask == DMA_32BIT_MASK) {
11497                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
11498                 if (err) {
11499                         printk(KERN_ERR PFX "No usable DMA configuration, "
11500                                "aborting.\n");
11501                         goto err_out_iounmap;
11502                 }
11503         }
11504
11505         tg3_init_bufmgr_config(tp);
11506
11507 #if TG3_TSO_SUPPORT != 0
11508         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
11509                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
11510         }
11511         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11512             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
11513             tp->pci_chip_rev_id == CHIPREV_ID_5705_A0 ||
11514             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
11515                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
11516         } else {
11517                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
11518         }
11519
11520         /* TSO is on by default on chips that support hardware TSO.
11521          * Firmware TSO on older chips gives lower performance, so it
11522          * is off by default, but can be enabled using ethtool.
11523          */
11524         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
11525                 dev->features |= NETIF_F_TSO;
11526                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_2)
11527                         dev->features |= NETIF_F_TSO6;
11528         }
11529
11530 #endif
11531
11532         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 &&
11533             !(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) &&
11534             !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
11535                 tp->tg3_flags2 |= TG3_FLG2_MAX_RXPEND_64;
11536                 tp->rx_pending = 63;
11537         }
11538
11539         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
11540             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714))
11541                 tp->pdev_peer = tg3_find_peer(tp);
11542
11543         err = tg3_get_device_address(tp);
11544         if (err) {
11545                 printk(KERN_ERR PFX "Could not obtain valid ethernet address, "
11546                        "aborting.\n");
11547                 goto err_out_iounmap;
11548         }
11549
11550         /*
11551          * Reset chip in case UNDI or EFI driver did not shutdown
11552          * DMA self test will enable WDMAC and we'll see (spurious)
11553          * pending DMA on the PCI bus at that point.
11554          */
11555         if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
11556             (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
11557                 pci_save_state(tp->pdev);
11558                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
11559                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11560         }
11561
11562         err = tg3_test_dma(tp);
11563         if (err) {
11564                 printk(KERN_ERR PFX "DMA engine test failed, aborting.\n");
11565                 goto err_out_iounmap;
11566         }
11567
11568         /* Tigon3 can do ipv4 only... and some chips have buggy
11569          * checksumming.
11570          */
11571         if ((tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) == 0) {
11572                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
11573                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
11574                         dev->features |= NETIF_F_HW_CSUM;
11575                 else
11576                         dev->features |= NETIF_F_IP_CSUM;
11577                 dev->features |= NETIF_F_SG;
11578                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
11579         } else
11580                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
11581
11582         /* flow control autonegotiation is default behavior */
11583         tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
11584
11585         tg3_init_coal(tp);
11586
11587         /* Now that we have fully setup the chip, save away a snapshot
11588          * of the PCI config space.  We need to restore this after
11589          * GRC_MISC_CFG core clock resets and some resume events.
11590          */
11591         pci_save_state(tp->pdev);
11592
11593         err = register_netdev(dev);
11594         if (err) {
11595                 printk(KERN_ERR PFX "Cannot register net device, "
11596                        "aborting.\n");
11597                 goto err_out_iounmap;
11598         }
11599
11600         pci_set_drvdata(pdev, dev);
11601
11602         printk(KERN_INFO "%s: Tigon3 [partno(%s) rev %04x PHY(%s)] (%s) %sBaseT Ethernet ",
11603                dev->name,
11604                tp->board_part_number,
11605                tp->pci_chip_rev_id,
11606                tg3_phy_string(tp),
11607                tg3_bus_string(tp, str),
11608                (tp->tg3_flags & TG3_FLAG_10_100_ONLY) ? "10/100" : "10/100/1000");
11609
11610         for (i = 0; i < 6; i++)
11611                 printk("%2.2x%c", dev->dev_addr[i],
11612                        i == 5 ? '\n' : ':');
11613
11614         printk(KERN_INFO "%s: RXcsums[%d] LinkChgREG[%d] "
11615                "MIirq[%d] ASF[%d] Split[%d] WireSpeed[%d] "
11616                "TSOcap[%d] \n",
11617                dev->name,
11618                (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0,
11619                (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) != 0,
11620                (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) != 0,
11621                (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0,
11622                (tp->tg3_flags & TG3_FLAG_SPLIT_MODE) != 0,
11623                (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED) == 0,
11624                (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) != 0);
11625         printk(KERN_INFO "%s: dma_rwctrl[%08x] dma_mask[%d-bit]\n",
11626                dev->name, tp->dma_rwctrl,
11627                (pdev->dma_mask == DMA_32BIT_MASK) ? 32 :
11628                 (((u64) pdev->dma_mask == DMA_40BIT_MASK) ? 40 : 64));
11629
11630         netif_carrier_off(tp->dev);
11631
11632         return 0;
11633
11634 err_out_iounmap:
11635         if (tp->regs) {
11636                 iounmap(tp->regs);
11637                 tp->regs = NULL;
11638         }
11639
11640 err_out_free_dev:
11641         free_netdev(dev);
11642
11643 err_out_free_res:
11644         pci_release_regions(pdev);
11645
11646 err_out_disable_pdev:
11647         pci_disable_device(pdev);
11648         pci_set_drvdata(pdev, NULL);
11649         return err;
11650 }
11651
11652 static void __devexit tg3_remove_one(struct pci_dev *pdev)
11653 {
11654         struct net_device *dev = pci_get_drvdata(pdev);
11655
11656         if (dev) {
11657                 struct tg3 *tp = netdev_priv(dev);
11658
11659                 flush_scheduled_work();
11660                 unregister_netdev(dev);
11661                 if (tp->regs) {
11662                         iounmap(tp->regs);
11663                         tp->regs = NULL;
11664                 }
11665                 free_netdev(dev);
11666                 pci_release_regions(pdev);
11667                 pci_disable_device(pdev);
11668                 pci_set_drvdata(pdev, NULL);
11669         }
11670 }
11671
11672 static int tg3_suspend(struct pci_dev *pdev, pm_message_t state)
11673 {
11674         struct net_device *dev = pci_get_drvdata(pdev);
11675         struct tg3 *tp = netdev_priv(dev);
11676         int err;
11677
11678         if (!netif_running(dev))
11679                 return 0;
11680
11681         flush_scheduled_work();
11682         tg3_netif_stop(tp);
11683
11684         del_timer_sync(&tp->timer);
11685
11686         tg3_full_lock(tp, 1);
11687         tg3_disable_ints(tp);
11688         tg3_full_unlock(tp);
11689
11690         netif_device_detach(dev);
11691
11692         tg3_full_lock(tp, 0);
11693         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11694         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
11695         tg3_full_unlock(tp);
11696
11697         err = tg3_set_power_state(tp, pci_choose_state(pdev, state));
11698         if (err) {
11699                 tg3_full_lock(tp, 0);
11700
11701                 tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
11702                 tg3_init_hw(tp, 1);
11703
11704                 tp->timer.expires = jiffies + tp->timer_offset;
11705                 add_timer(&tp->timer);
11706
11707                 netif_device_attach(dev);
11708                 tg3_netif_start(tp);
11709
11710                 tg3_full_unlock(tp);
11711         }
11712
11713         return err;
11714 }
11715
11716 static int tg3_resume(struct pci_dev *pdev)
11717 {
11718         struct net_device *dev = pci_get_drvdata(pdev);
11719         struct tg3 *tp = netdev_priv(dev);
11720         int err;
11721
11722         if (!netif_running(dev))
11723                 return 0;
11724
11725         pci_restore_state(tp->pdev);
11726
11727         err = tg3_set_power_state(tp, PCI_D0);
11728         if (err)
11729                 return err;
11730
11731         netif_device_attach(dev);
11732
11733         tg3_full_lock(tp, 0);
11734
11735         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
11736         tg3_init_hw(tp, 1);
11737
11738         tp->timer.expires = jiffies + tp->timer_offset;
11739         add_timer(&tp->timer);
11740
11741         tg3_netif_start(tp);
11742
11743         tg3_full_unlock(tp);
11744
11745         return 0;
11746 }
11747
11748 static struct pci_driver tg3_driver = {
11749         .name           = DRV_MODULE_NAME,
11750         .id_table       = tg3_pci_tbl,
11751         .probe          = tg3_init_one,
11752         .remove         = __devexit_p(tg3_remove_one),
11753         .suspend        = tg3_suspend,
11754         .resume         = tg3_resume
11755 };
11756
11757 static int __init tg3_init(void)
11758 {
11759         return pci_module_init(&tg3_driver);
11760 }
11761
11762 static void __exit tg3_cleanup(void)
11763 {
11764         pci_unregister_driver(&tg3_driver);
11765 }
11766
11767 module_init(tg3_init);
11768 module_exit(tg3_cleanup);