git.oblomov.eu Git - linux-2.6/blob - drivers/net/wireless/bcm43xx/bcm43xx_dma.c

   1 /*
   2
   3   Broadcom BCM43xx wireless driver
   4
   5   DMA ringbuffer and descriptor allocation/management
   6
   7   Copyright (c) 2005 Michael Buesch <mbuesch@freenet.de>
   8
   9   Some code in this file is derived from the b44.c driver
  10   Copyright (C) 2002 David S. Miller
  11   Copyright (C) Pekka Pietikainen
  12
  13   This program is free software; you can redistribute it and/or modify
  14   it under the terms of the GNU General Public License as published by
  15   the Free Software Foundation; either version 2 of the License, or
  16   (at your option) any later version.
  17
  18   This program is distributed in the hope that it will be useful,
  19   but WITHOUT ANY WARRANTY; without even the implied warranty of
  20   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21   GNU General Public License for more details.
  22
  23   You should have received a copy of the GNU General Public License
  24   along with this program; see the file COPYING.  If not, write to
  25   the Free Software Foundation, Inc., 51 Franklin Steet, Fifth Floor,
  26   Boston, MA 02110-1301, USA.
  27
  28 */
  29
  30 #include "bcm43xx.h"
  31 #include "bcm43xx_dma.h"
  32 #include "bcm43xx_main.h"
  33 #include "bcm43xx_debugfs.h"
  34 #include "bcm43xx_power.h"
  35 #include "bcm43xx_xmit.h"
  36
  37 #include <linux/dmapool.h>
  38 #include <linux/pci.h>
  39 #include <linux/delay.h>
  40 #include <linux/skbuff.h>
  41 #include <asm/semaphore.h>
  42
  43
  44 static inline int free_slots(struct bcm43xx_dmaring *ring)
  45 {
  46         return (ring->nr_slots - ring->used_slots);
  47 }
  48
  49 static inline int next_slot(struct bcm43xx_dmaring *ring, int slot)
  50 {
  51         assert(slot >= -1 && slot <= ring->nr_slots - 1);
  52         if (slot == ring->nr_slots - 1)
  53                 return 0;
  54         return slot + 1;
  55 }
  56
  57 static inline int prev_slot(struct bcm43xx_dmaring *ring, int slot)
  58 {
  59         assert(slot >= 0 && slot <= ring->nr_slots - 1);
  60         if (slot == 0)
  61                 return ring->nr_slots - 1;
  62         return slot - 1;
  63 }
  64
  65 /* Request a slot for usage. */
  66 static inline
  67 int request_slot(struct bcm43xx_dmaring *ring)
  68 {
  69         int slot;
  70
  71         assert(ring->tx);
  72         assert(!ring->suspended);
  73         assert(free_slots(ring) != 0);
  74
  75         slot = next_slot(ring, ring->current_slot);
  76         ring->current_slot = slot;
  77         ring->used_slots++;
  78
  79         /* Check the number of available slots and suspend TX,
  80          * if we are running low on free slots.
  81          */
  82         if (unlikely(free_slots(ring) < ring->suspend_mark)) {
  83                 netif_stop_queue(ring->bcm->net_dev);
  84                 ring->suspended = 1;
  85         }
  86 #ifdef CONFIG_BCM43XX_DEBUG
  87         if (ring->used_slots > ring->max_used_slots)
  88                 ring->max_used_slots = ring->used_slots;
  89 #endif /* CONFIG_BCM43XX_DEBUG*/
  90
  91         return slot;
  92 }
  93
  94 /* Return a slot to the free slots. */
  95 static inline
  96 void return_slot(struct bcm43xx_dmaring *ring, int slot)
  97 {
  98         assert(ring->tx);
  99
 100         ring->used_slots--;
 101
 102         /* Check if TX is suspended and check if we have
 103          * enough free slots to resume it again.
 104          */
 105         if (unlikely(ring->suspended)) {
 106                 if (free_slots(ring) >= ring->resume_mark) {
 107                         ring->suspended = 0;
 108                         netif_wake_queue(ring->bcm->net_dev);
 109                 }
 110         }
 111 }
 112
 113 static inline
 114 dma_addr_t map_descbuffer(struct bcm43xx_dmaring *ring,
 115                           unsigned char *buf,
 116                           size_t len,
 117                           int tx)
 118 {
 119         dma_addr_t dmaaddr;
 120
 121         if (tx) {
 122                 dmaaddr = dma_map_single(&ring->bcm->pci_dev->dev,
 123                                          buf, len,
 124                                          DMA_TO_DEVICE);
 125         } else {
 126                 dmaaddr = dma_map_single(&ring->bcm->pci_dev->dev,
 127                                          buf, len,
 128                                          DMA_FROM_DEVICE);
 129         }
 130
 131         return dmaaddr;
 132 }
 133
 134 static inline
 135 void unmap_descbuffer(struct bcm43xx_dmaring *ring,
 136                       dma_addr_t addr,
 137                       size_t len,
 138                       int tx)
 139 {
 140         if (tx) {
 141                 dma_unmap_single(&ring->bcm->pci_dev->dev,
 142                                  addr, len,
 143                                  DMA_TO_DEVICE);
 144         } else {
 145                 dma_unmap_single(&ring->bcm->pci_dev->dev,
 146                                  addr, len,
 147                                  DMA_FROM_DEVICE);
 148         }
 149 }
 150
 151 static inline
 152 void sync_descbuffer_for_cpu(struct bcm43xx_dmaring *ring,
 153                              dma_addr_t addr,
 154                              size_t len)
 155 {
 156         assert(!ring->tx);
 157
 158         dma_sync_single_for_cpu(&ring->bcm->pci_dev->dev,
 159                                 addr, len, DMA_FROM_DEVICE);
 160 }
 161
 162 static inline
 163 void sync_descbuffer_for_device(struct bcm43xx_dmaring *ring,
 164                                 dma_addr_t addr,
 165                                 size_t len)
 166 {
 167         assert(!ring->tx);
 168
 169         dma_sync_single_for_device(&ring->bcm->pci_dev->dev,
 170                                    addr, len, DMA_FROM_DEVICE);
 171 }
 172
 173 static inline
 174 void mark_skb_mustfree(struct sk_buff *skb,
 175                        char mustfree)
 176 {
 177         skb->cb[0] = mustfree;
 178 }
 179
 180 static inline
 181 int skb_mustfree(struct sk_buff *skb)
 182 {
 183         return (skb->cb[0] != 0);
 184 }
 185
 186 /* Unmap and free a descriptor buffer. */
 187 static inline
 188 void free_descriptor_buffer(struct bcm43xx_dmaring *ring,
 189                             struct bcm43xx_dmadesc *desc,
 190                             struct bcm43xx_dmadesc_meta *meta,
 191                             int irq_context)
 192 {
 193         assert(meta->skb);
 194         if (skb_mustfree(meta->skb)) {
 195                 if (irq_context)
 196                         dev_kfree_skb_irq(meta->skb);
 197                 else
 198                         dev_kfree_skb(meta->skb);
 199         }
 200         meta->skb = NULL;
 201         if (meta->txb) {
 202                 ieee80211_txb_free(meta->txb);
 203                 meta->txb = NULL;
 204         }
 205 }
 206
 207 static int alloc_ringmemory(struct bcm43xx_dmaring *ring)
 208 {
 209         struct device *dev = &(ring->bcm->pci_dev->dev);
 210
 211         ring->vbase = dma_alloc_coherent(dev, BCM43xx_DMA_RINGMEMSIZE,
 212                                          &(ring->dmabase), GFP_KERNEL);
 213         if (!ring->vbase) {
 214                 printk(KERN_ERR PFX "DMA ringmemory allocation failed\n");
 215                 return -ENOMEM;
 216         }
 217         if (ring->dmabase + BCM43xx_DMA_RINGMEMSIZE > BCM43xx_DMA_BUSADDRMAX) {
 218                 printk(KERN_ERR PFX ">>>FATAL ERROR<<<  DMA RINGMEMORY >1G "
 219                                     "(0x%08x, len: %lu)\n",
 220                        ring->dmabase, BCM43xx_DMA_RINGMEMSIZE);
 221                 dma_free_coherent(dev, BCM43xx_DMA_RINGMEMSIZE,
 222                                   ring->vbase, ring->dmabase);
 223                 return -ENOMEM;
 224         }
 225         assert(!(ring->dmabase & 0x000003FF));
 226         memset(ring->vbase, 0, BCM43xx_DMA_RINGMEMSIZE);
 227
 228         return 0;
 229 }
 230
 231 static void free_ringmemory(struct bcm43xx_dmaring *ring)
 232 {
 233         struct device *dev = &(ring->bcm->pci_dev->dev);
 234
 235         dma_free_coherent(dev, BCM43xx_DMA_RINGMEMSIZE,
 236                           ring->vbase, ring->dmabase);
 237 }
 238
 239 /* Reset the RX DMA channel */
 240 int bcm43xx_dmacontroller_rx_reset(struct bcm43xx_private *bcm,
 241                                    u16 mmio_base)
 242 {
 243         int i;
 244         u32 value;
 245
 246         bcm43xx_write32(bcm,
 247                         mmio_base + BCM43xx_DMA_RX_CONTROL,
 248                         0x00000000);
 249         for (i = 0; i < 1000; i++) {
 250                 value = bcm43xx_read32(bcm,
 251                                        mmio_base + BCM43xx_DMA_RX_STATUS);
 252                 value &= BCM43xx_DMA_RXSTAT_STAT_MASK;
 253                 if (value == BCM43xx_DMA_RXSTAT_STAT_DISABLED) {
 254                         i = -1;
 255                         break;
 256                 }
 257                 udelay(10);
 258         }
 259         if (i != -1) {
 260                 printk(KERN_ERR PFX "Error: Wait on DMA RX status timed out.\n");
 261                 return -ENODEV;
 262         }
 263
 264         return 0;
 265 }
 266
 267 /* Reset the RX DMA channel */
 268 int bcm43xx_dmacontroller_tx_reset(struct bcm43xx_private *bcm,
 269                                    u16 mmio_base)
 270 {
 271         int i;
 272         u32 value;
 273
 274         for (i = 0; i < 1000; i++) {
 275                 value = bcm43xx_read32(bcm,
 276                                        mmio_base + BCM43xx_DMA_TX_STATUS);
 277                 value &= BCM43xx_DMA_TXSTAT_STAT_MASK;
 278                 if (value == BCM43xx_DMA_TXSTAT_STAT_DISABLED ||
 279                     value == BCM43xx_DMA_TXSTAT_STAT_IDLEWAIT ||
 280                     value == BCM43xx_DMA_TXSTAT_STAT_STOPPED)
 281                         break;
 282                 udelay(10);
 283         }
 284         bcm43xx_write32(bcm,
 285                         mmio_base + BCM43xx_DMA_TX_CONTROL,
 286                         0x00000000);
 287         for (i = 0; i < 1000; i++) {
 288                 value = bcm43xx_read32(bcm,
 289                                        mmio_base + BCM43xx_DMA_TX_STATUS);
 290                 value &= BCM43xx_DMA_TXSTAT_STAT_MASK;
 291                 if (value == BCM43xx_DMA_TXSTAT_STAT_DISABLED) {
 292                         i = -1;
 293                         break;
 294                 }
 295                 udelay(10);
 296         }
 297         if (i != -1) {
 298                 printk(KERN_ERR PFX "Error: Wait on DMA TX status timed out.\n");
 299                 return -ENODEV;
 300         }
 301         /* ensure the reset is completed. */
 302         udelay(300);
 303
 304         return 0;
 305 }
 306
 307 static int setup_rx_descbuffer(struct bcm43xx_dmaring *ring,
 308                                struct bcm43xx_dmadesc *desc,
 309                                struct bcm43xx_dmadesc_meta *meta,
 310                                gfp_t gfp_flags)
 311 {
 312         struct bcm43xx_rxhdr *rxhdr;
 313         dma_addr_t dmaaddr;
 314         u32 desc_addr;
 315         u32 desc_ctl;
 316         const int slot = (int)(desc - ring->vbase);
 317         struct sk_buff *skb;
 318
 319         assert(slot >= 0 && slot < ring->nr_slots);
 320         assert(!ring->tx);
 321
 322         skb = __dev_alloc_skb(ring->rx_buffersize, gfp_flags);
 323         if (unlikely(!skb))
 324                 return -ENOMEM;
 325         dmaaddr = map_descbuffer(ring, skb->data, ring->rx_buffersize, 0);
 326         if (unlikely(dmaaddr + ring->rx_buffersize > BCM43xx_DMA_BUSADDRMAX)) {
 327                 unmap_descbuffer(ring, dmaaddr, ring->rx_buffersize, 0);
 328                 dev_kfree_skb_any(skb);
 329                 printk(KERN_ERR PFX ">>>FATAL ERROR<<<  DMA RX SKB >1G "
 330                                     "(0x%08x, len: %u)\n",
 331                        dmaaddr, ring->rx_buffersize);
 332                 return -ENOMEM;
 333         }
 334         meta->skb = skb;
 335         meta->dmaaddr = dmaaddr;
 336         skb->dev = ring->bcm->net_dev;
 337         mark_skb_mustfree(skb, 1);
 338         desc_addr = (u32)(dmaaddr + ring->memoffset);
 339         desc_ctl = (BCM43xx_DMADTOR_BYTECNT_MASK &
 340                     (u32)(ring->rx_buffersize - ring->frameoffset));
 341         if (slot == ring->nr_slots - 1)
 342                 desc_ctl |= BCM43xx_DMADTOR_DTABLEEND;
 343         set_desc_addr(desc, desc_addr);
 344         set_desc_ctl(desc, desc_ctl);
 345
 346         rxhdr = (struct bcm43xx_rxhdr *)(skb->data);
 347         rxhdr->frame_length = 0;
 348         rxhdr->flags1 = 0;
 349
 350         return 0;
 351 }
 352
 353 /* Allocate the initial descbuffers.
 354  * This is used for an RX ring only.
 355  */
 356 static int alloc_initial_descbuffers(struct bcm43xx_dmaring *ring)
 357 {
 358         int i, err = -ENOMEM;
 359         struct bcm43xx_dmadesc *desc;
 360         struct bcm43xx_dmadesc_meta *meta;
 361
 362         for (i = 0; i < ring->nr_slots; i++) {
 363                 desc = ring->vbase + i;
 364                 meta = ring->meta + i;
 365
 366                 err = setup_rx_descbuffer(ring, desc, meta, GFP_KERNEL);
 367                 if (err)
 368                         goto err_unwind;
 369         }
 370         ring->used_slots = ring->nr_slots;
 371         err = 0;
 372 out:
 373         return err;
 374
 375 err_unwind:
 376         for (i--; i >= 0; i--) {
 377                 desc = ring->vbase + i;
 378                 meta = ring->meta + i;
 379
 380                 unmap_descbuffer(ring, meta->dmaaddr, ring->rx_buffersize, 0);
 381                 dev_kfree_skb(meta->skb);
 382         }
 383         goto out;
 384 }
 385
 386 /* Do initial setup of the DMA controller.
 387  * Reset the controller, write the ring busaddress
 388  * and switch the "enable" bit on.
 389  */
 390 static int dmacontroller_setup(struct bcm43xx_dmaring *ring)
 391 {
 392         int err = 0;
 393         u32 value;
 394
 395         if (ring->tx) {
 396                 /* Set Transmit Control register to "transmit enable" */
 397                 bcm43xx_write32(ring->bcm,
 398                                 ring->mmio_base + BCM43xx_DMA_TX_CONTROL,
 399                                 BCM43xx_DMA_TXCTRL_ENABLE);
 400                 /* Set Transmit Descriptor ring address. */
 401                 bcm43xx_write32(ring->bcm,
 402                                 ring->mmio_base + BCM43xx_DMA_TX_DESC_RING,
 403                                 ring->dmabase + ring->memoffset);
 404         } else {
 405                 err = alloc_initial_descbuffers(ring);
 406                 if (err)
 407                         goto out;
 408                 /* Set Receive Control "receive enable" and frame offset */
 409                 value = (ring->frameoffset << BCM43xx_DMA_RXCTRL_FRAMEOFF_SHIFT);
 410                 value |= BCM43xx_DMA_RXCTRL_ENABLE;
 411                 bcm43xx_write32(ring->bcm,
 412                                 ring->mmio_base + BCM43xx_DMA_RX_CONTROL,
 413                                 value);
 414                 /* Set Receive Descriptor ring address. */
 415                 bcm43xx_write32(ring->bcm,
 416                                 ring->mmio_base + BCM43xx_DMA_RX_DESC_RING,
 417                                 ring->dmabase + ring->memoffset);
 418                 /* Init the descriptor pointer. */
 419                 bcm43xx_write32(ring->bcm,
 420                                 ring->mmio_base + BCM43xx_DMA_RX_DESC_INDEX,
 421                                 200);
 422         }
 423
 424 out:
 425         return err;
 426 }
 427
 428 /* Shutdown the DMA controller. */
 429 static void dmacontroller_cleanup(struct bcm43xx_dmaring *ring)
 430 {
 431         if (ring->tx) {
 432                 bcm43xx_dmacontroller_tx_reset(ring->bcm, ring->mmio_base);
 433                 /* Zero out Transmit Descriptor ring address. */
 434                 bcm43xx_write32(ring->bcm,
 435                                 ring->mmio_base + BCM43xx_DMA_TX_DESC_RING,
 436                                 0x00000000);
 437         } else {
 438                 bcm43xx_dmacontroller_rx_reset(ring->bcm, ring->mmio_base);
 439                 /* Zero out Receive Descriptor ring address. */
 440                 bcm43xx_write32(ring->bcm,
 441                                 ring->mmio_base + BCM43xx_DMA_RX_DESC_RING,
 442                                 0x00000000);
 443         }
 444 }
 445
 446 static void free_all_descbuffers(struct bcm43xx_dmaring *ring)
 447 {
 448         struct bcm43xx_dmadesc *desc;
 449         struct bcm43xx_dmadesc_meta *meta;
 450         int i;
 451
 452         if (!ring->used_slots)
 453                 return;
 454         for (i = 0; i < ring->nr_slots; i++) {
 455                 desc = ring->vbase + i;
 456                 meta = ring->meta + i;
 457
 458                 if (!meta->skb) {
 459                         assert(ring->tx);
 460                         assert(!meta->txb);
 461                         continue;
 462                 }
 463                 if (ring->tx) {
 464                         unmap_descbuffer(ring, meta->dmaaddr,
 465                                          meta->skb->len, 1);
 466                 } else {
 467                         unmap_descbuffer(ring, meta->dmaaddr,
 468                                          ring->rx_buffersize, 0);
 469                 }
 470                 free_descriptor_buffer(ring, desc, meta, 0);
 471         }
 472 }
 473
 474 /* Main initialization function. */
 475 static
 476 struct bcm43xx_dmaring * bcm43xx_setup_dmaring(struct bcm43xx_private *bcm,
 477                                                u16 dma_controller_base,
 478                                                int nr_descriptor_slots,
 479                                                int tx)
 480 {
 481         struct bcm43xx_dmaring *ring;
 482         int err;
 483
 484         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
 485         if (!ring)
 486                 goto out;
 487
 488         ring->meta = kzalloc(sizeof(*ring->meta) * nr_descriptor_slots,
 489                              GFP_KERNEL);
 490         if (!ring->meta)
 491                 goto err_kfree_ring;
 492
 493         ring->memoffset = BCM43xx_DMA_DMABUSADDROFFSET;
 494 #ifdef CONFIG_BCM947XX
 495         if (bcm->pci_dev->bus->number == 0)
 496                 ring->memoffset = 0;
 497 #endif
 498
 499         ring->bcm = bcm;
 500         ring->nr_slots = nr_descriptor_slots;
 501         ring->suspend_mark = ring->nr_slots * BCM43xx_TXSUSPEND_PERCENT / 100;
 502         ring->resume_mark = ring->nr_slots * BCM43xx_TXRESUME_PERCENT / 100;
 503         assert(ring->suspend_mark < ring->resume_mark);
 504         ring->mmio_base = dma_controller_base;
 505         if (tx) {
 506                 ring->tx = 1;
 507                 ring->current_slot = -1;
 508         } else {
 509                 switch (dma_controller_base) {
 510                 case BCM43xx_MMIO_DMA1_BASE:
 511                         ring->rx_buffersize = BCM43xx_DMA1_RXBUFFERSIZE;
 512                         ring->frameoffset = BCM43xx_DMA1_RX_FRAMEOFFSET;
 513                         break;
 514                 case BCM43xx_MMIO_DMA4_BASE:
 515                         ring->rx_buffersize = BCM43xx_DMA4_RXBUFFERSIZE;
 516                         ring->frameoffset = BCM43xx_DMA4_RX_FRAMEOFFSET;
 517                         break;
 518                 default:
 519                         assert(0);
 520                 }
 521         }
 522
 523         err = alloc_ringmemory(ring);
 524         if (err)
 525                 goto err_kfree_meta;
 526         err = dmacontroller_setup(ring);
 527         if (err)
 528                 goto err_free_ringmemory;
 529
 530 out:
 531         return ring;
 532
 533 err_free_ringmemory:
 534         free_ringmemory(ring);
 535 err_kfree_meta:
 536         kfree(ring->meta);
 537 err_kfree_ring:
 538         kfree(ring);
 539         ring = NULL;
 540         goto out;
 541 }
 542
 543 /* Main cleanup function. */
 544 static void bcm43xx_destroy_dmaring(struct bcm43xx_dmaring *ring)
 545 {
 546         if (!ring)
 547                 return;
 548
 549         dprintk(KERN_INFO PFX "DMA 0x%04x (%s) max used slots: %d/%d\n",
 550                 ring->mmio_base,
 551                 (ring->tx) ? "TX" : "RX",
 552                 ring->max_used_slots, ring->nr_slots);
 553         /* Device IRQs are disabled prior entering this function,
 554          * so no need to take care of concurrency with rx handler stuff.
 555          */
 556         dmacontroller_cleanup(ring);
 557         free_all_descbuffers(ring);
 558         free_ringmemory(ring);
 559
 560         kfree(ring->meta);
 561         kfree(ring);
 562 }
 563
 564 void bcm43xx_dma_free(struct bcm43xx_private *bcm)
 565 {
 566         struct bcm43xx_dma *dma = bcm->current_core->dma;
 567
 568         bcm43xx_destroy_dmaring(dma->rx_ring1);
 569         dma->rx_ring1 = NULL;
 570         bcm43xx_destroy_dmaring(dma->rx_ring0);
 571         dma->rx_ring0 = NULL;
 572         bcm43xx_destroy_dmaring(dma->tx_ring3);
 573         dma->tx_ring3 = NULL;
 574         bcm43xx_destroy_dmaring(dma->tx_ring2);
 575         dma->tx_ring2 = NULL;
 576         bcm43xx_destroy_dmaring(dma->tx_ring1);
 577         dma->tx_ring1 = NULL;
 578         bcm43xx_destroy_dmaring(dma->tx_ring0);
 579         dma->tx_ring0 = NULL;
 580 }
 581
 582 int bcm43xx_dma_init(struct bcm43xx_private *bcm)
 583 {
 584         struct bcm43xx_dma *dma = bcm->current_core->dma;
 585         struct bcm43xx_dmaring *ring;
 586         int err = -ENOMEM;
 587
 588         /* setup TX DMA channels. */
 589         ring = bcm43xx_setup_dmaring(bcm, BCM43xx_MMIO_DMA1_BASE,
 590                                      BCM43xx_TXRING_SLOTS, 1);
 591         if (!ring)
 592                 goto out;
 593         dma->tx_ring0 = ring;
 594
 595         ring = bcm43xx_setup_dmaring(bcm, BCM43xx_MMIO_DMA2_BASE,
 596                                      BCM43xx_TXRING_SLOTS, 1);
 597         if (!ring)
 598                 goto err_destroy_tx0;
 599         dma->tx_ring1 = ring;
 600
 601         ring = bcm43xx_setup_dmaring(bcm, BCM43xx_MMIO_DMA3_BASE,
 602                                      BCM43xx_TXRING_SLOTS, 1);
 603         if (!ring)
 604                 goto err_destroy_tx1;
 605         dma->tx_ring2 = ring;
 606
 607         ring = bcm43xx_setup_dmaring(bcm, BCM43xx_MMIO_DMA4_BASE,
 608                                      BCM43xx_TXRING_SLOTS, 1);
 609         if (!ring)
 610                 goto err_destroy_tx2;
 611         dma->tx_ring3 = ring;
 612
 613         /* setup RX DMA channels. */
 614         ring = bcm43xx_setup_dmaring(bcm, BCM43xx_MMIO_DMA1_BASE,
 615                                      BCM43xx_RXRING_SLOTS, 0);
 616         if (!ring)
 617                 goto err_destroy_tx3;
 618         dma->rx_ring0 = ring;
 619
 620         if (bcm->current_core->rev < 5) {
 621                 ring = bcm43xx_setup_dmaring(bcm, BCM43xx_MMIO_DMA4_BASE,
 622                                              BCM43xx_RXRING_SLOTS, 0);
 623                 if (!ring)
 624                         goto err_destroy_rx0;
 625                 dma->rx_ring1 = ring;
 626         }
 627
 628         dprintk(KERN_INFO PFX "DMA initialized\n");
 629         err = 0;
 630 out:
 631         return err;
 632
 633 err_destroy_rx0:
 634         bcm43xx_destroy_dmaring(dma->rx_ring0);
 635         dma->rx_ring0 = NULL;
 636 err_destroy_tx3:
 637         bcm43xx_destroy_dmaring(dma->tx_ring3);
 638         dma->tx_ring3 = NULL;
 639 err_destroy_tx2:
 640         bcm43xx_destroy_dmaring(dma->tx_ring2);
 641         dma->tx_ring2 = NULL;
 642 err_destroy_tx1:
 643         bcm43xx_destroy_dmaring(dma->tx_ring1);
 644         dma->tx_ring1 = NULL;
 645 err_destroy_tx0:
 646         bcm43xx_destroy_dmaring(dma->tx_ring0);
 647         dma->tx_ring0 = NULL;
 648         goto out;
 649 }
 650
 651 /* Generate a cookie for the TX header. */
 652 static u16 generate_cookie(struct bcm43xx_dmaring *ring,
 653                            int slot)
 654 {
 655         u16 cookie = 0x0000;
 656
 657         /* Use the upper 4 bits of the cookie as
 658          * DMA controller ID and store the slot number
 659          * in the lower 12 bits
 660          */
 661         switch (ring->mmio_base) {
 662         default:
 663                 assert(0);
 664         case BCM43xx_MMIO_DMA1_BASE:
 665                 break;
 666         case BCM43xx_MMIO_DMA2_BASE:
 667                 cookie = 0x1000;
 668                 break;
 669         case BCM43xx_MMIO_DMA3_BASE:
 670                 cookie = 0x2000;
 671                 break;
 672         case BCM43xx_MMIO_DMA4_BASE:
 673                 cookie = 0x3000;
 674                 break;
 675         }
 676         assert(((u16)slot & 0xF000) == 0x0000);
 677         cookie |= (u16)slot;
 678
 679         return cookie;
 680 }
 681
 682 /* Inspect a cookie and find out to which controller/slot it belongs. */
 683 static
 684 struct bcm43xx_dmaring * parse_cookie(struct bcm43xx_private *bcm,
 685                                       u16 cookie, int *slot)
 686 {
 687         struct bcm43xx_dma *dma = bcm->current_core->dma;
 688         struct bcm43xx_dmaring *ring = NULL;
 689
 690         switch (cookie & 0xF000) {
 691         case 0x0000:
 692                 ring = dma->tx_ring0;
 693                 break;
 694         case 0x1000:
 695                 ring = dma->tx_ring1;
 696                 break;
 697         case 0x2000:
 698                 ring = dma->tx_ring2;
 699                 break;
 700         case 0x3000:
 701                 ring = dma->tx_ring3;
 702                 break;
 703         default:
 704                 assert(0);
 705         }
 706         *slot = (cookie & 0x0FFF);
 707         assert(*slot >= 0 && *slot < ring->nr_slots);
 708
 709         return ring;
 710 }
 711
 712 static void dmacontroller_poke_tx(struct bcm43xx_dmaring *ring,
 713                                   int slot)
 714 {
 715         /* Everything is ready to start. Buffers are DMA mapped and
 716          * associated with slots.
 717          * "slot" is the last slot of the new frame we want to transmit.
 718          * Close your seat belts now, please.
 719          */
 720         wmb();
 721         slot = next_slot(ring, slot);
 722         bcm43xx_write32(ring->bcm,
 723                         ring->mmio_base + BCM43xx_DMA_TX_DESC_INDEX,
 724                         (u32)(slot * sizeof(struct bcm43xx_dmadesc)));
 725 }
 726
 727 static int dma_tx_fragment(struct bcm43xx_dmaring *ring,
 728                            struct sk_buff *skb,
 729                            struct ieee80211_txb *txb,
 730                            u8 cur_frag)
 731 {
 732         int slot;
 733         struct bcm43xx_dmadesc *desc;
 734         struct bcm43xx_dmadesc_meta *meta;
 735         u32 desc_ctl;
 736         u32 desc_addr;
 737
 738         assert(skb_shinfo(skb)->nr_frags == 0);
 739
 740         slot = request_slot(ring);
 741         desc = ring->vbase + slot;
 742         meta = ring->meta + slot;
 743
 744         if (cur_frag == 0) {
 745                 /* Save the txb pointer for freeing in xmitstatus IRQ */
 746                 meta->txb = txb;
 747         }
 748
 749         /* Add a device specific TX header. */
 750         assert(skb_headroom(skb) >= sizeof(struct bcm43xx_txhdr));
 751         /* Reserve enough headroom for the device tx header. */
 752         __skb_push(skb, sizeof(struct bcm43xx_txhdr));
 753         /* Now calculate and add the tx header.
 754          * The tx header includes the PLCP header.
 755          */
 756         bcm43xx_generate_txhdr(ring->bcm,
 757                                (struct bcm43xx_txhdr *)skb->data,
 758                                skb->data + sizeof(struct bcm43xx_txhdr),
 759                                skb->len - sizeof(struct bcm43xx_txhdr),
 760                                (cur_frag == 0),
 761                                generate_cookie(ring, slot));
 762
 763         meta->skb = skb;
 764         meta->dmaaddr = map_descbuffer(ring, skb->data, skb->len, 1);
 765         if (unlikely(meta->dmaaddr + skb->len > BCM43xx_DMA_BUSADDRMAX)) {
 766                 return_slot(ring, slot);
 767                 printk(KERN_ERR PFX ">>>FATAL ERROR<<<  DMA TX SKB >1G "
 768                                     "(0x%08x, len: %u)\n",
 769                        meta->dmaaddr, skb->len);
 770                 return -ENOMEM;
 771         }
 772
 773         desc_addr = (u32)(meta->dmaaddr + ring->memoffset);
 774         desc_ctl = BCM43xx_DMADTOR_FRAMESTART | BCM43xx_DMADTOR_FRAMEEND;
 775         desc_ctl |= BCM43xx_DMADTOR_COMPIRQ;
 776         desc_ctl |= (BCM43xx_DMADTOR_BYTECNT_MASK &
 777                      (u32)(meta->skb->len - ring->frameoffset));
 778         if (slot == ring->nr_slots - 1)
 779                 desc_ctl |= BCM43xx_DMADTOR_DTABLEEND;
 780
 781         set_desc_ctl(desc, desc_ctl);
 782         set_desc_addr(desc, desc_addr);
 783         /* Now transfer the whole frame. */
 784         dmacontroller_poke_tx(ring, slot);
 785
 786         return 0;
 787 }
 788
 789 int bcm43xx_dma_tx(struct bcm43xx_private *bcm,
 790                    struct ieee80211_txb *txb)
 791 {
 792         /* We just received a packet from the kernel network subsystem.
 793          * Add headers and DMA map the memory. Poke
 794          * the device to send the stuff.
 795          * Note that this is called from atomic context.
 796          */
 797         struct bcm43xx_dmaring *ring = bcm->current_core->dma->tx_ring1;
 798         u8 i;
 799         struct sk_buff *skb;
 800
 801         assert(ring->tx);
 802         if (unlikely(free_slots(ring) < txb->nr_frags)) {
 803                 /* The queue should be stopped,
 804                  * if we are low on free slots.
 805                  * If this ever triggers, we have to lower the suspend_mark.
 806                  */
 807                 dprintkl(KERN_ERR PFX "Out of DMA descriptor slots!\n");
 808                 return -ENOMEM;
 809         }
 810
 811         for (i = 0; i < txb->nr_frags; i++) {
 812                 skb = txb->fragments[i];
 813                 /* We do not free the skb, as it is freed as
 814                  * part of the txb freeing.
 815                  */
 816                 mark_skb_mustfree(skb, 0);
 817                 dma_tx_fragment(ring, skb, txb, i);
 818                 //TODO: handle failure of dma_tx_fragment
 819         }
 820
 821         return 0;
 822 }
 823
 824 void bcm43xx_dma_handle_xmitstatus(struct bcm43xx_private *bcm,
 825                                    struct bcm43xx_xmitstatus *status)
 826 {
 827         struct bcm43xx_dmaring *ring;
 828         struct bcm43xx_dmadesc *desc;
 829         struct bcm43xx_dmadesc_meta *meta;
 830         int is_last_fragment;
 831         int slot;
 832
 833         ring = parse_cookie(bcm, status->cookie, &slot);
 834         assert(ring);
 835         assert(ring->tx);
 836         assert(get_desc_ctl(ring->vbase + slot) & BCM43xx_DMADTOR_FRAMESTART);
 837         while (1) {
 838                 assert(slot >= 0 && slot < ring->nr_slots);
 839                 desc = ring->vbase + slot;
 840                 meta = ring->meta + slot;
 841
 842                 is_last_fragment = !!(get_desc_ctl(desc) & BCM43xx_DMADTOR_FRAMEEND);
 843                 unmap_descbuffer(ring, meta->dmaaddr, meta->skb->len, 1);
 844                 free_descriptor_buffer(ring, desc, meta, 1);
 845                 /* Everything belonging to the slot is unmapped
 846                  * and freed, so we can return it.
 847                  */
 848                 return_slot(ring, slot);
 849
 850                 if (is_last_fragment)
 851                         break;
 852                 slot = next_slot(ring, slot);
 853         }
 854         bcm->stats.last_tx = jiffies;
 855 }
 856
 857 static void dma_rx(struct bcm43xx_dmaring *ring,
 858                    int *slot)
 859 {
 860         struct bcm43xx_dmadesc *desc;
 861         struct bcm43xx_dmadesc_meta *meta;
 862         struct bcm43xx_rxhdr *rxhdr;
 863         struct sk_buff *skb;
 864         u16 len;
 865         int err;
 866         dma_addr_t dmaaddr;
 867
 868         desc = ring->vbase + *slot;
 869         meta = ring->meta + *slot;
 870
 871         sync_descbuffer_for_cpu(ring, meta->dmaaddr, ring->rx_buffersize);
 872         skb = meta->skb;
 873
 874         if (ring->mmio_base == BCM43xx_MMIO_DMA4_BASE) {
 875                 /* We received an xmit status. */
 876                 struct bcm43xx_hwxmitstatus *hw = (struct bcm43xx_hwxmitstatus *)skb->data;
 877                 struct bcm43xx_xmitstatus stat;
 878
 879                 stat.cookie = le16_to_cpu(hw->cookie);
 880                 stat.flags = hw->flags;
 881                 stat.cnt1 = hw->cnt1;
 882                 stat.cnt2 = hw->cnt2;
 883                 stat.seq = le16_to_cpu(hw->seq);
 884                 stat.unknown = le16_to_cpu(hw->unknown);
 885
 886                 bcm43xx_debugfs_log_txstat(ring->bcm, &stat);
 887                 bcm43xx_dma_handle_xmitstatus(ring->bcm, &stat);
 888                 /* recycle the descriptor buffer. */
 889                 sync_descbuffer_for_device(ring, meta->dmaaddr, ring->rx_buffersize);
 890
 891                 return;
 892         }
 893         rxhdr = (struct bcm43xx_rxhdr *)skb->data;
 894         len = le16_to_cpu(rxhdr->frame_length);
 895         if (len == 0) {
 896                 int i = 0;
 897
 898                 do {
 899                         udelay(2);
 900                         barrier();
 901                         len = le16_to_cpu(rxhdr->frame_length);
 902                 } while (len == 0 && i++ < 5);
 903                 if (unlikely(len == 0)) {
 904                         /* recycle the descriptor buffer. */
 905                         sync_descbuffer_for_device(ring, meta->dmaaddr,
 906                                                    ring->rx_buffersize);
 907                         goto drop;
 908                 }
 909         }
 910         if (unlikely(len > ring->rx_buffersize)) {
 911                 /* The data did not fit into one descriptor buffer
 912                  * and is split over multiple buffers.
 913                  * This should never happen, as we try to allocate buffers
 914                  * big enough. So simply ignore this packet.
 915                  */
 916                 int cnt = 0;
 917                 s32 tmp = len;
 918
 919                 while (1) {
 920                         desc = ring->vbase + *slot;
 921                         meta = ring->meta + *slot;
 922                         /* recycle the descriptor buffer. */
 923                         sync_descbuffer_for_device(ring, meta->dmaaddr,
 924                                                    ring->rx_buffersize);
 925                         *slot = next_slot(ring, *slot);
 926                         cnt++;
 927                         tmp -= ring->rx_buffersize;
 928                         if (tmp <= 0)
 929                                 break;
 930                 }
 931                 printkl(KERN_ERR PFX "DMA RX buffer too small "
 932                                      "(len: %u, buffer: %u, nr-dropped: %d)\n",
 933                         len, ring->rx_buffersize, cnt);
 934                 goto drop;
 935         }
 936         len -= IEEE80211_FCS_LEN;
 937
 938         dmaaddr = meta->dmaaddr;
 939         err = setup_rx_descbuffer(ring, desc, meta, GFP_ATOMIC);
 940         if (unlikely(err)) {
 941                 dprintkl(KERN_ERR PFX "DMA RX: setup_rx_descbuffer() failed\n");
 942                 sync_descbuffer_for_device(ring, dmaaddr,
 943                                            ring->rx_buffersize);
 944                 goto drop;
 945         }
 946
 947         unmap_descbuffer(ring, dmaaddr, ring->rx_buffersize, 0);
 948         skb_put(skb, len + ring->frameoffset);
 949         skb_pull(skb, ring->frameoffset);
 950
 951         err = bcm43xx_rx(ring->bcm, skb, rxhdr);
 952         if (err) {
 953                 dev_kfree_skb_irq(skb);
 954                 goto drop;
 955         }
 956
 957 drop:
 958         return;
 959 }
 960
 961 void bcm43xx_dma_rx(struct bcm43xx_dmaring *ring)
 962 {
 963         u32 status;
 964         u16 descptr;
 965         int slot, current_slot;
 966 #ifdef CONFIG_BCM43XX_DEBUG
 967         int used_slots = 0;
 968 #endif
 969
 970         assert(!ring->tx);
 971         status = bcm43xx_read32(ring->bcm, ring->mmio_base + BCM43xx_DMA_RX_STATUS);
 972         descptr = (status & BCM43xx_DMA_RXSTAT_DPTR_MASK);
 973         current_slot = descptr / sizeof(struct bcm43xx_dmadesc);
 974         assert(current_slot >= 0 && current_slot < ring->nr_slots);
 975
 976         slot = ring->current_slot;
 977         for ( ; slot != current_slot; slot = next_slot(ring, slot)) {
 978                 dma_rx(ring, &slot);
 979 #ifdef CONFIG_BCM43XX_DEBUG
 980                 if (++used_slots > ring->max_used_slots)
 981                         ring->max_used_slots = used_slots;
 982 #endif
 983         }
 984         bcm43xx_write32(ring->bcm,
 985                         ring->mmio_base + BCM43xx_DMA_RX_DESC_INDEX,
 986                         (u32)(slot * sizeof(struct bcm43xx_dmadesc)));
 987         ring->current_slot = slot;
 988 }
 989
 990 /* vim: set ts=8 sw=8 sts=8: */