git.oblomov.eu Git - linux-2.6/blob - drivers/infiniband/hw/ipath/ipath_sdma.c

   1 /*
   2  * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/spinlock.h>
  34
  35 #include "ipath_kernel.h"
  36 #include "ipath_verbs.h"
  37 #include "ipath_common.h"
  38
  39 #define SDMA_DESCQ_SZ PAGE_SIZE /* 256 entries per 4KB page */
  40
  41 static void vl15_watchdog_enq(struct ipath_devdata *dd)
  42 {
  43         /* ipath_sdma_lock must already be held */
  44         if (atomic_inc_return(&dd->ipath_sdma_vl15_count) == 1) {
  45                 unsigned long interval = (HZ + 19) / 20;
  46                 dd->ipath_sdma_vl15_timer.expires = jiffies + interval;
  47                 add_timer(&dd->ipath_sdma_vl15_timer);
  48         }
  49 }
  50
  51 static void vl15_watchdog_deq(struct ipath_devdata *dd)
  52 {
  53         /* ipath_sdma_lock must already be held */
  54         if (atomic_dec_return(&dd->ipath_sdma_vl15_count) != 0) {
  55                 unsigned long interval = (HZ + 19) / 20;
  56                 mod_timer(&dd->ipath_sdma_vl15_timer, jiffies + interval);
  57         } else {
  58                 del_timer(&dd->ipath_sdma_vl15_timer);
  59         }
  60 }
  61
  62 static void vl15_watchdog_timeout(unsigned long opaque)
  63 {
  64         struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
  65
  66         if (atomic_read(&dd->ipath_sdma_vl15_count) != 0) {
  67                 ipath_dbg("vl15 watchdog timeout - clearing\n");
  68                 ipath_cancel_sends(dd, 1);
  69                 ipath_hol_down(dd);
  70         } else {
  71                 ipath_dbg("vl15 watchdog timeout - "
  72                           "condition already cleared\n");
  73         }
  74 }
  75
  76 static void unmap_desc(struct ipath_devdata *dd, unsigned head)
  77 {
  78         __le64 *descqp = &dd->ipath_sdma_descq[head].qw[0];
  79         u64 desc[2];
  80         dma_addr_t addr;
  81         size_t len;
  82
  83         desc[0] = le64_to_cpu(descqp[0]);
  84         desc[1] = le64_to_cpu(descqp[1]);
  85
  86         addr = (desc[1] << 32) | (desc[0] >> 32);
  87         len = (desc[0] >> 14) & (0x7ffULL << 2);
  88         dma_unmap_single(&dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
  89 }
  90
  91 /*
  92  * ipath_sdma_lock should be locked before calling this.
  93  */
  94 int ipath_sdma_make_progress(struct ipath_devdata *dd)
  95 {
  96         struct list_head *lp = NULL;
  97         struct ipath_sdma_txreq *txp = NULL;
  98         u16 dmahead;
  99         u16 start_idx = 0;
 100         int progress = 0;
 101
 102         if (!list_empty(&dd->ipath_sdma_activelist)) {
 103                 lp = dd->ipath_sdma_activelist.next;
 104                 txp = list_entry(lp, struct ipath_sdma_txreq, list);
 105                 start_idx = txp->start_idx;
 106         }
 107
 108         /*
 109          * Read the SDMA head register in order to know that the
 110          * interrupt clear has been written to the chip.
 111          * Otherwise, we may not get an interrupt for the last
 112          * descriptor in the queue.
 113          */
 114         dmahead = (u16)ipath_read_kreg32(dd, dd->ipath_kregs->kr_senddmahead);
 115         /* sanity check return value for error handling (chip reset, etc.) */
 116         if (dmahead >= dd->ipath_sdma_descq_cnt)
 117                 goto done;
 118
 119         while (dd->ipath_sdma_descq_head != dmahead) {
 120                 if (txp && txp->flags & IPATH_SDMA_TXREQ_F_FREEDESC &&
 121                     dd->ipath_sdma_descq_head == start_idx) {
 122                         unmap_desc(dd, dd->ipath_sdma_descq_head);
 123                         start_idx++;
 124                         if (start_idx == dd->ipath_sdma_descq_cnt)
 125                                 start_idx = 0;
 126                 }
 127
 128                 /* increment free count and head */
 129                 dd->ipath_sdma_descq_removed++;
 130                 if (++dd->ipath_sdma_descq_head == dd->ipath_sdma_descq_cnt)
 131                         dd->ipath_sdma_descq_head = 0;
 132
 133                 if (txp && txp->next_descq_idx == dd->ipath_sdma_descq_head) {
 134                         /* move to notify list */
 135                         if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
 136                                 vl15_watchdog_deq(dd);
 137                         list_move_tail(lp, &dd->ipath_sdma_notifylist);
 138                         if (!list_empty(&dd->ipath_sdma_activelist)) {
 139                                 lp = dd->ipath_sdma_activelist.next;
 140                                 txp = list_entry(lp, struct ipath_sdma_txreq,
 141                                                  list);
 142                                 start_idx = txp->start_idx;
 143                         } else {
 144                                 lp = NULL;
 145                                 txp = NULL;
 146                         }
 147                 }
 148                 progress = 1;
 149         }
 150
 151         if (progress)
 152                 tasklet_hi_schedule(&dd->ipath_sdma_notify_task);
 153
 154 done:
 155         return progress;
 156 }
 157
 158 static void ipath_sdma_notify(struct ipath_devdata *dd, struct list_head *list)
 159 {
 160         struct ipath_sdma_txreq *txp, *txp_next;
 161
 162         list_for_each_entry_safe(txp, txp_next, list, list) {
 163                 list_del_init(&txp->list);
 164
 165                 if (txp->callback)
 166                         (*txp->callback)(txp->callback_cookie,
 167                                          txp->callback_status);
 168         }
 169 }
 170
 171 static void sdma_notify_taskbody(struct ipath_devdata *dd)
 172 {
 173         unsigned long flags;
 174         struct list_head list;
 175
 176         INIT_LIST_HEAD(&list);
 177
 178         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 179
 180         list_splice_init(&dd->ipath_sdma_notifylist, &list);
 181
 182         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 183
 184         ipath_sdma_notify(dd, &list);
 185
 186         /*
 187          * The IB verbs layer needs to see the callback before getting
 188          * the call to ipath_ib_piobufavail() because the callback
 189          * handles releasing resources the next send will need.
 190          * Otherwise, we could do these calls in
 191          * ipath_sdma_make_progress().
 192          */
 193         ipath_ib_piobufavail(dd->verbs_dev);
 194 }
 195
 196 static void sdma_notify_task(unsigned long opaque)
 197 {
 198         struct ipath_devdata *dd = (struct ipath_devdata *)opaque;
 199
 200         if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 201                 sdma_notify_taskbody(dd);
 202 }
 203
 204 static void dump_sdma_state(struct ipath_devdata *dd)
 205 {
 206         unsigned long reg;
 207
 208         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmastatus);
 209         ipath_cdbg(VERBOSE, "kr_senddmastatus: 0x%016lx\n", reg);
 210
 211         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendctrl);
 212         ipath_cdbg(VERBOSE, "kr_sendctrl: 0x%016lx\n", reg);
 213
 214         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask0);
 215         ipath_cdbg(VERBOSE, "kr_senddmabufmask0: 0x%016lx\n", reg);
 216
 217         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask1);
 218         ipath_cdbg(VERBOSE, "kr_senddmabufmask1: 0x%016lx\n", reg);
 219
 220         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask2);
 221         ipath_cdbg(VERBOSE, "kr_senddmabufmask2: 0x%016lx\n", reg);
 222
 223         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmatail);
 224         ipath_cdbg(VERBOSE, "kr_senddmatail: 0x%016lx\n", reg);
 225
 226         reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmahead);
 227         ipath_cdbg(VERBOSE, "kr_senddmahead: 0x%016lx\n", reg);
 228 }
 229
 230 static void sdma_abort_task(unsigned long opaque)
 231 {
 232         struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
 233         u64 status;
 234         unsigned long flags;
 235
 236         if (test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 237                 return;
 238
 239         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 240
 241         status = dd->ipath_sdma_status & IPATH_SDMA_ABORT_MASK;
 242
 243         /* nothing to do */
 244         if (status == IPATH_SDMA_ABORT_NONE)
 245                 goto unlock;
 246
 247         /* ipath_sdma_abort() is done, waiting for interrupt */
 248         if (status == IPATH_SDMA_ABORT_DISARMED) {
 249                 if (jiffies < dd->ipath_sdma_abort_intr_timeout)
 250                         goto resched_noprint;
 251                 /* give up, intr got lost somewhere */
 252                 ipath_dbg("give up waiting for SDMADISABLED intr\n");
 253                 __set_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
 254                 status = IPATH_SDMA_ABORT_ABORTED;
 255         }
 256
 257         /* everything is stopped, time to clean up and restart */
 258         if (status == IPATH_SDMA_ABORT_ABORTED) {
 259                 struct ipath_sdma_txreq *txp, *txpnext;
 260                 u64 hwstatus;
 261                 int notify = 0;
 262
 263                 hwstatus = ipath_read_kreg64(dd,
 264                                 dd->ipath_kregs->kr_senddmastatus);
 265
 266                 if (/* ScoreBoardDrainInProg */
 267                     test_bit(63, &hwstatus) ||
 268                     /* AbortInProg */
 269                     test_bit(62, &hwstatus) ||
 270                     /* InternalSDmaEnable */
 271                     test_bit(61, &hwstatus) ||
 272                     /* ScbEmpty */
 273                     !test_bit(30, &hwstatus)) {
 274                         if (dd->ipath_sdma_reset_wait > 0) {
 275                                 /* not done shutting down sdma */
 276                                 --dd->ipath_sdma_reset_wait;
 277                                 goto resched;
 278                         }
 279                         ipath_cdbg(VERBOSE, "gave up waiting for quiescent "
 280                                 "status after SDMA reset, continuing\n");
 281                         dump_sdma_state(dd);
 282                 }
 283
 284                 /* dequeue all "sent" requests */
 285                 list_for_each_entry_safe(txp, txpnext,
 286                                          &dd->ipath_sdma_activelist, list) {
 287                         txp->callback_status = IPATH_SDMA_TXREQ_S_ABORTED;
 288                         if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
 289                                 vl15_watchdog_deq(dd);
 290                         list_move_tail(&txp->list, &dd->ipath_sdma_notifylist);
 291                         notify = 1;
 292                 }
 293                 if (notify)
 294                         tasklet_hi_schedule(&dd->ipath_sdma_notify_task);
 295
 296                 /* reset our notion of head and tail */
 297                 dd->ipath_sdma_descq_tail = 0;
 298                 dd->ipath_sdma_descq_head = 0;
 299                 dd->ipath_sdma_head_dma[0] = 0;
 300                 dd->ipath_sdma_generation = 0;
 301                 dd->ipath_sdma_descq_removed = dd->ipath_sdma_descq_added;
 302
 303                 /* Reset SendDmaLenGen */
 304                 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen,
 305                         (u64) dd->ipath_sdma_descq_cnt | (1ULL << 18));
 306
 307                 /* done with sdma state for a bit */
 308                 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 309
 310                 /*
 311                  * Don't restart sdma here (with the exception
 312                  * below). Wait until link is up to ACTIVE.  VL15 MADs
 313                  * used to bring the link up use PIO, and multiple link
 314                  * transitions otherwise cause the sdma engine to be
 315                  * stopped and started multiple times.
 316                  * The disable is done here, including the shadow,
 317                  * so the state is kept consistent.
 318                  * See ipath_restart_sdma() for the actual starting
 319                  * of sdma.
 320                  */
 321                 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 322                 dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
 323                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
 324                                  dd->ipath_sendctrl);
 325                 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 326                 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 327
 328                 /* make sure I see next message */
 329                 dd->ipath_sdma_abort_jiffies = 0;
 330
 331                 /*
 332                  * Not everything that takes SDMA offline is a link
 333                  * status change.  If the link was up, restart SDMA.
 334                  */
 335                 if (dd->ipath_flags & IPATH_LINKACTIVE)
 336                         ipath_restart_sdma(dd);
 337
 338                 goto done;
 339         }
 340
 341 resched:
 342         /*
 343          * for now, keep spinning
 344          * JAG - this is bad to just have default be a loop without
 345          * state change
 346          */
 347         if (jiffies > dd->ipath_sdma_abort_jiffies) {
 348                 ipath_dbg("looping with status 0x%016llx\n",
 349                           dd->ipath_sdma_status);
 350                 dd->ipath_sdma_abort_jiffies = jiffies + 5 * HZ;
 351         }
 352 resched_noprint:
 353         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 354         if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 355                 tasklet_hi_schedule(&dd->ipath_sdma_abort_task);
 356         return;
 357
 358 unlock:
 359         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 360 done:
 361         return;
 362 }
 363
 364 /*
 365  * This is called from interrupt context.
 366  */
 367 void ipath_sdma_intr(struct ipath_devdata *dd)
 368 {
 369         unsigned long flags;
 370
 371         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 372
 373         (void) ipath_sdma_make_progress(dd);
 374
 375         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 376 }
 377
 378 static int alloc_sdma(struct ipath_devdata *dd)
 379 {
 380         int ret = 0;
 381
 382         /* Allocate memory for SendDMA descriptor FIFO */
 383         dd->ipath_sdma_descq = dma_alloc_coherent(&dd->pcidev->dev,
 384                 SDMA_DESCQ_SZ, &dd->ipath_sdma_descq_phys, GFP_KERNEL);
 385
 386         if (!dd->ipath_sdma_descq) {
 387                 ipath_dev_err(dd, "failed to allocate SendDMA descriptor "
 388                         "FIFO memory\n");
 389                 ret = -ENOMEM;
 390                 goto done;
 391         }
 392
 393         dd->ipath_sdma_descq_cnt =
 394                 SDMA_DESCQ_SZ / sizeof(struct ipath_sdma_desc);
 395
 396         /* Allocate memory for DMA of head register to memory */
 397         dd->ipath_sdma_head_dma = dma_alloc_coherent(&dd->pcidev->dev,
 398                 PAGE_SIZE, &dd->ipath_sdma_head_phys, GFP_KERNEL);
 399         if (!dd->ipath_sdma_head_dma) {
 400                 ipath_dev_err(dd, "failed to allocate SendDMA head memory\n");
 401                 ret = -ENOMEM;
 402                 goto cleanup_descq;
 403         }
 404         dd->ipath_sdma_head_dma[0] = 0;
 405
 406         init_timer(&dd->ipath_sdma_vl15_timer);
 407         dd->ipath_sdma_vl15_timer.function = vl15_watchdog_timeout;
 408         dd->ipath_sdma_vl15_timer.data = (unsigned long)dd;
 409         atomic_set(&dd->ipath_sdma_vl15_count, 0);
 410
 411         goto done;
 412
 413 cleanup_descq:
 414         dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ,
 415                 (void *)dd->ipath_sdma_descq, dd->ipath_sdma_descq_phys);
 416         dd->ipath_sdma_descq = NULL;
 417         dd->ipath_sdma_descq_phys = 0;
 418 done:
 419         return ret;
 420 }
 421
 422 int setup_sdma(struct ipath_devdata *dd)
 423 {
 424         int ret = 0;
 425         unsigned i, n;
 426         u64 tmp64;
 427         u64 senddmabufmask[3] = { 0 };
 428         unsigned long flags;
 429
 430         ret = alloc_sdma(dd);
 431         if (ret)
 432                 goto done;
 433
 434         if (!dd->ipath_sdma_descq) {
 435                 ipath_dev_err(dd, "SendDMA memory not allocated\n");
 436                 goto done;
 437         }
 438
 439         /*
 440          * Set initial status as if we had been up, then gone down.
 441          * This lets initial start on transition to ACTIVE be the
 442          * same as restart after link flap.
 443          */
 444         dd->ipath_sdma_status = IPATH_SDMA_ABORT_ABORTED;
 445         dd->ipath_sdma_abort_jiffies = 0;
 446         dd->ipath_sdma_generation = 0;
 447         dd->ipath_sdma_descq_tail = 0;
 448         dd->ipath_sdma_descq_head = 0;
 449         dd->ipath_sdma_descq_removed = 0;
 450         dd->ipath_sdma_descq_added = 0;
 451
 452         /* Set SendDmaBase */
 453         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase,
 454                          dd->ipath_sdma_descq_phys);
 455         /* Set SendDmaLenGen */
 456         tmp64 = dd->ipath_sdma_descq_cnt;
 457         tmp64 |= 1<<18; /* enable generation checking */
 458         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, tmp64);
 459         /* Set SendDmaTail */
 460         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail,
 461                          dd->ipath_sdma_descq_tail);
 462         /* Set SendDmaHeadAddr */
 463         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr,
 464                          dd->ipath_sdma_head_phys);
 465
 466         /*
 467          * Reserve all the former "kernel" piobufs, using high number range
 468          * so we get as many 4K buffers as possible
 469          */
 470         n = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k;
 471         i = dd->ipath_lastport_piobuf + dd->ipath_pioreserved;
 472         ipath_chg_pioavailkernel(dd, i, n - i , 0);
 473         for (; i < n; ++i) {
 474                 unsigned word = i / 64;
 475                 unsigned bit = i & 63;
 476                 BUG_ON(word >= 3);
 477                 senddmabufmask[word] |= 1ULL << bit;
 478         }
 479         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0,
 480                          senddmabufmask[0]);
 481         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1,
 482                          senddmabufmask[1]);
 483         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2,
 484                          senddmabufmask[2]);
 485
 486         INIT_LIST_HEAD(&dd->ipath_sdma_activelist);
 487         INIT_LIST_HEAD(&dd->ipath_sdma_notifylist);
 488
 489         tasklet_init(&dd->ipath_sdma_notify_task, sdma_notify_task,
 490                      (unsigned long) dd);
 491         tasklet_init(&dd->ipath_sdma_abort_task, sdma_abort_task,
 492                      (unsigned long) dd);
 493
 494         /*
 495          * No use to turn on SDMA here, as link is probably not ACTIVE
 496          * Just mark it RUNNING and enable the interrupt, and let the
 497          * ipath_restart_sdma() on link transition to ACTIVE actually
 498          * enable it.
 499          */
 500         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 501         dd->ipath_sendctrl |= INFINIPATH_S_SDMAINTENABLE;
 502         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
 503         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 504         __set_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status);
 505         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 506
 507 done:
 508         return ret;
 509 }
 510
 511 void teardown_sdma(struct ipath_devdata *dd)
 512 {
 513         struct ipath_sdma_txreq *txp, *txpnext;
 514         unsigned long flags;
 515         dma_addr_t sdma_head_phys = 0;
 516         dma_addr_t sdma_descq_phys = 0;
 517         void *sdma_descq = NULL;
 518         void *sdma_head_dma = NULL;
 519
 520         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 521         __clear_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status);
 522         __set_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
 523         __set_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status);
 524         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 525
 526         tasklet_kill(&dd->ipath_sdma_abort_task);
 527         tasklet_kill(&dd->ipath_sdma_notify_task);
 528
 529         /* turn off sdma */
 530         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 531         dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
 532         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
 533                 dd->ipath_sendctrl);
 534         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 535         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 536
 537         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 538         /* dequeue all "sent" requests */
 539         list_for_each_entry_safe(txp, txpnext, &dd->ipath_sdma_activelist,
 540                                  list) {
 541                 txp->callback_status = IPATH_SDMA_TXREQ_S_SHUTDOWN;
 542                 if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
 543                         vl15_watchdog_deq(dd);
 544                 list_move_tail(&txp->list, &dd->ipath_sdma_notifylist);
 545         }
 546         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 547
 548         sdma_notify_taskbody(dd);
 549
 550         del_timer_sync(&dd->ipath_sdma_vl15_timer);
 551
 552         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 553
 554         dd->ipath_sdma_abort_jiffies = 0;
 555
 556         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase, 0);
 557         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, 0);
 558         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, 0);
 559         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr, 0);
 560         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0, 0);
 561         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1, 0);
 562         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2, 0);
 563
 564         if (dd->ipath_sdma_head_dma) {
 565                 sdma_head_dma = (void *) dd->ipath_sdma_head_dma;
 566                 sdma_head_phys = dd->ipath_sdma_head_phys;
 567                 dd->ipath_sdma_head_dma = NULL;
 568                 dd->ipath_sdma_head_phys = 0;
 569         }
 570
 571         if (dd->ipath_sdma_descq) {
 572                 sdma_descq = dd->ipath_sdma_descq;
 573                 sdma_descq_phys = dd->ipath_sdma_descq_phys;
 574                 dd->ipath_sdma_descq = NULL;
 575                 dd->ipath_sdma_descq_phys = 0;
 576         }
 577
 578         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 579
 580         if (sdma_head_dma)
 581                 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
 582                                   sdma_head_dma, sdma_head_phys);
 583
 584         if (sdma_descq)
 585                 dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ,
 586                                   sdma_descq, sdma_descq_phys);
 587 }
 588
 589 /*
 590  * [Re]start SDMA, if we use it, and it's not already OK.
 591  * This is called on transition to link ACTIVE, either the first or
 592  * subsequent times.
 593  */
 594 void ipath_restart_sdma(struct ipath_devdata *dd)
 595 {
 596         unsigned long flags;
 597         int needed = 1;
 598
 599         if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA))
 600                 goto bail;
 601
 602         /*
 603          * First, make sure we should, which is to say,
 604          * check that we are "RUNNING" (not in teardown)
 605          * and not "SHUTDOWN"
 606          */
 607         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 608         if (!test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status)
 609                 || test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
 610                         needed = 0;
 611         else {
 612                 __clear_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
 613                 __clear_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
 614                 __clear_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
 615         }
 616         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 617         if (!needed) {
 618                 ipath_dbg("invalid attempt to restart SDMA, status 0x%016llx\n",
 619                         dd->ipath_sdma_status);
 620                 goto bail;
 621         }
 622         spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
 623         /*
 624          * First clear, just to be safe. Enable is only done
 625          * in chip on 0->1 transition
 626          */
 627         dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
 628         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
 629         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 630         dd->ipath_sendctrl |= INFINIPATH_S_SDMAENABLE;
 631         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
 632         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
 633         spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
 634
 635         /* notify upper layers */
 636         ipath_ib_piobufavail(dd->verbs_dev);
 637
 638 bail:
 639         return;
 640 }
 641
 642 static inline void make_sdma_desc(struct ipath_devdata *dd,
 643         u64 *sdmadesc, u64 addr, u64 dwlen, u64 dwoffset)
 644 {
 645         WARN_ON(addr & 3);
 646         /* SDmaPhyAddr[47:32] */
 647         sdmadesc[1] = addr >> 32;
 648         /* SDmaPhyAddr[31:0] */
 649         sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
 650         /* SDmaGeneration[1:0] */
 651         sdmadesc[0] |= (dd->ipath_sdma_generation & 3ULL) << 30;
 652         /* SDmaDwordCount[10:0] */
 653         sdmadesc[0] |= (dwlen & 0x7ffULL) << 16;
 654         /* SDmaBufOffset[12:2] */
 655         sdmadesc[0] |= dwoffset & 0x7ffULL;
 656 }
 657
 658 /*
 659  * This function queues one IB packet onto the send DMA queue per call.
 660  * The caller is responsible for checking:
 661  * 1) The number of send DMA descriptor entries is less than the size of
 662  *    the descriptor queue.
 663  * 2) The IB SGE addresses and lengths are 32-bit aligned
 664  *    (except possibly the last SGE's length)
 665  * 3) The SGE addresses are suitable for passing to dma_map_single().
 666  */
 667 int ipath_sdma_verbs_send(struct ipath_devdata *dd,
 668         struct ipath_sge_state *ss, u32 dwords,
 669         struct ipath_verbs_txreq *tx)
 670 {
 671
 672         unsigned long flags;
 673         struct ipath_sge *sge;
 674         int ret = 0;
 675         u16 tail;
 676         __le64 *descqp;
 677         u64 sdmadesc[2];
 678         u32 dwoffset;
 679         dma_addr_t addr;
 680
 681         if ((tx->map_len + (dwords<<2)) > dd->ipath_ibmaxlen) {
 682                 ipath_dbg("packet size %X > ibmax %X, fail\n",
 683                         tx->map_len + (dwords<<2), dd->ipath_ibmaxlen);
 684                 ret = -EMSGSIZE;
 685                 goto fail;
 686         }
 687
 688         spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
 689
 690 retry:
 691         if (unlikely(test_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status))) {
 692                 ret = -EBUSY;
 693                 goto unlock;
 694         }
 695
 696         if (tx->txreq.sg_count > ipath_sdma_descq_freecnt(dd)) {
 697                 if (ipath_sdma_make_progress(dd))
 698                         goto retry;
 699                 ret = -ENOBUFS;
 700                 goto unlock;
 701         }
 702
 703         addr = dma_map_single(&dd->pcidev->dev, tx->txreq.map_addr,
 704                               tx->map_len, DMA_TO_DEVICE);
 705         if (dma_mapping_error(addr)) {
 706                 ret = -EIO;
 707                 goto unlock;
 708         }
 709
 710         dwoffset = tx->map_len >> 2;
 711         make_sdma_desc(dd, sdmadesc, (u64) addr, dwoffset, 0);
 712
 713         /* SDmaFirstDesc */
 714         sdmadesc[0] |= 1ULL << 12;
 715         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF)
 716                 sdmadesc[0] |= 1ULL << 14;      /* SDmaUseLargeBuf */
 717
 718         /* write to the descq */
 719         tail = dd->ipath_sdma_descq_tail;
 720         descqp = &dd->ipath_sdma_descq[tail].qw[0];
 721         *descqp++ = cpu_to_le64(sdmadesc[0]);
 722         *descqp++ = cpu_to_le64(sdmadesc[1]);
 723
 724         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEDESC)
 725                 tx->txreq.start_idx = tail;
 726
 727         /* increment the tail */
 728         if (++tail == dd->ipath_sdma_descq_cnt) {
 729                 tail = 0;
 730                 descqp = &dd->ipath_sdma_descq[0].qw[0];
 731                 ++dd->ipath_sdma_generation;
 732         }
 733
 734         sge = &ss->sge;
 735         while (dwords) {
 736                 u32 dw;
 737                 u32 len;
 738
 739                 len = dwords << 2;
 740                 if (len > sge->length)
 741                         len = sge->length;
 742                 if (len > sge->sge_length)
 743                         len = sge->sge_length;
 744                 BUG_ON(len == 0);
 745                 dw = (len + 3) >> 2;
 746                 addr = dma_map_single(&dd->pcidev->dev, sge->vaddr, dw << 2,
 747                                       DMA_TO_DEVICE);
 748                 make_sdma_desc(dd, sdmadesc, (u64) addr, dw, dwoffset);
 749                 /* SDmaUseLargeBuf has to be set in every descriptor */
 750                 if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF)
 751                         sdmadesc[0] |= 1ULL << 14;
 752                 /* write to the descq */
 753                 *descqp++ = cpu_to_le64(sdmadesc[0]);
 754                 *descqp++ = cpu_to_le64(sdmadesc[1]);
 755
 756                 /* increment the tail */
 757                 if (++tail == dd->ipath_sdma_descq_cnt) {
 758                         tail = 0;
 759                         descqp = &dd->ipath_sdma_descq[0].qw[0];
 760                         ++dd->ipath_sdma_generation;
 761                 }
 762                 sge->vaddr += len;
 763                 sge->length -= len;
 764                 sge->sge_length -= len;
 765                 if (sge->sge_length == 0) {
 766                         if (--ss->num_sge)
 767                                 *sge = *ss->sg_list++;
 768                 } else if (sge->length == 0 && sge->mr != NULL) {
 769                         if (++sge->n >= IPATH_SEGSZ) {
 770                                 if (++sge->m >= sge->mr->mapsz)
 771                                         break;
 772                                 sge->n = 0;
 773                         }
 774                         sge->vaddr =
 775                                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
 776                         sge->length =
 777                                 sge->mr->map[sge->m]->segs[sge->n].length;
 778                 }
 779
 780                 dwoffset += dw;
 781                 dwords -= dw;
 782         }
 783
 784         if (!tail)
 785                 descqp = &dd->ipath_sdma_descq[dd->ipath_sdma_descq_cnt].qw[0];
 786         descqp -= 2;
 787         /* SDmaLastDesc */
 788         descqp[0] |= __constant_cpu_to_le64(1ULL << 11);
 789         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_INTREQ) {
 790                 /* SDmaIntReq */
 791                 descqp[0] |= __constant_cpu_to_le64(1ULL << 15);
 792         }
 793
 794         /* Commit writes to memory and advance the tail on the chip */
 795         wmb();
 796         ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, tail);
 797
 798         tx->txreq.next_descq_idx = tail;
 799         tx->txreq.callback_status = IPATH_SDMA_TXREQ_S_OK;
 800         dd->ipath_sdma_descq_tail = tail;
 801         dd->ipath_sdma_descq_added += tx->txreq.sg_count;
 802         list_add_tail(&tx->txreq.list, &dd->ipath_sdma_activelist);
 803         if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_VL15)
 804                 vl15_watchdog_enq(dd);
 805
 806 unlock:
 807         spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
 808 fail:
 809         return ret;
 810 }