git.oblomov.eu Git - linux-2.6/blob - drivers/spi/spi_imx.c

   1 /*
   2  * drivers/spi/spi_imx.c
   3  *
   4  * Copyright (C) 2006 SWAPP
   5  *      Andrea Paterniani <a.paterniani@swapp-eng.it>
   6  *
   7  * Initial version inspired by:
   8  *      linux-2.6.17-rc3-mm1/drivers/spi/pxa2xx_spi.c
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  */
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/device.h>
  24 #include <linux/ioport.h>
  25 #include <linux/errno.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/dma-mapping.h>
  29 #include <linux/spi/spi.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/delay.h>
  32 #include <linux/clk.h>
  33
  34 #include <asm/io.h>
  35 #include <asm/irq.h>
  36 #include <asm/delay.h>
  37
  38 #include <mach/hardware.h>
  39 #include <mach/imx-dma.h>
  40 #include <mach/spi_imx.h>
  41
  42 /*-------------------------------------------------------------------------*/
  43 /* SPI Registers offsets from peripheral base address */
  44 #define SPI_RXDATA              (0x00)
  45 #define SPI_TXDATA              (0x04)
  46 #define SPI_CONTROL             (0x08)
  47 #define SPI_INT_STATUS          (0x0C)
  48 #define SPI_TEST                (0x10)
  49 #define SPI_PERIOD              (0x14)
  50 #define SPI_DMA                 (0x18)
  51 #define SPI_RESET               (0x1C)
  52
  53 /* SPI Control Register Bit Fields & Masks */
  54 #define SPI_CONTROL_BITCOUNT_MASK       (0xF)           /* Bit Count Mask */
  55 #define SPI_CONTROL_BITCOUNT(n)         (((n) - 1) & SPI_CONTROL_BITCOUNT_MASK)
  56 #define SPI_CONTROL_POL                 (0x1 << 4)      /* Clock Polarity Mask */
  57 #define SPI_CONTROL_POL_ACT_HIGH        (0x0 << 4)      /* Active high pol. (0=idle) */
  58 #define SPI_CONTROL_POL_ACT_LOW         (0x1 << 4)      /* Active low pol. (1=idle) */
  59 #define SPI_CONTROL_PHA                 (0x1 << 5)      /* Clock Phase Mask */
  60 #define SPI_CONTROL_PHA_0               (0x0 << 5)      /* Clock Phase 0 */
  61 #define SPI_CONTROL_PHA_1               (0x1 << 5)      /* Clock Phase 1 */
  62 #define SPI_CONTROL_SSCTL               (0x1 << 6)      /* /SS Waveform Select Mask */
  63 #define SPI_CONTROL_SSCTL_0             (0x0 << 6)      /* Master: /SS stays low between SPI burst
  64                                                            Slave: RXFIFO advanced by BIT_COUNT */
  65 #define SPI_CONTROL_SSCTL_1             (0x1 << 6)      /* Master: /SS insert pulse between SPI burst
  66                                                            Slave: RXFIFO advanced by /SS rising edge */
  67 #define SPI_CONTROL_SSPOL               (0x1 << 7)      /* /SS Polarity Select Mask */
  68 #define SPI_CONTROL_SSPOL_ACT_LOW       (0x0 << 7)      /* /SS Active low */
  69 #define SPI_CONTROL_SSPOL_ACT_HIGH      (0x1 << 7)      /* /SS Active high */
  70 #define SPI_CONTROL_XCH                 (0x1 << 8)      /* Exchange */
  71 #define SPI_CONTROL_SPIEN               (0x1 << 9)      /* SPI Module Enable */
  72 #define SPI_CONTROL_MODE                (0x1 << 10)     /* SPI Mode Select Mask */
  73 #define SPI_CONTROL_MODE_SLAVE          (0x0 << 10)     /* SPI Mode Slave */
  74 #define SPI_CONTROL_MODE_MASTER         (0x1 << 10)     /* SPI Mode Master */
  75 #define SPI_CONTROL_DRCTL               (0x3 << 11)     /* /SPI_RDY Control Mask */
  76 #define SPI_CONTROL_DRCTL_0             (0x0 << 11)     /* Ignore /SPI_RDY */
  77 #define SPI_CONTROL_DRCTL_1             (0x1 << 11)     /* /SPI_RDY falling edge triggers input */
  78 #define SPI_CONTROL_DRCTL_2             (0x2 << 11)     /* /SPI_RDY active low level triggers input */
  79 #define SPI_CONTROL_DATARATE            (0x7 << 13)     /* Data Rate Mask */
  80 #define SPI_PERCLK2_DIV_MIN             (0)             /* PERCLK2:4 */
  81 #define SPI_PERCLK2_DIV_MAX             (7)             /* PERCLK2:512 */
  82 #define SPI_CONTROL_DATARATE_MIN        (SPI_PERCLK2_DIV_MAX << 13)
  83 #define SPI_CONTROL_DATARATE_MAX        (SPI_PERCLK2_DIV_MIN << 13)
  84 #define SPI_CONTROL_DATARATE_BAD        (SPI_CONTROL_DATARATE_MIN + 1)
  85
  86 /* SPI Interrupt/Status Register Bit Fields & Masks */
  87 #define SPI_STATUS_TE   (0x1 << 0)      /* TXFIFO Empty Status */
  88 #define SPI_STATUS_TH   (0x1 << 1)      /* TXFIFO Half Status */
  89 #define SPI_STATUS_TF   (0x1 << 2)      /* TXFIFO Full Status */
  90 #define SPI_STATUS_RR   (0x1 << 3)      /* RXFIFO Data Ready Status */
  91 #define SPI_STATUS_RH   (0x1 << 4)      /* RXFIFO Half Status */
  92 #define SPI_STATUS_RF   (0x1 << 5)      /* RXFIFO Full Status */
  93 #define SPI_STATUS_RO   (0x1 << 6)      /* RXFIFO Overflow */
  94 #define SPI_STATUS_BO   (0x1 << 7)      /* Bit Count Overflow */
  95 #define SPI_STATUS      (0xFF)          /* SPI Status Mask */
  96 #define SPI_INTEN_TE    (0x1 << 8)      /* TXFIFO Empty Interrupt Enable */
  97 #define SPI_INTEN_TH    (0x1 << 9)      /* TXFIFO Half Interrupt Enable */
  98 #define SPI_INTEN_TF    (0x1 << 10)     /* TXFIFO Full Interrupt Enable */
  99 #define SPI_INTEN_RE    (0x1 << 11)     /* RXFIFO Data Ready Interrupt Enable */
 100 #define SPI_INTEN_RH    (0x1 << 12)     /* RXFIFO Half Interrupt Enable */
 101 #define SPI_INTEN_RF    (0x1 << 13)     /* RXFIFO Full Interrupt Enable */
 102 #define SPI_INTEN_RO    (0x1 << 14)     /* RXFIFO Overflow Interrupt Enable */
 103 #define SPI_INTEN_BO    (0x1 << 15)     /* Bit Count Overflow Interrupt Enable */
 104 #define SPI_INTEN       (0xFF << 8)     /* SPI Interrupt Enable Mask */
 105
 106 /* SPI Test Register Bit Fields & Masks */
 107 #define SPI_TEST_TXCNT          (0xF << 0)      /* TXFIFO Counter */
 108 #define SPI_TEST_RXCNT_LSB      (4)             /* RXFIFO Counter LSB */
 109 #define SPI_TEST_RXCNT          (0xF << 4)      /* RXFIFO Counter */
 110 #define SPI_TEST_SSTATUS        (0xF << 8)      /* State Machine Status */
 111 #define SPI_TEST_LBC            (0x1 << 14)     /* Loop Back Control */
 112
 113 /* SPI Period Register Bit Fields & Masks */
 114 #define SPI_PERIOD_WAIT         (0x7FFF << 0)   /* Wait Between Transactions */
 115 #define SPI_PERIOD_MAX_WAIT     (0x7FFF)        /* Max Wait Between
 116                                                         Transactions */
 117 #define SPI_PERIOD_CSRC         (0x1 << 15)     /* Period Clock Source Mask */
 118 #define SPI_PERIOD_CSRC_BCLK    (0x0 << 15)     /* Period Clock Source is
 119                                                         Bit Clock */
 120 #define SPI_PERIOD_CSRC_32768   (0x1 << 15)     /* Period Clock Source is
 121                                                         32.768 KHz Clock */
 122
 123 /* SPI DMA Register Bit Fields & Masks */
 124 #define SPI_DMA_RHDMA   (0x1 << 4)      /* RXFIFO Half Status */
 125 #define SPI_DMA_RFDMA   (0x1 << 5)      /* RXFIFO Full Status */
 126 #define SPI_DMA_TEDMA   (0x1 << 6)      /* TXFIFO Empty Status */
 127 #define SPI_DMA_THDMA   (0x1 << 7)      /* TXFIFO Half Status */
 128 #define SPI_DMA_RHDEN   (0x1 << 12)     /* RXFIFO Half DMA Request Enable */
 129 #define SPI_DMA_RFDEN   (0x1 << 13)     /* RXFIFO Full DMA Request Enable */
 130 #define SPI_DMA_TEDEN   (0x1 << 14)     /* TXFIFO Empty DMA Request Enable */
 131 #define SPI_DMA_THDEN   (0x1 << 15)     /* TXFIFO Half DMA Request Enable */
 132
 133 /* SPI Soft Reset Register Bit Fields & Masks */
 134 #define SPI_RESET_START (0x1)           /* Start */
 135
 136 /* Default SPI configuration values */
 137 #define SPI_DEFAULT_CONTROL             \
 138 (                                       \
 139         SPI_CONTROL_BITCOUNT(16) |      \
 140         SPI_CONTROL_POL_ACT_HIGH |      \
 141         SPI_CONTROL_PHA_0 |             \
 142         SPI_CONTROL_SPIEN |             \
 143         SPI_CONTROL_SSCTL_1 |           \
 144         SPI_CONTROL_MODE_MASTER |       \
 145         SPI_CONTROL_DRCTL_0 |           \
 146         SPI_CONTROL_DATARATE_MIN        \
 147 )
 148 #define SPI_DEFAULT_ENABLE_LOOPBACK     (0)
 149 #define SPI_DEFAULT_ENABLE_DMA          (0)
 150 #define SPI_DEFAULT_PERIOD_WAIT         (8)
 151 /*-------------------------------------------------------------------------*/
 152
 153
 154 /*-------------------------------------------------------------------------*/
 155 /* TX/RX SPI FIFO size */
 156 #define SPI_FIFO_DEPTH                  (8)
 157 #define SPI_FIFO_BYTE_WIDTH             (2)
 158 #define SPI_FIFO_OVERFLOW_MARGIN        (2)
 159
 160 /* DMA burst length for half full/empty request trigger */
 161 #define SPI_DMA_BLR                     (SPI_FIFO_DEPTH * SPI_FIFO_BYTE_WIDTH / 2)
 162
 163 /* Dummy char output to achieve reads.
 164    Choosing something different from all zeroes may help pattern recogition
 165    for oscilloscope analysis, but may break some drivers. */
 166 #define SPI_DUMMY_u8                    0
 167 #define SPI_DUMMY_u16                   ((SPI_DUMMY_u8 << 8) | SPI_DUMMY_u8)
 168 #define SPI_DUMMY_u32                   ((SPI_DUMMY_u16 << 16) | SPI_DUMMY_u16)
 169
 170 /**
 171  * Macro to change a u32 field:
 172  * @r : register to edit
 173  * @m : bit mask
 174  * @v : new value for the field correctly bit-alligned
 175 */
 176 #define u32_EDIT(r, m, v)               r = (r & ~(m)) | (v)
 177
 178 /* Message state */
 179 #define START_STATE                     ((void*)0)
 180 #define RUNNING_STATE                   ((void*)1)
 181 #define DONE_STATE                      ((void*)2)
 182 #define ERROR_STATE                     ((void*)-1)
 183
 184 /* Queue state */
 185 #define QUEUE_RUNNING                   (0)
 186 #define QUEUE_STOPPED                   (1)
 187
 188 #define IS_DMA_ALIGNED(x)               (((u32)(x) & 0x03) == 0)
 189 /*-------------------------------------------------------------------------*/
 190
 191
 192 /*-------------------------------------------------------------------------*/
 193 /* Driver data structs */
 194
 195 /* Context */
 196 struct driver_data {
 197         /* Driver model hookup */
 198         struct platform_device *pdev;
 199
 200         /* SPI framework hookup */
 201         struct spi_master *master;
 202
 203         /* IMX hookup */
 204         struct spi_imx_master *master_info;
 205
 206         /* Memory resources and SPI regs virtual address */
 207         struct resource *ioarea;
 208         void __iomem *regs;
 209
 210         /* SPI RX_DATA physical address */
 211         dma_addr_t rd_data_phys;
 212
 213         /* Driver message queue */
 214         struct workqueue_struct *workqueue;
 215         struct work_struct work;
 216         spinlock_t lock;
 217         struct list_head queue;
 218         int busy;
 219         int run;
 220
 221         /* Message Transfer pump */
 222         struct tasklet_struct pump_transfers;
 223
 224         /* Current message, transfer and state */
 225         struct spi_message *cur_msg;
 226         struct spi_transfer *cur_transfer;
 227         struct chip_data *cur_chip;
 228
 229         /* Rd / Wr buffers pointers */
 230         size_t len;
 231         void *tx;
 232         void *tx_end;
 233         void *rx;
 234         void *rx_end;
 235
 236         u8 rd_only;
 237         u8 n_bytes;
 238         int cs_change;
 239
 240         /* Function pointers */
 241         irqreturn_t (*transfer_handler)(struct driver_data *drv_data);
 242         void (*cs_control)(u32 command);
 243
 244         /* DMA setup */
 245         int rx_channel;
 246         int tx_channel;
 247         dma_addr_t rx_dma;
 248         dma_addr_t tx_dma;
 249         int rx_dma_needs_unmap;
 250         int tx_dma_needs_unmap;
 251         size_t tx_map_len;
 252         u32 dummy_dma_buf ____cacheline_aligned;
 253
 254         struct clk *clk;
 255 };
 256
 257 /* Runtime state */
 258 struct chip_data {
 259         u32 control;
 260         u32 period;
 261         u32 test;
 262
 263         u8 enable_dma:1;
 264         u8 bits_per_word;
 265         u8 n_bytes;
 266         u32 max_speed_hz;
 267
 268         void (*cs_control)(u32 command);
 269 };
 270 /*-------------------------------------------------------------------------*/
 271
 272
 273 static void pump_messages(struct work_struct *work);
 274
 275 static void flush(struct driver_data *drv_data)
 276 {
 277         void __iomem *regs = drv_data->regs;
 278         u32 control;
 279
 280         dev_dbg(&drv_data->pdev->dev, "flush\n");
 281
 282         /* Wait for end of transaction */
 283         do {
 284                 control = readl(regs + SPI_CONTROL);
 285         } while (control & SPI_CONTROL_XCH);
 286
 287         /* Release chip select if requested, transfer delays are
 288            handled in pump_transfers */
 289         if (drv_data->cs_change)
 290                 drv_data->cs_control(SPI_CS_DEASSERT);
 291
 292         /* Disable SPI to flush FIFOs */
 293         writel(control & ~SPI_CONTROL_SPIEN, regs + SPI_CONTROL);
 294         writel(control, regs + SPI_CONTROL);
 295 }
 296
 297 static void restore_state(struct driver_data *drv_data)
 298 {
 299         void __iomem *regs = drv_data->regs;
 300         struct chip_data *chip = drv_data->cur_chip;
 301
 302         /* Load chip registers */
 303         dev_dbg(&drv_data->pdev->dev,
 304                 "restore_state\n"
 305                 "    test    = 0x%08X\n"
 306                 "    control = 0x%08X\n",
 307                 chip->test,
 308                 chip->control);
 309         writel(chip->test, regs + SPI_TEST);
 310         writel(chip->period, regs + SPI_PERIOD);
 311         writel(0, regs + SPI_INT_STATUS);
 312         writel(chip->control, regs + SPI_CONTROL);
 313 }
 314
 315 static void null_cs_control(u32 command)
 316 {
 317 }
 318
 319 static inline u32 data_to_write(struct driver_data *drv_data)
 320 {
 321         return ((u32)(drv_data->tx_end - drv_data->tx)) / drv_data->n_bytes;
 322 }
 323
 324 static inline u32 data_to_read(struct driver_data *drv_data)
 325 {
 326         return ((u32)(drv_data->rx_end - drv_data->rx)) / drv_data->n_bytes;
 327 }
 328
 329 static int write(struct driver_data *drv_data)
 330 {
 331         void __iomem *regs = drv_data->regs;
 332         void *tx = drv_data->tx;
 333         void *tx_end = drv_data->tx_end;
 334         u8 n_bytes = drv_data->n_bytes;
 335         u32 remaining_writes;
 336         u32 fifo_avail_space;
 337         u32 n;
 338         u16 d;
 339
 340         /* Compute how many fifo writes to do */
 341         remaining_writes = (u32)(tx_end - tx) / n_bytes;
 342         fifo_avail_space = SPI_FIFO_DEPTH -
 343                                 (readl(regs + SPI_TEST) & SPI_TEST_TXCNT);
 344         if (drv_data->rx && (fifo_avail_space > SPI_FIFO_OVERFLOW_MARGIN))
 345                 /* Fix misunderstood receive overflow */
 346                 fifo_avail_space -= SPI_FIFO_OVERFLOW_MARGIN;
 347         n = min(remaining_writes, fifo_avail_space);
 348
 349         dev_dbg(&drv_data->pdev->dev,
 350                 "write type %s\n"
 351                 "    remaining writes = %d\n"
 352                 "    fifo avail space = %d\n"
 353                 "    fifo writes      = %d\n",
 354                 (n_bytes == 1) ? "u8" : "u16",
 355                 remaining_writes,
 356                 fifo_avail_space,
 357                 n);
 358
 359         if (n > 0) {
 360                 /* Fill SPI TXFIFO */
 361                 if (drv_data->rd_only) {
 362                         tx += n * n_bytes;
 363                         while (n--)
 364                                 writel(SPI_DUMMY_u16, regs + SPI_TXDATA);
 365                 } else {
 366                         if (n_bytes == 1) {
 367                                 while (n--) {
 368                                         d = *(u8*)tx;
 369                                         writel(d, regs + SPI_TXDATA);
 370                                         tx += 1;
 371                                 }
 372                         } else {
 373                                 while (n--) {
 374                                         d = *(u16*)tx;
 375                                         writel(d, regs + SPI_TXDATA);
 376                                         tx += 2;
 377                                 }
 378                         }
 379                 }
 380
 381                 /* Trigger transfer */
 382                 writel(readl(regs + SPI_CONTROL) | SPI_CONTROL_XCH,
 383                         regs + SPI_CONTROL);
 384
 385                 /* Update tx pointer */
 386                 drv_data->tx = tx;
 387         }
 388
 389         return (tx >= tx_end);
 390 }
 391
 392 static int read(struct driver_data *drv_data)
 393 {
 394         void __iomem *regs = drv_data->regs;
 395         void *rx = drv_data->rx;
 396         void *rx_end = drv_data->rx_end;
 397         u8 n_bytes = drv_data->n_bytes;
 398         u32 remaining_reads;
 399         u32 fifo_rxcnt;
 400         u32 n;
 401         u16 d;
 402
 403         /* Compute how many fifo reads to do */
 404         remaining_reads = (u32)(rx_end - rx) / n_bytes;
 405         fifo_rxcnt = (readl(regs + SPI_TEST) & SPI_TEST_RXCNT) >>
 406                         SPI_TEST_RXCNT_LSB;
 407         n = min(remaining_reads, fifo_rxcnt);
 408
 409         dev_dbg(&drv_data->pdev->dev,
 410                 "read type %s\n"
 411                 "    remaining reads = %d\n"
 412                 "    fifo rx count   = %d\n"
 413                 "    fifo reads      = %d\n",
 414                 (n_bytes == 1) ? "u8" : "u16",
 415                 remaining_reads,
 416                 fifo_rxcnt,
 417                 n);
 418
 419         if (n > 0) {
 420                 /* Read SPI RXFIFO */
 421                 if (n_bytes == 1) {
 422                         while (n--) {
 423                                 d = readl(regs + SPI_RXDATA);
 424                                 *((u8*)rx) = d;
 425                                 rx += 1;
 426                         }
 427                 } else {
 428                         while (n--) {
 429                                 d = readl(regs + SPI_RXDATA);
 430                                 *((u16*)rx) = d;
 431                                 rx += 2;
 432                         }
 433                 }
 434
 435                 /* Update rx pointer */
 436                 drv_data->rx = rx;
 437         }
 438
 439         return (rx >= rx_end);
 440 }
 441
 442 static void *next_transfer(struct driver_data *drv_data)
 443 {
 444         struct spi_message *msg = drv_data->cur_msg;
 445         struct spi_transfer *trans = drv_data->cur_transfer;
 446
 447         /* Move to next transfer */
 448         if (trans->transfer_list.next != &msg->transfers) {
 449                 drv_data->cur_transfer =
 450                         list_entry(trans->transfer_list.next,
 451                                         struct spi_transfer,
 452                                         transfer_list);
 453                 return RUNNING_STATE;
 454         }
 455
 456         return DONE_STATE;
 457 }
 458
 459 static int map_dma_buffers(struct driver_data *drv_data)
 460 {
 461         struct spi_message *msg;
 462         struct device *dev;
 463         void *buf;
 464
 465         drv_data->rx_dma_needs_unmap = 0;
 466         drv_data->tx_dma_needs_unmap = 0;
 467
 468         if (!drv_data->master_info->enable_dma ||
 469                 !drv_data->cur_chip->enable_dma)
 470                         return -1;
 471
 472         msg = drv_data->cur_msg;
 473         dev = &msg->spi->dev;
 474         if (msg->is_dma_mapped) {
 475                 if (drv_data->tx_dma)
 476                         /* The caller provided at least dma and cpu virtual
 477                            address for write; pump_transfers() will consider the
 478                            transfer as write only if cpu rx virtual address is
 479                            NULL */
 480                         return 0;
 481
 482                 if (drv_data->rx_dma) {
 483                         /* The caller provided dma and cpu virtual address to
 484                            performe read only transfer -->
 485                            use drv_data->dummy_dma_buf for dummy writes to
 486                            achive reads */
 487                         buf = &drv_data->dummy_dma_buf;
 488                         drv_data->tx_map_len = sizeof(drv_data->dummy_dma_buf);
 489                         drv_data->tx_dma = dma_map_single(dev,
 490                                                         buf,
 491                                                         drv_data->tx_map_len,
 492                                                         DMA_TO_DEVICE);
 493                         if (dma_mapping_error(dev, drv_data->tx_dma))
 494                                 return -1;
 495
 496                         drv_data->tx_dma_needs_unmap = 1;
 497
 498                         /* Flags transfer as rd_only for pump_transfers() DMA
 499                            regs programming (should be redundant) */
 500                         drv_data->tx = NULL;
 501
 502                         return 0;
 503                 }
 504         }
 505
 506         if (!IS_DMA_ALIGNED(drv_data->rx) || !IS_DMA_ALIGNED(drv_data->tx))
 507                 return -1;
 508
 509         if (drv_data->tx == NULL) {
 510                 /* Read only message --> use drv_data->dummy_dma_buf for dummy
 511                    writes to achive reads */
 512                 buf = &drv_data->dummy_dma_buf;
 513                 drv_data->tx_map_len = sizeof(drv_data->dummy_dma_buf);
 514         } else {
 515                 buf = drv_data->tx;
 516                 drv_data->tx_map_len = drv_data->len;
 517         }
 518         drv_data->tx_dma = dma_map_single(dev,
 519                                         buf,
 520                                         drv_data->tx_map_len,
 521                                         DMA_TO_DEVICE);
 522         if (dma_mapping_error(dev, drv_data->tx_dma))
 523                 return -1;
 524         drv_data->tx_dma_needs_unmap = 1;
 525
 526         /* NULL rx means write-only transfer and no map needed
 527          * since rx DMA will not be used */
 528         if (drv_data->rx) {
 529                 buf = drv_data->rx;
 530                 drv_data->rx_dma = dma_map_single(dev,
 531                                                 buf,
 532                                                 drv_data->len,
 533                                                 DMA_FROM_DEVICE);
 534                 if (dma_mapping_error(dev, drv_data->rx_dma)) {
 535                         if (drv_data->tx_dma) {
 536                                 dma_unmap_single(dev,
 537                                                 drv_data->tx_dma,
 538                                                 drv_data->tx_map_len,
 539                                                 DMA_TO_DEVICE);
 540                                 drv_data->tx_dma_needs_unmap = 0;
 541                         }
 542                         return -1;
 543                 }
 544                 drv_data->rx_dma_needs_unmap = 1;
 545         }
 546
 547         return 0;
 548 }
 549
 550 static void unmap_dma_buffers(struct driver_data *drv_data)
 551 {
 552         struct spi_message *msg = drv_data->cur_msg;
 553         struct device *dev = &msg->spi->dev;
 554
 555         if (drv_data->rx_dma_needs_unmap) {
 556                 dma_unmap_single(dev,
 557                                 drv_data->rx_dma,
 558                                 drv_data->len,
 559                                 DMA_FROM_DEVICE);
 560                 drv_data->rx_dma_needs_unmap = 0;
 561         }
 562         if (drv_data->tx_dma_needs_unmap) {
 563                 dma_unmap_single(dev,
 564                                 drv_data->tx_dma,
 565                                 drv_data->tx_map_len,
 566                                 DMA_TO_DEVICE);
 567                 drv_data->tx_dma_needs_unmap = 0;
 568         }
 569 }
 570
 571 /* Caller already set message->status (dma is already blocked) */
 572 static void giveback(struct spi_message *message, struct driver_data *drv_data)
 573 {
 574         void __iomem *regs = drv_data->regs;
 575
 576         /* Bring SPI to sleep; restore_state() and pump_transfer()
 577            will do new setup */
 578         writel(0, regs + SPI_INT_STATUS);
 579         writel(0, regs + SPI_DMA);
 580
 581         /* Unconditioned deselct */
 582         drv_data->cs_control(SPI_CS_DEASSERT);
 583
 584         message->state = NULL;
 585         if (message->complete)
 586                 message->complete(message->context);
 587
 588         drv_data->cur_msg = NULL;
 589         drv_data->cur_transfer = NULL;
 590         drv_data->cur_chip = NULL;
 591         queue_work(drv_data->workqueue, &drv_data->work);
 592 }
 593
 594 static void dma_err_handler(int channel, void *data, int errcode)
 595 {
 596         struct driver_data *drv_data = data;
 597         struct spi_message *msg = drv_data->cur_msg;
 598
 599         dev_dbg(&drv_data->pdev->dev, "dma_err_handler\n");
 600
 601         /* Disable both rx and tx dma channels */
 602         imx_dma_disable(drv_data->rx_channel);
 603         imx_dma_disable(drv_data->tx_channel);
 604         unmap_dma_buffers(drv_data);
 605
 606         flush(drv_data);
 607
 608         msg->state = ERROR_STATE;
 609         tasklet_schedule(&drv_data->pump_transfers);
 610 }
 611
 612 static void dma_tx_handler(int channel, void *data)
 613 {
 614         struct driver_data *drv_data = data;
 615
 616         dev_dbg(&drv_data->pdev->dev, "dma_tx_handler\n");
 617
 618         imx_dma_disable(channel);
 619
 620         /* Now waits for TX FIFO empty */
 621         writel(SPI_INTEN_TE, drv_data->regs + SPI_INT_STATUS);
 622 }
 623
 624 static irqreturn_t dma_transfer(struct driver_data *drv_data)
 625 {
 626         u32 status;
 627         struct spi_message *msg = drv_data->cur_msg;
 628         void __iomem *regs = drv_data->regs;
 629
 630         status = readl(regs + SPI_INT_STATUS);
 631
 632         if ((status & (SPI_INTEN_RO | SPI_STATUS_RO))
 633                         == (SPI_INTEN_RO | SPI_STATUS_RO)) {
 634                 writel(status & ~SPI_INTEN, regs + SPI_INT_STATUS);
 635
 636                 imx_dma_disable(drv_data->tx_channel);
 637                 imx_dma_disable(drv_data->rx_channel);
 638                 unmap_dma_buffers(drv_data);
 639
 640                 flush(drv_data);
 641
 642                 dev_warn(&drv_data->pdev->dev,
 643                                 "dma_transfer - fifo overun\n");
 644
 645                 msg->state = ERROR_STATE;
 646                 tasklet_schedule(&drv_data->pump_transfers);
 647
 648                 return IRQ_HANDLED;
 649         }
 650
 651         if (status & SPI_STATUS_TE) {
 652                 writel(status & ~SPI_INTEN_TE, regs + SPI_INT_STATUS);
 653
 654                 if (drv_data->rx) {
 655                         /* Wait end of transfer before read trailing data */
 656                         while (readl(regs + SPI_CONTROL) & SPI_CONTROL_XCH)
 657                                 cpu_relax();
 658
 659                         imx_dma_disable(drv_data->rx_channel);
 660                         unmap_dma_buffers(drv_data);
 661
 662                         /* Release chip select if requested, transfer delays are
 663                            handled in pump_transfers() */
 664                         if (drv_data->cs_change)
 665                                 drv_data->cs_control(SPI_CS_DEASSERT);
 666
 667                         /* Calculate number of trailing data and read them */
 668                         dev_dbg(&drv_data->pdev->dev,
 669                                 "dma_transfer - test = 0x%08X\n",
 670                                 readl(regs + SPI_TEST));
 671                         drv_data->rx = drv_data->rx_end -
 672                                         ((readl(regs + SPI_TEST) &
 673                                         SPI_TEST_RXCNT) >>
 674                                         SPI_TEST_RXCNT_LSB)*drv_data->n_bytes;
 675                         read(drv_data);
 676                 } else {
 677                         /* Write only transfer */
 678                         unmap_dma_buffers(drv_data);
 679
 680                         flush(drv_data);
 681                 }
 682
 683                 /* End of transfer, update total byte transfered */
 684                 msg->actual_length += drv_data->len;
 685
 686                 /* Move to next transfer */
 687                 msg->state = next_transfer(drv_data);
 688
 689                 /* Schedule transfer tasklet */
 690                 tasklet_schedule(&drv_data->pump_transfers);
 691
 692                 return IRQ_HANDLED;
 693         }
 694
 695         /* Opps problem detected */
 696         return IRQ_NONE;
 697 }
 698
 699 static irqreturn_t interrupt_wronly_transfer(struct driver_data *drv_data)
 700 {
 701         struct spi_message *msg = drv_data->cur_msg;
 702         void __iomem *regs = drv_data->regs;
 703         u32 status;
 704         irqreturn_t handled = IRQ_NONE;
 705
 706         status = readl(regs + SPI_INT_STATUS);
 707
 708         if (status & SPI_INTEN_TE) {
 709                 /* TXFIFO Empty Interrupt on the last transfered word */
 710                 writel(status & ~SPI_INTEN, regs + SPI_INT_STATUS);
 711                 dev_dbg(&drv_data->pdev->dev,
 712                         "interrupt_wronly_transfer - end of tx\n");
 713
 714                 flush(drv_data);
 715
 716                 /* Update total byte transfered */
 717                 msg->actual_length += drv_data->len;
 718
 719                 /* Move to next transfer */
 720                 msg->state = next_transfer(drv_data);
 721
 722                 /* Schedule transfer tasklet */
 723                 tasklet_schedule(&drv_data->pump_transfers);
 724
 725                 return IRQ_HANDLED;
 726         } else {
 727                 while (status & SPI_STATUS_TH) {
 728                         dev_dbg(&drv_data->pdev->dev,
 729                                 "interrupt_wronly_transfer - status = 0x%08X\n",
 730                                 status);
 731
 732                         /* Pump data */
 733                         if (write(drv_data)) {
 734                                 /* End of TXFIFO writes,
 735                                    now wait until TXFIFO is empty */
 736                                 writel(SPI_INTEN_TE, regs + SPI_INT_STATUS);
 737                                 return IRQ_HANDLED;
 738                         }
 739
 740                         status = readl(regs + SPI_INT_STATUS);
 741
 742                         /* We did something */
 743                         handled = IRQ_HANDLED;
 744                 }
 745         }
 746
 747         return handled;
 748 }
 749
 750 static irqreturn_t interrupt_transfer(struct driver_data *drv_data)
 751 {
 752         struct spi_message *msg = drv_data->cur_msg;
 753         void __iomem *regs = drv_data->regs;
 754         u32 status, control;
 755         irqreturn_t handled = IRQ_NONE;
 756         unsigned long limit;
 757
 758         status = readl(regs + SPI_INT_STATUS);
 759
 760         if (status & SPI_INTEN_TE) {
 761                 /* TXFIFO Empty Interrupt on the last transfered word */
 762                 writel(status & ~SPI_INTEN, regs + SPI_INT_STATUS);
 763                 dev_dbg(&drv_data->pdev->dev,
 764                         "interrupt_transfer - end of tx\n");
 765
 766                 if (msg->state == ERROR_STATE) {
 767                         /* RXFIFO overrun was detected and message aborted */
 768                         flush(drv_data);
 769                 } else {
 770                         /* Wait for end of transaction */
 771                         do {
 772                                 control = readl(regs + SPI_CONTROL);
 773                         } while (control & SPI_CONTROL_XCH);
 774
 775                         /* Release chip select if requested, transfer delays are
 776                            handled in pump_transfers */
 777                         if (drv_data->cs_change)
 778                                 drv_data->cs_control(SPI_CS_DEASSERT);
 779
 780                         /* Read trailing bytes */
 781                         limit = loops_per_jiffy << 1;
 782                         while ((read(drv_data) == 0) && limit--);
 783
 784                         if (limit == 0)
 785                                 dev_err(&drv_data->pdev->dev,
 786                                         "interrupt_transfer - "
 787                                         "trailing byte read failed\n");
 788                         else
 789                                 dev_dbg(&drv_data->pdev->dev,
 790                                         "interrupt_transfer - end of rx\n");
 791
 792                         /* Update total byte transfered */
 793                         msg->actual_length += drv_data->len;
 794
 795                         /* Move to next transfer */
 796                         msg->state = next_transfer(drv_data);
 797                 }
 798
 799                 /* Schedule transfer tasklet */
 800                 tasklet_schedule(&drv_data->pump_transfers);
 801
 802                 return IRQ_HANDLED;
 803         } else {
 804                 while (status & (SPI_STATUS_TH | SPI_STATUS_RO)) {
 805                         dev_dbg(&drv_data->pdev->dev,
 806                                 "interrupt_transfer - status = 0x%08X\n",
 807                                 status);
 808
 809                         if (status & SPI_STATUS_RO) {
 810                                 /* RXFIFO overrun, abort message end wait
 811                                    until TXFIFO is empty */
 812                                 writel(SPI_INTEN_TE, regs + SPI_INT_STATUS);
 813
 814                                 dev_warn(&drv_data->pdev->dev,
 815                                         "interrupt_transfer - fifo overun\n"
 816                                         "    data not yet written = %d\n"
 817                                         "    data not yet read    = %d\n",
 818                                         data_to_write(drv_data),
 819                                         data_to_read(drv_data));
 820
 821                                 msg->state = ERROR_STATE;
 822
 823                                 return IRQ_HANDLED;
 824                         }
 825
 826                         /* Pump data */
 827                         read(drv_data);
 828                         if (write(drv_data)) {
 829                                 /* End of TXFIFO writes,
 830                                    now wait until TXFIFO is empty */
 831                                 writel(SPI_INTEN_TE, regs + SPI_INT_STATUS);
 832                                 return IRQ_HANDLED;
 833                         }
 834
 835                         status = readl(regs + SPI_INT_STATUS);
 836
 837                         /* We did something */
 838                         handled = IRQ_HANDLED;
 839                 }
 840         }
 841
 842         return handled;
 843 }
 844
 845 static irqreturn_t spi_int(int irq, void *dev_id)
 846 {
 847         struct driver_data *drv_data = (struct driver_data *)dev_id;
 848
 849         if (!drv_data->cur_msg) {
 850                 dev_err(&drv_data->pdev->dev,
 851                         "spi_int - bad message state\n");
 852                 /* Never fail */
 853                 return IRQ_HANDLED;
 854         }
 855
 856         return drv_data->transfer_handler(drv_data);
 857 }
 858
 859 static inline u32 spi_speed_hz(struct driver_data *drv_data, u32 data_rate)
 860 {
 861         return clk_get_rate(drv_data->clk) / (4 << ((data_rate) >> 13));
 862 }
 863
 864 static u32 spi_data_rate(struct driver_data *drv_data, u32 speed_hz)
 865 {
 866         u32 div;
 867         u32 quantized_hz = clk_get_rate(drv_data->clk) >> 2;
 868
 869         for (div = SPI_PERCLK2_DIV_MIN;
 870                 div <= SPI_PERCLK2_DIV_MAX;
 871                 div++, quantized_hz >>= 1) {
 872                         if (quantized_hz <= speed_hz)
 873                                 /* Max available speed LEQ required speed */
 874                                 return div << 13;
 875         }
 876         return SPI_CONTROL_DATARATE_BAD;
 877 }
 878
 879 static void pump_transfers(unsigned long data)
 880 {
 881         struct driver_data *drv_data = (struct driver_data *)data;
 882         struct spi_message *message;
 883         struct spi_transfer *transfer, *previous;
 884         struct chip_data *chip;
 885         void __iomem *regs;
 886         u32 tmp, control;
 887
 888         dev_dbg(&drv_data->pdev->dev, "pump_transfer\n");
 889
 890         message = drv_data->cur_msg;
 891
 892         /* Handle for abort */
 893         if (message->state == ERROR_STATE) {
 894                 message->status = -EIO;
 895                 giveback(message, drv_data);
 896                 return;
 897         }
 898
 899         /* Handle end of message */
 900         if (message->state == DONE_STATE) {
 901                 message->status = 0;
 902                 giveback(message, drv_data);
 903                 return;
 904         }
 905
 906         chip = drv_data->cur_chip;
 907
 908         /* Delay if requested at end of transfer*/
 909         transfer = drv_data->cur_transfer;
 910         if (message->state == RUNNING_STATE) {
 911                 previous = list_entry(transfer->transfer_list.prev,
 912                                         struct spi_transfer,
 913                                         transfer_list);
 914                 if (previous->delay_usecs)
 915                         udelay(previous->delay_usecs);
 916         } else {
 917                 /* START_STATE */
 918                 message->state = RUNNING_STATE;
 919                 drv_data->cs_control = chip->cs_control;
 920         }
 921
 922         transfer = drv_data->cur_transfer;
 923         drv_data->tx = (void *)transfer->tx_buf;
 924         drv_data->tx_end = drv_data->tx + transfer->len;
 925         drv_data->rx = transfer->rx_buf;
 926         drv_data->rx_end = drv_data->rx + transfer->len;
 927         drv_data->rx_dma = transfer->rx_dma;
 928         drv_data->tx_dma = transfer->tx_dma;
 929         drv_data->len = transfer->len;
 930         drv_data->cs_change = transfer->cs_change;
 931         drv_data->rd_only = (drv_data->tx == NULL);
 932
 933         regs = drv_data->regs;
 934         control = readl(regs + SPI_CONTROL);
 935
 936         /* Bits per word setup */
 937         tmp = transfer->bits_per_word;
 938         if (tmp == 0) {
 939                 /* Use device setup */
 940                 tmp = chip->bits_per_word;
 941                 drv_data->n_bytes = chip->n_bytes;
 942         } else
 943                 /* Use per-transfer setup */
 944                 drv_data->n_bytes = (tmp <= 8) ? 1 : 2;
 945         u32_EDIT(control, SPI_CONTROL_BITCOUNT_MASK, tmp - 1);
 946
 947         /* Speed setup (surely valid because already checked) */
 948         tmp = transfer->speed_hz;
 949         if (tmp == 0)
 950                 tmp = chip->max_speed_hz;
 951         tmp = spi_data_rate(drv_data, tmp);
 952         u32_EDIT(control, SPI_CONTROL_DATARATE, tmp);
 953
 954         writel(control, regs + SPI_CONTROL);
 955
 956         /* Assert device chip-select */
 957         drv_data->cs_control(SPI_CS_ASSERT);
 958
 959         /* DMA cannot read/write SPI FIFOs other than 16 bits at a time; hence
 960            if bits_per_word is less or equal 8 PIO transfers are performed.
 961            Moreover DMA is convinient for transfer length bigger than FIFOs
 962            byte size. */
 963         if ((drv_data->n_bytes == 2) &&
 964                 (drv_data->len > SPI_FIFO_DEPTH*SPI_FIFO_BYTE_WIDTH) &&
 965                 (map_dma_buffers(drv_data) == 0)) {
 966                 dev_dbg(&drv_data->pdev->dev,
 967                         "pump dma transfer\n"
 968                         "    tx      = %p\n"
 969                         "    tx_dma  = %08X\n"
 970                         "    rx      = %p\n"
 971                         "    rx_dma  = %08X\n"
 972                         "    len     = %d\n",
 973                         drv_data->tx,
 974                         (unsigned int)drv_data->tx_dma,
 975                         drv_data->rx,
 976                         (unsigned int)drv_data->rx_dma,
 977                         drv_data->len);
 978
 979                 /* Ensure we have the correct interrupt handler */
 980                 drv_data->transfer_handler = dma_transfer;
 981
 982                 /* Trigger transfer */
 983                 writel(readl(regs + SPI_CONTROL) | SPI_CONTROL_XCH,
 984                         regs + SPI_CONTROL);
 985
 986                 /* Setup tx DMA */
 987                 if (drv_data->tx)
 988                         /* Linear source address */
 989                         CCR(drv_data->tx_channel) =
 990                                 CCR_DMOD_FIFO |
 991                                 CCR_SMOD_LINEAR |
 992                                 CCR_SSIZ_32 | CCR_DSIZ_16 |
 993                                 CCR_REN;
 994                 else
 995                         /* Read only transfer -> fixed source address for
 996                            dummy write to achive read */
 997                         CCR(drv_data->tx_channel) =
 998                                 CCR_DMOD_FIFO |
 999                                 CCR_SMOD_FIFO |
1000                                 CCR_SSIZ_32 | CCR_DSIZ_16 |
1001                                 CCR_REN;
1002
1003                 imx_dma_setup_single(
1004                         drv_data->tx_channel,
1005                         drv_data->tx_dma,
1006                         drv_data->len,
1007                         drv_data->rd_data_phys + 4,
1008                         DMA_MODE_WRITE);
1009
1010                 if (drv_data->rx) {
1011                         /* Setup rx DMA for linear destination address */
1012                         CCR(drv_data->rx_channel) =
1013                                 CCR_DMOD_LINEAR |
1014                                 CCR_SMOD_FIFO |
1015                                 CCR_DSIZ_32 | CCR_SSIZ_16 |
1016                                 CCR_REN;
1017                         imx_dma_setup_single(
1018                                 drv_data->rx_channel,
1019                                 drv_data->rx_dma,
1020                                 drv_data->len,
1021                                 drv_data->rd_data_phys,
1022                                 DMA_MODE_READ);
1023                         imx_dma_enable(drv_data->rx_channel);
1024
1025                         /* Enable SPI interrupt */
1026                         writel(SPI_INTEN_RO, regs + SPI_INT_STATUS);
1027
1028                         /* Set SPI to request DMA service on both
1029                            Rx and Tx half fifo watermark */
1030                         writel(SPI_DMA_RHDEN | SPI_DMA_THDEN, regs + SPI_DMA);
1031                 } else
1032                         /* Write only access -> set SPI to request DMA
1033                            service on Tx half fifo watermark */
1034                         writel(SPI_DMA_THDEN, regs + SPI_DMA);
1035
1036                 imx_dma_enable(drv_data->tx_channel);
1037         } else {
1038                 dev_dbg(&drv_data->pdev->dev,
1039                         "pump pio transfer\n"
1040                         "    tx      = %p\n"
1041                         "    rx      = %p\n"
1042                         "    len     = %d\n",
1043                         drv_data->tx,
1044                         drv_data->rx,
1045                         drv_data->len);
1046
1047                 /* Ensure we have the correct interrupt handler */
1048                 if (drv_data->rx)
1049                         drv_data->transfer_handler = interrupt_transfer;
1050                 else
1051                         drv_data->transfer_handler = interrupt_wronly_transfer;
1052
1053                 /* Enable SPI interrupt */
1054                 if (drv_data->rx)
1055                         writel(SPI_INTEN_TH | SPI_INTEN_RO,
1056                                 regs + SPI_INT_STATUS);
1057                 else
1058                         writel(SPI_INTEN_TH, regs + SPI_INT_STATUS);
1059         }
1060 }
1061
1062 static void pump_messages(struct work_struct *work)
1063 {
1064         struct driver_data *drv_data =
1065                                 container_of(work, struct driver_data, work);
1066         unsigned long flags;
1067
1068         /* Lock queue and check for queue work */
1069         spin_lock_irqsave(&drv_data->lock, flags);
1070         if (list_empty(&drv_data->queue) || drv_data->run == QUEUE_STOPPED) {
1071                 drv_data->busy = 0;
1072                 spin_unlock_irqrestore(&drv_data->lock, flags);
1073                 return;
1074         }
1075
1076         /* Make sure we are not already running a message */
1077         if (drv_data->cur_msg) {
1078                 spin_unlock_irqrestore(&drv_data->lock, flags);
1079                 return;
1080         }
1081
1082         /* Extract head of queue */
1083         drv_data->cur_msg = list_entry(drv_data->queue.next,
1084                                         struct spi_message, queue);
1085         list_del_init(&drv_data->cur_msg->queue);
1086         drv_data->busy = 1;
1087         spin_unlock_irqrestore(&drv_data->lock, flags);
1088
1089         /* Initial message state */
1090         drv_data->cur_msg->state = START_STATE;
1091         drv_data->cur_transfer = list_entry(drv_data->cur_msg->transfers.next,
1092                                                 struct spi_transfer,
1093                                                 transfer_list);
1094
1095         /* Setup the SPI using the per chip configuration */
1096         drv_data->cur_chip = spi_get_ctldata(drv_data->cur_msg->spi);
1097         restore_state(drv_data);
1098
1099         /* Mark as busy and launch transfers */
1100         tasklet_schedule(&drv_data->pump_transfers);
1101 }
1102
1103 static int transfer(struct spi_device *spi, struct spi_message *msg)
1104 {
1105         struct driver_data *drv_data = spi_master_get_devdata(spi->master);
1106         u32 min_speed_hz, max_speed_hz, tmp;
1107         struct spi_transfer *trans;
1108         unsigned long flags;
1109
1110         msg->actual_length = 0;
1111
1112         /* Per transfer setup check */
1113         min_speed_hz = spi_speed_hz(drv_data, SPI_CONTROL_DATARATE_MIN);
1114         max_speed_hz = spi->max_speed_hz;
1115         list_for_each_entry(trans, &msg->transfers, transfer_list) {
1116                 tmp = trans->bits_per_word;
1117                 if (tmp > 16) {
1118                         dev_err(&drv_data->pdev->dev,
1119                                 "message rejected : "
1120                                 "invalid transfer bits_per_word (%d bits)\n",
1121                                 tmp);
1122                         goto msg_rejected;
1123                 }
1124                 tmp = trans->speed_hz;
1125                 if (tmp) {
1126                         if (tmp < min_speed_hz) {
1127                                 dev_err(&drv_data->pdev->dev,
1128                                         "message rejected : "
1129                                         "device min speed (%d Hz) exceeds "
1130                                         "required transfer speed (%d Hz)\n",
1131                                         min_speed_hz,
1132                                         tmp);
1133                                 goto msg_rejected;
1134                         } else if (tmp > max_speed_hz) {
1135                                 dev_err(&drv_data->pdev->dev,
1136                                         "message rejected : "
1137                                         "transfer speed (%d Hz) exceeds "
1138                                         "device max speed (%d Hz)\n",
1139                                         tmp,
1140                                         max_speed_hz);
1141                                 goto msg_rejected;
1142                         }
1143                 }
1144         }
1145
1146         /* Message accepted */
1147         msg->status = -EINPROGRESS;
1148         msg->state = START_STATE;
1149
1150         spin_lock_irqsave(&drv_data->lock, flags);
1151         if (drv_data->run == QUEUE_STOPPED) {
1152                 spin_unlock_irqrestore(&drv_data->lock, flags);
1153                 return -ESHUTDOWN;
1154         }
1155
1156         list_add_tail(&msg->queue, &drv_data->queue);
1157         if (drv_data->run == QUEUE_RUNNING && !drv_data->busy)
1158                 queue_work(drv_data->workqueue, &drv_data->work);
1159
1160         spin_unlock_irqrestore(&drv_data->lock, flags);
1161         return 0;
1162
1163 msg_rejected:
1164         /* Message rejected and not queued */
1165         msg->status = -EINVAL;
1166         msg->state = ERROR_STATE;
1167         if (msg->complete)
1168                 msg->complete(msg->context);
1169         return -EINVAL;
1170 }
1171
1172 /* the spi->mode bits understood by this driver: */
1173 #define MODEBITS (SPI_CPOL | SPI_CPHA | SPI_CS_HIGH)
1174
1175 /* On first setup bad values must free chip_data memory since will cause
1176    spi_new_device to fail. Bad value setup from protocol driver are simply not
1177    applied and notified to the calling driver. */
1178 static int setup(struct spi_device *spi)
1179 {
1180         struct driver_data *drv_data = spi_master_get_devdata(spi->master);
1181         struct spi_imx_chip *chip_info;
1182         struct chip_data *chip;
1183         int first_setup = 0;
1184         u32 tmp;
1185         int status = 0;
1186
1187         if (spi->mode & ~MODEBITS) {
1188                 dev_dbg(&spi->dev, "setup: unsupported mode bits %x\n",
1189                         spi->mode & ~MODEBITS);
1190                 return -EINVAL;
1191         }
1192
1193         /* Get controller data */
1194         chip_info = spi->controller_data;
1195
1196         /* Get controller_state */
1197         chip = spi_get_ctldata(spi);
1198         if (chip == NULL) {
1199                 first_setup = 1;
1200
1201                 chip = kzalloc(sizeof(struct chip_data), GFP_KERNEL);
1202                 if (!chip) {
1203                         dev_err(&spi->dev,
1204                                 "setup - cannot allocate controller state\n");
1205                         return -ENOMEM;
1206                 }
1207                 chip->control = SPI_DEFAULT_CONTROL;
1208
1209                 if (chip_info == NULL) {
1210                         /* spi_board_info.controller_data not is supplied */
1211                         chip_info = kzalloc(sizeof(struct spi_imx_chip),
1212                                                 GFP_KERNEL);
1213                         if (!chip_info) {
1214                                 dev_err(&spi->dev,
1215                                         "setup - "
1216                                         "cannot allocate controller data\n");
1217                                 status = -ENOMEM;
1218                                 goto err_first_setup;
1219                         }
1220                         /* Set controller data default value */
1221                         chip_info->enable_loopback =
1222                                                 SPI_DEFAULT_ENABLE_LOOPBACK;
1223                         chip_info->enable_dma = SPI_DEFAULT_ENABLE_DMA;
1224                         chip_info->ins_ss_pulse = 1;
1225                         chip_info->bclk_wait = SPI_DEFAULT_PERIOD_WAIT;
1226                         chip_info->cs_control = null_cs_control;
1227                 }
1228         }
1229
1230         /* Now set controller state based on controller data */
1231
1232         if (first_setup) {
1233                 /* SPI loopback */
1234                 if (chip_info->enable_loopback)
1235                         chip->test = SPI_TEST_LBC;
1236                 else
1237                         chip->test = 0;
1238
1239                 /* SPI dma driven */
1240                 chip->enable_dma = chip_info->enable_dma;
1241
1242                 /* SPI /SS pulse between spi burst */
1243                 if (chip_info->ins_ss_pulse)
1244                         u32_EDIT(chip->control,
1245                                 SPI_CONTROL_SSCTL, SPI_CONTROL_SSCTL_1);
1246                 else
1247                         u32_EDIT(chip->control,
1248                                 SPI_CONTROL_SSCTL, SPI_CONTROL_SSCTL_0);
1249
1250                 /* SPI bclk waits between each bits_per_word spi burst */
1251                 if (chip_info->bclk_wait > SPI_PERIOD_MAX_WAIT) {
1252                         dev_err(&spi->dev,
1253                                 "setup - "
1254                                 "bclk_wait exceeds max allowed (%d)\n",
1255                                 SPI_PERIOD_MAX_WAIT);
1256                         goto err_first_setup;
1257                 }
1258                 chip->period = SPI_PERIOD_CSRC_BCLK |
1259                                 (chip_info->bclk_wait & SPI_PERIOD_WAIT);
1260         }
1261
1262         /* SPI mode */
1263         tmp = spi->mode;
1264         if (tmp & SPI_CS_HIGH) {
1265                 u32_EDIT(chip->control,
1266                                 SPI_CONTROL_SSPOL, SPI_CONTROL_SSPOL_ACT_HIGH);
1267         }
1268         switch (tmp & SPI_MODE_3) {
1269         case SPI_MODE_0:
1270                 tmp = 0;
1271                 break;
1272         case SPI_MODE_1:
1273                 tmp = SPI_CONTROL_PHA_1;
1274                 break;
1275         case SPI_MODE_2:
1276                 tmp = SPI_CONTROL_POL_ACT_LOW;
1277                 break;
1278         default:
1279                 /* SPI_MODE_3 */
1280                 tmp = SPI_CONTROL_PHA_1 | SPI_CONTROL_POL_ACT_LOW;
1281                 break;
1282         }
1283         u32_EDIT(chip->control, SPI_CONTROL_POL | SPI_CONTROL_PHA, tmp);
1284
1285         /* SPI word width */
1286         tmp = spi->bits_per_word;
1287         if (tmp == 0) {
1288                 tmp = 8;
1289                 spi->bits_per_word = 8;
1290         } else if (tmp > 16) {
1291                 status = -EINVAL;
1292                 dev_err(&spi->dev,
1293                         "setup - "
1294                         "invalid bits_per_word (%d)\n",
1295                         tmp);
1296                 if (first_setup)
1297                         goto err_first_setup;
1298                 else {
1299                         /* Undo setup using chip as backup copy */
1300                         tmp = chip->bits_per_word;
1301                         spi->bits_per_word = tmp;
1302                 }
1303         }
1304         chip->bits_per_word = tmp;
1305         u32_EDIT(chip->control, SPI_CONTROL_BITCOUNT_MASK, tmp - 1);
1306         chip->n_bytes = (tmp <= 8) ? 1 : 2;
1307
1308         /* SPI datarate */
1309         tmp = spi_data_rate(drv_data, spi->max_speed_hz);
1310         if (tmp == SPI_CONTROL_DATARATE_BAD) {
1311                 status = -EINVAL;
1312                 dev_err(&spi->dev,
1313                         "setup - "
1314                         "HW min speed (%d Hz) exceeds required "
1315                         "max speed (%d Hz)\n",
1316                         spi_speed_hz(drv_data, SPI_CONTROL_DATARATE_MIN),
1317                         spi->max_speed_hz);
1318                 if (first_setup)
1319                         goto err_first_setup;
1320                 else
1321                         /* Undo setup using chip as backup copy */
1322                         spi->max_speed_hz = chip->max_speed_hz;
1323         } else {
1324                 u32_EDIT(chip->control, SPI_CONTROL_DATARATE, tmp);
1325                 /* Actual rounded max_speed_hz */
1326                 tmp = spi_speed_hz(drv_data, tmp);
1327                 spi->max_speed_hz = tmp;
1328                 chip->max_speed_hz = tmp;
1329         }
1330
1331         /* SPI chip-select management */
1332         if (chip_info->cs_control)
1333                 chip->cs_control = chip_info->cs_control;
1334         else
1335                 chip->cs_control = null_cs_control;
1336
1337         /* Save controller_state */
1338         spi_set_ctldata(spi, chip);
1339
1340         /* Summary */
1341         dev_dbg(&spi->dev,
1342                 "setup succeded\n"
1343                 "    loopback enable   = %s\n"
1344                 "    dma enable        = %s\n"
1345                 "    insert /ss pulse  = %s\n"
1346                 "    period wait       = %d\n"
1347                 "    mode              = %d\n"
1348                 "    bits per word     = %d\n"
1349                 "    min speed         = %d Hz\n"
1350                 "    rounded max speed = %d Hz\n",
1351                 chip->test & SPI_TEST_LBC ? "Yes" : "No",
1352                 chip->enable_dma ? "Yes" : "No",
1353                 chip->control & SPI_CONTROL_SSCTL ? "Yes" : "No",
1354                 chip->period & SPI_PERIOD_WAIT,
1355                 spi->mode,
1356                 spi->bits_per_word,
1357                 spi_speed_hz(drv_data, SPI_CONTROL_DATARATE_MIN),
1358                 spi->max_speed_hz);
1359         return status;
1360
1361 err_first_setup:
1362         kfree(chip);
1363         return status;
1364 }
1365
1366 static void cleanup(struct spi_device *spi)
1367 {
1368         kfree(spi_get_ctldata(spi));
1369 }
1370
1371 static int __init init_queue(struct driver_data *drv_data)
1372 {
1373         INIT_LIST_HEAD(&drv_data->queue);
1374         spin_lock_init(&drv_data->lock);
1375
1376         drv_data->run = QUEUE_STOPPED;
1377         drv_data->busy = 0;
1378
1379         tasklet_init(&drv_data->pump_transfers,
1380                         pump_transfers, (unsigned long)drv_data);
1381
1382         INIT_WORK(&drv_data->work, pump_messages);
1383         drv_data->workqueue = create_singlethread_workqueue(
1384                                         drv_data->master->dev.parent->bus_id);
1385         if (drv_data->workqueue == NULL)
1386                 return -EBUSY;
1387
1388         return 0;
1389 }
1390
1391 static int start_queue(struct driver_data *drv_data)
1392 {
1393         unsigned long flags;
1394
1395         spin_lock_irqsave(&drv_data->lock, flags);
1396
1397         if (drv_data->run == QUEUE_RUNNING || drv_data->busy) {
1398                 spin_unlock_irqrestore(&drv_data->lock, flags);
1399                 return -EBUSY;
1400         }
1401
1402         drv_data->run = QUEUE_RUNNING;
1403         drv_data->cur_msg = NULL;
1404         drv_data->cur_transfer = NULL;
1405         drv_data->cur_chip = NULL;
1406         spin_unlock_irqrestore(&drv_data->lock, flags);
1407
1408         queue_work(drv_data->workqueue, &drv_data->work);
1409
1410         return 0;
1411 }
1412
1413 static int stop_queue(struct driver_data *drv_data)
1414 {
1415         unsigned long flags;
1416         unsigned limit = 500;
1417         int status = 0;
1418
1419         spin_lock_irqsave(&drv_data->lock, flags);
1420
1421         /* This is a bit lame, but is optimized for the common execution path.
1422          * A wait_queue on the drv_data->busy could be used, but then the common
1423          * execution path (pump_messages) would be required to call wake_up or
1424          * friends on every SPI message. Do this instead */
1425         drv_data->run = QUEUE_STOPPED;
1426         while (!list_empty(&drv_data->queue) && drv_data->busy && limit--) {
1427                 spin_unlock_irqrestore(&drv_data->lock, flags);
1428                 msleep(10);
1429                 spin_lock_irqsave(&drv_data->lock, flags);
1430         }
1431
1432         if (!list_empty(&drv_data->queue) || drv_data->busy)
1433                 status = -EBUSY;
1434
1435         spin_unlock_irqrestore(&drv_data->lock, flags);
1436
1437         return status;
1438 }
1439
1440 static int destroy_queue(struct driver_data *drv_data)
1441 {
1442         int status;
1443
1444         status = stop_queue(drv_data);
1445         if (status != 0)
1446                 return status;
1447
1448         if (drv_data->workqueue)
1449                 destroy_workqueue(drv_data->workqueue);
1450
1451         return 0;
1452 }
1453
1454 static int __init spi_imx_probe(struct platform_device *pdev)
1455 {
1456         struct device *dev = &pdev->dev;
1457         struct spi_imx_master *platform_info;
1458         struct spi_master *master;
1459         struct driver_data *drv_data;
1460         struct resource *res;
1461         int irq, status = 0;
1462
1463         platform_info = dev->platform_data;
1464         if (platform_info == NULL) {
1465                 dev_err(&pdev->dev, "probe - no platform data supplied\n");
1466                 status = -ENODEV;
1467                 goto err_no_pdata;
1468         }
1469
1470         /* Allocate master with space for drv_data */
1471         master = spi_alloc_master(dev, sizeof(struct driver_data));
1472         if (!master) {
1473                 dev_err(&pdev->dev, "probe - cannot alloc spi_master\n");
1474                 status = -ENOMEM;
1475                 goto err_no_mem;
1476         }
1477         drv_data = spi_master_get_devdata(master);
1478         drv_data->master = master;
1479         drv_data->master_info = platform_info;
1480         drv_data->pdev = pdev;
1481
1482         master->bus_num = pdev->id;
1483         master->num_chipselect = platform_info->num_chipselect;
1484         master->cleanup = cleanup;
1485         master->setup = setup;
1486         master->transfer = transfer;
1487
1488         drv_data->dummy_dma_buf = SPI_DUMMY_u32;
1489
1490         drv_data->clk = clk_get(&pdev->dev, "perclk2");
1491         if (IS_ERR(drv_data->clk)) {
1492                 dev_err(&pdev->dev, "probe - cannot get clock\n");
1493                 status = PTR_ERR(drv_data->clk);
1494                 goto err_no_clk;
1495         }
1496         clk_enable(drv_data->clk);
1497
1498         /* Find and map resources */
1499         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1500         if (!res) {
1501                 dev_err(&pdev->dev, "probe - MEM resources not defined\n");
1502                 status = -ENODEV;
1503                 goto err_no_iores;
1504         }
1505         drv_data->ioarea = request_mem_region(res->start,
1506                                                 res->end - res->start + 1,
1507                                                 pdev->name);
1508         if (drv_data->ioarea == NULL) {
1509                 dev_err(&pdev->dev, "probe - cannot reserve region\n");
1510                 status = -ENXIO;
1511                 goto err_no_iores;
1512         }
1513         drv_data->regs = ioremap(res->start, res->end - res->start + 1);
1514         if (drv_data->regs == NULL) {
1515                 dev_err(&pdev->dev, "probe - cannot map IO\n");
1516                 status = -ENXIO;
1517                 goto err_no_iomap;
1518         }
1519         drv_data->rd_data_phys = (dma_addr_t)res->start;
1520
1521         /* Attach to IRQ */
1522         irq = platform_get_irq(pdev, 0);
1523         if (irq < 0) {
1524                 dev_err(&pdev->dev, "probe - IRQ resource not defined\n");
1525                 status = -ENODEV;
1526                 goto err_no_irqres;
1527         }
1528         status = request_irq(irq, spi_int, IRQF_DISABLED, dev->bus_id, drv_data);
1529         if (status < 0) {
1530                 dev_err(&pdev->dev, "probe - cannot get IRQ (%d)\n", status);
1531                 goto err_no_irqres;
1532         }
1533
1534         /* Setup DMA if requested */
1535         drv_data->tx_channel = -1;
1536         drv_data->rx_channel = -1;
1537         if (platform_info->enable_dma) {
1538                 /* Get rx DMA channel */
1539                 drv_data->rx_channel = imx_dma_request_by_prio("spi_imx_rx",
1540                                                                DMA_PRIO_HIGH);
1541                 if (drv_data->rx_channel < 0) {
1542                         dev_err(dev,
1543                                 "probe - problem (%d) requesting rx channel\n",
1544                                 drv_data->rx_channel);
1545                         goto err_no_rxdma;
1546                 } else
1547                         imx_dma_setup_handlers(drv_data->rx_channel, NULL,
1548                                                 dma_err_handler, drv_data);
1549
1550                 /* Get tx DMA channel */
1551                 drv_data->tx_channel = imx_dma_request_by_prio("spi_imx_tx",
1552                                                                DMA_PRIO_MEDIUM);
1553                 if (drv_data->tx_channel < 0) {
1554                         dev_err(dev,
1555                                 "probe - problem (%d) requesting tx channel\n",
1556                                 drv_data->tx_channel);
1557                         imx_dma_free(drv_data->rx_channel);
1558                         goto err_no_txdma;
1559                 } else
1560                         imx_dma_setup_handlers(drv_data->tx_channel,
1561                                                 dma_tx_handler, dma_err_handler,
1562                                                 drv_data);
1563
1564                 /* Set request source and burst length for allocated channels */
1565                 switch (drv_data->pdev->id) {
1566                 case 1:
1567                         /* Using SPI1 */
1568                         RSSR(drv_data->rx_channel) = DMA_REQ_SPI1_R;
1569                         RSSR(drv_data->tx_channel) = DMA_REQ_SPI1_T;
1570                         break;
1571                 case 2:
1572                         /* Using SPI2 */
1573                         RSSR(drv_data->rx_channel) = DMA_REQ_SPI2_R;
1574                         RSSR(drv_data->tx_channel) = DMA_REQ_SPI2_T;
1575                         break;
1576                 default:
1577                         dev_err(dev, "probe - bad SPI Id\n");
1578                         imx_dma_free(drv_data->rx_channel);
1579                         imx_dma_free(drv_data->tx_channel);
1580                         status = -ENODEV;
1581                         goto err_no_devid;
1582                 }
1583                 BLR(drv_data->rx_channel) = SPI_DMA_BLR;
1584                 BLR(drv_data->tx_channel) = SPI_DMA_BLR;
1585         }
1586
1587         /* Load default SPI configuration */
1588         writel(SPI_RESET_START, drv_data->regs + SPI_RESET);
1589         writel(0, drv_data->regs + SPI_RESET);
1590         writel(SPI_DEFAULT_CONTROL, drv_data->regs + SPI_CONTROL);
1591
1592         /* Initial and start queue */
1593         status = init_queue(drv_data);
1594         if (status != 0) {
1595                 dev_err(&pdev->dev, "probe - problem initializing queue\n");
1596                 goto err_init_queue;
1597         }
1598         status = start_queue(drv_data);
1599         if (status != 0) {
1600                 dev_err(&pdev->dev, "probe - problem starting queue\n");
1601                 goto err_start_queue;
1602         }
1603
1604         /* Register with the SPI framework */
1605         platform_set_drvdata(pdev, drv_data);
1606         status = spi_register_master(master);
1607         if (status != 0) {
1608                 dev_err(&pdev->dev, "probe - problem registering spi master\n");
1609                 goto err_spi_register;
1610         }
1611
1612         dev_dbg(dev, "probe succeded\n");
1613         return 0;
1614
1615 err_init_queue:
1616 err_start_queue:
1617 err_spi_register:
1618         destroy_queue(drv_data);
1619
1620 err_no_rxdma:
1621 err_no_txdma:
1622 err_no_devid:
1623         free_irq(irq, drv_data);
1624
1625 err_no_irqres:
1626         iounmap(drv_data->regs);
1627
1628 err_no_iomap:
1629         release_resource(drv_data->ioarea);
1630         kfree(drv_data->ioarea);
1631
1632 err_no_iores:
1633         clk_disable(drv_data->clk);
1634         clk_put(drv_data->clk);
1635
1636 err_no_clk:
1637         spi_master_put(master);
1638
1639 err_no_pdata:
1640 err_no_mem:
1641         return status;
1642 }
1643
1644 static int __exit spi_imx_remove(struct platform_device *pdev)
1645 {
1646         struct driver_data *drv_data = platform_get_drvdata(pdev);
1647         int irq;
1648         int status = 0;
1649
1650         if (!drv_data)
1651                 return 0;
1652
1653         tasklet_kill(&drv_data->pump_transfers);
1654
1655         /* Remove the queue */
1656         status = destroy_queue(drv_data);
1657         if (status != 0) {
1658                 dev_err(&pdev->dev, "queue remove failed (%d)\n", status);
1659                 return status;
1660         }
1661
1662         /* Reset SPI */
1663         writel(SPI_RESET_START, drv_data->regs + SPI_RESET);
1664         writel(0, drv_data->regs + SPI_RESET);
1665
1666         /* Release DMA */
1667         if (drv_data->master_info->enable_dma) {
1668                 RSSR(drv_data->rx_channel) = 0;
1669                 RSSR(drv_data->tx_channel) = 0;
1670                 imx_dma_free(drv_data->tx_channel);
1671                 imx_dma_free(drv_data->rx_channel);
1672         }
1673
1674         /* Release IRQ */
1675         irq = platform_get_irq(pdev, 0);
1676         if (irq >= 0)
1677                 free_irq(irq, drv_data);
1678
1679         clk_disable(drv_data->clk);
1680         clk_put(drv_data->clk);
1681
1682         /* Release map resources */
1683         iounmap(drv_data->regs);
1684         release_resource(drv_data->ioarea);
1685         kfree(drv_data->ioarea);
1686
1687         /* Disconnect from the SPI framework */
1688         spi_unregister_master(drv_data->master);
1689         spi_master_put(drv_data->master);
1690
1691         /* Prevent double remove */
1692         platform_set_drvdata(pdev, NULL);
1693
1694         dev_dbg(&pdev->dev, "remove succeded\n");
1695
1696         return 0;
1697 }
1698
1699 static void spi_imx_shutdown(struct platform_device *pdev)
1700 {
1701         struct driver_data *drv_data = platform_get_drvdata(pdev);
1702
1703         /* Reset SPI */
1704         writel(SPI_RESET_START, drv_data->regs + SPI_RESET);
1705         writel(0, drv_data->regs + SPI_RESET);
1706
1707         dev_dbg(&pdev->dev, "shutdown succeded\n");
1708 }
1709
1710 #ifdef CONFIG_PM
1711
1712 static int spi_imx_suspend(struct platform_device *pdev, pm_message_t state)
1713 {
1714         struct driver_data *drv_data = platform_get_drvdata(pdev);
1715         int status = 0;
1716
1717         status = stop_queue(drv_data);
1718         if (status != 0) {
1719                 dev_warn(&pdev->dev, "suspend cannot stop queue\n");
1720                 return status;
1721         }
1722
1723         dev_dbg(&pdev->dev, "suspended\n");
1724
1725         return 0;
1726 }
1727
1728 static int spi_imx_resume(struct platform_device *pdev)
1729 {
1730         struct driver_data *drv_data = platform_get_drvdata(pdev);
1731         int status = 0;
1732
1733         /* Start the queue running */
1734         status = start_queue(drv_data);
1735         if (status != 0)
1736                 dev_err(&pdev->dev, "problem starting queue (%d)\n", status);
1737         else
1738                 dev_dbg(&pdev->dev, "resumed\n");
1739
1740         return status;
1741 }
1742 #else
1743 #define spi_imx_suspend NULL
1744 #define spi_imx_resume NULL
1745 #endif /* CONFIG_PM */
1746
1747 /* work with hotplug and coldplug */
1748 MODULE_ALIAS("platform:spi_imx");
1749
1750 static struct platform_driver driver = {
1751         .driver = {
1752                 .name = "spi_imx",
1753                 .owner = THIS_MODULE,
1754         },
1755         .remove = __exit_p(spi_imx_remove),
1756         .shutdown = spi_imx_shutdown,
1757         .suspend = spi_imx_suspend,
1758         .resume = spi_imx_resume,
1759 };
1760
1761 static int __init spi_imx_init(void)
1762 {
1763         return platform_driver_probe(&driver, spi_imx_probe);
1764 }
1765 module_init(spi_imx_init);
1766
1767 static void __exit spi_imx_exit(void)
1768 {
1769         platform_driver_unregister(&driver);
1770 }
1771 module_exit(spi_imx_exit);
1772
1773 MODULE_AUTHOR("Andrea Paterniani, <a.paterniani@swapp-eng.it>");
1774 MODULE_DESCRIPTION("iMX SPI Controller Driver");
1775 MODULE_LICENSE("GPL");