git.oblomov.eu Git - nouveau/blob - src/nv_exa.c

   1 /*
   2  * Copyright 2003 NVIDIA, Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #include "nv_include.h"
  24 #include "exa.h"
  25
  26 #include "nv_dma.h"
  27 #include "nv_local.h"
  28
  29 #include <sys/time.h>
  30
  31 const int NVCopyROP[16] =
  32 {
  33    0x00,            /* GXclear */
  34    0x88,            /* GXand */
  35    0x44,            /* GXandReverse */
  36    0xCC,            /* GXcopy */
  37    0x22,            /* GXandInverted */
  38    0xAA,            /* GXnoop */
  39    0x66,            /* GXxor */
  40    0xEE,            /* GXor */
  41    0x11,            /* GXnor */
  42    0x99,            /* GXequiv */
  43    0x55,            /* GXinvert*/
  44    0xDD,            /* GXorReverse */
  45    0x33,            /* GXcopyInverted */
  46    0xBB,            /* GXorInverted */
  47    0x77,            /* GXnand */
  48    0xFF             /* GXset */
  49 };
  50
  51 static void
  52 NVSetPattern(ScrnInfoPtr pScrn, CARD32 clr0, CARD32 clr1,
  53                                 CARD32 pat0, CARD32 pat1)
  54 {
  55         NVPtr pNv = NVPTR(pScrn);
  56         struct nouveau_channel *chan = pNv->chan;
  57         struct nouveau_grobj *patt = pNv->NvImagePattern;
  58
  59         BEGIN_RING(chan, patt, NV04_IMAGE_PATTERN_MONOCHROME_COLOR0, 4);
  60         OUT_RING  (chan, clr0);
  61         OUT_RING  (chan, clr1);
  62         OUT_RING  (chan, pat0);
  63         OUT_RING  (chan, pat1);
  64 }
  65
  66 static void
  67 NVSetROP(ScrnInfoPtr pScrn, CARD32 alu, CARD32 planemask)
  68 {
  69         NVPtr pNv = NVPTR(pScrn);
  70         struct nouveau_channel *chan = pNv->chan;
  71         struct nouveau_grobj *objrop = pNv->NvRop;
  72         int rop = NVCopyROP[alu] & 0xf0;
  73
  74         if (planemask != ~0) {
  75                 NVSetPattern(pScrn, 0, planemask, ~0, ~0);
  76                 if (pNv->currentRop != (alu + 32)) {
  77                         BEGIN_RING(chan, objrop, NV03_CONTEXT_ROP_ROP, 1);
  78                         OUT_RING  (chan, rop | 0x0a);
  79                         pNv->currentRop = alu + 32;
  80                 }
  81         } else
  82         if (pNv->currentRop != alu) {
  83                 if(pNv->currentRop >= 16)
  84                         NVSetPattern(pScrn, ~0, ~0, ~0, ~0);
  85                 BEGIN_RING(chan, objrop, NV03_CONTEXT_ROP_ROP, 1);
  86                 OUT_RING  (chan, rop | (rop >> 4));
  87                 pNv->currentRop = alu;
  88         }
  89 }
  90
  91 /* EXA acceleration hooks */
  92 static void NVExaWaitMarker(ScreenPtr pScreen, int marker)
  93 {
  94         NVSync(xf86Screens[pScreen->myNum]);
  95 }
  96
  97 static Bool NVExaPrepareSolid(PixmapPtr pPixmap,
  98                               int   alu,
  99                               Pixel planemask,
 100                               Pixel fg)
 101 {
 102         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 103         NVPtr pNv = NVPTR(pScrn);
 104         struct nouveau_channel *chan = pNv->chan;
 105         struct nouveau_grobj *surf2d = pNv->NvContextSurfaces;
 106         struct nouveau_grobj *rect = pNv->NvRectangle;
 107         unsigned int fmt, pitch, color;
 108
 109         planemask |= ~0 << pPixmap->drawable.bitsPerPixel;
 110         if (planemask != ~0 || alu != GXcopy) {
 111                 if (pPixmap->drawable.bitsPerPixel == 32)
 112                         return FALSE;
 113                 BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_OPERATION, 1);
 114                 OUT_RING  (chan, 1); /* ROP_AND */
 115                 NVSetROP(pScrn, alu, planemask);
 116         } else {
 117                 BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_OPERATION, 1);
 118                 OUT_RING  (chan, 3); /* SRCCOPY */
 119         }
 120
 121         if (!NVAccelGetCtxSurf2DFormatFromPixmap(pPixmap, (int*)&fmt))
 122                 return FALSE;
 123         pitch = exaGetPixmapPitch(pPixmap);
 124
 125         if (pPixmap->drawable.bitsPerPixel == 16) {
 126                 /* convert to 32bpp */
 127                 uint32_t r =  (fg&0x1F)          * 255 / 31;
 128                 uint32_t g = ((fg&0x7E0) >> 5)   * 255 / 63;
 129                 uint32_t b = ((fg&0xF100) >> 11) * 255 / 31;
 130                 color = b<<16 | g<<8 | r;
 131         } else
 132                 color = fg;
 133
 134         /* When SURFACE_FORMAT_A8R8G8B8 is used with GDI_RECTANGLE_TEXT, the
 135          * alpha channel gets forced to 0xFF for some reason.  We're using
 136          * SURFACE_FORMAT_Y32 as a workaround
 137          */
 138         if (fmt == NV04_CONTEXT_SURFACES_2D_FORMAT_A8R8G8B8)
 139                 fmt = NV04_CONTEXT_SURFACES_2D_FORMAT_Y32;
 140
 141         BEGIN_RING(chan, surf2d, NV04_CONTEXT_SURFACES_2D_FORMAT, 4);
 142         OUT_RING  (chan, fmt);
 143         OUT_RING  (chan, (pitch << 16) | pitch);
 144         OUT_PIXMAPl(chan, pPixmap, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 145         OUT_PIXMAPl(chan, pPixmap, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 146
 147         BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_COLOR_FORMAT, 1);
 148         OUT_RING  (chan, NV04_GDI_RECTANGLE_TEXT_COLOR_FORMAT_A8R8G8B8);
 149         BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_COLOR1_A, 1);
 150         OUT_RING (chan, color);
 151
 152         return TRUE;
 153 }
 154
 155 static void NVExaSolid (PixmapPtr pPixmap, int x1, int y1, int x2, int y2)
 156 {
 157         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 158         NVPtr pNv = NVPTR(pScrn);
 159         struct nouveau_channel *chan = pNv->chan;
 160         struct nouveau_grobj *rect = pNv->NvRectangle;
 161         int width = x2-x1;
 162         int height = y2-y1;
 163
 164         BEGIN_RING(chan, rect,
 165                    NV04_GDI_RECTANGLE_TEXT_UNCLIPPED_RECTANGLE_POINT(0), 2);
 166         OUT_RING  (chan, (x1 << 16) | y1);
 167         OUT_RING  (chan, (width << 16) | height);
 168
 169         if((width * height) >= 512)
 170                 FIRE_RING (chan);
 171 }
 172
 173 static void NVExaDoneSolid (PixmapPtr pPixmap)
 174 {
 175 }
 176
 177 static Bool NVExaPrepareCopy(PixmapPtr pSrcPixmap,
 178                              PixmapPtr pDstPixmap,
 179                              int       dx,
 180                              int       dy,
 181                              int       alu,
 182                              Pixel     planemask)
 183 {
 184         ScrnInfoPtr pScrn = xf86Screens[pSrcPixmap->drawable.pScreen->myNum];
 185         NVPtr pNv = NVPTR(pScrn);
 186         struct nouveau_channel *chan = pNv->chan;
 187         struct nouveau_grobj *surf2d = pNv->NvContextSurfaces;
 188         struct nouveau_grobj *blit = pNv->NvImageBlit;
 189         int fmt;
 190
 191         if (pSrcPixmap->drawable.bitsPerPixel !=
 192                         pDstPixmap->drawable.bitsPerPixel)
 193                 return FALSE;
 194
 195         planemask |= ~0 << pDstPixmap->drawable.bitsPerPixel;
 196         if (planemask != ~0 || alu != GXcopy) {
 197                 if (pDstPixmap->drawable.bitsPerPixel == 32)
 198                         return FALSE;
 199                 BEGIN_RING(chan, blit, NV04_IMAGE_BLIT_OPERATION, 1);
 200                 OUT_RING  (chan, 1); /* ROP_AND */
 201                 NVSetROP(pScrn, alu, planemask);
 202         } else {
 203                 BEGIN_RING(chan, blit, NV04_IMAGE_BLIT_OPERATION, 1);
 204                 OUT_RING  (chan, 3); /* SRCCOPY */
 205         }
 206
 207         if (!NVAccelGetCtxSurf2DFormatFromPixmap(pDstPixmap, &fmt))
 208                 return FALSE;
 209
 210         BEGIN_RING(chan, surf2d, NV04_CONTEXT_SURFACES_2D_FORMAT, 4);
 211         OUT_RING  (chan, fmt);
 212         OUT_RING  (chan, (exaGetPixmapPitch(pDstPixmap) << 16) |
 213                    (exaGetPixmapPitch(pSrcPixmap)));
 214         OUT_PIXMAPl(chan, pSrcPixmap, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
 215         OUT_PIXMAPl(chan, pDstPixmap, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 216
 217         return TRUE;
 218 }
 219
 220 static void NVExaCopy(PixmapPtr pDstPixmap,
 221                       int       srcX,
 222                       int       srcY,
 223                       int       dstX,
 224                       int       dstY,
 225                       int       width,
 226                       int       height)
 227 {
 228         ScrnInfoPtr pScrn = xf86Screens[pDstPixmap->drawable.pScreen->myNum];
 229         NVPtr pNv = NVPTR(pScrn);
 230         struct nouveau_channel *chan = pNv->chan;
 231         struct nouveau_grobj *blit = pNv->NvImageBlit;
 232
 233         BEGIN_RING(chan, blit, NV01_IMAGE_BLIT_POINT_IN, 3);
 234         OUT_RING  (chan, (srcY << 16) | srcX);
 235         OUT_RING  (chan, (dstY << 16) | dstX);
 236         OUT_RING  (chan, (height  << 16) | width);
 237
 238         if((width * height) >= 512)
 239                 FIRE_RING (chan);
 240 }
 241
 242 static void NVExaDoneCopy (PixmapPtr pDstPixmap) {}
 243
 244 static inline Bool NVAccelMemcpyRect(char *dst, const char *src, int height,
 245                        int dst_pitch, int src_pitch, int line_len)
 246 {
 247         if ((src_pitch == line_len) && (src_pitch == dst_pitch)) {
 248                 memcpy(dst, src, line_len*height);
 249         } else {
 250                 while (height--) {
 251                         memcpy(dst, src, line_len);
 252                         src += src_pitch;
 253                         dst += dst_pitch;
 254                 }
 255         }
 256
 257         return TRUE;
 258 }
 259
 260 static inline Bool
 261 NVAccelDownloadM2MF(PixmapPtr pspix, int x, int y, int w, int h,
 262                     char *dst, unsigned dst_pitch)
 263 {
 264         ScrnInfoPtr pScrn = xf86Screens[pspix->drawable.pScreen->myNum];
 265         NVPtr pNv = NVPTR(pScrn);
 266         struct nouveau_channel *chan = pNv->chan;
 267         struct nouveau_grobj *m2mf = pNv->NvMemFormat;
 268         unsigned cpp = pspix->drawable.bitsPerPixel / 8;
 269         unsigned line_len = w * cpp;
 270         unsigned src_pitch = 0, src_offset = 0, linear = 0;
 271
 272         BEGIN_RING(chan, m2mf, 0x184, 2);
 273         OUT_PIXMAPo(chan, pspix,
 274                     NOUVEAU_BO_GART | NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
 275         OUT_RELOCo(chan, pNv->GART, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 276
 277         if (pNv->Architecture < NV_ARCH_50 ||
 278             exaGetPixmapOffset(pspix) < pNv->EXADriverPtr->offScreenBase) {
 279                 linear     = 1;
 280                 src_pitch  = exaGetPixmapPitch(pspix);
 281                 src_offset = (y * src_pitch) + (x * cpp);
 282         }
 283
 284         if (pNv->Architecture >= NV_ARCH_50) {
 285                 if (linear) {
 286                         BEGIN_RING(chan, m2mf, 0x0200, 1);
 287                         OUT_RING  (chan, 1);
 288                 } else {
 289                         BEGIN_RING(chan, m2mf, 0x0200, 6);
 290                         OUT_RING  (chan, 0);
 291                         OUT_RING  (chan, 0);
 292                         OUT_RING  (chan, exaGetPixmapPitch(pspix));
 293                         OUT_RING  (chan, pspix->drawable.height);
 294                         OUT_RING  (chan, 1);
 295                         OUT_RING  (chan, 0);
 296                 }
 297
 298                 BEGIN_RING(chan, m2mf, 0x021c, 1);
 299                 OUT_RING  (chan, 1);
 300         }
 301
 302         while (h) {
 303                 char *src = pNv->GART->map;
 304                 int line_count, i;
 305
 306                 if (h * line_len <= pNv->GART->size) {
 307                         line_count = h;
 308                 } else {
 309                         line_count = pNv->GART->size / line_len;
 310                         if (line_count > h)
 311                                 line_count = h;
 312                 }
 313
 314                 /* HW limitations */
 315                 if (line_count > 2047)
 316                         line_count = 2047;
 317
 318                 if (pNv->Architecture >= NV_ARCH_50) {
 319                         if (!linear) {
 320                                 BEGIN_RING(chan, m2mf, 0x0218, 1);
 321                                 OUT_RING  (chan, (y << 16) | (x * cpp));
 322                         }
 323
 324                         BEGIN_RING(chan, m2mf, 0x238, 2);
 325                         OUT_PIXMAPh(chan, pspix, src_offset, NOUVEAU_BO_GART |
 326                                     NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
 327                         OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO_GART |
 328                                    NOUVEAU_BO_WR);
 329                 }
 330
 331                 BEGIN_RING(chan, m2mf,
 332                            NV04_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
 333                 OUT_PIXMAPl(chan, pspix, src_offset, NOUVEAU_BO_GART |
 334                             NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
 335                 OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 336                 OUT_RING  (chan, src_pitch);
 337                 OUT_RING  (chan, line_len);
 338                 OUT_RING  (chan, line_len);
 339                 OUT_RING  (chan, line_count);
 340                 OUT_RING  (chan, (1<<8)|1);
 341                 OUT_RING  (chan, 0);
 342
 343                 nouveau_notifier_reset(pNv->notify0, 0);
 344                 BEGIN_RING(chan, m2mf, NV04_MEMORY_TO_MEMORY_FORMAT_NOTIFY, 1);
 345                 OUT_RING  (chan, 0);
 346                 BEGIN_RING(chan, m2mf, 0x100, 1);
 347                 OUT_RING  (chan, 0);
 348                 FIRE_RING (chan);
 349                 if (nouveau_notifier_wait_status(pNv->notify0, 0, 0, 2000))
 350                         return FALSE;
 351
 352                 if (dst_pitch == line_len) {
 353                         memcpy(dst, src, dst_pitch * line_count);
 354                         dst += dst_pitch * line_count;
 355                 } else {
 356                         for (i = 0; i < line_count; i++) {
 357                                 memcpy(dst, src, line_len);
 358                                 src += line_len;
 359                                 dst += dst_pitch;
 360                         }
 361                 }
 362
 363                 if (linear)
 364                         src_offset += line_count * src_pitch;
 365                 h -= line_count;
 366                 y += line_count;
 367         }
 368
 369         return TRUE;
 370 }
 371
 372 static inline void *
 373 NVExaPixmapMap(PixmapPtr pPix)
 374 {
 375         void *map;
 376 #if NOUVEAU_EXA_PIXMAPS
 377         struct nouveau_pixmap *nvpix;
 378
 379         nvpix = exaGetPixmapDriverPrivate(pPix);
 380         if (!nvpix || !nvpix->bo)
 381                 return NULL;
 382
 383         map = nvpix->bo->map;
 384 #else
 385         ScrnInfoPtr pScrn = xf86Screens[pPix->drawable.pScreen->myNum];
 386         NVPtr pNv = NVPTR(pScrn);
 387         map = pNv->FB->map + exaGetPixmapOffset(pPix);
 388 #endif /* NOUVEAU_EXA_PIXMAPS */
 389         return map;
 390 }
 391
 392 static Bool NVDownloadFromScreen(PixmapPtr pSrc,
 393                                  int x,  int y,
 394                                  int w,  int h,
 395                                  char *dst,  int dst_pitch)
 396 {
 397         ScrnInfoPtr pScrn = xf86Screens[pSrc->drawable.pScreen->myNum];
 398         NVPtr pNv = NVPTR(pScrn);
 399         int src_pitch, cpp, offset;
 400         const char *src;
 401
 402         src_pitch  = exaGetPixmapPitch(pSrc);
 403         cpp = pSrc->drawable.bitsPerPixel >> 3;
 404         offset = (y * src_pitch) + (x * cpp);
 405
 406         if (pNv->GART) {
 407                 if (NVAccelDownloadM2MF(pSrc, x, y, w, h, dst, dst_pitch))
 408                         return TRUE;
 409         }
 410
 411         src = NVExaPixmapMap(pSrc);
 412         if (!src)
 413                 return FALSE;
 414         src += offset;
 415         exaWaitSync(pSrc->drawable.pScreen);
 416         if (NVAccelMemcpyRect(dst, src, h, dst_pitch, src_pitch, w*cpp))
 417                 return TRUE;
 418
 419         return FALSE;
 420 }
 421
 422 static inline Bool
 423 NVAccelUploadIFC(ScrnInfoPtr pScrn, const char *src, int src_pitch,
 424                  PixmapPtr pDst, int x, int y, int w, int h, int cpp)
 425 {
 426         NVPtr pNv = NVPTR(pScrn);
 427         struct nouveau_channel *chan = pNv->chan;
 428         struct nouveau_grobj *surf2d = pNv->NvContextSurfaces;
 429         struct nouveau_grobj *clip = pNv->NvClipRectangle;
 430         struct nouveau_grobj *ifc = pNv->NvImageFromCpu;
 431         int line_len = w * cpp;
 432         int iw, id, surf_fmt, ifc_fmt;
 433         int padbytes;
 434
 435         if (pNv->Architecture >= NV_ARCH_50)
 436                 return FALSE;
 437
 438         if (h > 1024)
 439                 return FALSE;
 440
 441         if (line_len<4)
 442                 return FALSE;
 443
 444         switch (cpp) {
 445         case 2: ifc_fmt = 1; break;
 446         case 4: ifc_fmt = 4; break;
 447         default:
 448                 return FALSE;
 449         }
 450
 451         if (!NVAccelGetCtxSurf2DFormatFromPixmap(pDst, &surf_fmt))
 452                 return FALSE;
 453
 454         BEGIN_RING(chan, surf2d, NV04_CONTEXT_SURFACES_2D_FORMAT, 4);
 455         OUT_RING  (chan, surf_fmt);
 456         OUT_RING  (chan, (exaGetPixmapPitch(pDst) << 16) | exaGetPixmapPitch(pDst));
 457         OUT_PIXMAPl(chan, pDst, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 458         OUT_PIXMAPl(chan, pDst, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 459
 460         /* Pad out input width to cover both COLORA() and COLORB() */
 461         iw  = (line_len + 7) & ~7;
 462         padbytes = iw - line_len;
 463         id  = iw / 4; /* line push size */
 464         iw /= cpp;
 465
 466         /* Don't support lines longer than max push size yet.. */
 467         if (id > 1792)
 468                 return FALSE;
 469
 470         BEGIN_RING(chan, clip, NV01_CONTEXT_CLIP_RECTANGLE_POINT, 2);
 471         OUT_RING  (chan, 0x0);
 472         OUT_RING  (chan, 0x7FFF7FFF);
 473
 474         BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_OPERATION, 2);
 475         OUT_RING  (chan, NV01_IMAGE_FROM_CPU_OPERATION_SRCCOPY);
 476         OUT_RING  (chan, ifc_fmt);
 477         BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_POINT, 3);
 478         OUT_RING  (chan, (y << 16) | x); /* dst point */
 479         OUT_RING  (chan, (h << 16) | w); /* width/height out */
 480         OUT_RING  (chan, (h << 16) | iw); /* width/height in */
 481
 482         if (padbytes)
 483                 h--;
 484         while (h--) {
 485                 /* send a line */
 486                 BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_COLOR(0), id);
 487                 OUT_RINGp (chan, src, id);
 488
 489                 src += src_pitch;
 490         }
 491         if (padbytes) {
 492                 char padding[8];
 493                 int aux = (padbytes + 7) >> 2;
 494                 BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_COLOR(0), id);
 495                 OUT_RINGp (chan, src, id - aux);
 496                 memcpy(padding, src + (id - aux) * 4, padbytes);
 497                 OUT_RINGp (chan, padding, aux);
 498         }
 499
 500         return TRUE;
 501 }
 502
 503 static inline Bool
 504 NVAccelUploadM2MF(PixmapPtr pdpix, int x, int y, int w, int h,
 505                   const char *src, int src_pitch)
 506 {
 507         ScrnInfoPtr pScrn = xf86Screens[pdpix->drawable.pScreen->myNum];
 508         NVPtr pNv = NVPTR(pScrn);
 509         struct nouveau_channel *chan = pNv->chan;
 510         struct nouveau_grobj *m2mf = pNv->NvMemFormat;
 511         unsigned cpp = pdpix->drawable.bitsPerPixel / 8;
 512         unsigned line_len = w * cpp;
 513         unsigned dst_pitch = 0, dst_offset = 0, linear = 0;
 514
 515         BEGIN_RING(chan, m2mf, 0x184, 2);
 516         OUT_RELOCo(chan, pNv->GART, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 517         OUT_PIXMAPo(chan, pdpix,
 518                     NOUVEAU_BO_VRAM | NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 519
 520         if (pNv->Architecture < NV_ARCH_50 ||
 521             exaGetPixmapOffset(pdpix) < pNv->EXADriverPtr->offScreenBase) {
 522                 linear     = 1;
 523                 dst_pitch  = exaGetPixmapPitch(pdpix);
 524                 dst_offset = (y * dst_pitch) + (x * cpp);
 525         }
 526
 527         if (pNv->Architecture >= NV_ARCH_50) {
 528                 BEGIN_RING(chan, m2mf, 0x0200, 1);
 529                 OUT_RING  (chan, 1);
 530
 531                 if (linear) {
 532                         BEGIN_RING(chan, m2mf, 0x021c, 1);
 533                         OUT_RING  (chan, 1);
 534                 } else {
 535                         BEGIN_RING(chan, m2mf, 0x021c, 6);
 536                         OUT_RING  (chan, 0);
 537                         OUT_RING  (chan, 0);
 538                         OUT_RING  (chan, exaGetPixmapPitch(pdpix));
 539                         OUT_RING  (chan, pdpix->drawable.height);
 540                         OUT_RING  (chan, 1);
 541                         OUT_RING  (chan, 0);
 542                 }
 543         }
 544
 545         while (h) {
 546                 char *dst = pNv->GART->map;
 547                 int line_count, i;
 548
 549                 /* Determine max amount of data we can DMA at once */
 550                 if (h * line_len <= pNv->GART->size) {
 551                         line_count = h;
 552                 } else {
 553                         line_count = pNv->GART->size / line_len;
 554                         if (line_count > h)
 555                                 line_count = h;
 556                 }
 557
 558                 /* HW limitations */
 559                 if (line_count > 2047)
 560                         line_count = 2047;
 561
 562                 /* Upload to GART */
 563                 if (src_pitch == line_len) {
 564                         memcpy(dst, src, src_pitch * line_count);
 565                         src += src_pitch * line_count;
 566                 } else {
 567                         for (i = 0; i < line_count; i++) {
 568                                 memcpy(dst, src, line_len);
 569                                 src += src_pitch;
 570                                 dst += line_len;
 571                         }
 572                 }
 573
 574                 if (pNv->Architecture >= NV_ARCH_50) {
 575                         if (!linear) {
 576                                 BEGIN_RING(chan, m2mf, 0x0234, 1);
 577                                 OUT_RING  (chan, (y << 16) | (x * cpp));
 578                         }
 579
 580                         BEGIN_RING(chan, m2mf, 0x0238, 2);
 581                         OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO_GART |
 582                                    NOUVEAU_BO_RD);
 583                         OUT_PIXMAPh(chan, pdpix, dst_offset, NOUVEAU_BO_VRAM |
 584                                     NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 585                 }
 586
 587                 /* DMA to VRAM */
 588                 BEGIN_RING(chan, m2mf,
 589                            NV04_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
 590                 OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 591                 OUT_PIXMAPl(chan, pdpix, dst_offset, NOUVEAU_BO_VRAM |
 592                             NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 593                 OUT_RING  (chan, line_len);
 594                 OUT_RING  (chan, dst_pitch);
 595                 OUT_RING  (chan, line_len);
 596                 OUT_RING  (chan, line_count);
 597                 OUT_RING  (chan, (1<<8)|1);
 598                 OUT_RING  (chan, 0);
 599
 600                 nouveau_notifier_reset(pNv->notify0, 0);
 601                 BEGIN_RING(chan, m2mf, NV04_MEMORY_TO_MEMORY_FORMAT_NOTIFY, 1);
 602                 OUT_RING  (chan, 0);
 603                 BEGIN_RING(chan, m2mf, 0x100, 1);
 604                 OUT_RING  (chan, 0);
 605                 FIRE_RING (chan);
 606                 if (nouveau_notifier_wait_status(pNv->notify0, 0, 0, 2000))
 607                         return FALSE;
 608
 609                 if (linear)
 610                         dst_offset += line_count * dst_pitch;
 611                 h -= line_count;
 612                 y += line_count;
 613         }
 614
 615         return TRUE;
 616 }
 617
 618 static Bool NVUploadToScreen(PixmapPtr pDst,
 619                              int x, int y, int w, int h,
 620                              char *src, int src_pitch)
 621 {
 622         ScrnInfoPtr pScrn = xf86Screens[pDst->drawable.pScreen->myNum];
 623         NVPtr pNv = NVPTR(pScrn);
 624         int dst_pitch, cpp;
 625         char *dst;
 626
 627         dst_pitch  = exaGetPixmapPitch(pDst);
 628         cpp = pDst->drawable.bitsPerPixel >> 3;
 629
 630         /* try hostdata transfer */
 631         if (w * h * cpp < 16*1024) /* heuristic */
 632         {
 633                 if (pNv->Architecture < NV_ARCH_50) {
 634                         if (NVAccelUploadIFC(pScrn, src, src_pitch, pDst,
 635                                              x, y, w, h, cpp)) {
 636                                 exaMarkSync(pDst->drawable.pScreen);
 637                                 return TRUE;
 638                         }
 639                 } else {
 640                         if (NV50EXAUploadSIFC(src, src_pitch, pDst,
 641                                               x, y, w, h, cpp)) {
 642                                 exaMarkSync(pDst->drawable.pScreen);
 643                                 return TRUE;
 644                         }
 645                 }
 646         }
 647
 648         /* try gart-based transfer */
 649         if (pNv->GART) {
 650                 if (NVAccelUploadM2MF(pDst, x, y, w, h, src, src_pitch))
 651                         return TRUE;
 652         }
 653
 654         /* fallback to memcpy-based transfer */
 655         dst = NVExaPixmapMap(pDst);
 656         if (!dst)
 657                 return FALSE;
 658         dst += (y * dst_pitch) + (x * cpp);
 659         exaWaitSync(pDst->drawable.pScreen);
 660         if (NVAccelMemcpyRect(dst, src, h, dst_pitch, src_pitch, w*cpp))
 661                 return TRUE;
 662
 663         return FALSE;
 664 }
 665
 666 #if NOUVEAU_EXA_PIXMAPS
 667 static Bool
 668 NVExaPrepareAccess(PixmapPtr pPix, int index)
 669 {
 670         ScrnInfoPtr pScrn = xf86Screens[pPix->drawable.pScreen->myNum];
 671         NVPtr pNv = NVPTR(pScrn);
 672         struct nouveau_pixmap *nvpix;
 673         (void)pNv;
 674
 675         nvpix = exaGetPixmapDriverPrivate(pPix);
 676         if (!nvpix || !nvpix->bo)
 677                 return FALSE;
 678
 679         /*XXX: ho hum.. sync if needed */
 680
 681         if (nvpix->mapped)
 682                 return TRUE;
 683
 684         if (nouveau_bo_map(nvpix->bo, NOUVEAU_BO_RDWR))
 685                 return FALSE;
 686         pPix->devPrivate.ptr = nvpix->bo->map;
 687         nvpix->mapped = TRUE;
 688         return TRUE;
 689 }
 690
 691 static void
 692 NVExaFinishAccess(PixmapPtr pPix, int index)
 693 {
 694         ScrnInfoPtr pScrn = xf86Screens[pPix->drawable.pScreen->myNum];
 695         NVPtr pNv = NVPTR(pScrn);
 696         struct nouveau_pixmap *nvpix;
 697         (void)pNv;
 698
 699         nvpix = exaGetPixmapDriverPrivate(pPix);
 700         if (!nvpix || !nvpix->bo || !nvpix->mapped)
 701                 return;
 702
 703         nouveau_bo_unmap(nvpix->bo);
 704         pPix->devPrivate.ptr = NULL;
 705         nvpix->mapped = FALSE;
 706 }
 707
 708 static Bool
 709 NVExaPixmapIsOffscreen(PixmapPtr pPix)
 710 {
 711         ScrnInfoPtr pScrn = xf86Screens[pPix->drawable.pScreen->myNum];
 712         NVPtr pNv = NVPTR(pScrn);
 713         struct nouveau_pixmap *nvpix;
 714         (void)pNv;
 715
 716         nvpix = exaGetPixmapDriverPrivate(pPix);
 717         if (!nvpix || !nvpix->bo)
 718                 return FALSE;
 719
 720         return TRUE;
 721 }
 722
 723 static void *
 724 NVExaCreatePixmap(ScreenPtr pScreen, int size, int align)
 725 {
 726         ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum];
 727         NVPtr pNv = NVPTR(pScrn);
 728         struct nouveau_pixmap *nvpix;
 729
 730         nvpix = xcalloc(1, sizeof(struct nouveau_pixmap));
 731         if (!nvpix)
 732                 return NULL;
 733
 734         if (size) {
 735                 if (nouveau_bo_new(pNv->dev, NOUVEAU_BO_VRAM, 0, size,
 736                                    &nvpix->bo)) {
 737                         xfree(nvpix);
 738                         return NULL;
 739                 }
 740         }
 741
 742         return nvpix;
 743 }
 744
 745 static void
 746 NVExaDestroyPixmap(ScreenPtr pScreen, void *driverPriv)
 747 {
 748         ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum];
 749         NVPtr pNv = NVPTR(pScrn);
 750         struct nouveau_pixmap *nvpix = driverPriv;
 751
 752         if (!driverPriv)
 753                 return;
 754
 755         /*XXX: only if pending relocs reference this buffer..*/
 756         FIRE_RING (chan);
 757
 758         nouveau_bo_del(&nvpix->bo);
 759         xfree(nvpix);
 760 }
 761
 762 static Bool
 763 NVExaModifyPixmapHeader(PixmapPtr pPixmap, int width, int height, int depth,
 764                         int bitsPerPixel, int devKind, pointer pPixData)
 765 {
 766         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 767         NVPtr pNv = NVPTR(pScrn);
 768         struct nouveau_pixmap *nvpix;
 769
 770         if (pPixData == pNv->FB->map) {
 771                 nvpix = exaGetPixmapDriverPrivate(pPixmap);
 772                 if (!nvpix)
 773                         return FALSE;
 774
 775                 if (nouveau_bo_ref(pNv->dev, pNv->FB->handle, &nvpix->bo))
 776                         return FALSE;
 777
 778                 miModifyPixmapHeader(pPixmap, width, height, depth,
 779                                      bitsPerPixel, devKind, NULL);
 780                 return TRUE;
 781         }
 782
 783         return FALSE;
 784 }
 785 #endif
 786
 787 #if !NOUVEAU_EXA_PIXMAPS
 788 static Bool
 789 nouveau_exa_pixmap_is_offscreen(PixmapPtr pPixmap)
 790 {
 791         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 792         NVPtr pNv = NVPTR(pScrn);
 793         void *addr = (void *)pPixmap->devPrivate.ptr;
 794
 795         if (addr >= pNv->FB->map && addr < (pNv->FB->map + pNv->FB->size))
 796                 return TRUE;
 797
 798         if (pNv->shadow[0] && (addr >= pNv->shadow[0]->map && addr < (pNv->shadow[0]->map + pNv->shadow[0]->size)))
 799                 return TRUE;
 800
 801         if (pNv->shadow[1] && (addr >= pNv->shadow[1]->map && addr < (pNv->shadow[1]->map + pNv->shadow[1]->size)))
 802                 return TRUE;
 803
 804         return FALSE;
 805 }
 806
 807 static Bool
 808 nouveau_exa_prepare_access(PixmapPtr ppix, int index)
 809 {
 810         ScrnInfoPtr pScrn = xf86Screens[ppix->drawable.pScreen->myNum];
 811         NVPtr pNv = NVPTR(pScrn);
 812         unsigned long offset = exaGetPixmapOffset(ppix);
 813
 814         if (pNv->Architecture < NV_ARCH_50) /* not tiled */
 815                 return TRUE;
 816
 817         if (offset < pNv->EXADriverPtr->offScreenBase) /* not tiled */
 818                 return TRUE;
 819
 820         return FALSE; /* cannot cpu access tiled memory */
 821 }
 822
 823 static void
 824 nouveau_exa_finish_access(PixmapPtr ppix, int index)
 825 {
 826 }
 827
 828 #endif /* !NOUVEAU_EXA_PIXMAPS */
 829
 830 Bool
 831 NVExaPixmapIsOnscreen(PixmapPtr pPixmap)
 832 {
 833         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 834         NVPtr pNv = NVPTR(pScrn);
 835
 836 #if NOUVEAU_EXA_PIXMAPS
 837         struct nouveau_pixmap *nvpix;
 838         nvpix = exaGetPixmapDriverPrivate(pPixmap);
 839
 840         if (nvpix && nvpix->bo == pNv->FB)
 841                 return TRUE;
 842
 843 #else
 844         unsigned long offset = exaGetPixmapOffset(pPixmap);
 845
 846         if (offset < pNv->EXADriverPtr->offScreenBase)
 847                 return TRUE;
 848 #endif /* NOUVEAU_EXA_PIXMAPS */
 849
 850         return FALSE;
 851 }
 852
 853 Bool
 854 NVExaInit(ScreenPtr pScreen)
 855 {
 856         ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum];
 857         NVPtr pNv = NVPTR(pScrn);
 858
 859         if(!(pNv->EXADriverPtr = (ExaDriverPtr) xnfcalloc(sizeof(ExaDriverRec), 1))) {
 860                 pNv->NoAccel = TRUE;
 861                 return FALSE;
 862         }
 863
 864         pNv->EXADriverPtr->exa_major = EXA_VERSION_MAJOR;
 865         pNv->EXADriverPtr->exa_minor = EXA_VERSION_MINOR;
 866
 867         if (pNv->Architecture < NV_ARCH_50)
 868                 pNv->EXADriverPtr->pixmapOffsetAlign = 256;
 869         else
 870                 pNv->EXADriverPtr->pixmapOffsetAlign = 65536; /* fuck me! */
 871         pNv->EXADriverPtr->pixmapPitchAlign = 64;
 872
 873 #if NOUVEAU_EXA_PIXMAPS
 874         if (NOUVEAU_EXA_PIXMAPS) {
 875                 pNv->EXADriverPtr->flags = EXA_OFFSCREEN_PIXMAPS |
 876                                            EXA_HANDLES_PIXMAPS;
 877                 pNv->EXADriverPtr->PrepareAccess = NVExaPrepareAccess;
 878                 pNv->EXADriverPtr->FinishAccess = NVExaFinishAccess;
 879                 pNv->EXADriverPtr->PixmapIsOffscreen = NVExaPixmapIsOffscreen;
 880                 pNv->EXADriverPtr->CreatePixmap = NVExaCreatePixmap;
 881                 pNv->EXADriverPtr->DestroyPixmap = NVExaDestroyPixmap;
 882                 pNv->EXADriverPtr->ModifyPixmapHeader = NVExaModifyPixmapHeader;
 883         } else
 884 #endif
 885         {
 886                 pNv->EXADriverPtr->flags = EXA_OFFSCREEN_PIXMAPS;
 887                 pNv->EXADriverPtr->memoryBase = pNv->FB->map;
 888                 pNv->EXADriverPtr->offScreenBase = NOUVEAU_ALIGN(pScrn->displayWidth * pScrn->virtualY *
 889                         (pScrn->bitsPerPixel / 8), pNv->EXADriverPtr->pixmapOffsetAlign);
 890                 pNv->EXADriverPtr->memorySize           = pNv->FB->size;
 891 #if EXA_VERSION_MINOR >= 2
 892                 pNv->EXADriverPtr->PixmapIsOffscreen = nouveau_exa_pixmap_is_offscreen;
 893                 /* Needed to avoid cpu access on offscreen memory. */
 894                 /* Can be disabled because of a bug that exists in a lot of xserver's. */
 895                 /* It caused the xserver to exit when failing PrepareAccess on non-PINNED memory. */
 896                 /* While it allowed pinned memory to migrated. This bug had existed for about 3 years. */
 897                 /* Any server-1.5-branch or master xserver after 18-08-2008 doesn't have this bug. */
 898                 if (pNv->Architecture >= NV_ARCH_50) {
 899                         pNv->EXADriverPtr->PrepareAccess = nouveau_exa_prepare_access;
 900                         pNv->EXADriverPtr->FinishAccess = nouveau_exa_finish_access;
 901
 902                         if (!xf86ReturnOptValBool(pNv->Options, OPTION_PREP_FIN_ACCESS, TRUE)) {
 903                                 pNv->EXADriverPtr->PrepareAccess = NULL;
 904                                 pNv->EXADriverPtr->FinishAccess = NULL;
 905                         }
 906
 907                         if (pNv->EXADriverPtr->PrepareAccess)
 908                                 xf86DrvMsg(pScrn->scrnIndex, X_INFO, "PrepareAccess and FinishAccess hooks added\n");
 909                         else
 910                                 xf86DrvMsg(pScrn->scrnIndex, X_INFO, "PrepareAccess and FinishAccess hooks NOT added\n");
 911                 }
 912 #endif
 913         }
 914
 915         if (pNv->Architecture >= NV_ARCH_50) {
 916                 struct nouveau_device_priv *nvdev = nouveau_device(pNv->dev);
 917                 struct nouveau_bo_priv *nvbo = nouveau_bo(pNv->FB);
 918                 struct drm_nouveau_mem_tile t;
 919
 920                 t.offset = nvbo->drm.offset;
 921                 t.flags  = nvbo->drm.flags | NOUVEAU_MEM_TILE;
 922                 t.delta  = pNv->EXADriverPtr->offScreenBase;
 923                 t.size   = pNv->EXADriverPtr->memorySize -
 924                            pNv->EXADriverPtr->offScreenBase;
 925                 drmCommandWrite(nvdev->fd, DRM_NOUVEAU_MEM_TILE, &t, sizeof(t));
 926
 927                 pNv->EXADriverPtr->maxX = 8192;
 928                 pNv->EXADriverPtr->maxY = 8192;
 929         } else
 930         if (pNv->Architecture >= NV_ARCH_20) {
 931                 pNv->EXADriverPtr->maxX = 4096;
 932                 pNv->EXADriverPtr->maxY = 4096;
 933         } else {
 934                 pNv->EXADriverPtr->maxX = 2048;
 935                 pNv->EXADriverPtr->maxY = 2048;
 936         }
 937
 938         pNv->EXADriverPtr->WaitMarker = NVExaWaitMarker;
 939
 940         /* Install default hooks */
 941         pNv->EXADriverPtr->DownloadFromScreen = NVDownloadFromScreen;
 942         pNv->EXADriverPtr->UploadToScreen = NVUploadToScreen;
 943
 944         if (pNv->Architecture < NV_ARCH_50) {
 945                 pNv->EXADriverPtr->PrepareCopy = NVExaPrepareCopy;
 946                 pNv->EXADriverPtr->Copy = NVExaCopy;
 947                 pNv->EXADriverPtr->DoneCopy = NVExaDoneCopy;
 948
 949                 pNv->EXADriverPtr->PrepareSolid = NVExaPrepareSolid;
 950                 pNv->EXADriverPtr->Solid = NVExaSolid;
 951                 pNv->EXADriverPtr->DoneSolid = NVExaDoneSolid;
 952         } else {
 953                 pNv->EXADriverPtr->PrepareCopy = NV50EXAPrepareCopy;
 954                 pNv->EXADriverPtr->Copy = NV50EXACopy;
 955                 pNv->EXADriverPtr->DoneCopy = NV50EXADoneCopy;
 956
 957                 pNv->EXADriverPtr->PrepareSolid = NV50EXAPrepareSolid;
 958                 pNv->EXADriverPtr->Solid = NV50EXASolid;
 959                 pNv->EXADriverPtr->DoneSolid = NV50EXADoneSolid;
 960         }
 961
 962         switch (pNv->Architecture) {
 963         case NV_ARCH_10:
 964         case NV_ARCH_20:
 965                 pNv->EXADriverPtr->CheckComposite   = NV10CheckComposite;
 966                 pNv->EXADriverPtr->PrepareComposite = NV10PrepareComposite;
 967                 pNv->EXADriverPtr->Composite        = NV10Composite;
 968                 pNv->EXADriverPtr->DoneComposite    = NV10DoneComposite;
 969                 break;
 970         case NV_ARCH_30:
 971                 pNv->EXADriverPtr->CheckComposite   = NV30EXACheckComposite;
 972                 pNv->EXADriverPtr->PrepareComposite = NV30EXAPrepareComposite;
 973                 pNv->EXADriverPtr->Composite        = NV30EXAComposite;
 974                 pNv->EXADriverPtr->DoneComposite    = NV30EXADoneComposite;
 975                 break;
 976         case NV_ARCH_40:
 977                 pNv->EXADriverPtr->CheckComposite   = NV40EXACheckComposite;
 978                 pNv->EXADriverPtr->PrepareComposite = NV40EXAPrepareComposite;
 979                 pNv->EXADriverPtr->Composite        = NV40EXAComposite;
 980                 pNv->EXADriverPtr->DoneComposite    = NV40EXADoneComposite;
 981                 break;
 982         case NV_ARCH_50:
 983                 pNv->EXADriverPtr->CheckComposite   = NV50EXACheckComposite;
 984                 pNv->EXADriverPtr->PrepareComposite = NV50EXAPrepareComposite;
 985                 pNv->EXADriverPtr->Composite        = NV50EXAComposite;
 986                 pNv->EXADriverPtr->DoneComposite    = NV50EXADoneComposite;
 987                 break;
 988         default:
 989                 break;
 990         }
 991
 992         if (!exaDriverInit(pScreen, pNv->EXADriverPtr))
 993                 return FALSE;
 994         else
 995                 /* EXA init catches this, but only for xserver >= 1.4 */
 996                 if (pNv->VRAMPhysicalSize / 2 < NOUVEAU_ALIGN(pScrn->virtualX, 64) * NOUVEAU_ALIGN(pScrn->virtualY, 64) * (pScrn->bitsPerPixel >> 3)) {
 997                         xf86DrvMsg(pScrn->scrnIndex, X_ERROR, "The virtual screen size's resolution is too big for the video RAM framebuffer at this colour depth.\n");
 998                         return FALSE;
 999                 }
1000
1001         return TRUE;
1002 }