git.oblomov.eu Git - nouveau/blob - src/nv_exa.c

   1 /*
   2  * Copyright 2003 NVIDIA, Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #include "nv_include.h"
  24 #include "exa.h"
  25
  26 #include "nv_dma.h"
  27 #include "nv_local.h"
  28 #include "nv_rop.h"
  29 #include <sys/time.h>
  30
  31 static void
  32 NVSetPattern(ScrnInfoPtr pScrn, CARD32 clr0, CARD32 clr1,
  33                                 CARD32 pat0, CARD32 pat1)
  34 {
  35         NVPtr pNv = NVPTR(pScrn);
  36         struct nouveau_channel *chan = pNv->chan;
  37         struct nouveau_grobj *patt = pNv->NvImagePattern;
  38
  39         BEGIN_RING(chan, patt, NV04_IMAGE_PATTERN_MONOCHROME_COLOR0, 4);
  40         OUT_RING  (chan, clr0);
  41         OUT_RING  (chan, clr1);
  42         OUT_RING  (chan, pat0);
  43         OUT_RING  (chan, pat1);
  44 }
  45
  46 static void
  47 NVSetROP(ScrnInfoPtr pScrn, CARD32 alu, CARD32 planemask)
  48 {
  49         NVPtr pNv = NVPTR(pScrn);
  50         struct nouveau_channel *chan = pNv->chan;
  51         struct nouveau_grobj *rop = pNv->NvRop;
  52
  53         if (planemask != ~0) {
  54                 NVSetPattern(pScrn, 0, planemask, ~0, ~0);
  55                 if (pNv->currentRop != (alu + 32)) {
  56                         BEGIN_RING(chan, rop, NV03_CONTEXT_ROP_ROP, 1);
  57                         OUT_RING  (chan, NVROP[alu].copy_planemask);
  58                         pNv->currentRop = alu + 32;
  59                 }
  60         } else
  61         if (pNv->currentRop != alu) {
  62                 if(pNv->currentRop >= 16)
  63                         NVSetPattern(pScrn, ~0, ~0, ~0, ~0);
  64                 BEGIN_RING(chan, rop, NV03_CONTEXT_ROP_ROP, 1);
  65                 OUT_RING  (chan, NVROP[alu].copy);
  66                 pNv->currentRop = alu;
  67         }
  68 }
  69
  70 /* EXA acceleration hooks */
  71 static int
  72 NVExaMarkSync(ScreenPtr pScreen)
  73 {
  74         return 0;
  75 }
  76
  77 static void
  78 NVExaWaitMarker(ScreenPtr pScreen, int marker)
  79 {
  80         /* Nothing required - synchronisation for CPU acess to buffers will
  81          * be handled as required when buffers are mapped.
  82          */
  83 }
  84
  85 static unsigned rect_colour = 0;
  86
  87 static Bool NVExaPrepareSolid(PixmapPtr pPixmap,
  88                               int   alu,
  89                               Pixel planemask,
  90                               Pixel fg)
  91 {
  92         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
  93         NVPtr pNv = NVPTR(pScrn);
  94         struct nouveau_channel *chan = pNv->chan;
  95         struct nouveau_grobj *surf2d = pNv->NvContextSurfaces;
  96         struct nouveau_grobj *rect = pNv->NvRectangle;
  97         struct nouveau_pixmap *dst = nouveau_pixmap(pPixmap);
  98         unsigned int fmt, pitch, color;
  99
 100         planemask |= ~0 << pPixmap->drawable.bitsPerPixel;
 101         if (planemask != ~0 || alu != GXcopy) {
 102                 if (pPixmap->drawable.bitsPerPixel == 32)
 103                         return FALSE;
 104                 BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_OPERATION, 1);
 105                 OUT_RING  (chan, 1); /* ROP_AND */
 106                 NVSetROP(pScrn, alu, planemask);
 107         } else {
 108                 BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_OPERATION, 1);
 109                 OUT_RING  (chan, 3); /* SRCCOPY */
 110         }
 111
 112         if (!NVAccelGetCtxSurf2DFormatFromPixmap(pPixmap, (int*)&fmt))
 113                 return FALSE;
 114         pitch = exaGetPixmapPitch(pPixmap);
 115
 116         if (pPixmap->drawable.bitsPerPixel == 16) {
 117                 /* convert to 32bpp */
 118                 uint32_t r =  (fg&0x1F)          * 255 / 31;
 119                 uint32_t g = ((fg&0x7E0) >> 5)   * 255 / 63;
 120                 uint32_t b = ((fg&0xF100) >> 11) * 255 / 31;
 121                 color = b<<16 | g<<8 | r;
 122         } else
 123                 color = fg;
 124
 125         /* When SURFACE_FORMAT_A8R8G8B8 is used with GDI_RECTANGLE_TEXT, the
 126          * alpha channel gets forced to 0xFF for some reason.  We're using
 127          * SURFACE_FORMAT_Y32 as a workaround
 128          */
 129         if (fmt == NV04_CONTEXT_SURFACES_2D_FORMAT_A8R8G8B8)
 130                 fmt = NV04_CONTEXT_SURFACES_2D_FORMAT_Y32;
 131
 132         BEGIN_RING(chan, surf2d, NV04_CONTEXT_SURFACES_2D_FORMAT, 4);
 133         OUT_RING  (chan, fmt);
 134         OUT_RING  (chan, (pitch << 16) | pitch);
 135         OUT_RELOCl(chan, dst->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 136         OUT_RELOCl(chan, dst->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 137
 138         rect_colour = color;
 139         return TRUE;
 140 }
 141
 142 static void NVExaSolid (PixmapPtr pPixmap, int x1, int y1, int x2, int y2)
 143 {
 144         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 145         NVPtr pNv = NVPTR(pScrn);
 146         struct nouveau_channel *chan = pNv->chan;
 147         struct nouveau_grobj *rect = pNv->NvRectangle;
 148         int width = x2-x1;
 149         int height = y2-y1;
 150
 151         if (chan->pushbuf->remaining < 10)
 152                 FIRE_RING (chan);
 153
 154         BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_COLOR_FORMAT, 1);
 155         OUT_RING  (chan, NV04_GDI_RECTANGLE_TEXT_COLOR_FORMAT_A8R8G8B8);
 156         BEGIN_RING(chan, rect, NV04_GDI_RECTANGLE_TEXT_COLOR1_A, 1);
 157         OUT_RING  (chan, rect_colour);
 158
 159         BEGIN_RING(chan, rect,
 160                    NV04_GDI_RECTANGLE_TEXT_UNCLIPPED_RECTANGLE_POINT(0), 2);
 161         OUT_RING  (chan, (x1 << 16) | y1);
 162         OUT_RING  (chan, (width << 16) | height);
 163
 164         if((width * height) >= 512)
 165                 FIRE_RING (chan);
 166 }
 167
 168 static void NVExaDoneSolid (PixmapPtr pPixmap)
 169 {
 170 }
 171
 172 static Bool NVExaPrepareCopy(PixmapPtr pSrcPixmap,
 173                              PixmapPtr pDstPixmap,
 174                              int       dx,
 175                              int       dy,
 176                              int       alu,
 177                              Pixel     planemask)
 178 {
 179         ScrnInfoPtr pScrn = xf86Screens[pSrcPixmap->drawable.pScreen->myNum];
 180         NVPtr pNv = NVPTR(pScrn);
 181         struct nouveau_channel *chan = pNv->chan;
 182         struct nouveau_grobj *surf2d = pNv->NvContextSurfaces;
 183         struct nouveau_grobj *blit = pNv->NvImageBlit;
 184         struct nouveau_pixmap *src = nouveau_pixmap(pSrcPixmap);
 185         struct nouveau_pixmap *dst = nouveau_pixmap(pDstPixmap);
 186         int fmt;
 187
 188         if (pSrcPixmap->drawable.bitsPerPixel !=
 189                         pDstPixmap->drawable.bitsPerPixel)
 190                 return FALSE;
 191
 192         planemask |= ~0 << pDstPixmap->drawable.bitsPerPixel;
 193         if (planemask != ~0 || alu != GXcopy) {
 194                 if (pDstPixmap->drawable.bitsPerPixel == 32)
 195                         return FALSE;
 196                 BEGIN_RING(chan, blit, NV04_IMAGE_BLIT_OPERATION, 1);
 197                 OUT_RING  (chan, 1); /* ROP_AND */
 198                 NVSetROP(pScrn, alu, planemask);
 199         } else {
 200                 BEGIN_RING(chan, blit, NV04_IMAGE_BLIT_OPERATION, 1);
 201                 OUT_RING  (chan, 3); /* SRCCOPY */
 202         }
 203
 204         if (!NVAccelGetCtxSurf2DFormatFromPixmap(pDstPixmap, &fmt))
 205                 return FALSE;
 206
 207         BEGIN_RING(chan, surf2d, NV04_CONTEXT_SURFACES_2D_FORMAT, 4);
 208         OUT_RING  (chan, fmt);
 209         OUT_RING  (chan, (exaGetPixmapPitch(pDstPixmap) << 16) |
 210                          (exaGetPixmapPitch(pSrcPixmap)));
 211         OUT_RELOCl(chan, src->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
 212         OUT_RELOCl(chan, dst->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 213
 214         return TRUE;
 215 }
 216
 217 static void NVExaCopy(PixmapPtr pDstPixmap,
 218                       int       srcX,
 219                       int       srcY,
 220                       int       dstX,
 221                       int       dstY,
 222                       int       width,
 223                       int       height)
 224 {
 225         ScrnInfoPtr pScrn = xf86Screens[pDstPixmap->drawable.pScreen->myNum];
 226         NVPtr pNv = NVPTR(pScrn);
 227         struct nouveau_channel *chan = pNv->chan;
 228         struct nouveau_grobj *blit = pNv->NvImageBlit;
 229
 230         BEGIN_RING(chan, blit, NV01_IMAGE_BLIT_POINT_IN, 3);
 231         OUT_RING  (chan, (srcY << 16) | srcX);
 232         OUT_RING  (chan, (dstY << 16) | dstX);
 233         OUT_RING  (chan, (height  << 16) | width);
 234
 235         if((width * height) >= 512)
 236                 FIRE_RING (chan);
 237 }
 238
 239 static void NVExaDoneCopy (PixmapPtr pDstPixmap) {}
 240
 241 static inline Bool NVAccelMemcpyRect(char *dst, const char *src, int height,
 242                        int dst_pitch, int src_pitch, int line_len)
 243 {
 244         if ((src_pitch == line_len) && (src_pitch == dst_pitch)) {
 245                 memcpy(dst, src, line_len*height);
 246         } else {
 247                 while (height--) {
 248                         memcpy(dst, src, line_len);
 249                         src += src_pitch;
 250                         dst += dst_pitch;
 251                 }
 252         }
 253
 254         return TRUE;
 255 }
 256
 257 static inline Bool
 258 NVAccelDownloadM2MF(PixmapPtr pspix, int x, int y, int w, int h,
 259                     char *dst, unsigned dst_pitch)
 260 {
 261         ScrnInfoPtr pScrn = xf86Screens[pspix->drawable.pScreen->myNum];
 262         NVPtr pNv = NVPTR(pScrn);
 263         struct nouveau_channel *chan = pNv->chan;
 264         struct nouveau_grobj *m2mf = pNv->NvMemFormat;
 265         unsigned cpp = pspix->drawable.bitsPerPixel / 8;
 266         unsigned line_len = w * cpp;
 267         unsigned src_pitch = 0, src_offset = 0, linear = 0;
 268         struct nouveau_pixmap *nvpix = nouveau_pixmap(pspix);
 269
 270         BEGIN_RING(chan, m2mf, 0x184, 2);
 271         OUT_RELOCo(chan, nvpix->bo,
 272                          NOUVEAU_BO_GART | NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
 273         OUT_RELOCo(chan, pNv->GART, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 274
 275         if (!nvpix->bo->tiled) {
 276                 linear     = 1;
 277                 src_pitch  = exaGetPixmapPitch(pspix);
 278                 src_offset = (y * src_pitch) + (x * cpp);
 279         }
 280
 281         if (pNv->Architecture >= NV_ARCH_50) {
 282                 if (linear) {
 283                         BEGIN_RING(chan, m2mf, 0x0200, 1);
 284                         OUT_RING  (chan, 1);
 285                 } else {
 286                         BEGIN_RING(chan, m2mf, 0x0200, 6);
 287                         OUT_RING  (chan, 0);
 288                         OUT_RING  (chan, 0);
 289                         OUT_RING  (chan, pspix->drawable.width * cpp);
 290                         OUT_RING  (chan, pspix->drawable.height);
 291                         OUT_RING  (chan, 1);
 292                         OUT_RING  (chan, 0);
 293                 }
 294
 295                 BEGIN_RING(chan, m2mf, 0x021c, 1);
 296                 OUT_RING  (chan, 1);
 297         }
 298
 299         while (h) {
 300                 int line_count, i;
 301
 302                 if (h * line_len <= pNv->GART->size) {
 303                         line_count = h;
 304                 } else {
 305                         line_count = pNv->GART->size / line_len;
 306                         if (line_count > h)
 307                                 line_count = h;
 308                 }
 309
 310                 /* HW limitations */
 311                 if (line_count > 2047)
 312                         line_count = 2047;
 313
 314                 if (pNv->Architecture >= NV_ARCH_50) {
 315                         if (!linear) {
 316                                 BEGIN_RING(chan, m2mf, 0x0218, 1);
 317                                 OUT_RING  (chan, (y << 16) | (x * cpp));
 318                         }
 319
 320                         BEGIN_RING(chan, m2mf, 0x238, 2);
 321                         OUT_RELOCh(chan, nvpix->bo, src_offset,
 322                                          NOUVEAU_BO_GART | NOUVEAU_BO_VRAM |
 323                                          NOUVEAU_BO_RD);
 324                         OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO_GART |
 325                                    NOUVEAU_BO_WR);
 326                 }
 327
 328                 BEGIN_RING(chan, m2mf,
 329                            NV04_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
 330                 OUT_RELOCl(chan, nvpix->bo, src_offset, NOUVEAU_BO_GART |
 331                                  NOUVEAU_BO_VRAM | NOUVEAU_BO_RD);
 332                 OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 333                 OUT_RING  (chan, src_pitch);
 334                 OUT_RING  (chan, line_len);
 335                 OUT_RING  (chan, line_len);
 336                 OUT_RING  (chan, line_count);
 337                 OUT_RING  (chan, (1<<8)|1);
 338                 OUT_RING  (chan, 0);
 339
 340                 nouveau_bo_map(pNv->GART, NOUVEAU_BO_RD);
 341                 if (dst_pitch == line_len) {
 342                         memcpy(dst, pNv->GART->map, dst_pitch * line_count);
 343                         dst += dst_pitch * line_count;
 344                 } else {
 345                         const void *src = pNv->GART->map;
 346
 347                         for (i = 0; i < line_count; i++) {
 348                                 memcpy(dst, src, line_len);
 349                                 src += line_len;
 350                                 dst += dst_pitch;
 351                         }
 352                 }
 353                 nouveau_bo_unmap(pNv->GART);
 354
 355                 if (linear)
 356                         src_offset += line_count * src_pitch;
 357                 h -= line_count;
 358                 y += line_count;
 359         }
 360
 361         return TRUE;
 362 }
 363
 364 static inline void *
 365 NVExaPixmapMap(PixmapPtr pPix)
 366 {
 367         struct nouveau_pixmap *nvpix = nouveau_pixmap(pPix);
 368
 369         if (!nvpix || !nvpix->bo)
 370                 return NULL;
 371
 372         nouveau_bo_map(nvpix->bo, NOUVEAU_BO_RDWR);
 373         return nvpix->bo->map;
 374 }
 375
 376 static inline void
 377 NVExaPixmapUnmap(PixmapPtr pPix)
 378 {
 379         struct nouveau_pixmap *nvpix = nouveau_pixmap(pPix);
 380
 381         if (!nvpix || !nvpix->bo)
 382                 return;
 383
 384         nouveau_bo_unmap(nvpix->bo);
 385 }
 386
 387 static Bool NVDownloadFromScreen(PixmapPtr pSrc,
 388                                  int x,  int y,
 389                                  int w,  int h,
 390                                  char *dst,  int dst_pitch)
 391 {
 392         ScrnInfoPtr pScrn = xf86Screens[pSrc->drawable.pScreen->myNum];
 393         NVPtr pNv = NVPTR(pScrn);
 394         int src_pitch, cpp, offset;
 395         const char *src;
 396
 397         src_pitch  = exaGetPixmapPitch(pSrc);
 398         cpp = pSrc->drawable.bitsPerPixel >> 3;
 399         offset = (y * src_pitch) + (x * cpp);
 400
 401         if (pNv->GART) {
 402                 if (NVAccelDownloadM2MF(pSrc, x, y, w, h, dst, dst_pitch))
 403                         return TRUE;
 404         }
 405
 406         src = NVExaPixmapMap(pSrc);
 407         if (!src)
 408                 return FALSE;
 409         src += offset;
 410         exaWaitSync(pSrc->drawable.pScreen);
 411         if (NVAccelMemcpyRect(dst, src, h, dst_pitch, src_pitch, w*cpp)) {
 412                 NVExaPixmapUnmap(pSrc);
 413                 return TRUE;
 414         }
 415         NVExaPixmapUnmap(pSrc);
 416
 417         return FALSE;
 418 }
 419
 420 static inline Bool
 421 NVAccelUploadIFC(ScrnInfoPtr pScrn, const char *src, int src_pitch,
 422                  PixmapPtr pDst, int x, int y, int w, int h, int cpp)
 423 {
 424         NVPtr pNv = NVPTR(pScrn);
 425         struct nouveau_channel *chan = pNv->chan;
 426         struct nouveau_grobj *surf2d = pNv->NvContextSurfaces;
 427         struct nouveau_grobj *clip = pNv->NvClipRectangle;
 428         struct nouveau_grobj *ifc = pNv->NvImageFromCpu;
 429         struct nouveau_pixmap *dst = nouveau_pixmap(pDst);
 430         int line_len = w * cpp;
 431         int iw, id, surf_fmt, ifc_fmt;
 432         int padbytes;
 433
 434         if (pNv->Architecture >= NV_ARCH_50)
 435                 return FALSE;
 436
 437         if (h > 1024)
 438                 return FALSE;
 439
 440         if (line_len<4)
 441                 return FALSE;
 442
 443         switch (cpp) {
 444         case 2: ifc_fmt = 1; break;
 445         case 4: ifc_fmt = 4; break;
 446         default:
 447                 return FALSE;
 448         }
 449
 450         if (!NVAccelGetCtxSurf2DFormatFromPixmap(pDst, &surf_fmt))
 451                 return FALSE;
 452
 453         BEGIN_RING(chan, surf2d, NV04_CONTEXT_SURFACES_2D_FORMAT, 4);
 454         OUT_RING  (chan, surf_fmt);
 455         OUT_RING  (chan, (exaGetPixmapPitch(pDst) << 16) | exaGetPixmapPitch(pDst));
 456         OUT_RELOCl(chan, dst->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 457         OUT_RELOCl(chan, dst->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR);
 458
 459         /* Pad out input width to cover both COLORA() and COLORB() */
 460         iw  = (line_len + 7) & ~7;
 461         padbytes = iw - line_len;
 462         id  = iw / 4; /* line push size */
 463         iw /= cpp;
 464
 465         /* Don't support lines longer than max push size yet.. */
 466         if (id > 1792)
 467                 return FALSE;
 468
 469         BEGIN_RING(chan, clip, NV01_CONTEXT_CLIP_RECTANGLE_POINT, 2);
 470         OUT_RING  (chan, 0x0);
 471         OUT_RING  (chan, 0x7FFF7FFF);
 472
 473         BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_OPERATION, 2);
 474         OUT_RING  (chan, NV01_IMAGE_FROM_CPU_OPERATION_SRCCOPY);
 475         OUT_RING  (chan, ifc_fmt);
 476
 477         if (padbytes)
 478                 h--;
 479         while (h--) {
 480                 /* send a line */
 481                 RING_SPACE(chan, 5 + id);
 482                 BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_POINT, 3);
 483                 OUT_RING  (chan, (y++ << 16) | x); /* dst point */
 484                 OUT_RING  (chan, (1 << 16) | w); /* width/height out */
 485                 OUT_RING  (chan, (1 << 16) | iw); /* width/height in */
 486                 BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_COLOR(0), id);
 487                 OUT_RINGp (chan, src, id);
 488
 489                 src += src_pitch;
 490         }
 491         if (padbytes) {
 492                 char padding[8];
 493                 int aux = (padbytes + 7) >> 2;
 494                 RING_SPACE(chan, 5 + id + 1);
 495                 BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_POINT, 3);
 496                 OUT_RING  (chan, (y++ << 16) | x); /* dst point */
 497                 OUT_RING  (chan, (1 << 16) | w); /* width/height out */
 498                 OUT_RING  (chan, (1 << 16) | iw); /* width/height in */
 499                 BEGIN_RING(chan, ifc, NV01_IMAGE_FROM_CPU_COLOR(0), id);
 500                 OUT_RINGp (chan, src, id - aux);
 501                 memcpy(padding, src + (id - aux) * 4, padbytes);
 502                 OUT_RINGp (chan, padding, aux);
 503         }
 504
 505         return TRUE;
 506 }
 507
 508 static inline Bool
 509 NVAccelUploadM2MF(PixmapPtr pdpix, int x, int y, int w, int h,
 510                   const char *src, int src_pitch)
 511 {
 512         ScrnInfoPtr pScrn = xf86Screens[pdpix->drawable.pScreen->myNum];
 513         NVPtr pNv = NVPTR(pScrn);
 514         struct nouveau_channel *chan = pNv->chan;
 515         struct nouveau_grobj *m2mf = pNv->NvMemFormat;
 516         unsigned cpp = pdpix->drawable.bitsPerPixel / 8;
 517         unsigned line_len = w * cpp;
 518         unsigned dst_pitch = 0, dst_offset = 0, linear = 0;
 519         struct nouveau_pixmap *nvpix = nouveau_pixmap(pdpix);
 520
 521         if (!nvpix || !nvpix->bo)
 522                 return FALSE;
 523
 524         BEGIN_RING(chan, m2mf, 0x184, 2);
 525         OUT_RELOCo(chan, pNv->GART, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 526         OUT_RELOCo(chan, nvpix->bo,
 527                          NOUVEAU_BO_VRAM | NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 528
 529         if (!nvpix->bo->tiled) {
 530                 linear     = 1;
 531                 dst_pitch  = exaGetPixmapPitch(pdpix);
 532                 dst_offset = (y * dst_pitch) + (x * cpp);
 533         }
 534
 535         if (pNv->Architecture >= NV_ARCH_50) {
 536                 BEGIN_RING(chan, m2mf, 0x0200, 1);
 537                 OUT_RING  (chan, 1);
 538
 539                 if (linear) {
 540                         BEGIN_RING(chan, m2mf, 0x021c, 1);
 541                         OUT_RING  (chan, 1);
 542                 } else {
 543                         BEGIN_RING(chan, m2mf, 0x021c, 6);
 544                         OUT_RING  (chan, 0);
 545                         OUT_RING  (chan, 0);
 546                         OUT_RING  (chan, pdpix->drawable.width * cpp);
 547                         OUT_RING  (chan, pdpix->drawable.height);
 548                         OUT_RING  (chan, 1);
 549                         OUT_RING  (chan, 0);
 550                 }
 551         }
 552
 553         while (h) {
 554                 int line_count, i;
 555
 556                 /* Determine max amount of data we can DMA at once */
 557                 if (h * line_len <= pNv->GART->size) {
 558                         line_count = h;
 559                 } else {
 560                         line_count = pNv->GART->size / line_len;
 561                         if (line_count > h)
 562                                 line_count = h;
 563                 }
 564
 565                 /* HW limitations */
 566                 if (line_count > 2047)
 567                         line_count = 2047;
 568
 569                 /* Upload to GART */
 570                 nouveau_bo_map(pNv->GART, NOUVEAU_BO_WR);
 571                 if (src_pitch == line_len) {
 572                         memcpy(pNv->GART->map, src, src_pitch * line_count);
 573                         src += src_pitch * line_count;
 574                 } else {
 575                         void *dst = pNv->GART->map;
 576
 577                         for (i = 0; i < line_count; i++) {
 578                                 memcpy(dst, src, line_len);
 579                                 src += src_pitch;
 580                                 dst += line_len;
 581                         }
 582                 }
 583                 nouveau_bo_unmap(pNv->GART);
 584
 585                 if (pNv->Architecture >= NV_ARCH_50) {
 586                         if (!linear) {
 587                                 BEGIN_RING(chan, m2mf, 0x0234, 1);
 588                                 OUT_RING  (chan, (y << 16) | (x * cpp));
 589                         }
 590
 591                         BEGIN_RING(chan, m2mf, 0x0238, 2);
 592                         OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO_GART |
 593                                    NOUVEAU_BO_RD);
 594                         OUT_RELOCh(chan, nvpix->bo, dst_offset,
 595                                          NOUVEAU_BO_VRAM | NOUVEAU_BO_GART |
 596                                          NOUVEAU_BO_WR);
 597                 }
 598
 599                 /* DMA to VRAM */
 600                 BEGIN_RING(chan, m2mf,
 601                            NV04_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
 602                 OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 603                 OUT_RELOCl(chan, nvpix->bo, dst_offset, NOUVEAU_BO_VRAM |
 604                                  NOUVEAU_BO_GART | NOUVEAU_BO_WR);
 605                 OUT_RING  (chan, line_len);
 606                 OUT_RING  (chan, dst_pitch);
 607                 OUT_RING  (chan, line_len);
 608                 OUT_RING  (chan, line_count);
 609                 OUT_RING  (chan, (1<<8)|1);
 610                 OUT_RING  (chan, 0);
 611
 612                 if (linear)
 613                         dst_offset += line_count * dst_pitch;
 614                 h -= line_count;
 615                 y += line_count;
 616         }
 617
 618         return TRUE;
 619 }
 620
 621 static Bool NVUploadToScreen(PixmapPtr pDst,
 622                              int x, int y, int w, int h,
 623                              char *src, int src_pitch)
 624 {
 625         ScrnInfoPtr pScrn = xf86Screens[pDst->drawable.pScreen->myNum];
 626         NVPtr pNv = NVPTR(pScrn);
 627         int dst_pitch, cpp;
 628         char *dst;
 629
 630         dst_pitch  = exaGetPixmapPitch(pDst);
 631         cpp = pDst->drawable.bitsPerPixel >> 3;
 632
 633         /* try hostdata transfer */
 634         if (w * h * cpp < 16*1024) /* heuristic */
 635         {
 636                 if (pNv->Architecture < NV_ARCH_50) {
 637                         if (NVAccelUploadIFC(pScrn, src, src_pitch, pDst,
 638                                              x, y, w, h, cpp)) {
 639                                 exaMarkSync(pDst->drawable.pScreen);
 640                                 return TRUE;
 641                         }
 642                 } else {
 643                         if (NV50EXAUploadSIFC(src, src_pitch, pDst,
 644                                               x, y, w, h, cpp)) {
 645                                 exaMarkSync(pDst->drawable.pScreen);
 646                                 return TRUE;
 647                         }
 648                 }
 649         }
 650
 651         /* try gart-based transfer */
 652         if (pNv->GART) {
 653                 if (NVAccelUploadM2MF(pDst, x, y, w, h, src, src_pitch)) {
 654                         exaMarkSync(pDst->drawable.pScreen);
 655                         return TRUE;
 656                 }
 657         }
 658
 659         /* fallback to memcpy-based transfer */
 660         dst = NVExaPixmapMap(pDst);
 661         if (!dst)
 662                 return FALSE;
 663         dst += (y * dst_pitch) + (x * cpp);
 664         exaWaitSync(pDst->drawable.pScreen);
 665         if (NVAccelMemcpyRect(dst, src, h, dst_pitch, src_pitch, w*cpp)) {
 666                 NVExaPixmapUnmap(pDst);
 667                 return TRUE;
 668         }
 669         NVExaPixmapUnmap(pDst);
 670
 671         return FALSE;
 672 }
 673
 674 static Bool
 675 NVExaPrepareAccess(PixmapPtr pPix, int index)
 676 {
 677         ScrnInfoPtr pScrn = xf86Screens[pPix->drawable.pScreen->myNum];
 678         NVPtr pNv = NVPTR(pScrn);
 679         struct nouveau_pixmap *nvpix;
 680         (void)pNv;
 681
 682         nvpix = exaGetPixmapDriverPrivate(pPix);
 683         if (!nvpix || !nvpix->bo)
 684                 return FALSE;
 685
 686         if (!nvpix->bo->map) {
 687                 if (nouveau_bo_map(nvpix->bo, NOUVEAU_BO_RDWR))
 688                         return FALSE;
 689         }
 690
 691         pPix->devPrivate.ptr = nvpix->bo->map;
 692         return TRUE;
 693 }
 694
 695 static void
 696 NVExaFinishAccess(PixmapPtr pPix, int index)
 697 {
 698         ScrnInfoPtr pScrn = xf86Screens[pPix->drawable.pScreen->myNum];
 699         NVPtr pNv = NVPTR(pScrn);
 700         struct nouveau_pixmap *nvpix;
 701         (void)pNv;
 702
 703         nvpix = exaGetPixmapDriverPrivate(pPix);
 704         if (!nvpix || !nvpix->bo)
 705                 return;
 706
 707         nouveau_bo_unmap(nvpix->bo);
 708         pPix->devPrivate.ptr = NULL;
 709 }
 710
 711 static Bool
 712 NVExaPixmapIsOffscreen(PixmapPtr pPix)
 713 {
 714         ScrnInfoPtr pScrn = xf86Screens[pPix->drawable.pScreen->myNum];
 715         NVPtr pNv = NVPTR(pScrn);
 716         struct nouveau_pixmap *nvpix;
 717         (void)pNv;
 718
 719         nvpix = exaGetPixmapDriverPrivate(pPix);
 720         if (!nvpix || !nvpix->bo)
 721                 return FALSE;
 722
 723         return TRUE;
 724 }
 725
 726 static void *
 727 NVExaCreatePixmap(ScreenPtr pScreen, int size, int align)
 728 {
 729         ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum];
 730         NVPtr pNv = NVPTR(pScrn);
 731         struct nouveau_pixmap *nvpix;
 732
 733         nvpix = xcalloc(1, sizeof(struct nouveau_pixmap));
 734         if (!nvpix)
 735                 return NULL;
 736
 737         if (size) {
 738                 uint32_t flags = NOUVEAU_BO_VRAM;
 739
 740                 if (pNv->Architecture >= NV_ARCH_50)
 741                         flags |= NOUVEAU_BO_TILED;
 742
 743                 if (nouveau_bo_new(pNv->dev, flags, 0, size,
 744                                    &nvpix->bo)) {
 745                         xfree(nvpix);
 746                         return NULL;
 747                 }
 748         }
 749
 750         return nvpix;
 751 }
 752
 753 static void
 754 NVExaDestroyPixmap(ScreenPtr pScreen, void *driverPriv)
 755 {
 756         struct nouveau_pixmap *nvpix = driverPriv;
 757
 758         if (!driverPriv)
 759                 return;
 760
 761         nouveau_bo_ref(NULL, &nvpix->bo);
 762         xfree(nvpix);
 763 }
 764
 765 static Bool
 766 NVExaModifyPixmapHeader(PixmapPtr pPixmap, int width, int height, int depth,
 767                         int bitsPerPixel, int devKind, pointer pPixData)
 768 {
 769         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 770         NVPtr pNv = NVPTR(pScrn);
 771         struct nouveau_pixmap *nvpix;
 772
 773         if (pPixData == pNv->FBMap) {
 774                 nvpix = exaGetPixmapDriverPrivate(pPixmap);
 775                 if (!nvpix)
 776                         return FALSE;
 777
 778                 if (nouveau_bo_ref(pNv->FB, &nvpix->bo))
 779                         return FALSE;
 780
 781                 miModifyPixmapHeader(pPixmap, width, height, depth,
 782                                      bitsPerPixel, devKind, NULL);
 783                 return TRUE;
 784         }
 785
 786         return FALSE;
 787 }
 788
 789 Bool
 790 NVExaPixmapIsOnscreen(PixmapPtr pPixmap)
 791 {
 792         ScrnInfoPtr pScrn = xf86Screens[pPixmap->drawable.pScreen->myNum];
 793         NVPtr pNv = NVPTR(pScrn);
 794         struct nouveau_pixmap *nvpix;
 795
 796         nvpix = exaGetPixmapDriverPrivate(pPixmap);
 797         if (nvpix && nvpix->bo == pNv->FB)
 798                 return TRUE;
 799
 800         return FALSE;
 801 }
 802
 803 Bool
 804 NVExaInit(ScreenPtr pScreen)
 805 {
 806         ScrnInfoPtr pScrn = xf86Screens[pScreen->myNum];
 807         NVPtr pNv = NVPTR(pScrn);
 808
 809         pNv->EXADriverPtr = exaDriverAlloc();
 810         if (!pNv->EXADriverPtr) {
 811                 pNv->NoAccel = TRUE;
 812                 return FALSE;
 813         }
 814
 815         pNv->EXADriverPtr->exa_major = EXA_VERSION_MAJOR;
 816         pNv->EXADriverPtr->exa_minor = EXA_VERSION_MINOR;
 817         pNv->EXADriverPtr->flags = EXA_OFFSCREEN_PIXMAPS | EXA_HANDLES_PIXMAPS;
 818
 819         pNv->EXADriverPtr->pixmapOffsetAlign = 256;
 820         pNv->EXADriverPtr->pixmapPitchAlign = 64;
 821
 822         pNv->EXADriverPtr->PrepareAccess = NVExaPrepareAccess;
 823         pNv->EXADriverPtr->FinishAccess = NVExaFinishAccess;
 824         pNv->EXADriverPtr->PixmapIsOffscreen = NVExaPixmapIsOffscreen;
 825         pNv->EXADriverPtr->CreatePixmap = NVExaCreatePixmap;
 826         pNv->EXADriverPtr->DestroyPixmap = NVExaDestroyPixmap;
 827         pNv->EXADriverPtr->ModifyPixmapHeader = NVExaModifyPixmapHeader;
 828
 829         if (pNv->Architecture >= NV_ARCH_50) {
 830                 pNv->EXADriverPtr->maxX = 8192;
 831                 pNv->EXADriverPtr->maxY = 8192;
 832         } else
 833         if (pNv->Architecture >= NV_ARCH_20) {
 834                 pNv->EXADriverPtr->maxX = 4096;
 835                 pNv->EXADriverPtr->maxY = 4096;
 836         } else {
 837                 pNv->EXADriverPtr->maxX = 2048;
 838                 pNv->EXADriverPtr->maxY = 2048;
 839         }
 840
 841         pNv->EXADriverPtr->MarkSync   = NVExaMarkSync;
 842         pNv->EXADriverPtr->WaitMarker = NVExaWaitMarker;
 843
 844         /* Install default hooks */
 845         pNv->EXADriverPtr->DownloadFromScreen = NVDownloadFromScreen;
 846         pNv->EXADriverPtr->UploadToScreen = NVUploadToScreen;
 847
 848         if (pNv->Architecture < NV_ARCH_50) {
 849                 pNv->EXADriverPtr->PrepareCopy = NVExaPrepareCopy;
 850                 pNv->EXADriverPtr->Copy = NVExaCopy;
 851                 pNv->EXADriverPtr->DoneCopy = NVExaDoneCopy;
 852
 853                 pNv->EXADriverPtr->PrepareSolid = NVExaPrepareSolid;
 854                 pNv->EXADriverPtr->Solid = NVExaSolid;
 855                 pNv->EXADriverPtr->DoneSolid = NVExaDoneSolid;
 856         } else {
 857                 pNv->EXADriverPtr->PrepareCopy = NV50EXAPrepareCopy;
 858                 pNv->EXADriverPtr->Copy = NV50EXACopy;
 859                 pNv->EXADriverPtr->DoneCopy = NV50EXADoneCopy;
 860
 861                 pNv->EXADriverPtr->PrepareSolid = NV50EXAPrepareSolid;
 862                 pNv->EXADriverPtr->Solid = NV50EXASolid;
 863                 pNv->EXADriverPtr->DoneSolid = NV50EXADoneSolid;
 864         }
 865
 866         switch (pNv->Architecture) {
 867         case NV_ARCH_10:
 868         case NV_ARCH_20:
 869                 pNv->EXADriverPtr->CheckComposite   = NV10CheckComposite;
 870                 pNv->EXADriverPtr->PrepareComposite = NV10PrepareComposite;
 871                 pNv->EXADriverPtr->Composite        = NV10Composite;
 872                 pNv->EXADriverPtr->DoneComposite    = NV10DoneComposite;
 873                 break;
 874         case NV_ARCH_30:
 875                 pNv->EXADriverPtr->CheckComposite   = NV30EXACheckComposite;
 876                 pNv->EXADriverPtr->PrepareComposite = NV30EXAPrepareComposite;
 877                 pNv->EXADriverPtr->Composite        = NV30EXAComposite;
 878                 pNv->EXADriverPtr->DoneComposite    = NV30EXADoneComposite;
 879                 break;
 880         case NV_ARCH_40:
 881                 pNv->EXADriverPtr->CheckComposite   = NV40EXACheckComposite;
 882                 pNv->EXADriverPtr->PrepareComposite = NV40EXAPrepareComposite;
 883                 pNv->EXADriverPtr->Composite        = NV40EXAComposite;
 884                 pNv->EXADriverPtr->DoneComposite    = NV40EXADoneComposite;
 885                 break;
 886         case NV_ARCH_50:
 887                 pNv->EXADriverPtr->CheckComposite   = NV50EXACheckComposite;
 888                 pNv->EXADriverPtr->PrepareComposite = NV50EXAPrepareComposite;
 889                 pNv->EXADriverPtr->Composite        = NV50EXAComposite;
 890                 pNv->EXADriverPtr->DoneComposite    = NV50EXADoneComposite;
 891                 break;
 892         default:
 893                 break;
 894         }
 895
 896         if (!exaDriverInit(pScreen, pNv->EXADriverPtr))
 897                 return FALSE;
 898
 899         return TRUE;
 900 }