git.oblomov.eu Git - linux-2.6/blob - drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include <linux/swap.h>
  33
  34 static int
  35 i915_gem_object_set_domain(struct drm_gem_object *obj,
  36                             uint32_t read_domains,
  37                             uint32_t write_domain);
  38 static int
  39 i915_gem_object_set_domain_range(struct drm_gem_object *obj,
  40                                  uint64_t offset,
  41                                  uint64_t size,
  42                                  uint32_t read_domains,
  43                                  uint32_t write_domain);
  44 static int
  45 i915_gem_set_domain(struct drm_gem_object *obj,
  46                     struct drm_file *file_priv,
  47                     uint32_t read_domains,
  48                     uint32_t write_domain);
  49 static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
  50 static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
  51 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  52
  53 static void
  54 i915_gem_cleanup_ringbuffer(struct drm_device *dev);
  55
  56 int
  57 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  58                     struct drm_file *file_priv)
  59 {
  60         drm_i915_private_t *dev_priv = dev->dev_private;
  61         struct drm_i915_gem_init *args = data;
  62
  63         mutex_lock(&dev->struct_mutex);
  64
  65         if (args->gtt_start >= args->gtt_end ||
  66             (args->gtt_start & (PAGE_SIZE - 1)) != 0 ||
  67             (args->gtt_end & (PAGE_SIZE - 1)) != 0) {
  68                 mutex_unlock(&dev->struct_mutex);
  69                 return -EINVAL;
  70         }
  71
  72         drm_mm_init(&dev_priv->mm.gtt_space, args->gtt_start,
  73             args->gtt_end - args->gtt_start);
  74
  75         dev->gtt_total = (uint32_t) (args->gtt_end - args->gtt_start);
  76
  77         mutex_unlock(&dev->struct_mutex);
  78
  79         return 0;
  80 }
  81
  82
  83 /**
  84  * Creates a new mm object and returns a handle to it.
  85  */
  86 int
  87 i915_gem_create_ioctl(struct drm_device *dev, void *data,
  88                       struct drm_file *file_priv)
  89 {
  90         struct drm_i915_gem_create *args = data;
  91         struct drm_gem_object *obj;
  92         int handle, ret;
  93
  94         args->size = roundup(args->size, PAGE_SIZE);
  95
  96         /* Allocate the new object */
  97         obj = drm_gem_object_alloc(dev, args->size);
  98         if (obj == NULL)
  99                 return -ENOMEM;
 100
 101         ret = drm_gem_handle_create(file_priv, obj, &handle);
 102         mutex_lock(&dev->struct_mutex);
 103         drm_gem_object_handle_unreference(obj);
 104         mutex_unlock(&dev->struct_mutex);
 105
 106         if (ret)
 107                 return ret;
 108
 109         args->handle = handle;
 110
 111         return 0;
 112 }
 113
 114 /**
 115  * Reads data from the object referenced by handle.
 116  *
 117  * On error, the contents of *data are undefined.
 118  */
 119 int
 120 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 121                      struct drm_file *file_priv)
 122 {
 123         struct drm_i915_gem_pread *args = data;
 124         struct drm_gem_object *obj;
 125         struct drm_i915_gem_object *obj_priv;
 126         ssize_t read;
 127         loff_t offset;
 128         int ret;
 129
 130         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 131         if (obj == NULL)
 132                 return -EBADF;
 133         obj_priv = obj->driver_private;
 134
 135         /* Bounds check source.
 136          *
 137          * XXX: This could use review for overflow issues...
 138          */
 139         if (args->offset > obj->size || args->size > obj->size ||
 140             args->offset + args->size > obj->size) {
 141                 drm_gem_object_unreference(obj);
 142                 return -EINVAL;
 143         }
 144
 145         mutex_lock(&dev->struct_mutex);
 146
 147         ret = i915_gem_object_set_domain_range(obj, args->offset, args->size,
 148                                                I915_GEM_DOMAIN_CPU, 0);
 149         if (ret != 0) {
 150                 drm_gem_object_unreference(obj);
 151                 mutex_unlock(&dev->struct_mutex);
 152                 return ret;
 153         }
 154
 155         offset = args->offset;
 156
 157         read = vfs_read(obj->filp, (char __user *)(uintptr_t)args->data_ptr,
 158                         args->size, &offset);
 159         if (read != args->size) {
 160                 drm_gem_object_unreference(obj);
 161                 mutex_unlock(&dev->struct_mutex);
 162                 if (read < 0)
 163                         return read;
 164                 else
 165                         return -EINVAL;
 166         }
 167
 168         drm_gem_object_unreference(obj);
 169         mutex_unlock(&dev->struct_mutex);
 170
 171         return 0;
 172 }
 173
 174 static int
 175 i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
 176                     struct drm_i915_gem_pwrite *args,
 177                     struct drm_file *file_priv)
 178 {
 179         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 180         ssize_t remain;
 181         loff_t offset;
 182         char __user *user_data;
 183         char __iomem *vaddr;
 184         char *vaddr_atomic;
 185         int i, o, l;
 186         int ret = 0;
 187         unsigned long pfn;
 188         unsigned long unwritten;
 189
 190         user_data = (char __user *) (uintptr_t) args->data_ptr;
 191         remain = args->size;
 192         if (!access_ok(VERIFY_READ, user_data, remain))
 193                 return -EFAULT;
 194
 195
 196         mutex_lock(&dev->struct_mutex);
 197         ret = i915_gem_object_pin(obj, 0);
 198         if (ret) {
 199                 mutex_unlock(&dev->struct_mutex);
 200                 return ret;
 201         }
 202         ret = i915_gem_set_domain(obj, file_priv,
 203                                   I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
 204         if (ret)
 205                 goto fail;
 206
 207         obj_priv = obj->driver_private;
 208         offset = obj_priv->gtt_offset + args->offset;
 209         obj_priv->dirty = 1;
 210
 211         while (remain > 0) {
 212                 /* Operation in this page
 213                  *
 214                  * i = page number
 215                  * o = offset within page
 216                  * l = bytes to copy
 217                  */
 218                 i = offset >> PAGE_SHIFT;
 219                 o = offset & (PAGE_SIZE-1);
 220                 l = remain;
 221                 if ((o + l) > PAGE_SIZE)
 222                         l = PAGE_SIZE - o;
 223
 224                 pfn = (dev->agp->base >> PAGE_SHIFT) + i;
 225
 226 #ifdef CONFIG_HIGHMEM
 227                 /* This is a workaround for the low performance of iounmap
 228                  * (approximate 10% cpu cost on normal 3D workloads).
 229                  * kmap_atomic on HIGHMEM kernels happens to let us map card
 230                  * memory without taking IPIs.  When the vmap rework lands
 231                  * we should be able to dump this hack.
 232                  */
 233                 vaddr_atomic = kmap_atomic_pfn(pfn, KM_USER0);
 234 #if WATCH_PWRITE
 235                 DRM_INFO("pwrite i %d o %d l %d pfn %ld vaddr %p\n",
 236                          i, o, l, pfn, vaddr_atomic);
 237 #endif
 238                 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + o,
 239                                                               user_data, l);
 240                 kunmap_atomic(vaddr_atomic, KM_USER0);
 241
 242                 if (unwritten)
 243 #endif /* CONFIG_HIGHMEM */
 244                 {
 245                         vaddr = ioremap_wc(pfn << PAGE_SHIFT, PAGE_SIZE);
 246 #if WATCH_PWRITE
 247                         DRM_INFO("pwrite slow i %d o %d l %d "
 248                                  "pfn %ld vaddr %p\n",
 249                                  i, o, l, pfn, vaddr);
 250 #endif
 251                         if (vaddr == NULL) {
 252                                 ret = -EFAULT;
 253                                 goto fail;
 254                         }
 255                         unwritten = __copy_from_user(vaddr + o, user_data, l);
 256 #if WATCH_PWRITE
 257                         DRM_INFO("unwritten %ld\n", unwritten);
 258 #endif
 259                         iounmap(vaddr);
 260                         if (unwritten) {
 261                                 ret = -EFAULT;
 262                                 goto fail;
 263                         }
 264                 }
 265
 266                 remain -= l;
 267                 user_data += l;
 268                 offset += l;
 269         }
 270 #if WATCH_PWRITE && 1
 271         i915_gem_clflush_object(obj);
 272         i915_gem_dump_object(obj, args->offset + args->size, __func__, ~0);
 273         i915_gem_clflush_object(obj);
 274 #endif
 275
 276 fail:
 277         i915_gem_object_unpin(obj);
 278         mutex_unlock(&dev->struct_mutex);
 279
 280         return ret;
 281 }
 282
 283 static int
 284 i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
 285                       struct drm_i915_gem_pwrite *args,
 286                       struct drm_file *file_priv)
 287 {
 288         int ret;
 289         loff_t offset;
 290         ssize_t written;
 291
 292         mutex_lock(&dev->struct_mutex);
 293
 294         ret = i915_gem_set_domain(obj, file_priv,
 295                                   I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
 296         if (ret) {
 297                 mutex_unlock(&dev->struct_mutex);
 298                 return ret;
 299         }
 300
 301         offset = args->offset;
 302
 303         written = vfs_write(obj->filp,
 304                             (char __user *)(uintptr_t) args->data_ptr,
 305                             args->size, &offset);
 306         if (written != args->size) {
 307                 mutex_unlock(&dev->struct_mutex);
 308                 if (written < 0)
 309                         return written;
 310                 else
 311                         return -EINVAL;
 312         }
 313
 314         mutex_unlock(&dev->struct_mutex);
 315
 316         return 0;
 317 }
 318
 319 /**
 320  * Writes data to the object referenced by handle.
 321  *
 322  * On error, the contents of the buffer that were to be modified are undefined.
 323  */
 324 int
 325 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 326                       struct drm_file *file_priv)
 327 {
 328         struct drm_i915_gem_pwrite *args = data;
 329         struct drm_gem_object *obj;
 330         struct drm_i915_gem_object *obj_priv;
 331         int ret = 0;
 332
 333         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 334         if (obj == NULL)
 335                 return -EBADF;
 336         obj_priv = obj->driver_private;
 337
 338         /* Bounds check destination.
 339          *
 340          * XXX: This could use review for overflow issues...
 341          */
 342         if (args->offset > obj->size || args->size > obj->size ||
 343             args->offset + args->size > obj->size) {
 344                 drm_gem_object_unreference(obj);
 345                 return -EINVAL;
 346         }
 347
 348         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 349          * it would end up going through the fenced access, and we'll get
 350          * different detiling behavior between reading and writing.
 351          * pread/pwrite currently are reading and writing from the CPU
 352          * perspective, requiring manual detiling by the client.
 353          */
 354         if (obj_priv->tiling_mode == I915_TILING_NONE &&
 355             dev->gtt_total != 0)
 356                 ret = i915_gem_gtt_pwrite(dev, obj, args, file_priv);
 357         else
 358                 ret = i915_gem_shmem_pwrite(dev, obj, args, file_priv);
 359
 360 #if WATCH_PWRITE
 361         if (ret)
 362                 DRM_INFO("pwrite failed %d\n", ret);
 363 #endif
 364
 365         drm_gem_object_unreference(obj);
 366
 367         return ret;
 368 }
 369
 370 /**
 371  * Called when user space prepares to use an object
 372  */
 373 int
 374 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 375                           struct drm_file *file_priv)
 376 {
 377         struct drm_i915_gem_set_domain *args = data;
 378         struct drm_gem_object *obj;
 379         int ret;
 380
 381         if (!(dev->driver->driver_features & DRIVER_GEM))
 382                 return -ENODEV;
 383
 384         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 385         if (obj == NULL)
 386                 return -EBADF;
 387
 388         mutex_lock(&dev->struct_mutex);
 389 #if WATCH_BUF
 390         DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
 391                  obj, obj->size, args->read_domains, args->write_domain);
 392 #endif
 393         ret = i915_gem_set_domain(obj, file_priv,
 394                                   args->read_domains, args->write_domain);
 395         drm_gem_object_unreference(obj);
 396         mutex_unlock(&dev->struct_mutex);
 397         return ret;
 398 }
 399
 400 /**
 401  * Called when user space has done writes to this buffer
 402  */
 403 int
 404 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 405                       struct drm_file *file_priv)
 406 {
 407         struct drm_i915_gem_sw_finish *args = data;
 408         struct drm_gem_object *obj;
 409         struct drm_i915_gem_object *obj_priv;
 410         int ret = 0;
 411
 412         if (!(dev->driver->driver_features & DRIVER_GEM))
 413                 return -ENODEV;
 414
 415         mutex_lock(&dev->struct_mutex);
 416         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 417         if (obj == NULL) {
 418                 mutex_unlock(&dev->struct_mutex);
 419                 return -EBADF;
 420         }
 421
 422 #if WATCH_BUF
 423         DRM_INFO("%s: sw_finish %d (%p %d)\n",
 424                  __func__, args->handle, obj, obj->size);
 425 #endif
 426         obj_priv = obj->driver_private;
 427
 428         /* Pinned buffers may be scanout, so flush the cache */
 429         if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) {
 430                 i915_gem_clflush_object(obj);
 431                 drm_agp_chipset_flush(dev);
 432         }
 433         drm_gem_object_unreference(obj);
 434         mutex_unlock(&dev->struct_mutex);
 435         return ret;
 436 }
 437
 438 /**
 439  * Maps the contents of an object, returning the address it is mapped
 440  * into.
 441  *
 442  * While the mapping holds a reference on the contents of the object, it doesn't
 443  * imply a ref on the object itself.
 444  */
 445 int
 446 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 447                    struct drm_file *file_priv)
 448 {
 449         struct drm_i915_gem_mmap *args = data;
 450         struct drm_gem_object *obj;
 451         loff_t offset;
 452         unsigned long addr;
 453
 454         if (!(dev->driver->driver_features & DRIVER_GEM))
 455                 return -ENODEV;
 456
 457         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 458         if (obj == NULL)
 459                 return -EBADF;
 460
 461         offset = args->offset;
 462
 463         down_write(&current->mm->mmap_sem);
 464         addr = do_mmap(obj->filp, 0, args->size,
 465                        PROT_READ | PROT_WRITE, MAP_SHARED,
 466                        args->offset);
 467         up_write(&current->mm->mmap_sem);
 468         mutex_lock(&dev->struct_mutex);
 469         drm_gem_object_unreference(obj);
 470         mutex_unlock(&dev->struct_mutex);
 471         if (IS_ERR((void *)addr))
 472                 return addr;
 473
 474         args->addr_ptr = (uint64_t) addr;
 475
 476         return 0;
 477 }
 478
 479 static void
 480 i915_gem_object_free_page_list(struct drm_gem_object *obj)
 481 {
 482         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 483         int page_count = obj->size / PAGE_SIZE;
 484         int i;
 485
 486         if (obj_priv->page_list == NULL)
 487                 return;
 488
 489
 490         for (i = 0; i < page_count; i++)
 491                 if (obj_priv->page_list[i] != NULL) {
 492                         if (obj_priv->dirty)
 493                                 set_page_dirty(obj_priv->page_list[i]);
 494                         mark_page_accessed(obj_priv->page_list[i]);
 495                         page_cache_release(obj_priv->page_list[i]);
 496                 }
 497         obj_priv->dirty = 0;
 498
 499         drm_free(obj_priv->page_list,
 500                  page_count * sizeof(struct page *),
 501                  DRM_MEM_DRIVER);
 502         obj_priv->page_list = NULL;
 503 }
 504
 505 static void
 506 i915_gem_object_move_to_active(struct drm_gem_object *obj)
 507 {
 508         struct drm_device *dev = obj->dev;
 509         drm_i915_private_t *dev_priv = dev->dev_private;
 510         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 511
 512         /* Add a reference if we're newly entering the active list. */
 513         if (!obj_priv->active) {
 514                 drm_gem_object_reference(obj);
 515                 obj_priv->active = 1;
 516         }
 517         /* Move from whatever list we were on to the tail of execution. */
 518         list_move_tail(&obj_priv->list,
 519                        &dev_priv->mm.active_list);
 520 }
 521
 522
 523 static void
 524 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
 525 {
 526         struct drm_device *dev = obj->dev;
 527         drm_i915_private_t *dev_priv = dev->dev_private;
 528         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 529
 530         i915_verify_inactive(dev, __FILE__, __LINE__);
 531         if (obj_priv->pin_count != 0)
 532                 list_del_init(&obj_priv->list);
 533         else
 534                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
 535
 536         if (obj_priv->active) {
 537                 obj_priv->active = 0;
 538                 drm_gem_object_unreference(obj);
 539         }
 540         i915_verify_inactive(dev, __FILE__, __LINE__);
 541 }
 542
 543 /**
 544  * Creates a new sequence number, emitting a write of it to the status page
 545  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 546  *
 547  * Must be called with struct_lock held.
 548  *
 549  * Returned sequence numbers are nonzero on success.
 550  */
 551 static uint32_t
 552 i915_add_request(struct drm_device *dev, uint32_t flush_domains)
 553 {
 554         drm_i915_private_t *dev_priv = dev->dev_private;
 555         struct drm_i915_gem_request *request;
 556         uint32_t seqno;
 557         int was_empty;
 558         RING_LOCALS;
 559
 560         request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
 561         if (request == NULL)
 562                 return 0;
 563
 564         /* Grab the seqno we're going to make this request be, and bump the
 565          * next (skipping 0 so it can be the reserved no-seqno value).
 566          */
 567         seqno = dev_priv->mm.next_gem_seqno;
 568         dev_priv->mm.next_gem_seqno++;
 569         if (dev_priv->mm.next_gem_seqno == 0)
 570                 dev_priv->mm.next_gem_seqno++;
 571
 572         BEGIN_LP_RING(4);
 573         OUT_RING(MI_STORE_DWORD_INDEX);
 574         OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 575         OUT_RING(seqno);
 576
 577         OUT_RING(MI_USER_INTERRUPT);
 578         ADVANCE_LP_RING();
 579
 580         DRM_DEBUG("%d\n", seqno);
 581
 582         request->seqno = seqno;
 583         request->emitted_jiffies = jiffies;
 584         request->flush_domains = flush_domains;
 585         was_empty = list_empty(&dev_priv->mm.request_list);
 586         list_add_tail(&request->list, &dev_priv->mm.request_list);
 587
 588         if (was_empty && !dev_priv->mm.suspended)
 589                 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
 590         return seqno;
 591 }
 592
 593 /**
 594  * Command execution barrier
 595  *
 596  * Ensures that all commands in the ring are finished
 597  * before signalling the CPU
 598  */
 599 static uint32_t
 600 i915_retire_commands(struct drm_device *dev)
 601 {
 602         drm_i915_private_t *dev_priv = dev->dev_private;
 603         uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 604         uint32_t flush_domains = 0;
 605         RING_LOCALS;
 606
 607         /* The sampler always gets flushed on i965 (sigh) */
 608         if (IS_I965G(dev))
 609                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
 610         BEGIN_LP_RING(2);
 611         OUT_RING(cmd);
 612         OUT_RING(0); /* noop */
 613         ADVANCE_LP_RING();
 614         return flush_domains;
 615 }
 616
 617 /**
 618  * Moves buffers associated only with the given active seqno from the active
 619  * to inactive list, potentially freeing them.
 620  */
 621 static void
 622 i915_gem_retire_request(struct drm_device *dev,
 623                         struct drm_i915_gem_request *request)
 624 {
 625         drm_i915_private_t *dev_priv = dev->dev_private;
 626
 627         /* Move any buffers on the active list that are no longer referenced
 628          * by the ringbuffer to the flushing/inactive lists as appropriate.
 629          */
 630         while (!list_empty(&dev_priv->mm.active_list)) {
 631                 struct drm_gem_object *obj;
 632                 struct drm_i915_gem_object *obj_priv;
 633
 634                 obj_priv = list_first_entry(&dev_priv->mm.active_list,
 635                                             struct drm_i915_gem_object,
 636                                             list);
 637                 obj = obj_priv->obj;
 638
 639                 /* If the seqno being retired doesn't match the oldest in the
 640                  * list, then the oldest in the list must still be newer than
 641                  * this seqno.
 642                  */
 643                 if (obj_priv->last_rendering_seqno != request->seqno)
 644                         return;
 645 #if WATCH_LRU
 646                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
 647                          __func__, request->seqno, obj);
 648 #endif
 649
 650                 if (obj->write_domain != 0) {
 651                         list_move_tail(&obj_priv->list,
 652                                        &dev_priv->mm.flushing_list);
 653                 } else {
 654                         i915_gem_object_move_to_inactive(obj);
 655                 }
 656         }
 657
 658         if (request->flush_domains != 0) {
 659                 struct drm_i915_gem_object *obj_priv, *next;
 660
 661                 /* Clear the write domain and activity from any buffers
 662                  * that are just waiting for a flush matching the one retired.
 663                  */
 664                 list_for_each_entry_safe(obj_priv, next,
 665                                          &dev_priv->mm.flushing_list, list) {
 666                         struct drm_gem_object *obj = obj_priv->obj;
 667
 668                         if (obj->write_domain & request->flush_domains) {
 669                                 obj->write_domain = 0;
 670                                 i915_gem_object_move_to_inactive(obj);
 671                         }
 672                 }
 673
 674         }
 675 }
 676
 677 /**
 678  * Returns true if seq1 is later than seq2.
 679  */
 680 static int
 681 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
 682 {
 683         return (int32_t)(seq1 - seq2) >= 0;
 684 }
 685
 686 uint32_t
 687 i915_get_gem_seqno(struct drm_device *dev)
 688 {
 689         drm_i915_private_t *dev_priv = dev->dev_private;
 690
 691         return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
 692 }
 693
 694 /**
 695  * This function clears the request list as sequence numbers are passed.
 696  */
 697 void
 698 i915_gem_retire_requests(struct drm_device *dev)
 699 {
 700         drm_i915_private_t *dev_priv = dev->dev_private;
 701         uint32_t seqno;
 702
 703         seqno = i915_get_gem_seqno(dev);
 704
 705         while (!list_empty(&dev_priv->mm.request_list)) {
 706                 struct drm_i915_gem_request *request;
 707                 uint32_t retiring_seqno;
 708
 709                 request = list_first_entry(&dev_priv->mm.request_list,
 710                                            struct drm_i915_gem_request,
 711                                            list);
 712                 retiring_seqno = request->seqno;
 713
 714                 if (i915_seqno_passed(seqno, retiring_seqno) ||
 715                     dev_priv->mm.wedged) {
 716                         i915_gem_retire_request(dev, request);
 717
 718                         list_del(&request->list);
 719                         drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
 720                 } else
 721                         break;
 722         }
 723 }
 724
 725 void
 726 i915_gem_retire_work_handler(struct work_struct *work)
 727 {
 728         drm_i915_private_t *dev_priv;
 729         struct drm_device *dev;
 730
 731         dev_priv = container_of(work, drm_i915_private_t,
 732                                 mm.retire_work.work);
 733         dev = dev_priv->dev;
 734
 735         mutex_lock(&dev->struct_mutex);
 736         i915_gem_retire_requests(dev);
 737         if (!dev_priv->mm.suspended &&
 738             !list_empty(&dev_priv->mm.request_list))
 739                 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
 740         mutex_unlock(&dev->struct_mutex);
 741 }
 742
 743 /**
 744  * Waits for a sequence number to be signaled, and cleans up the
 745  * request and object lists appropriately for that event.
 746  */
 747 static int
 748 i915_wait_request(struct drm_device *dev, uint32_t seqno)
 749 {
 750         drm_i915_private_t *dev_priv = dev->dev_private;
 751         int ret = 0;
 752
 753         BUG_ON(seqno == 0);
 754
 755         if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
 756                 dev_priv->mm.waiting_gem_seqno = seqno;
 757                 i915_user_irq_get(dev);
 758                 ret = wait_event_interruptible(dev_priv->irq_queue,
 759                                                i915_seqno_passed(i915_get_gem_seqno(dev),
 760                                                                  seqno) ||
 761                                                dev_priv->mm.wedged);
 762                 i915_user_irq_put(dev);
 763                 dev_priv->mm.waiting_gem_seqno = 0;
 764         }
 765         if (dev_priv->mm.wedged)
 766                 ret = -EIO;
 767
 768         if (ret && ret != -ERESTARTSYS)
 769                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
 770                           __func__, ret, seqno, i915_get_gem_seqno(dev));
 771
 772         /* Directly dispatch request retiring.  While we have the work queue
 773          * to handle this, the waiter on a request often wants an associated
 774          * buffer to have made it to the inactive list, and we would need
 775          * a separate wait queue to handle that.
 776          */
 777         if (ret == 0)
 778                 i915_gem_retire_requests(dev);
 779
 780         return ret;
 781 }
 782
 783 static void
 784 i915_gem_flush(struct drm_device *dev,
 785                uint32_t invalidate_domains,
 786                uint32_t flush_domains)
 787 {
 788         drm_i915_private_t *dev_priv = dev->dev_private;
 789         uint32_t cmd;
 790         RING_LOCALS;
 791
 792 #if WATCH_EXEC
 793         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
 794                   invalidate_domains, flush_domains);
 795 #endif
 796
 797         if (flush_domains & I915_GEM_DOMAIN_CPU)
 798                 drm_agp_chipset_flush(dev);
 799
 800         if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
 801                                                      I915_GEM_DOMAIN_GTT)) {
 802                 /*
 803                  * read/write caches:
 804                  *
 805                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
 806                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
 807                  * also flushed at 2d versus 3d pipeline switches.
 808                  *
 809                  * read-only caches:
 810                  *
 811                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 812                  * MI_READ_FLUSH is set, and is always flushed on 965.
 813                  *
 814                  * I915_GEM_DOMAIN_COMMAND may not exist?
 815                  *
 816                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 817                  * invalidated when MI_EXE_FLUSH is set.
 818                  *
 819                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 820                  * invalidated with every MI_FLUSH.
 821                  *
 822                  * TLBs:
 823                  *
 824                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 825                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 826                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 827                  * are flushed at any MI_FLUSH.
 828                  */
 829
 830                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 831                 if ((invalidate_domains|flush_domains) &
 832                     I915_GEM_DOMAIN_RENDER)
 833                         cmd &= ~MI_NO_WRITE_FLUSH;
 834                 if (!IS_I965G(dev)) {
 835                         /*
 836                          * On the 965, the sampler cache always gets flushed
 837                          * and this bit is reserved.
 838                          */
 839                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 840                                 cmd |= MI_READ_FLUSH;
 841                 }
 842                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 843                         cmd |= MI_EXE_FLUSH;
 844
 845 #if WATCH_EXEC
 846                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 847 #endif
 848                 BEGIN_LP_RING(2);
 849                 OUT_RING(cmd);
 850                 OUT_RING(0); /* noop */
 851                 ADVANCE_LP_RING();
 852         }
 853 }
 854
 855 /**
 856  * Ensures that all rendering to the object has completed and the object is
 857  * safe to unbind from the GTT or access from the CPU.
 858  */
 859 static int
 860 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
 861 {
 862         struct drm_device *dev = obj->dev;
 863         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 864         int ret;
 865
 866         /* If there are writes queued to the buffer, flush and
 867          * create a new seqno to wait for.
 868          */
 869         if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) {
 870                 uint32_t write_domain = obj->write_domain;
 871 #if WATCH_BUF
 872                 DRM_INFO("%s: flushing object %p from write domain %08x\n",
 873                           __func__, obj, write_domain);
 874 #endif
 875                 i915_gem_flush(dev, 0, write_domain);
 876
 877                 i915_gem_object_move_to_active(obj);
 878                 obj_priv->last_rendering_seqno = i915_add_request(dev,
 879                                                                   write_domain);
 880                 BUG_ON(obj_priv->last_rendering_seqno == 0);
 881 #if WATCH_LRU
 882                 DRM_INFO("%s: flush moves to exec list %p\n", __func__, obj);
 883 #endif
 884         }
 885
 886         /* If there is rendering queued on the buffer being evicted, wait for
 887          * it.
 888          */
 889         if (obj_priv->active) {
 890 #if WATCH_BUF
 891                 DRM_INFO("%s: object %p wait for seqno %08x\n",
 892                           __func__, obj, obj_priv->last_rendering_seqno);
 893 #endif
 894                 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
 895                 if (ret != 0)
 896                         return ret;
 897         }
 898
 899         return 0;
 900 }
 901
 902 /**
 903  * Unbinds an object from the GTT aperture.
 904  */
 905 static int
 906 i915_gem_object_unbind(struct drm_gem_object *obj)
 907 {
 908         struct drm_device *dev = obj->dev;
 909         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 910         int ret = 0;
 911
 912 #if WATCH_BUF
 913         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
 914         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
 915 #endif
 916         if (obj_priv->gtt_space == NULL)
 917                 return 0;
 918
 919         if (obj_priv->pin_count != 0) {
 920                 DRM_ERROR("Attempting to unbind pinned buffer\n");
 921                 return -EINVAL;
 922         }
 923
 924         /* Wait for any rendering to complete
 925          */
 926         ret = i915_gem_object_wait_rendering(obj);
 927         if (ret) {
 928                 DRM_ERROR("wait_rendering failed: %d\n", ret);
 929                 return ret;
 930         }
 931
 932         /* Move the object to the CPU domain to ensure that
 933          * any possible CPU writes while it's not in the GTT
 934          * are flushed when we go to remap it. This will
 935          * also ensure that all pending GPU writes are finished
 936          * before we unbind.
 937          */
 938         ret = i915_gem_object_set_domain(obj, I915_GEM_DOMAIN_CPU,
 939                                          I915_GEM_DOMAIN_CPU);
 940         if (ret) {
 941                 DRM_ERROR("set_domain failed: %d\n", ret);
 942                 return ret;
 943         }
 944
 945         if (obj_priv->agp_mem != NULL) {
 946                 drm_unbind_agp(obj_priv->agp_mem);
 947                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
 948                 obj_priv->agp_mem = NULL;
 949         }
 950
 951         BUG_ON(obj_priv->active);
 952
 953         i915_gem_object_free_page_list(obj);
 954
 955         if (obj_priv->gtt_space) {
 956                 atomic_dec(&dev->gtt_count);
 957                 atomic_sub(obj->size, &dev->gtt_memory);
 958
 959                 drm_mm_put_block(obj_priv->gtt_space);
 960                 obj_priv->gtt_space = NULL;
 961         }
 962
 963         /* Remove ourselves from the LRU list if present. */
 964         if (!list_empty(&obj_priv->list))
 965                 list_del_init(&obj_priv->list);
 966
 967         return 0;
 968 }
 969
 970 static int
 971 i915_gem_evict_something(struct drm_device *dev)
 972 {
 973         drm_i915_private_t *dev_priv = dev->dev_private;
 974         struct drm_gem_object *obj;
 975         struct drm_i915_gem_object *obj_priv;
 976         int ret = 0;
 977
 978         for (;;) {
 979                 /* If there's an inactive buffer available now, grab it
 980                  * and be done.
 981                  */
 982                 if (!list_empty(&dev_priv->mm.inactive_list)) {
 983                         obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
 984                                                     struct drm_i915_gem_object,
 985                                                     list);
 986                         obj = obj_priv->obj;
 987                         BUG_ON(obj_priv->pin_count != 0);
 988 #if WATCH_LRU
 989                         DRM_INFO("%s: evicting %p\n", __func__, obj);
 990 #endif
 991                         BUG_ON(obj_priv->active);
 992
 993                         /* Wait on the rendering and unbind the buffer. */
 994                         ret = i915_gem_object_unbind(obj);
 995                         break;
 996                 }
 997
 998                 /* If we didn't get anything, but the ring is still processing
 999                  * things, wait for one of those things to finish and hopefully
1000                  * leave us a buffer to evict.
1001                  */
1002                 if (!list_empty(&dev_priv->mm.request_list)) {
1003                         struct drm_i915_gem_request *request;
1004
1005                         request = list_first_entry(&dev_priv->mm.request_list,
1006                                                    struct drm_i915_gem_request,
1007                                                    list);
1008
1009                         ret = i915_wait_request(dev, request->seqno);
1010                         if (ret)
1011                                 break;
1012
1013                         /* if waiting caused an object to become inactive,
1014                          * then loop around and wait for it. Otherwise, we
1015                          * assume that waiting freed and unbound something,
1016                          * so there should now be some space in the GTT
1017                          */
1018                         if (!list_empty(&dev_priv->mm.inactive_list))
1019                                 continue;
1020                         break;
1021                 }
1022
1023                 /* If we didn't have anything on the request list but there
1024                  * are buffers awaiting a flush, emit one and try again.
1025                  * When we wait on it, those buffers waiting for that flush
1026                  * will get moved to inactive.
1027                  */
1028                 if (!list_empty(&dev_priv->mm.flushing_list)) {
1029                         obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1030                                                     struct drm_i915_gem_object,
1031                                                     list);
1032                         obj = obj_priv->obj;
1033
1034                         i915_gem_flush(dev,
1035                                        obj->write_domain,
1036                                        obj->write_domain);
1037                         i915_add_request(dev, obj->write_domain);
1038
1039                         obj = NULL;
1040                         continue;
1041                 }
1042
1043                 DRM_ERROR("inactive empty %d request empty %d "
1044                           "flushing empty %d\n",
1045                           list_empty(&dev_priv->mm.inactive_list),
1046                           list_empty(&dev_priv->mm.request_list),
1047                           list_empty(&dev_priv->mm.flushing_list));
1048                 /* If we didn't do any of the above, there's nothing to be done
1049                  * and we just can't fit it in.
1050                  */
1051                 return -ENOMEM;
1052         }
1053         return ret;
1054 }
1055
1056 static int
1057 i915_gem_object_get_page_list(struct drm_gem_object *obj)
1058 {
1059         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1060         int page_count, i;
1061         struct address_space *mapping;
1062         struct inode *inode;
1063         struct page *page;
1064         int ret;
1065
1066         if (obj_priv->page_list)
1067                 return 0;
1068
1069         /* Get the list of pages out of our struct file.  They'll be pinned
1070          * at this point until we release them.
1071          */
1072         page_count = obj->size / PAGE_SIZE;
1073         BUG_ON(obj_priv->page_list != NULL);
1074         obj_priv->page_list = drm_calloc(page_count, sizeof(struct page *),
1075                                          DRM_MEM_DRIVER);
1076         if (obj_priv->page_list == NULL) {
1077                 DRM_ERROR("Faled to allocate page list\n");
1078                 return -ENOMEM;
1079         }
1080
1081         inode = obj->filp->f_path.dentry->d_inode;
1082         mapping = inode->i_mapping;
1083         for (i = 0; i < page_count; i++) {
1084                 page = read_mapping_page(mapping, i, NULL);
1085                 if (IS_ERR(page)) {
1086                         ret = PTR_ERR(page);
1087                         DRM_ERROR("read_mapping_page failed: %d\n", ret);
1088                         i915_gem_object_free_page_list(obj);
1089                         return ret;
1090                 }
1091                 obj_priv->page_list[i] = page;
1092         }
1093         return 0;
1094 }
1095
1096 /**
1097  * Finds free space in the GTT aperture and binds the object there.
1098  */
1099 static int
1100 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
1101 {
1102         struct drm_device *dev = obj->dev;
1103         drm_i915_private_t *dev_priv = dev->dev_private;
1104         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1105         struct drm_mm_node *free_space;
1106         int page_count, ret;
1107
1108         if (alignment == 0)
1109                 alignment = PAGE_SIZE;
1110         if (alignment & (PAGE_SIZE - 1)) {
1111                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1112                 return -EINVAL;
1113         }
1114
1115  search_free:
1116         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1117                                         obj->size, alignment, 0);
1118         if (free_space != NULL) {
1119                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
1120                                                        alignment);
1121                 if (obj_priv->gtt_space != NULL) {
1122                         obj_priv->gtt_space->private = obj;
1123                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
1124                 }
1125         }
1126         if (obj_priv->gtt_space == NULL) {
1127                 /* If the gtt is empty and we're still having trouble
1128                  * fitting our object in, we're out of memory.
1129                  */
1130 #if WATCH_LRU
1131                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
1132 #endif
1133                 if (list_empty(&dev_priv->mm.inactive_list) &&
1134                     list_empty(&dev_priv->mm.flushing_list) &&
1135                     list_empty(&dev_priv->mm.active_list)) {
1136                         DRM_ERROR("GTT full, but LRU list empty\n");
1137                         return -ENOMEM;
1138                 }
1139
1140                 ret = i915_gem_evict_something(dev);
1141                 if (ret != 0) {
1142                         DRM_ERROR("Failed to evict a buffer %d\n", ret);
1143                         return ret;
1144                 }
1145                 goto search_free;
1146         }
1147
1148 #if WATCH_BUF
1149         DRM_INFO("Binding object of size %d at 0x%08x\n",
1150                  obj->size, obj_priv->gtt_offset);
1151 #endif
1152         ret = i915_gem_object_get_page_list(obj);
1153         if (ret) {
1154                 drm_mm_put_block(obj_priv->gtt_space);
1155                 obj_priv->gtt_space = NULL;
1156                 return ret;
1157         }
1158
1159         page_count = obj->size / PAGE_SIZE;
1160         /* Create an AGP memory structure pointing at our pages, and bind it
1161          * into the GTT.
1162          */
1163         obj_priv->agp_mem = drm_agp_bind_pages(dev,
1164                                                obj_priv->page_list,
1165                                                page_count,
1166                                                obj_priv->gtt_offset,
1167                                                obj_priv->agp_type);
1168         if (obj_priv->agp_mem == NULL) {
1169                 i915_gem_object_free_page_list(obj);
1170                 drm_mm_put_block(obj_priv->gtt_space);
1171                 obj_priv->gtt_space = NULL;
1172                 return -ENOMEM;
1173         }
1174         atomic_inc(&dev->gtt_count);
1175         atomic_add(obj->size, &dev->gtt_memory);
1176
1177         /* Assert that the object is not currently in any GPU domain. As it
1178          * wasn't in the GTT, there shouldn't be any way it could have been in
1179          * a GPU cache
1180          */
1181         BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1182         BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1183
1184         return 0;
1185 }
1186
1187 void
1188 i915_gem_clflush_object(struct drm_gem_object *obj)
1189 {
1190         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
1191
1192         /* If we don't have a page list set up, then we're not pinned
1193          * to GPU, and we can ignore the cache flush because it'll happen
1194          * again at bind time.
1195          */
1196         if (obj_priv->page_list == NULL)
1197                 return;
1198
1199         drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1200 }
1201
1202 /*
1203  * Set the next domain for the specified object. This
1204  * may not actually perform the necessary flushing/invaliding though,
1205  * as that may want to be batched with other set_domain operations
1206  *
1207  * This is (we hope) the only really tricky part of gem. The goal
1208  * is fairly simple -- track which caches hold bits of the object
1209  * and make sure they remain coherent. A few concrete examples may
1210  * help to explain how it works. For shorthand, we use the notation
1211  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
1212  * a pair of read and write domain masks.
1213  *
1214  * Case 1: the batch buffer
1215  *
1216  *      1. Allocated
1217  *      2. Written by CPU
1218  *      3. Mapped to GTT
1219  *      4. Read by GPU
1220  *      5. Unmapped from GTT
1221  *      6. Freed
1222  *
1223  *      Let's take these a step at a time
1224  *
1225  *      1. Allocated
1226  *              Pages allocated from the kernel may still have
1227  *              cache contents, so we set them to (CPU, CPU) always.
1228  *      2. Written by CPU (using pwrite)
1229  *              The pwrite function calls set_domain (CPU, CPU) and
1230  *              this function does nothing (as nothing changes)
1231  *      3. Mapped by GTT
1232  *              This function asserts that the object is not
1233  *              currently in any GPU-based read or write domains
1234  *      4. Read by GPU
1235  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
1236  *              As write_domain is zero, this function adds in the
1237  *              current read domains (CPU+COMMAND, 0).
1238  *              flush_domains is set to CPU.
1239  *              invalidate_domains is set to COMMAND
1240  *              clflush is run to get data out of the CPU caches
1241  *              then i915_dev_set_domain calls i915_gem_flush to
1242  *              emit an MI_FLUSH and drm_agp_chipset_flush
1243  *      5. Unmapped from GTT
1244  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
1245  *              flush_domains and invalidate_domains end up both zero
1246  *              so no flushing/invalidating happens
1247  *      6. Freed
1248  *              yay, done
1249  *
1250  * Case 2: The shared render buffer
1251  *
1252  *      1. Allocated
1253  *      2. Mapped to GTT
1254  *      3. Read/written by GPU
1255  *      4. set_domain to (CPU,CPU)
1256  *      5. Read/written by CPU
1257  *      6. Read/written by GPU
1258  *
1259  *      1. Allocated
1260  *              Same as last example, (CPU, CPU)
1261  *      2. Mapped to GTT
1262  *              Nothing changes (assertions find that it is not in the GPU)
1263  *      3. Read/written by GPU
1264  *              execbuffer calls set_domain (RENDER, RENDER)
1265  *              flush_domains gets CPU
1266  *              invalidate_domains gets GPU
1267  *              clflush (obj)
1268  *              MI_FLUSH and drm_agp_chipset_flush
1269  *      4. set_domain (CPU, CPU)
1270  *              flush_domains gets GPU
1271  *              invalidate_domains gets CPU
1272  *              wait_rendering (obj) to make sure all drawing is complete.
1273  *              This will include an MI_FLUSH to get the data from GPU
1274  *              to memory
1275  *              clflush (obj) to invalidate the CPU cache
1276  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
1277  *      5. Read/written by CPU
1278  *              cache lines are loaded and dirtied
1279  *      6. Read written by GPU
1280  *              Same as last GPU access
1281  *
1282  * Case 3: The constant buffer
1283  *
1284  *      1. Allocated
1285  *      2. Written by CPU
1286  *      3. Read by GPU
1287  *      4. Updated (written) by CPU again
1288  *      5. Read by GPU
1289  *
1290  *      1. Allocated
1291  *              (CPU, CPU)
1292  *      2. Written by CPU
1293  *              (CPU, CPU)
1294  *      3. Read by GPU
1295  *              (CPU+RENDER, 0)
1296  *              flush_domains = CPU
1297  *              invalidate_domains = RENDER
1298  *              clflush (obj)
1299  *              MI_FLUSH
1300  *              drm_agp_chipset_flush
1301  *      4. Updated (written) by CPU again
1302  *              (CPU, CPU)
1303  *              flush_domains = 0 (no previous write domain)
1304  *              invalidate_domains = 0 (no new read domains)
1305  *      5. Read by GPU
1306  *              (CPU+RENDER, 0)
1307  *              flush_domains = CPU
1308  *              invalidate_domains = RENDER
1309  *              clflush (obj)
1310  *              MI_FLUSH
1311  *              drm_agp_chipset_flush
1312  */
1313 static int
1314 i915_gem_object_set_domain(struct drm_gem_object *obj,
1315                             uint32_t read_domains,
1316                             uint32_t write_domain)
1317 {
1318         struct drm_device               *dev = obj->dev;
1319         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
1320         uint32_t                        invalidate_domains = 0;
1321         uint32_t                        flush_domains = 0;
1322         int                             ret;
1323
1324 #if WATCH_BUF
1325         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
1326                  __func__, obj,
1327                  obj->read_domains, read_domains,
1328                  obj->write_domain, write_domain);
1329 #endif
1330         /*
1331          * If the object isn't moving to a new write domain,
1332          * let the object stay in multiple read domains
1333          */
1334         if (write_domain == 0)
1335                 read_domains |= obj->read_domains;
1336         else
1337                 obj_priv->dirty = 1;
1338
1339         /*
1340          * Flush the current write domain if
1341          * the new read domains don't match. Invalidate
1342          * any read domains which differ from the old
1343          * write domain
1344          */
1345         if (obj->write_domain && obj->write_domain != read_domains) {
1346                 flush_domains |= obj->write_domain;
1347                 invalidate_domains |= read_domains & ~obj->write_domain;
1348         }
1349         /*
1350          * Invalidate any read caches which may have
1351          * stale data. That is, any new read domains.
1352          */
1353         invalidate_domains |= read_domains & ~obj->read_domains;
1354         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
1355 #if WATCH_BUF
1356                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
1357                          __func__, flush_domains, invalidate_domains);
1358 #endif
1359                 /*
1360                  * If we're invaliding the CPU cache and flushing a GPU cache,
1361                  * then pause for rendering so that the GPU caches will be
1362                  * flushed before the cpu cache is invalidated
1363                  */
1364                 if ((invalidate_domains & I915_GEM_DOMAIN_CPU) &&
1365                     (flush_domains & ~(I915_GEM_DOMAIN_CPU |
1366                                        I915_GEM_DOMAIN_GTT))) {
1367                         ret = i915_gem_object_wait_rendering(obj);
1368                         if (ret)
1369                                 return ret;
1370                 }
1371                 i915_gem_clflush_object(obj);
1372         }
1373
1374         if ((write_domain | flush_domains) != 0)
1375                 obj->write_domain = write_domain;
1376
1377         /* If we're invalidating the CPU domain, clear the per-page CPU
1378          * domain list as well.
1379          */
1380         if (obj_priv->page_cpu_valid != NULL &&
1381             (write_domain != 0 ||
1382              read_domains & I915_GEM_DOMAIN_CPU)) {
1383                 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1384                          DRM_MEM_DRIVER);
1385                 obj_priv->page_cpu_valid = NULL;
1386         }
1387         obj->read_domains = read_domains;
1388
1389         dev->invalidate_domains |= invalidate_domains;
1390         dev->flush_domains |= flush_domains;
1391 #if WATCH_BUF
1392         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
1393                  __func__,
1394                  obj->read_domains, obj->write_domain,
1395                  dev->invalidate_domains, dev->flush_domains);
1396 #endif
1397         return 0;
1398 }
1399
1400 /**
1401  * Set the read/write domain on a range of the object.
1402  *
1403  * Currently only implemented for CPU reads, otherwise drops to normal
1404  * i915_gem_object_set_domain().
1405  */
1406 static int
1407 i915_gem_object_set_domain_range(struct drm_gem_object *obj,
1408                                  uint64_t offset,
1409                                  uint64_t size,
1410                                  uint32_t read_domains,
1411                                  uint32_t write_domain)
1412 {
1413         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1414         int ret, i;
1415
1416         if (obj->read_domains & I915_GEM_DOMAIN_CPU)
1417                 return 0;
1418
1419         if (read_domains != I915_GEM_DOMAIN_CPU ||
1420             write_domain != 0)
1421                 return i915_gem_object_set_domain(obj,
1422                                                   read_domains, write_domain);
1423
1424         /* Wait on any GPU rendering to the object to be flushed. */
1425         if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) {
1426                 ret = i915_gem_object_wait_rendering(obj);
1427                 if (ret)
1428                         return ret;
1429         }
1430
1431         if (obj_priv->page_cpu_valid == NULL) {
1432                 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1433                                                       DRM_MEM_DRIVER);
1434         }
1435
1436         /* Flush the cache on any pages that are still invalid from the CPU's
1437          * perspective.
1438          */
1439         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) {
1440                 if (obj_priv->page_cpu_valid[i])
1441                         continue;
1442
1443                 drm_clflush_pages(obj_priv->page_list + i, 1);
1444
1445                 obj_priv->page_cpu_valid[i] = 1;
1446         }
1447
1448         return 0;
1449 }
1450
1451 /**
1452  * Once all of the objects have been set in the proper domain,
1453  * perform the necessary flush and invalidate operations.
1454  *
1455  * Returns the write domains flushed, for use in flush tracking.
1456  */
1457 static uint32_t
1458 i915_gem_dev_set_domain(struct drm_device *dev)
1459 {
1460         uint32_t flush_domains = dev->flush_domains;
1461
1462         /*
1463          * Now that all the buffers are synced to the proper domains,
1464          * flush and invalidate the collected domains
1465          */
1466         if (dev->invalidate_domains | dev->flush_domains) {
1467 #if WATCH_EXEC
1468                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
1469                           __func__,
1470                          dev->invalidate_domains,
1471                          dev->flush_domains);
1472 #endif
1473                 i915_gem_flush(dev,
1474                                dev->invalidate_domains,
1475                                dev->flush_domains);
1476                 dev->invalidate_domains = 0;
1477                 dev->flush_domains = 0;
1478         }
1479
1480         return flush_domains;
1481 }
1482
1483 /**
1484  * Pin an object to the GTT and evaluate the relocations landing in it.
1485  */
1486 static int
1487 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1488                                  struct drm_file *file_priv,
1489                                  struct drm_i915_gem_exec_object *entry)
1490 {
1491         struct drm_device *dev = obj->dev;
1492         struct drm_i915_gem_relocation_entry reloc;
1493         struct drm_i915_gem_relocation_entry __user *relocs;
1494         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1495         int i, ret;
1496         uint32_t last_reloc_offset = -1;
1497         void __iomem *reloc_page = NULL;
1498
1499         /* Choose the GTT offset for our buffer and put it there. */
1500         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
1501         if (ret)
1502                 return ret;
1503
1504         entry->offset = obj_priv->gtt_offset;
1505
1506         relocs = (struct drm_i915_gem_relocation_entry __user *)
1507                  (uintptr_t) entry->relocs_ptr;
1508         /* Apply the relocations, using the GTT aperture to avoid cache
1509          * flushing requirements.
1510          */
1511         for (i = 0; i < entry->relocation_count; i++) {
1512                 struct drm_gem_object *target_obj;
1513                 struct drm_i915_gem_object *target_obj_priv;
1514                 uint32_t reloc_val, reloc_offset;
1515                 uint32_t __iomem *reloc_entry;
1516
1517                 ret = copy_from_user(&reloc, relocs + i, sizeof(reloc));
1518                 if (ret != 0) {
1519                         i915_gem_object_unpin(obj);
1520                         return ret;
1521                 }
1522
1523                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
1524                                                    reloc.target_handle);
1525                 if (target_obj == NULL) {
1526                         i915_gem_object_unpin(obj);
1527                         return -EBADF;
1528                 }
1529                 target_obj_priv = target_obj->driver_private;
1530
1531                 /* The target buffer should have appeared before us in the
1532                  * exec_object list, so it should have a GTT space bound by now.
1533                  */
1534                 if (target_obj_priv->gtt_space == NULL) {
1535                         DRM_ERROR("No GTT space found for object %d\n",
1536                                   reloc.target_handle);
1537                         drm_gem_object_unreference(target_obj);
1538                         i915_gem_object_unpin(obj);
1539                         return -EINVAL;
1540                 }
1541
1542                 if (reloc.offset > obj->size - 4) {
1543                         DRM_ERROR("Relocation beyond object bounds: "
1544                                   "obj %p target %d offset %d size %d.\n",
1545                                   obj, reloc.target_handle,
1546                                   (int) reloc.offset, (int) obj->size);
1547                         drm_gem_object_unreference(target_obj);
1548                         i915_gem_object_unpin(obj);
1549                         return -EINVAL;
1550                 }
1551                 if (reloc.offset & 3) {
1552                         DRM_ERROR("Relocation not 4-byte aligned: "
1553                                   "obj %p target %d offset %d.\n",
1554                                   obj, reloc.target_handle,
1555                                   (int) reloc.offset);
1556                         drm_gem_object_unreference(target_obj);
1557                         i915_gem_object_unpin(obj);
1558                         return -EINVAL;
1559                 }
1560
1561                 if (reloc.write_domain && target_obj->pending_write_domain &&
1562                     reloc.write_domain != target_obj->pending_write_domain) {
1563                         DRM_ERROR("Write domain conflict: "
1564                                   "obj %p target %d offset %d "
1565                                   "new %08x old %08x\n",
1566                                   obj, reloc.target_handle,
1567                                   (int) reloc.offset,
1568                                   reloc.write_domain,
1569                                   target_obj->pending_write_domain);
1570                         drm_gem_object_unreference(target_obj);
1571                         i915_gem_object_unpin(obj);
1572                         return -EINVAL;
1573                 }
1574
1575 #if WATCH_RELOC
1576                 DRM_INFO("%s: obj %p offset %08x target %d "
1577                          "read %08x write %08x gtt %08x "
1578                          "presumed %08x delta %08x\n",
1579                          __func__,
1580                          obj,
1581                          (int) reloc.offset,
1582                          (int) reloc.target_handle,
1583                          (int) reloc.read_domains,
1584                          (int) reloc.write_domain,
1585                          (int) target_obj_priv->gtt_offset,
1586                          (int) reloc.presumed_offset,
1587                          reloc.delta);
1588 #endif
1589
1590                 target_obj->pending_read_domains |= reloc.read_domains;
1591                 target_obj->pending_write_domain |= reloc.write_domain;
1592
1593                 /* If the relocation already has the right value in it, no
1594                  * more work needs to be done.
1595                  */
1596                 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
1597                         drm_gem_object_unreference(target_obj);
1598                         continue;
1599                 }
1600
1601                 /* Now that we're going to actually write some data in,
1602                  * make sure that any rendering using this buffer's contents
1603                  * is completed.
1604                  */
1605                 i915_gem_object_wait_rendering(obj);
1606
1607                 /* As we're writing through the gtt, flush
1608                  * any CPU writes before we write the relocations
1609                  */
1610                 if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
1611                         i915_gem_clflush_object(obj);
1612                         drm_agp_chipset_flush(dev);
1613                         obj->write_domain = 0;
1614                 }
1615
1616                 /* Map the page containing the relocation we're going to
1617                  * perform.
1618                  */
1619                 reloc_offset = obj_priv->gtt_offset + reloc.offset;
1620                 if (reloc_page == NULL ||
1621                     (last_reloc_offset & ~(PAGE_SIZE - 1)) !=
1622                     (reloc_offset & ~(PAGE_SIZE - 1))) {
1623                         if (reloc_page != NULL)
1624                                 iounmap(reloc_page);
1625
1626                         reloc_page = ioremap_wc(dev->agp->base +
1627                                                 (reloc_offset &
1628                                                  ~(PAGE_SIZE - 1)),
1629                                                 PAGE_SIZE);
1630                         last_reloc_offset = reloc_offset;
1631                         if (reloc_page == NULL) {
1632                                 drm_gem_object_unreference(target_obj);
1633                                 i915_gem_object_unpin(obj);
1634                                 return -ENOMEM;
1635                         }
1636                 }
1637
1638                 reloc_entry = (uint32_t __iomem *)(reloc_page +
1639                                            (reloc_offset & (PAGE_SIZE - 1)));
1640                 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1641
1642 #if WATCH_BUF
1643                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
1644                           obj, (unsigned int) reloc.offset,
1645                           readl(reloc_entry), reloc_val);
1646 #endif
1647                 writel(reloc_val, reloc_entry);
1648
1649                 /* Write the updated presumed offset for this entry back out
1650                  * to the user.
1651                  */
1652                 reloc.presumed_offset = target_obj_priv->gtt_offset;
1653                 ret = copy_to_user(relocs + i, &reloc, sizeof(reloc));
1654                 if (ret != 0) {
1655                         drm_gem_object_unreference(target_obj);
1656                         i915_gem_object_unpin(obj);
1657                         return ret;
1658                 }
1659
1660                 drm_gem_object_unreference(target_obj);
1661         }
1662
1663         if (reloc_page != NULL)
1664                 iounmap(reloc_page);
1665
1666 #if WATCH_BUF
1667         if (0)
1668                 i915_gem_dump_object(obj, 128, __func__, ~0);
1669 #endif
1670         return 0;
1671 }
1672
1673 /** Dispatch a batchbuffer to the ring
1674  */
1675 static int
1676 i915_dispatch_gem_execbuffer(struct drm_device *dev,
1677                               struct drm_i915_gem_execbuffer *exec,
1678                               uint64_t exec_offset)
1679 {
1680         drm_i915_private_t *dev_priv = dev->dev_private;
1681         struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
1682                                              (uintptr_t) exec->cliprects_ptr;
1683         int nbox = exec->num_cliprects;
1684         int i = 0, count;
1685         uint32_t        exec_start, exec_len;
1686         RING_LOCALS;
1687
1688         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
1689         exec_len = (uint32_t) exec->batch_len;
1690
1691         if ((exec_start | exec_len) & 0x7) {
1692                 DRM_ERROR("alignment\n");
1693                 return -EINVAL;
1694         }
1695
1696         if (!exec_start)
1697                 return -EINVAL;
1698
1699         count = nbox ? nbox : 1;
1700
1701         for (i = 0; i < count; i++) {
1702                 if (i < nbox) {
1703                         int ret = i915_emit_box(dev, boxes, i,
1704                                                 exec->DR1, exec->DR4);
1705                         if (ret)
1706                                 return ret;
1707                 }
1708
1709                 if (IS_I830(dev) || IS_845G(dev)) {
1710                         BEGIN_LP_RING(4);
1711                         OUT_RING(MI_BATCH_BUFFER);
1712                         OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1713                         OUT_RING(exec_start + exec_len - 4);
1714                         OUT_RING(0);
1715                         ADVANCE_LP_RING();
1716                 } else {
1717                         BEGIN_LP_RING(2);
1718                         if (IS_I965G(dev)) {
1719                                 OUT_RING(MI_BATCH_BUFFER_START |
1720                                          (2 << 6) |
1721                                          MI_BATCH_NON_SECURE_I965);
1722                                 OUT_RING(exec_start);
1723                         } else {
1724                                 OUT_RING(MI_BATCH_BUFFER_START |
1725                                          (2 << 6));
1726                                 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1727                         }
1728                         ADVANCE_LP_RING();
1729                 }
1730         }
1731
1732         /* XXX breadcrumb */
1733         return 0;
1734 }
1735
1736 /* Throttle our rendering by waiting until the ring has completed our requests
1737  * emitted over 20 msec ago.
1738  *
1739  * This should get us reasonable parallelism between CPU and GPU but also
1740  * relatively low latency when blocking on a particular request to finish.
1741  */
1742 static int
1743 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
1744 {
1745         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1746         int ret = 0;
1747         uint32_t seqno;
1748
1749         mutex_lock(&dev->struct_mutex);
1750         seqno = i915_file_priv->mm.last_gem_throttle_seqno;
1751         i915_file_priv->mm.last_gem_throttle_seqno =
1752                 i915_file_priv->mm.last_gem_seqno;
1753         if (seqno)
1754                 ret = i915_wait_request(dev, seqno);
1755         mutex_unlock(&dev->struct_mutex);
1756         return ret;
1757 }
1758
1759 int
1760 i915_gem_execbuffer(struct drm_device *dev, void *data,
1761                     struct drm_file *file_priv)
1762 {
1763         drm_i915_private_t *dev_priv = dev->dev_private;
1764         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1765         struct drm_i915_gem_execbuffer *args = data;
1766         struct drm_i915_gem_exec_object *exec_list = NULL;
1767         struct drm_gem_object **object_list = NULL;
1768         struct drm_gem_object *batch_obj;
1769         int ret, i, pinned = 0;
1770         uint64_t exec_offset;
1771         uint32_t seqno, flush_domains;
1772
1773 #if WATCH_EXEC
1774         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1775                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1776 #endif
1777
1778         if (args->buffer_count < 1) {
1779                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1780                 return -EINVAL;
1781         }
1782         /* Copy in the exec list from userland */
1783         exec_list = drm_calloc(sizeof(*exec_list), args->buffer_count,
1784                                DRM_MEM_DRIVER);
1785         object_list = drm_calloc(sizeof(*object_list), args->buffer_count,
1786                                  DRM_MEM_DRIVER);
1787         if (exec_list == NULL || object_list == NULL) {
1788                 DRM_ERROR("Failed to allocate exec or object list "
1789                           "for %d buffers\n",
1790                           args->buffer_count);
1791                 ret = -ENOMEM;
1792                 goto pre_mutex_err;
1793         }
1794         ret = copy_from_user(exec_list,
1795                              (struct drm_i915_relocation_entry __user *)
1796                              (uintptr_t) args->buffers_ptr,
1797                              sizeof(*exec_list) * args->buffer_count);
1798         if (ret != 0) {
1799                 DRM_ERROR("copy %d exec entries failed %d\n",
1800                           args->buffer_count, ret);
1801                 goto pre_mutex_err;
1802         }
1803
1804         mutex_lock(&dev->struct_mutex);
1805
1806         i915_verify_inactive(dev, __FILE__, __LINE__);
1807
1808         if (dev_priv->mm.wedged) {
1809                 DRM_ERROR("Execbuf while wedged\n");
1810                 mutex_unlock(&dev->struct_mutex);
1811                 return -EIO;
1812         }
1813
1814         if (dev_priv->mm.suspended) {
1815                 DRM_ERROR("Execbuf while VT-switched.\n");
1816                 mutex_unlock(&dev->struct_mutex);
1817                 return -EBUSY;
1818         }
1819
1820         /* Zero the gloabl flush/invalidate flags. These
1821          * will be modified as each object is bound to the
1822          * gtt
1823          */
1824         dev->invalidate_domains = 0;
1825         dev->flush_domains = 0;
1826
1827         /* Look up object handles and perform the relocations */
1828         for (i = 0; i < args->buffer_count; i++) {
1829                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
1830                                                        exec_list[i].handle);
1831                 if (object_list[i] == NULL) {
1832                         DRM_ERROR("Invalid object handle %d at index %d\n",
1833                                    exec_list[i].handle, i);
1834                         ret = -EBADF;
1835                         goto err;
1836                 }
1837
1838                 object_list[i]->pending_read_domains = 0;
1839                 object_list[i]->pending_write_domain = 0;
1840                 ret = i915_gem_object_pin_and_relocate(object_list[i],
1841                                                        file_priv,
1842                                                        &exec_list[i]);
1843                 if (ret) {
1844                         DRM_ERROR("object bind and relocate failed %d\n", ret);
1845                         goto err;
1846                 }
1847                 pinned = i + 1;
1848         }
1849
1850         /* Set the pending read domains for the batch buffer to COMMAND */
1851         batch_obj = object_list[args->buffer_count-1];
1852         batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
1853         batch_obj->pending_write_domain = 0;
1854
1855         i915_verify_inactive(dev, __FILE__, __LINE__);
1856
1857         for (i = 0; i < args->buffer_count; i++) {
1858                 struct drm_gem_object *obj = object_list[i];
1859                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1860
1861                 if (obj_priv->gtt_space == NULL) {
1862                         /* We evicted the buffer in the process of validating
1863                          * our set of buffers in.  We could try to recover by
1864                          * kicking them everything out and trying again from
1865                          * the start.
1866                          */
1867                         ret = -ENOMEM;
1868                         goto err;
1869                 }
1870
1871                 /* make sure all previous memory operations have passed */
1872                 ret = i915_gem_object_set_domain(obj,
1873                                                  obj->pending_read_domains,
1874                                                  obj->pending_write_domain);
1875                 if (ret)
1876                         goto err;
1877         }
1878
1879         i915_verify_inactive(dev, __FILE__, __LINE__);
1880
1881         /* Flush/invalidate caches and chipset buffer */
1882         flush_domains = i915_gem_dev_set_domain(dev);
1883
1884         i915_verify_inactive(dev, __FILE__, __LINE__);
1885
1886 #if WATCH_COHERENCY
1887         for (i = 0; i < args->buffer_count; i++) {
1888                 i915_gem_object_check_coherency(object_list[i],
1889                                                 exec_list[i].handle);
1890         }
1891 #endif
1892
1893         exec_offset = exec_list[args->buffer_count - 1].offset;
1894
1895 #if WATCH_EXEC
1896         i915_gem_dump_object(object_list[args->buffer_count - 1],
1897                               args->batch_len,
1898                               __func__,
1899                               ~0);
1900 #endif
1901
1902         (void)i915_add_request(dev, flush_domains);
1903
1904         /* Exec the batchbuffer */
1905         ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
1906         if (ret) {
1907                 DRM_ERROR("dispatch failed %d\n", ret);
1908                 goto err;
1909         }
1910
1911         /*
1912          * Ensure that the commands in the batch buffer are
1913          * finished before the interrupt fires
1914          */
1915         flush_domains = i915_retire_commands(dev);
1916
1917         i915_verify_inactive(dev, __FILE__, __LINE__);
1918
1919         /*
1920          * Get a seqno representing the execution of the current buffer,
1921          * which we can wait on.  We would like to mitigate these interrupts,
1922          * likely by only creating seqnos occasionally (so that we have
1923          * *some* interrupts representing completion of buffers that we can
1924          * wait on when trying to clear up gtt space).
1925          */
1926         seqno = i915_add_request(dev, flush_domains);
1927         BUG_ON(seqno == 0);
1928         i915_file_priv->mm.last_gem_seqno = seqno;
1929         for (i = 0; i < args->buffer_count; i++) {
1930                 struct drm_gem_object *obj = object_list[i];
1931                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1932
1933                 i915_gem_object_move_to_active(obj);
1934                 obj_priv->last_rendering_seqno = seqno;
1935 #if WATCH_LRU
1936                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
1937 #endif
1938         }
1939 #if WATCH_LRU
1940         i915_dump_lru(dev, __func__);
1941 #endif
1942
1943         i915_verify_inactive(dev, __FILE__, __LINE__);
1944
1945         /* Copy the new buffer offsets back to the user's exec list. */
1946         ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1947                            (uintptr_t) args->buffers_ptr,
1948                            exec_list,
1949                            sizeof(*exec_list) * args->buffer_count);
1950         if (ret)
1951                 DRM_ERROR("failed to copy %d exec entries "
1952                           "back to user (%d)\n",
1953                            args->buffer_count, ret);
1954 err:
1955         if (object_list != NULL) {
1956                 for (i = 0; i < pinned; i++)
1957                         i915_gem_object_unpin(object_list[i]);
1958
1959                 for (i = 0; i < args->buffer_count; i++)
1960                         drm_gem_object_unreference(object_list[i]);
1961         }
1962         mutex_unlock(&dev->struct_mutex);
1963
1964 pre_mutex_err:
1965         drm_free(object_list, sizeof(*object_list) * args->buffer_count,
1966                  DRM_MEM_DRIVER);
1967         drm_free(exec_list, sizeof(*exec_list) * args->buffer_count,
1968                  DRM_MEM_DRIVER);
1969
1970         return ret;
1971 }
1972
1973 int
1974 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
1975 {
1976         struct drm_device *dev = obj->dev;
1977         struct drm_i915_gem_object *obj_priv = obj->driver_private;
1978         int ret;
1979
1980         i915_verify_inactive(dev, __FILE__, __LINE__);
1981         if (obj_priv->gtt_space == NULL) {
1982                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
1983                 if (ret != 0) {
1984                         DRM_ERROR("Failure to bind: %d", ret);
1985                         return ret;
1986                 }
1987         }
1988         obj_priv->pin_count++;
1989
1990         /* If the object is not active and not pending a flush,
1991          * remove it from the inactive list
1992          */
1993         if (obj_priv->pin_count == 1) {
1994                 atomic_inc(&dev->pin_count);
1995                 atomic_add(obj->size, &dev->pin_memory);
1996                 if (!obj_priv->active &&
1997                     (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
1998                                            I915_GEM_DOMAIN_GTT)) == 0 &&
1999                     !list_empty(&obj_priv->list))
2000                         list_del_init(&obj_priv->list);
2001         }
2002         i915_verify_inactive(dev, __FILE__, __LINE__);
2003
2004         return 0;
2005 }
2006
2007 void
2008 i915_gem_object_unpin(struct drm_gem_object *obj)
2009 {
2010         struct drm_device *dev = obj->dev;
2011         drm_i915_private_t *dev_priv = dev->dev_private;
2012         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2013
2014         i915_verify_inactive(dev, __FILE__, __LINE__);
2015         obj_priv->pin_count--;
2016         BUG_ON(obj_priv->pin_count < 0);
2017         BUG_ON(obj_priv->gtt_space == NULL);
2018
2019         /* If the object is no longer pinned, and is
2020          * neither active nor being flushed, then stick it on
2021          * the inactive list
2022          */
2023         if (obj_priv->pin_count == 0) {
2024                 if (!obj_priv->active &&
2025                     (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2026                                            I915_GEM_DOMAIN_GTT)) == 0)
2027                         list_move_tail(&obj_priv->list,
2028                                        &dev_priv->mm.inactive_list);
2029                 atomic_dec(&dev->pin_count);
2030                 atomic_sub(obj->size, &dev->pin_memory);
2031         }
2032         i915_verify_inactive(dev, __FILE__, __LINE__);
2033 }
2034
2035 int
2036 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2037                    struct drm_file *file_priv)
2038 {
2039         struct drm_i915_gem_pin *args = data;
2040         struct drm_gem_object *obj;
2041         struct drm_i915_gem_object *obj_priv;
2042         int ret;
2043
2044         mutex_lock(&dev->struct_mutex);
2045
2046         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2047         if (obj == NULL) {
2048                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
2049                           args->handle);
2050                 mutex_unlock(&dev->struct_mutex);
2051                 return -EBADF;
2052         }
2053         obj_priv = obj->driver_private;
2054
2055         ret = i915_gem_object_pin(obj, args->alignment);
2056         if (ret != 0) {
2057                 drm_gem_object_unreference(obj);
2058                 mutex_unlock(&dev->struct_mutex);
2059                 return ret;
2060         }
2061
2062         /* XXX - flush the CPU caches for pinned objects
2063          * as the X server doesn't manage domains yet
2064          */
2065         if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
2066                 i915_gem_clflush_object(obj);
2067                 drm_agp_chipset_flush(dev);
2068                 obj->write_domain = 0;
2069         }
2070         args->offset = obj_priv->gtt_offset;
2071         drm_gem_object_unreference(obj);
2072         mutex_unlock(&dev->struct_mutex);
2073
2074         return 0;
2075 }
2076
2077 int
2078 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2079                      struct drm_file *file_priv)
2080 {
2081         struct drm_i915_gem_pin *args = data;
2082         struct drm_gem_object *obj;
2083
2084         mutex_lock(&dev->struct_mutex);
2085
2086         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2087         if (obj == NULL) {
2088                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
2089                           args->handle);
2090                 mutex_unlock(&dev->struct_mutex);
2091                 return -EBADF;
2092         }
2093
2094         i915_gem_object_unpin(obj);
2095
2096         drm_gem_object_unreference(obj);
2097         mutex_unlock(&dev->struct_mutex);
2098         return 0;
2099 }
2100
2101 int
2102 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2103                     struct drm_file *file_priv)
2104 {
2105         struct drm_i915_gem_busy *args = data;
2106         struct drm_gem_object *obj;
2107         struct drm_i915_gem_object *obj_priv;
2108
2109         mutex_lock(&dev->struct_mutex);
2110         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2111         if (obj == NULL) {
2112                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
2113                           args->handle);
2114                 mutex_unlock(&dev->struct_mutex);
2115                 return -EBADF;
2116         }
2117
2118         obj_priv = obj->driver_private;
2119         args->busy = obj_priv->active;
2120
2121         drm_gem_object_unreference(obj);
2122         mutex_unlock(&dev->struct_mutex);
2123         return 0;
2124 }
2125
2126 int
2127 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2128                         struct drm_file *file_priv)
2129 {
2130     return i915_gem_ring_throttle(dev, file_priv);
2131 }
2132
2133 int i915_gem_init_object(struct drm_gem_object *obj)
2134 {
2135         struct drm_i915_gem_object *obj_priv;
2136
2137         obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
2138         if (obj_priv == NULL)
2139                 return -ENOMEM;
2140
2141         /*
2142          * We've just allocated pages from the kernel,
2143          * so they've just been written by the CPU with
2144          * zeros. They'll need to be clflushed before we
2145          * use them with the GPU.
2146          */
2147         obj->write_domain = I915_GEM_DOMAIN_CPU;
2148         obj->read_domains = I915_GEM_DOMAIN_CPU;
2149
2150         obj_priv->agp_type = AGP_USER_MEMORY;
2151
2152         obj->driver_private = obj_priv;
2153         obj_priv->obj = obj;
2154         INIT_LIST_HEAD(&obj_priv->list);
2155         return 0;
2156 }
2157
2158 void i915_gem_free_object(struct drm_gem_object *obj)
2159 {
2160         struct drm_i915_gem_object *obj_priv = obj->driver_private;
2161
2162         while (obj_priv->pin_count > 0)
2163                 i915_gem_object_unpin(obj);
2164
2165         i915_gem_object_unbind(obj);
2166
2167         drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
2168         drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
2169 }
2170
2171 static int
2172 i915_gem_set_domain(struct drm_gem_object *obj,
2173                     struct drm_file *file_priv,
2174                     uint32_t read_domains,
2175                     uint32_t write_domain)
2176 {
2177         struct drm_device *dev = obj->dev;
2178         int ret;
2179         uint32_t flush_domains;
2180
2181         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
2182
2183         ret = i915_gem_object_set_domain(obj, read_domains, write_domain);
2184         if (ret)
2185                 return ret;
2186         flush_domains = i915_gem_dev_set_domain(obj->dev);
2187
2188         if (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))
2189                 (void) i915_add_request(dev, flush_domains);
2190
2191         return 0;
2192 }
2193
2194 /** Unbinds all objects that are on the given buffer list. */
2195 static int
2196 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
2197 {
2198         struct drm_gem_object *obj;
2199         struct drm_i915_gem_object *obj_priv;
2200         int ret;
2201
2202         while (!list_empty(head)) {
2203                 obj_priv = list_first_entry(head,
2204                                             struct drm_i915_gem_object,
2205                                             list);
2206                 obj = obj_priv->obj;
2207
2208                 if (obj_priv->pin_count != 0) {
2209                         DRM_ERROR("Pinned object in unbind list\n");
2210                         mutex_unlock(&dev->struct_mutex);
2211                         return -EINVAL;
2212                 }
2213
2214                 ret = i915_gem_object_unbind(obj);
2215                 if (ret != 0) {
2216                         DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
2217                                   ret);
2218                         mutex_unlock(&dev->struct_mutex);
2219                         return ret;
2220                 }
2221         }
2222
2223
2224         return 0;
2225 }
2226
2227 static int
2228 i915_gem_idle(struct drm_device *dev)
2229 {
2230         drm_i915_private_t *dev_priv = dev->dev_private;
2231         uint32_t seqno, cur_seqno, last_seqno;
2232         int stuck, ret;
2233
2234         mutex_lock(&dev->struct_mutex);
2235
2236         if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
2237                 mutex_unlock(&dev->struct_mutex);
2238                 return 0;
2239         }
2240
2241         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2242          * We need to replace this with a semaphore, or something.
2243          */
2244         dev_priv->mm.suspended = 1;
2245
2246         /* Cancel the retire work handler, wait for it to finish if running
2247          */
2248         mutex_unlock(&dev->struct_mutex);
2249         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2250         mutex_lock(&dev->struct_mutex);
2251
2252         i915_kernel_lost_context(dev);
2253
2254         /* Flush the GPU along with all non-CPU write domains
2255          */
2256         i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
2257                        ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2258         seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
2259                                         I915_GEM_DOMAIN_GTT));
2260
2261         if (seqno == 0) {
2262                 mutex_unlock(&dev->struct_mutex);
2263                 return -ENOMEM;
2264         }
2265
2266         dev_priv->mm.waiting_gem_seqno = seqno;
2267         last_seqno = 0;
2268         stuck = 0;
2269         for (;;) {
2270                 cur_seqno = i915_get_gem_seqno(dev);
2271                 if (i915_seqno_passed(cur_seqno, seqno))
2272                         break;
2273                 if (last_seqno == cur_seqno) {
2274                         if (stuck++ > 100) {
2275                                 DRM_ERROR("hardware wedged\n");
2276                                 dev_priv->mm.wedged = 1;
2277                                 DRM_WAKEUP(&dev_priv->irq_queue);
2278                                 break;
2279                         }
2280                 }
2281                 msleep(10);
2282                 last_seqno = cur_seqno;
2283         }
2284         dev_priv->mm.waiting_gem_seqno = 0;
2285
2286         i915_gem_retire_requests(dev);
2287
2288         /* Active and flushing should now be empty as we've
2289          * waited for a sequence higher than any pending execbuffer
2290          */
2291         BUG_ON(!list_empty(&dev_priv->mm.active_list));
2292         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2293
2294         /* Request should now be empty as we've also waited
2295          * for the last request in the list
2296          */
2297         BUG_ON(!list_empty(&dev_priv->mm.request_list));
2298
2299         /* Move all buffers out of the GTT. */
2300         ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
2301         if (ret) {
2302                 mutex_unlock(&dev->struct_mutex);
2303                 return ret;
2304         }
2305
2306         BUG_ON(!list_empty(&dev_priv->mm.active_list));
2307         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2308         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2309         BUG_ON(!list_empty(&dev_priv->mm.request_list));
2310
2311         i915_gem_cleanup_ringbuffer(dev);
2312         mutex_unlock(&dev->struct_mutex);
2313
2314         return 0;
2315 }
2316
2317 static int
2318 i915_gem_init_hws(struct drm_device *dev)
2319 {
2320         drm_i915_private_t *dev_priv = dev->dev_private;
2321         struct drm_gem_object *obj;
2322         struct drm_i915_gem_object *obj_priv;
2323         int ret;
2324
2325         /* If we need a physical address for the status page, it's already
2326          * initialized at driver load time.
2327          */
2328         if (!I915_NEED_GFX_HWS(dev))
2329                 return 0;
2330
2331         obj = drm_gem_object_alloc(dev, 4096);
2332         if (obj == NULL) {
2333                 DRM_ERROR("Failed to allocate status page\n");
2334                 return -ENOMEM;
2335         }
2336         obj_priv = obj->driver_private;
2337         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
2338
2339         ret = i915_gem_object_pin(obj, 4096);
2340         if (ret != 0) {
2341                 drm_gem_object_unreference(obj);
2342                 return ret;
2343         }
2344
2345         dev_priv->status_gfx_addr = obj_priv->gtt_offset;
2346
2347         dev_priv->hw_status_page = kmap(obj_priv->page_list[0]);
2348         if (dev_priv->hw_status_page == NULL) {
2349                 DRM_ERROR("Failed to map status page.\n");
2350                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2351                 drm_gem_object_unreference(obj);
2352                 return -EINVAL;
2353         }
2354         dev_priv->hws_obj = obj;
2355         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
2356         I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
2357         I915_READ(HWS_PGA); /* posting read */
2358         DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
2359
2360         return 0;
2361 }
2362
2363 static int
2364 i915_gem_init_ringbuffer(struct drm_device *dev)
2365 {
2366         drm_i915_private_t *dev_priv = dev->dev_private;
2367         struct drm_gem_object *obj;
2368         struct drm_i915_gem_object *obj_priv;
2369         int ret;
2370         u32 head;
2371
2372         ret = i915_gem_init_hws(dev);
2373         if (ret != 0)
2374                 return ret;
2375
2376         obj = drm_gem_object_alloc(dev, 128 * 1024);
2377         if (obj == NULL) {
2378                 DRM_ERROR("Failed to allocate ringbuffer\n");
2379                 return -ENOMEM;
2380         }
2381         obj_priv = obj->driver_private;
2382
2383         ret = i915_gem_object_pin(obj, 4096);
2384         if (ret != 0) {
2385                 drm_gem_object_unreference(obj);
2386                 return ret;
2387         }
2388
2389         /* Set up the kernel mapping for the ring. */
2390         dev_priv->ring.Size = obj->size;
2391         dev_priv->ring.tail_mask = obj->size - 1;
2392
2393         dev_priv->ring.map.offset = dev->agp->base + obj_priv->gtt_offset;
2394         dev_priv->ring.map.size = obj->size;
2395         dev_priv->ring.map.type = 0;
2396         dev_priv->ring.map.flags = 0;
2397         dev_priv->ring.map.mtrr = 0;
2398
2399         drm_core_ioremap_wc(&dev_priv->ring.map, dev);
2400         if (dev_priv->ring.map.handle == NULL) {
2401                 DRM_ERROR("Failed to map ringbuffer.\n");
2402                 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2403                 drm_gem_object_unreference(obj);
2404                 return -EINVAL;
2405         }
2406         dev_priv->ring.ring_obj = obj;
2407         dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
2408
2409         /* Stop the ring if it's running. */
2410         I915_WRITE(PRB0_CTL, 0);
2411         I915_WRITE(PRB0_TAIL, 0);
2412         I915_WRITE(PRB0_HEAD, 0);
2413
2414         /* Initialize the ring. */
2415         I915_WRITE(PRB0_START, obj_priv->gtt_offset);
2416         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2417
2418         /* G45 ring initialization fails to reset head to zero */
2419         if (head != 0) {
2420                 DRM_ERROR("Ring head not reset to zero "
2421                           "ctl %08x head %08x tail %08x start %08x\n",
2422                           I915_READ(PRB0_CTL),
2423                           I915_READ(PRB0_HEAD),
2424                           I915_READ(PRB0_TAIL),
2425                           I915_READ(PRB0_START));
2426                 I915_WRITE(PRB0_HEAD, 0);
2427
2428                 DRM_ERROR("Ring head forced to zero "
2429                           "ctl %08x head %08x tail %08x start %08x\n",
2430                           I915_READ(PRB0_CTL),
2431                           I915_READ(PRB0_HEAD),
2432                           I915_READ(PRB0_TAIL),
2433                           I915_READ(PRB0_START));
2434         }
2435
2436         I915_WRITE(PRB0_CTL,
2437                    ((obj->size - 4096) & RING_NR_PAGES) |
2438                    RING_NO_REPORT |
2439                    RING_VALID);
2440
2441         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2442
2443         /* If the head is still not zero, the ring is dead */
2444         if (head != 0) {
2445                 DRM_ERROR("Ring initialization failed "
2446                           "ctl %08x head %08x tail %08x start %08x\n",
2447                           I915_READ(PRB0_CTL),
2448                           I915_READ(PRB0_HEAD),
2449                           I915_READ(PRB0_TAIL),
2450                           I915_READ(PRB0_START));
2451                 return -EIO;
2452         }
2453
2454         /* Update our cache of the ring state */
2455         i915_kernel_lost_context(dev);
2456
2457         return 0;
2458 }
2459
2460 static void
2461 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2462 {
2463         drm_i915_private_t *dev_priv = dev->dev_private;
2464
2465         if (dev_priv->ring.ring_obj == NULL)
2466                 return;
2467
2468         drm_core_ioremapfree(&dev_priv->ring.map, dev);
2469
2470         i915_gem_object_unpin(dev_priv->ring.ring_obj);
2471         drm_gem_object_unreference(dev_priv->ring.ring_obj);
2472         dev_priv->ring.ring_obj = NULL;
2473         memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2474
2475         if (dev_priv->hws_obj != NULL) {
2476                 struct drm_gem_object *obj = dev_priv->hws_obj;
2477                 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2478
2479                 kunmap(obj_priv->page_list[0]);
2480                 i915_gem_object_unpin(obj);
2481                 drm_gem_object_unreference(obj);
2482                 dev_priv->hws_obj = NULL;
2483                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2484                 dev_priv->hw_status_page = NULL;
2485
2486                 /* Write high address into HWS_PGA when disabling. */
2487                 I915_WRITE(HWS_PGA, 0x1ffff000);
2488         }
2489 }
2490
2491 int
2492 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2493                        struct drm_file *file_priv)
2494 {
2495         drm_i915_private_t *dev_priv = dev->dev_private;
2496         int ret;
2497
2498         if (dev_priv->mm.wedged) {
2499                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2500                 dev_priv->mm.wedged = 0;
2501         }
2502
2503         ret = i915_gem_init_ringbuffer(dev);
2504         if (ret != 0)
2505                 return ret;
2506
2507         mutex_lock(&dev->struct_mutex);
2508         BUG_ON(!list_empty(&dev_priv->mm.active_list));
2509         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2510         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2511         BUG_ON(!list_empty(&dev_priv->mm.request_list));
2512         dev_priv->mm.suspended = 0;
2513         mutex_unlock(&dev->struct_mutex);
2514
2515         drm_irq_install(dev);
2516
2517         return 0;
2518 }
2519
2520 int
2521 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2522                        struct drm_file *file_priv)
2523 {
2524         int ret;
2525
2526         ret = i915_gem_idle(dev);
2527         drm_irq_uninstall(dev);
2528
2529         return ret;
2530 }
2531
2532 void
2533 i915_gem_lastclose(struct drm_device *dev)
2534 {
2535         int ret;
2536
2537         ret = i915_gem_idle(dev);
2538         if (ret)
2539                 DRM_ERROR("failed to idle hardware: %d\n", ret);
2540 }
2541
2542 void
2543 i915_gem_load(struct drm_device *dev)
2544 {
2545         drm_i915_private_t *dev_priv = dev->dev_private;
2546
2547         INIT_LIST_HEAD(&dev_priv->mm.active_list);
2548         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2549         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2550         INIT_LIST_HEAD(&dev_priv->mm.request_list);
2551         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2552                           i915_gem_retire_work_handler);
2553         INIT_WORK(&dev_priv->mm.vblank_work,
2554                   i915_gem_vblank_work_handler);
2555         dev_priv->mm.next_gem_seqno = 1;
2556
2557         i915_gem_detect_bit_6_swizzle(dev);
2558 }