Merge branch 'master' of /home/tglx/work/mtd/git/linux-2.6.git/
[linux-2.6] / drivers / char / drm / savage_state.c
1 /* savage_state.c -- State and drawing support for Savage
2  *
3  * Copyright 2004  Felix Kuehling
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sub license,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial portions
15  * of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20  * NON-INFRINGEMENT. IN NO EVENT SHALL FELIX KUEHLING BE LIABLE FOR
21  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
22  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  */
25 #include "drmP.h"
26 #include "savage_drm.h"
27 #include "savage_drv.h"
28
29 void savage_emit_clip_rect_s3d(drm_savage_private_t * dev_priv,
30                                drm_clip_rect_t * pbox)
31 {
32         uint32_t scstart = dev_priv->state.s3d.new_scstart;
33         uint32_t scend = dev_priv->state.s3d.new_scend;
34         scstart = (scstart & ~SAVAGE_SCISSOR_MASK_S3D) |
35             ((uint32_t) pbox->x1 & 0x000007ff) |
36             (((uint32_t) pbox->y1 << 16) & 0x07ff0000);
37         scend = (scend & ~SAVAGE_SCISSOR_MASK_S3D) |
38             (((uint32_t) pbox->x2 - 1) & 0x000007ff) |
39             ((((uint32_t) pbox->y2 - 1) << 16) & 0x07ff0000);
40         if (scstart != dev_priv->state.s3d.scstart ||
41             scend != dev_priv->state.s3d.scend) {
42                 DMA_LOCALS;
43                 BEGIN_DMA(4);
44                 DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
45                 DMA_SET_REGISTERS(SAVAGE_SCSTART_S3D, 2);
46                 DMA_WRITE(scstart);
47                 DMA_WRITE(scend);
48                 dev_priv->state.s3d.scstart = scstart;
49                 dev_priv->state.s3d.scend = scend;
50                 dev_priv->waiting = 1;
51                 DMA_COMMIT();
52         }
53 }
54
55 void savage_emit_clip_rect_s4(drm_savage_private_t * dev_priv,
56                               drm_clip_rect_t * pbox)
57 {
58         uint32_t drawctrl0 = dev_priv->state.s4.new_drawctrl0;
59         uint32_t drawctrl1 = dev_priv->state.s4.new_drawctrl1;
60         drawctrl0 = (drawctrl0 & ~SAVAGE_SCISSOR_MASK_S4) |
61             ((uint32_t) pbox->x1 & 0x000007ff) |
62             (((uint32_t) pbox->y1 << 12) & 0x00fff000);
63         drawctrl1 = (drawctrl1 & ~SAVAGE_SCISSOR_MASK_S4) |
64             (((uint32_t) pbox->x2 - 1) & 0x000007ff) |
65             ((((uint32_t) pbox->y2 - 1) << 12) & 0x00fff000);
66         if (drawctrl0 != dev_priv->state.s4.drawctrl0 ||
67             drawctrl1 != dev_priv->state.s4.drawctrl1) {
68                 DMA_LOCALS;
69                 BEGIN_DMA(4);
70                 DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
71                 DMA_SET_REGISTERS(SAVAGE_DRAWCTRL0_S4, 2);
72                 DMA_WRITE(drawctrl0);
73                 DMA_WRITE(drawctrl1);
74                 dev_priv->state.s4.drawctrl0 = drawctrl0;
75                 dev_priv->state.s4.drawctrl1 = drawctrl1;
76                 dev_priv->waiting = 1;
77                 DMA_COMMIT();
78         }
79 }
80
81 static int savage_verify_texaddr(drm_savage_private_t * dev_priv, int unit,
82                                  uint32_t addr)
83 {
84         if ((addr & 6) != 2) {  /* reserved bits */
85                 DRM_ERROR("bad texAddr%d %08x (reserved bits)\n", unit, addr);
86                 return DRM_ERR(EINVAL);
87         }
88         if (!(addr & 1)) {      /* local */
89                 addr &= ~7;
90                 if (addr < dev_priv->texture_offset ||
91                     addr >= dev_priv->texture_offset + dev_priv->texture_size) {
92                         DRM_ERROR
93                             ("bad texAddr%d %08x (local addr out of range)\n",
94                              unit, addr);
95                         return DRM_ERR(EINVAL);
96                 }
97         } else {                /* AGP */
98                 if (!dev_priv->agp_textures) {
99                         DRM_ERROR("bad texAddr%d %08x (AGP not available)\n",
100                                   unit, addr);
101                         return DRM_ERR(EINVAL);
102                 }
103                 addr &= ~7;
104                 if (addr < dev_priv->agp_textures->offset ||
105                     addr >= (dev_priv->agp_textures->offset +
106                              dev_priv->agp_textures->size)) {
107                         DRM_ERROR
108                             ("bad texAddr%d %08x (AGP addr out of range)\n",
109                              unit, addr);
110                         return DRM_ERR(EINVAL);
111                 }
112         }
113         return 0;
114 }
115
116 #define SAVE_STATE(reg,where)                   \
117         if(start <= reg && start+count > reg)   \
118                 DRM_GET_USER_UNCHECKED(dev_priv->state.where, &regs[reg-start])
119 #define SAVE_STATE_MASK(reg,where,mask) do {                    \
120         if(start <= reg && start+count > reg) {                 \
121                 uint32_t tmp;                                   \
122                 DRM_GET_USER_UNCHECKED(tmp, &regs[reg-start]);  \
123                 dev_priv->state.where = (tmp & (mask)) |        \
124                         (dev_priv->state.where & ~(mask));      \
125         }                                                       \
126 } while (0)
127 static int savage_verify_state_s3d(drm_savage_private_t * dev_priv,
128                                    unsigned int start, unsigned int count,
129                                    const uint32_t __user * regs)
130 {
131         if (start < SAVAGE_TEXPALADDR_S3D ||
132             start + count - 1 > SAVAGE_DESTTEXRWWATERMARK_S3D) {
133                 DRM_ERROR("invalid register range (0x%04x-0x%04x)\n",
134                           start, start + count - 1);
135                 return DRM_ERR(EINVAL);
136         }
137
138         SAVE_STATE_MASK(SAVAGE_SCSTART_S3D, s3d.new_scstart,
139                         ~SAVAGE_SCISSOR_MASK_S3D);
140         SAVE_STATE_MASK(SAVAGE_SCEND_S3D, s3d.new_scend,
141                         ~SAVAGE_SCISSOR_MASK_S3D);
142
143         /* if any texture regs were changed ... */
144         if (start <= SAVAGE_TEXCTRL_S3D &&
145             start + count > SAVAGE_TEXPALADDR_S3D) {
146                 /* ... check texture state */
147                 SAVE_STATE(SAVAGE_TEXCTRL_S3D, s3d.texctrl);
148                 SAVE_STATE(SAVAGE_TEXADDR_S3D, s3d.texaddr);
149                 if (dev_priv->state.s3d.texctrl & SAVAGE_TEXCTRL_TEXEN_MASK)
150                         return savage_verify_texaddr(dev_priv, 0,
151                                                      dev_priv->state.s3d.
152                                                      texaddr);
153         }
154
155         return 0;
156 }
157
158 static int savage_verify_state_s4(drm_savage_private_t * dev_priv,
159                                   unsigned int start, unsigned int count,
160                                   const uint32_t __user * regs)
161 {
162         int ret = 0;
163
164         if (start < SAVAGE_DRAWLOCALCTRL_S4 ||
165             start + count - 1 > SAVAGE_TEXBLENDCOLOR_S4) {
166                 DRM_ERROR("invalid register range (0x%04x-0x%04x)\n",
167                           start, start + count - 1);
168                 return DRM_ERR(EINVAL);
169         }
170
171         SAVE_STATE_MASK(SAVAGE_DRAWCTRL0_S4, s4.new_drawctrl0,
172                         ~SAVAGE_SCISSOR_MASK_S4);
173         SAVE_STATE_MASK(SAVAGE_DRAWCTRL1_S4, s4.new_drawctrl1,
174                         ~SAVAGE_SCISSOR_MASK_S4);
175
176         /* if any texture regs were changed ... */
177         if (start <= SAVAGE_TEXDESCR_S4 && start + count > SAVAGE_TEXPALADDR_S4) {
178                 /* ... check texture state */
179                 SAVE_STATE(SAVAGE_TEXDESCR_S4, s4.texdescr);
180                 SAVE_STATE(SAVAGE_TEXADDR0_S4, s4.texaddr0);
181                 SAVE_STATE(SAVAGE_TEXADDR1_S4, s4.texaddr1);
182                 if (dev_priv->state.s4.texdescr & SAVAGE_TEXDESCR_TEX0EN_MASK)
183                         ret |=
184                             savage_verify_texaddr(dev_priv, 0,
185                                                   dev_priv->state.s4.texaddr0);
186                 if (dev_priv->state.s4.texdescr & SAVAGE_TEXDESCR_TEX1EN_MASK)
187                         ret |=
188                             savage_verify_texaddr(dev_priv, 1,
189                                                   dev_priv->state.s4.texaddr1);
190         }
191
192         return ret;
193 }
194
195 #undef SAVE_STATE
196 #undef SAVE_STATE_MASK
197
198 static int savage_dispatch_state(drm_savage_private_t * dev_priv,
199                                  const drm_savage_cmd_header_t * cmd_header,
200                                  const uint32_t __user * regs)
201 {
202         unsigned int count = cmd_header->state.count;
203         unsigned int start = cmd_header->state.start;
204         unsigned int count2 = 0;
205         unsigned int bci_size;
206         int ret;
207         DMA_LOCALS;
208
209         if (!count)
210                 return 0;
211
212         if (DRM_VERIFYAREA_READ(regs, count * 4))
213                 return DRM_ERR(EFAULT);
214
215         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
216                 ret = savage_verify_state_s3d(dev_priv, start, count, regs);
217                 if (ret != 0)
218                         return ret;
219                 /* scissor regs are emitted in savage_dispatch_draw */
220                 if (start < SAVAGE_SCSTART_S3D) {
221                         if (start + count > SAVAGE_SCEND_S3D + 1)
222                                 count2 = count - (SAVAGE_SCEND_S3D + 1 - start);
223                         if (start + count > SAVAGE_SCSTART_S3D)
224                                 count = SAVAGE_SCSTART_S3D - start;
225                 } else if (start <= SAVAGE_SCEND_S3D) {
226                         if (start + count > SAVAGE_SCEND_S3D + 1) {
227                                 count -= SAVAGE_SCEND_S3D + 1 - start;
228                                 start = SAVAGE_SCEND_S3D + 1;
229                         } else
230                                 return 0;
231                 }
232         } else {
233                 ret = savage_verify_state_s4(dev_priv, start, count, regs);
234                 if (ret != 0)
235                         return ret;
236                 /* scissor regs are emitted in savage_dispatch_draw */
237                 if (start < SAVAGE_DRAWCTRL0_S4) {
238                         if (start + count > SAVAGE_DRAWCTRL1_S4 + 1)
239                                 count2 =
240                                     count - (SAVAGE_DRAWCTRL1_S4 + 1 - start);
241                         if (start + count > SAVAGE_DRAWCTRL0_S4)
242                                 count = SAVAGE_DRAWCTRL0_S4 - start;
243                 } else if (start <= SAVAGE_DRAWCTRL1_S4) {
244                         if (start + count > SAVAGE_DRAWCTRL1_S4 + 1) {
245                                 count -= SAVAGE_DRAWCTRL1_S4 + 1 - start;
246                                 start = SAVAGE_DRAWCTRL1_S4 + 1;
247                         } else
248                                 return 0;
249                 }
250         }
251
252         bci_size = count + (count + 254) / 255 + count2 + (count2 + 254) / 255;
253
254         if (cmd_header->state.global) {
255                 BEGIN_DMA(bci_size + 1);
256                 DMA_WRITE(BCI_CMD_WAIT | BCI_CMD_WAIT_3D);
257                 dev_priv->waiting = 1;
258         } else {
259                 BEGIN_DMA(bci_size);
260         }
261
262         do {
263                 while (count > 0) {
264                         unsigned int n = count < 255 ? count : 255;
265                         DMA_SET_REGISTERS(start, n);
266                         DMA_COPY_FROM_USER(regs, n);
267                         count -= n;
268                         start += n;
269                         regs += n;
270                 }
271                 start += 2;
272                 regs += 2;
273                 count = count2;
274                 count2 = 0;
275         } while (count);
276
277         DMA_COMMIT();
278
279         return 0;
280 }
281
282 static int savage_dispatch_dma_prim(drm_savage_private_t * dev_priv,
283                                     const drm_savage_cmd_header_t * cmd_header,
284                                     const drm_buf_t * dmabuf)
285 {
286         unsigned char reorder = 0;
287         unsigned int prim = cmd_header->prim.prim;
288         unsigned int skip = cmd_header->prim.skip;
289         unsigned int n = cmd_header->prim.count;
290         unsigned int start = cmd_header->prim.start;
291         unsigned int i;
292         BCI_LOCALS;
293
294         if (!dmabuf) {
295                 DRM_ERROR("called without dma buffers!\n");
296                 return DRM_ERR(EINVAL);
297         }
298
299         if (!n)
300                 return 0;
301
302         switch (prim) {
303         case SAVAGE_PRIM_TRILIST_201:
304                 reorder = 1;
305                 prim = SAVAGE_PRIM_TRILIST;
306         case SAVAGE_PRIM_TRILIST:
307                 if (n % 3 != 0) {
308                         DRM_ERROR("wrong number of vertices %u in TRILIST\n",
309                                   n);
310                         return DRM_ERR(EINVAL);
311                 }
312                 break;
313         case SAVAGE_PRIM_TRISTRIP:
314         case SAVAGE_PRIM_TRIFAN:
315                 if (n < 3) {
316                         DRM_ERROR
317                             ("wrong number of vertices %u in TRIFAN/STRIP\n",
318                              n);
319                         return DRM_ERR(EINVAL);
320                 }
321                 break;
322         default:
323                 DRM_ERROR("invalid primitive type %u\n", prim);
324                 return DRM_ERR(EINVAL);
325         }
326
327         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
328                 if (skip != 0) {
329                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
330                         return DRM_ERR(EINVAL);
331                 }
332         } else {
333                 unsigned int size = 10 - (skip & 1) - (skip >> 1 & 1) -
334                     (skip >> 2 & 1) - (skip >> 3 & 1) - (skip >> 4 & 1) -
335                     (skip >> 5 & 1) - (skip >> 6 & 1) - (skip >> 7 & 1);
336                 if (skip > SAVAGE_SKIP_ALL_S4 || size != 8) {
337                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
338                         return DRM_ERR(EINVAL);
339                 }
340                 if (reorder) {
341                         DRM_ERROR("TRILIST_201 used on Savage4 hardware\n");
342                         return DRM_ERR(EINVAL);
343                 }
344         }
345
346         if (start + n > dmabuf->total / 32) {
347                 DRM_ERROR("vertex indices (%u-%u) out of range (0-%u)\n",
348                           start, start + n - 1, dmabuf->total / 32);
349                 return DRM_ERR(EINVAL);
350         }
351
352         /* Vertex DMA doesn't work with command DMA at the same time,
353          * so we use BCI_... to submit commands here. Flush buffered
354          * faked DMA first. */
355         DMA_FLUSH();
356
357         if (dmabuf->bus_address != dev_priv->state.common.vbaddr) {
358                 BEGIN_BCI(2);
359                 BCI_SET_REGISTERS(SAVAGE_VERTBUFADDR, 1);
360                 BCI_WRITE(dmabuf->bus_address | dev_priv->dma_type);
361                 dev_priv->state.common.vbaddr = dmabuf->bus_address;
362         }
363         if (S3_SAVAGE3D_SERIES(dev_priv->chipset) && dev_priv->waiting) {
364                 /* Workaround for what looks like a hardware bug. If a
365                  * WAIT_3D_IDLE was emitted some time before the
366                  * indexed drawing command then the engine will lock
367                  * up. There are two known workarounds:
368                  * WAIT_IDLE_EMPTY or emit at least 63 NOPs. */
369                 BEGIN_BCI(63);
370                 for (i = 0; i < 63; ++i)
371                         BCI_WRITE(BCI_CMD_WAIT);
372                 dev_priv->waiting = 0;
373         }
374
375         prim <<= 25;
376         while (n != 0) {
377                 /* Can emit up to 255 indices (85 triangles) at once. */
378                 unsigned int count = n > 255 ? 255 : n;
379                 if (reorder) {
380                         /* Need to reorder indices for correct flat
381                          * shading while preserving the clock sense
382                          * for correct culling. Only on Savage3D. */
383                         int reorder[3] = { -1, -1, -1 };
384                         reorder[start % 3] = 2;
385
386                         BEGIN_BCI((count + 1 + 1) / 2);
387                         BCI_DRAW_INDICES_S3D(count, prim, start + 2);
388
389                         for (i = start + 1; i + 1 < start + count; i += 2)
390                                 BCI_WRITE((i + reorder[i % 3]) |
391                                           ((i + 1 +
392                                             reorder[(i + 1) % 3]) << 16));
393                         if (i < start + count)
394                                 BCI_WRITE(i + reorder[i % 3]);
395                 } else if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
396                         BEGIN_BCI((count + 1 + 1) / 2);
397                         BCI_DRAW_INDICES_S3D(count, prim, start);
398
399                         for (i = start + 1; i + 1 < start + count; i += 2)
400                                 BCI_WRITE(i | ((i + 1) << 16));
401                         if (i < start + count)
402                                 BCI_WRITE(i);
403                 } else {
404                         BEGIN_BCI((count + 2 + 1) / 2);
405                         BCI_DRAW_INDICES_S4(count, prim, skip);
406
407                         for (i = start; i + 1 < start + count; i += 2)
408                                 BCI_WRITE(i | ((i + 1) << 16));
409                         if (i < start + count)
410                                 BCI_WRITE(i);
411                 }
412
413                 start += count;
414                 n -= count;
415
416                 prim |= BCI_CMD_DRAW_CONT;
417         }
418
419         return 0;
420 }
421
422 static int savage_dispatch_vb_prim(drm_savage_private_t * dev_priv,
423                                    const drm_savage_cmd_header_t * cmd_header,
424                                    const uint32_t __user * vtxbuf,
425                                    unsigned int vb_size, unsigned int vb_stride)
426 {
427         unsigned char reorder = 0;
428         unsigned int prim = cmd_header->prim.prim;
429         unsigned int skip = cmd_header->prim.skip;
430         unsigned int n = cmd_header->prim.count;
431         unsigned int start = cmd_header->prim.start;
432         unsigned int vtx_size;
433         unsigned int i;
434         DMA_LOCALS;
435
436         if (!n)
437                 return 0;
438
439         switch (prim) {
440         case SAVAGE_PRIM_TRILIST_201:
441                 reorder = 1;
442                 prim = SAVAGE_PRIM_TRILIST;
443         case SAVAGE_PRIM_TRILIST:
444                 if (n % 3 != 0) {
445                         DRM_ERROR("wrong number of vertices %u in TRILIST\n",
446                                   n);
447                         return DRM_ERR(EINVAL);
448                 }
449                 break;
450         case SAVAGE_PRIM_TRISTRIP:
451         case SAVAGE_PRIM_TRIFAN:
452                 if (n < 3) {
453                         DRM_ERROR
454                             ("wrong number of vertices %u in TRIFAN/STRIP\n",
455                              n);
456                         return DRM_ERR(EINVAL);
457                 }
458                 break;
459         default:
460                 DRM_ERROR("invalid primitive type %u\n", prim);
461                 return DRM_ERR(EINVAL);
462         }
463
464         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
465                 if (skip > SAVAGE_SKIP_ALL_S3D) {
466                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
467                         return DRM_ERR(EINVAL);
468                 }
469                 vtx_size = 8;   /* full vertex */
470         } else {
471                 if (skip > SAVAGE_SKIP_ALL_S4) {
472                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
473                         return DRM_ERR(EINVAL);
474                 }
475                 vtx_size = 10;  /* full vertex */
476         }
477
478         vtx_size -= (skip & 1) + (skip >> 1 & 1) +
479             (skip >> 2 & 1) + (skip >> 3 & 1) + (skip >> 4 & 1) +
480             (skip >> 5 & 1) + (skip >> 6 & 1) + (skip >> 7 & 1);
481
482         if (vtx_size > vb_stride) {
483                 DRM_ERROR("vertex size greater than vb stride (%u > %u)\n",
484                           vtx_size, vb_stride);
485                 return DRM_ERR(EINVAL);
486         }
487
488         if (start + n > vb_size / (vb_stride * 4)) {
489                 DRM_ERROR("vertex indices (%u-%u) out of range (0-%u)\n",
490                           start, start + n - 1, vb_size / (vb_stride * 4));
491                 return DRM_ERR(EINVAL);
492         }
493
494         prim <<= 25;
495         while (n != 0) {
496                 /* Can emit up to 255 vertices (85 triangles) at once. */
497                 unsigned int count = n > 255 ? 255 : n;
498                 if (reorder) {
499                         /* Need to reorder vertices for correct flat
500                          * shading while preserving the clock sense
501                          * for correct culling. Only on Savage3D. */
502                         int reorder[3] = { -1, -1, -1 };
503                         reorder[start % 3] = 2;
504
505                         BEGIN_DMA(count * vtx_size + 1);
506                         DMA_DRAW_PRIMITIVE(count, prim, skip);
507
508                         for (i = start; i < start + count; ++i) {
509                                 unsigned int j = i + reorder[i % 3];
510                                 DMA_COPY_FROM_USER(&vtxbuf[vb_stride * j],
511                                                    vtx_size);
512                         }
513
514                         DMA_COMMIT();
515                 } else {
516                         BEGIN_DMA(count * vtx_size + 1);
517                         DMA_DRAW_PRIMITIVE(count, prim, skip);
518
519                         if (vb_stride == vtx_size) {
520                                 DMA_COPY_FROM_USER(&vtxbuf[vb_stride * start],
521                                                    vtx_size * count);
522                         } else {
523                                 for (i = start; i < start + count; ++i) {
524                                         DMA_COPY_FROM_USER(&vtxbuf
525                                                            [vb_stride * i],
526                                                            vtx_size);
527                                 }
528                         }
529
530                         DMA_COMMIT();
531                 }
532
533                 start += count;
534                 n -= count;
535
536                 prim |= BCI_CMD_DRAW_CONT;
537         }
538
539         return 0;
540 }
541
542 static int savage_dispatch_dma_idx(drm_savage_private_t * dev_priv,
543                                    const drm_savage_cmd_header_t * cmd_header,
544                                    const uint16_t __user * usr_idx,
545                                    const drm_buf_t * dmabuf)
546 {
547         unsigned char reorder = 0;
548         unsigned int prim = cmd_header->idx.prim;
549         unsigned int skip = cmd_header->idx.skip;
550         unsigned int n = cmd_header->idx.count;
551         unsigned int i;
552         BCI_LOCALS;
553
554         if (!dmabuf) {
555                 DRM_ERROR("called without dma buffers!\n");
556                 return DRM_ERR(EINVAL);
557         }
558
559         if (!n)
560                 return 0;
561
562         switch (prim) {
563         case SAVAGE_PRIM_TRILIST_201:
564                 reorder = 1;
565                 prim = SAVAGE_PRIM_TRILIST;
566         case SAVAGE_PRIM_TRILIST:
567                 if (n % 3 != 0) {
568                         DRM_ERROR("wrong number of indices %u in TRILIST\n", n);
569                         return DRM_ERR(EINVAL);
570                 }
571                 break;
572         case SAVAGE_PRIM_TRISTRIP:
573         case SAVAGE_PRIM_TRIFAN:
574                 if (n < 3) {
575                         DRM_ERROR
576                             ("wrong number of indices %u in TRIFAN/STRIP\n", n);
577                         return DRM_ERR(EINVAL);
578                 }
579                 break;
580         default:
581                 DRM_ERROR("invalid primitive type %u\n", prim);
582                 return DRM_ERR(EINVAL);
583         }
584
585         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
586                 if (skip != 0) {
587                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
588                         return DRM_ERR(EINVAL);
589                 }
590         } else {
591                 unsigned int size = 10 - (skip & 1) - (skip >> 1 & 1) -
592                     (skip >> 2 & 1) - (skip >> 3 & 1) - (skip >> 4 & 1) -
593                     (skip >> 5 & 1) - (skip >> 6 & 1) - (skip >> 7 & 1);
594                 if (skip > SAVAGE_SKIP_ALL_S4 || size != 8) {
595                         DRM_ERROR("invalid skip flags 0x%04x for DMA\n", skip);
596                         return DRM_ERR(EINVAL);
597                 }
598                 if (reorder) {
599                         DRM_ERROR("TRILIST_201 used on Savage4 hardware\n");
600                         return DRM_ERR(EINVAL);
601                 }
602         }
603
604         /* Vertex DMA doesn't work with command DMA at the same time,
605          * so we use BCI_... to submit commands here. Flush buffered
606          * faked DMA first. */
607         DMA_FLUSH();
608
609         if (dmabuf->bus_address != dev_priv->state.common.vbaddr) {
610                 BEGIN_BCI(2);
611                 BCI_SET_REGISTERS(SAVAGE_VERTBUFADDR, 1);
612                 BCI_WRITE(dmabuf->bus_address | dev_priv->dma_type);
613                 dev_priv->state.common.vbaddr = dmabuf->bus_address;
614         }
615         if (S3_SAVAGE3D_SERIES(dev_priv->chipset) && dev_priv->waiting) {
616                 /* Workaround for what looks like a hardware bug. If a
617                  * WAIT_3D_IDLE was emitted some time before the
618                  * indexed drawing command then the engine will lock
619                  * up. There are two known workarounds:
620                  * WAIT_IDLE_EMPTY or emit at least 63 NOPs. */
621                 BEGIN_BCI(63);
622                 for (i = 0; i < 63; ++i)
623                         BCI_WRITE(BCI_CMD_WAIT);
624                 dev_priv->waiting = 0;
625         }
626
627         prim <<= 25;
628         while (n != 0) {
629                 /* Can emit up to 255 indices (85 triangles) at once. */
630                 unsigned int count = n > 255 ? 255 : n;
631                 /* Is it ok to allocate 510 bytes on the stack in an ioctl? */
632                 uint16_t idx[255];
633
634                 /* Copy and check indices */
635                 DRM_COPY_FROM_USER_UNCHECKED(idx, usr_idx, count * 2);
636                 for (i = 0; i < count; ++i) {
637                         if (idx[i] > dmabuf->total / 32) {
638                                 DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
639                                           i, idx[i], dmabuf->total / 32);
640                                 return DRM_ERR(EINVAL);
641                         }
642                 }
643
644                 if (reorder) {
645                         /* Need to reorder indices for correct flat
646                          * shading while preserving the clock sense
647                          * for correct culling. Only on Savage3D. */
648                         int reorder[3] = { 2, -1, -1 };
649
650                         BEGIN_BCI((count + 1 + 1) / 2);
651                         BCI_DRAW_INDICES_S3D(count, prim, idx[2]);
652
653                         for (i = 1; i + 1 < count; i += 2)
654                                 BCI_WRITE(idx[i + reorder[i % 3]] |
655                                           (idx[i + 1 + reorder[(i + 1) % 3]] <<
656                                            16));
657                         if (i < count)
658                                 BCI_WRITE(idx[i + reorder[i % 3]]);
659                 } else if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
660                         BEGIN_BCI((count + 1 + 1) / 2);
661                         BCI_DRAW_INDICES_S3D(count, prim, idx[0]);
662
663                         for (i = 1; i + 1 < count; i += 2)
664                                 BCI_WRITE(idx[i] | (idx[i + 1] << 16));
665                         if (i < count)
666                                 BCI_WRITE(idx[i]);
667                 } else {
668                         BEGIN_BCI((count + 2 + 1) / 2);
669                         BCI_DRAW_INDICES_S4(count, prim, skip);
670
671                         for (i = 0; i + 1 < count; i += 2)
672                                 BCI_WRITE(idx[i] | (idx[i + 1] << 16));
673                         if (i < count)
674                                 BCI_WRITE(idx[i]);
675                 }
676
677                 usr_idx += count;
678                 n -= count;
679
680                 prim |= BCI_CMD_DRAW_CONT;
681         }
682
683         return 0;
684 }
685
686 static int savage_dispatch_vb_idx(drm_savage_private_t * dev_priv,
687                                   const drm_savage_cmd_header_t * cmd_header,
688                                   const uint16_t __user * usr_idx,
689                                   const uint32_t __user * vtxbuf,
690                                   unsigned int vb_size, unsigned int vb_stride)
691 {
692         unsigned char reorder = 0;
693         unsigned int prim = cmd_header->idx.prim;
694         unsigned int skip = cmd_header->idx.skip;
695         unsigned int n = cmd_header->idx.count;
696         unsigned int vtx_size;
697         unsigned int i;
698         DMA_LOCALS;
699
700         if (!n)
701                 return 0;
702
703         switch (prim) {
704         case SAVAGE_PRIM_TRILIST_201:
705                 reorder = 1;
706                 prim = SAVAGE_PRIM_TRILIST;
707         case SAVAGE_PRIM_TRILIST:
708                 if (n % 3 != 0) {
709                         DRM_ERROR("wrong number of indices %u in TRILIST\n", n);
710                         return DRM_ERR(EINVAL);
711                 }
712                 break;
713         case SAVAGE_PRIM_TRISTRIP:
714         case SAVAGE_PRIM_TRIFAN:
715                 if (n < 3) {
716                         DRM_ERROR
717                             ("wrong number of indices %u in TRIFAN/STRIP\n", n);
718                         return DRM_ERR(EINVAL);
719                 }
720                 break;
721         default:
722                 DRM_ERROR("invalid primitive type %u\n", prim);
723                 return DRM_ERR(EINVAL);
724         }
725
726         if (S3_SAVAGE3D_SERIES(dev_priv->chipset)) {
727                 if (skip > SAVAGE_SKIP_ALL_S3D) {
728                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
729                         return DRM_ERR(EINVAL);
730                 }
731                 vtx_size = 8;   /* full vertex */
732         } else {
733                 if (skip > SAVAGE_SKIP_ALL_S4) {
734                         DRM_ERROR("invalid skip flags 0x%04x\n", skip);
735                         return DRM_ERR(EINVAL);
736                 }
737                 vtx_size = 10;  /* full vertex */
738         }
739
740         vtx_size -= (skip & 1) + (skip >> 1 & 1) +
741             (skip >> 2 & 1) + (skip >> 3 & 1) + (skip >> 4 & 1) +
742             (skip >> 5 & 1) + (skip >> 6 & 1) + (skip >> 7 & 1);
743
744         if (vtx_size > vb_stride) {
745                 DRM_ERROR("vertex size greater than vb stride (%u > %u)\n",
746                           vtx_size, vb_stride);
747                 return DRM_ERR(EINVAL);
748         }
749
750         prim <<= 25;
751         while (n != 0) {
752                 /* Can emit up to 255 vertices (85 triangles) at once. */
753                 unsigned int count = n > 255 ? 255 : n;
754                 /* Is it ok to allocate 510 bytes on the stack in an ioctl? */
755                 uint16_t idx[255];
756
757                 /* Copy and check indices */
758                 DRM_COPY_FROM_USER_UNCHECKED(idx, usr_idx, count * 2);
759                 for (i = 0; i < count; ++i) {
760                         if (idx[i] > vb_size / (vb_stride * 4)) {
761                                 DRM_ERROR("idx[%u]=%u out of range (0-%u)\n",
762                                           i, idx[i], vb_size / (vb_stride * 4));
763                                 return DRM_ERR(EINVAL);
764                         }
765                 }
766
767                 if (reorder) {
768                         /* Need to reorder vertices for correct flat
769                          * shading while preserving the clock sense
770                          * for correct culling. Only on Savage3D. */
771                         int reorder[3] = { 2, -1, -1 };
772
773                         BEGIN_DMA(count * vtx_size + 1);
774                         DMA_DRAW_PRIMITIVE(count, prim, skip);
775
776                         for (i = 0; i < count; ++i) {
777                                 unsigned int j = idx[i + reorder[i % 3]];
778                                 DMA_COPY_FROM_USER(&vtxbuf[vb_stride * j],
779                                                    vtx_size);
780                         }
781
782                         DMA_COMMIT();
783                 } else {
784                         BEGIN_DMA(count * vtx_size + 1);
785                         DMA_DRAW_PRIMITIVE(count, prim, skip);
786
787                         for (i = 0; i < count; ++i) {
788                                 unsigned int j = idx[i];
789                                 DMA_COPY_FROM_USER(&vtxbuf[vb_stride * j],
790                                                    vtx_size);
791                         }
792
793                         DMA_COMMIT();
794                 }
795
796                 usr_idx += count;
797                 n -= count;
798
799                 prim |= BCI_CMD_DRAW_CONT;
800         }
801
802         return 0;
803 }
804
805 static int savage_dispatch_clear(drm_savage_private_t * dev_priv,
806                                  const drm_savage_cmd_header_t * cmd_header,
807                                  const drm_savage_cmd_header_t __user * data,
808                                  unsigned int nbox,
809                                  const drm_clip_rect_t __user * usr_boxes)
810 {
811         unsigned int flags = cmd_header->clear0.flags, mask, value;
812         unsigned int clear_cmd;
813         unsigned int i, nbufs;
814         DMA_LOCALS;
815
816         if (nbox == 0)
817                 return 0;
818
819         DRM_GET_USER_UNCHECKED(mask, &data->clear1.mask);
820         DRM_GET_USER_UNCHECKED(value, &data->clear1.value);
821
822         clear_cmd = BCI_CMD_RECT | BCI_CMD_RECT_XP | BCI_CMD_RECT_YP |
823             BCI_CMD_SEND_COLOR | BCI_CMD_DEST_PBD_NEW;
824         BCI_CMD_SET_ROP(clear_cmd, 0xCC);
825
826         nbufs = ((flags & SAVAGE_FRONT) ? 1 : 0) +
827             ((flags & SAVAGE_BACK) ? 1 : 0) + ((flags & SAVAGE_DEPTH) ? 1 : 0);
828         if (nbufs == 0)
829                 return 0;
830
831         if (mask != 0xffffffff) {
832                 /* set mask */
833                 BEGIN_DMA(2);
834                 DMA_SET_REGISTERS(SAVAGE_BITPLANEWTMASK, 1);
835                 DMA_WRITE(mask);
836                 DMA_COMMIT();
837         }
838         for (i = 0; i < nbox; ++i) {
839                 drm_clip_rect_t box;
840                 unsigned int x, y, w, h;
841                 unsigned int buf;
842                 DRM_COPY_FROM_USER_UNCHECKED(&box, &usr_boxes[i], sizeof(box));
843                 x = box.x1, y = box.y1;
844                 w = box.x2 - box.x1;
845                 h = box.y2 - box.y1;
846                 BEGIN_DMA(nbufs * 6);
847                 for (buf = SAVAGE_FRONT; buf <= SAVAGE_DEPTH; buf <<= 1) {
848                         if (!(flags & buf))
849                                 continue;
850                         DMA_WRITE(clear_cmd);
851                         switch (buf) {
852                         case SAVAGE_FRONT:
853                                 DMA_WRITE(dev_priv->front_offset);
854                                 DMA_WRITE(dev_priv->front_bd);
855                                 break;
856                         case SAVAGE_BACK:
857                                 DMA_WRITE(dev_priv->back_offset);
858                                 DMA_WRITE(dev_priv->back_bd);
859                                 break;
860                         case SAVAGE_DEPTH:
861                                 DMA_WRITE(dev_priv->depth_offset);
862                                 DMA_WRITE(dev_priv->depth_bd);
863                                 break;
864                         }
865                         DMA_WRITE(value);
866                         DMA_WRITE(BCI_X_Y(x, y));
867                         DMA_WRITE(BCI_W_H(w, h));
868                 }
869                 DMA_COMMIT();
870         }
871         if (mask != 0xffffffff) {
872                 /* reset mask */
873                 BEGIN_DMA(2);
874                 DMA_SET_REGISTERS(SAVAGE_BITPLANEWTMASK, 1);
875                 DMA_WRITE(0xffffffff);
876                 DMA_COMMIT();
877         }
878
879         return 0;
880 }
881
882 static int savage_dispatch_swap(drm_savage_private_t * dev_priv,
883                                 unsigned int nbox,
884                                 const drm_clip_rect_t __user * usr_boxes)
885 {
886         unsigned int swap_cmd;
887         unsigned int i;
888         DMA_LOCALS;
889
890         if (nbox == 0)
891                 return 0;
892
893         swap_cmd = BCI_CMD_RECT | BCI_CMD_RECT_XP | BCI_CMD_RECT_YP |
894             BCI_CMD_SRC_PBD_COLOR_NEW | BCI_CMD_DEST_GBD;
895         BCI_CMD_SET_ROP(swap_cmd, 0xCC);
896
897         for (i = 0; i < nbox; ++i) {
898                 drm_clip_rect_t box;
899                 DRM_COPY_FROM_USER_UNCHECKED(&box, &usr_boxes[i], sizeof(box));
900
901                 BEGIN_DMA(6);
902                 DMA_WRITE(swap_cmd);
903                 DMA_WRITE(dev_priv->back_offset);
904                 DMA_WRITE(dev_priv->back_bd);
905                 DMA_WRITE(BCI_X_Y(box.x1, box.y1));
906                 DMA_WRITE(BCI_X_Y(box.x1, box.y1));
907                 DMA_WRITE(BCI_W_H(box.x2 - box.x1, box.y2 - box.y1));
908                 DMA_COMMIT();
909         }
910
911         return 0;
912 }
913
914 static int savage_dispatch_draw(drm_savage_private_t * dev_priv,
915                                 const drm_savage_cmd_header_t __user * start,
916                                 const drm_savage_cmd_header_t __user * end,
917                                 const drm_buf_t * dmabuf,
918                                 const unsigned int __user * usr_vtxbuf,
919                                 unsigned int vb_size, unsigned int vb_stride,
920                                 unsigned int nbox,
921                                 const drm_clip_rect_t __user * usr_boxes)
922 {
923         unsigned int i, j;
924         int ret;
925
926         for (i = 0; i < nbox; ++i) {
927                 drm_clip_rect_t box;
928                 const drm_savage_cmd_header_t __user *usr_cmdbuf;
929                 DRM_COPY_FROM_USER_UNCHECKED(&box, &usr_boxes[i], sizeof(box));
930                 dev_priv->emit_clip_rect(dev_priv, &box);
931
932                 usr_cmdbuf = start;
933                 while (usr_cmdbuf < end) {
934                         drm_savage_cmd_header_t cmd_header;
935                         DRM_COPY_FROM_USER_UNCHECKED(&cmd_header, usr_cmdbuf,
936                                                      sizeof(cmd_header));
937                         usr_cmdbuf++;
938                         switch (cmd_header.cmd.cmd) {
939                         case SAVAGE_CMD_DMA_PRIM:
940                                 ret =
941                                     savage_dispatch_dma_prim(dev_priv,
942                                                              &cmd_header,
943                                                              dmabuf);
944                                 break;
945                         case SAVAGE_CMD_VB_PRIM:
946                                 ret =
947                                     savage_dispatch_vb_prim(dev_priv,
948                                                             &cmd_header,
949                                                             (const uint32_t
950                                                              __user *)
951                                                             usr_vtxbuf, vb_size,
952                                                             vb_stride);
953                                 break;
954                         case SAVAGE_CMD_DMA_IDX:
955                                 j = (cmd_header.idx.count + 3) / 4;
956                                 /* j was check in savage_bci_cmdbuf */
957                                 ret =
958                                     savage_dispatch_dma_idx(dev_priv,
959                                                             &cmd_header,
960                                                             (const uint16_t
961                                                              __user *)
962                                                             usr_cmdbuf, dmabuf);
963                                 usr_cmdbuf += j;
964                                 break;
965                         case SAVAGE_CMD_VB_IDX:
966                                 j = (cmd_header.idx.count + 3) / 4;
967                                 /* j was check in savage_bci_cmdbuf */
968                                 ret =
969                                     savage_dispatch_vb_idx(dev_priv,
970                                                            &cmd_header,
971                                                            (const uint16_t
972                                                             __user *)usr_cmdbuf,
973                                                            (const uint32_t
974                                                             __user *)usr_vtxbuf,
975                                                            vb_size, vb_stride);
976                                 usr_cmdbuf += j;
977                                 break;
978                         default:
979                                 /* What's the best return code? EFAULT? */
980                                 DRM_ERROR("IMPLEMENTATION ERROR: "
981                                           "non-drawing-command %d\n",
982                                           cmd_header.cmd.cmd);
983                                 return DRM_ERR(EINVAL);
984                         }
985
986                         if (ret != 0)
987                                 return ret;
988                 }
989         }
990
991         return 0;
992 }
993
994 int savage_bci_cmdbuf(DRM_IOCTL_ARGS)
995 {
996         DRM_DEVICE;
997         drm_savage_private_t *dev_priv = dev->dev_private;
998         drm_device_dma_t *dma = dev->dma;
999         drm_buf_t *dmabuf;
1000         drm_savage_cmdbuf_t cmdbuf;
1001         drm_savage_cmd_header_t __user *usr_cmdbuf;
1002         drm_savage_cmd_header_t __user *first_draw_cmd;
1003         unsigned int __user *usr_vtxbuf;
1004         drm_clip_rect_t __user *usr_boxes;
1005         unsigned int i, j;
1006         int ret = 0;
1007
1008         DRM_DEBUG("\n");
1009
1010         LOCK_TEST_WITH_RETURN(dev, filp);
1011
1012         DRM_COPY_FROM_USER_IOCTL(cmdbuf, (drm_savage_cmdbuf_t __user *) data,
1013                                  sizeof(cmdbuf));
1014
1015         if (dma && dma->buflist) {
1016                 if (cmdbuf.dma_idx > dma->buf_count) {
1017                         DRM_ERROR
1018                             ("vertex buffer index %u out of range (0-%u)\n",
1019                              cmdbuf.dma_idx, dma->buf_count - 1);
1020                         return DRM_ERR(EINVAL);
1021                 }
1022                 dmabuf = dma->buflist[cmdbuf.dma_idx];
1023         } else {
1024                 dmabuf = NULL;
1025         }
1026
1027         usr_cmdbuf = (drm_savage_cmd_header_t __user *) cmdbuf.cmd_addr;
1028         usr_vtxbuf = (unsigned int __user *)cmdbuf.vb_addr;
1029         usr_boxes = (drm_clip_rect_t __user *) cmdbuf.box_addr;
1030         if ((cmdbuf.size && DRM_VERIFYAREA_READ(usr_cmdbuf, cmdbuf.size * 8)) ||
1031             (cmdbuf.vb_size && DRM_VERIFYAREA_READ(usr_vtxbuf, cmdbuf.vb_size))
1032             || (cmdbuf.nbox
1033                 && DRM_VERIFYAREA_READ(usr_boxes,
1034                                        cmdbuf.nbox * sizeof(drm_clip_rect_t))))
1035                 return DRM_ERR(EFAULT);
1036
1037         /* Make sure writes to DMA buffers are finished before sending
1038          * DMA commands to the graphics hardware. */
1039         DRM_MEMORYBARRIER();
1040
1041         /* Coming from user space. Don't know if the Xserver has
1042          * emitted wait commands. Assuming the worst. */
1043         dev_priv->waiting = 1;
1044
1045         i = 0;
1046         first_draw_cmd = NULL;
1047         while (i < cmdbuf.size) {
1048                 drm_savage_cmd_header_t cmd_header;
1049                 DRM_COPY_FROM_USER_UNCHECKED(&cmd_header, usr_cmdbuf,
1050                                              sizeof(cmd_header));
1051                 usr_cmdbuf++;
1052                 i++;
1053
1054                 /* Group drawing commands with same state to minimize
1055                  * iterations over clip rects. */
1056                 j = 0;
1057                 switch (cmd_header.cmd.cmd) {
1058                 case SAVAGE_CMD_DMA_IDX:
1059                 case SAVAGE_CMD_VB_IDX:
1060                         j = (cmd_header.idx.count + 3) / 4;
1061                         if (i + j > cmdbuf.size) {
1062                                 DRM_ERROR("indexed drawing command extends "
1063                                           "beyond end of command buffer\n");
1064                                 DMA_FLUSH();
1065                                 return DRM_ERR(EINVAL);
1066                         }
1067                         /* fall through */
1068                 case SAVAGE_CMD_DMA_PRIM:
1069                 case SAVAGE_CMD_VB_PRIM:
1070                         if (!first_draw_cmd)
1071                                 first_draw_cmd = usr_cmdbuf - 1;
1072                         usr_cmdbuf += j;
1073                         i += j;
1074                         break;
1075                 default:
1076                         if (first_draw_cmd) {
1077                                 ret =
1078                                     savage_dispatch_draw(dev_priv,
1079                                                          first_draw_cmd,
1080                                                          usr_cmdbuf - 1, dmabuf,
1081                                                          usr_vtxbuf,
1082                                                          cmdbuf.vb_size,
1083                                                          cmdbuf.vb_stride,
1084                                                          cmdbuf.nbox,
1085                                                          usr_boxes);
1086                                 if (ret != 0)
1087                                         return ret;
1088                                 first_draw_cmd = NULL;
1089                         }
1090                 }
1091                 if (first_draw_cmd)
1092                         continue;
1093
1094                 switch (cmd_header.cmd.cmd) {
1095                 case SAVAGE_CMD_STATE:
1096                         j = (cmd_header.state.count + 1) / 2;
1097                         if (i + j > cmdbuf.size) {
1098                                 DRM_ERROR("command SAVAGE_CMD_STATE extends "
1099                                           "beyond end of command buffer\n");
1100                                 DMA_FLUSH();
1101                                 return DRM_ERR(EINVAL);
1102                         }
1103                         ret = savage_dispatch_state(dev_priv, &cmd_header,
1104                                                     (uint32_t __user *)
1105                                                     usr_cmdbuf);
1106                         usr_cmdbuf += j;
1107                         i += j;
1108                         break;
1109                 case SAVAGE_CMD_CLEAR:
1110                         if (i + 1 > cmdbuf.size) {
1111                                 DRM_ERROR("command SAVAGE_CMD_CLEAR extends "
1112                                           "beyond end of command buffer\n");
1113                                 DMA_FLUSH();
1114                                 return DRM_ERR(EINVAL);
1115                         }
1116                         ret = savage_dispatch_clear(dev_priv, &cmd_header,
1117                                                     usr_cmdbuf,
1118                                                     cmdbuf.nbox, usr_boxes);
1119                         usr_cmdbuf++;
1120                         i++;
1121                         break;
1122                 case SAVAGE_CMD_SWAP:
1123                         ret = savage_dispatch_swap(dev_priv,
1124                                                    cmdbuf.nbox, usr_boxes);
1125                         break;
1126                 default:
1127                         DRM_ERROR("invalid command 0x%x\n", cmd_header.cmd.cmd);
1128                         DMA_FLUSH();
1129                         return DRM_ERR(EINVAL);
1130                 }
1131
1132                 if (ret != 0) {
1133                         DMA_FLUSH();
1134                         return ret;
1135                 }
1136         }
1137
1138         if (first_draw_cmd) {
1139                 ret =
1140                     savage_dispatch_draw(dev_priv, first_draw_cmd, usr_cmdbuf,
1141                                          dmabuf, usr_vtxbuf, cmdbuf.vb_size,
1142                                          cmdbuf.vb_stride, cmdbuf.nbox,
1143                                          usr_boxes);
1144                 if (ret != 0) {
1145                         DMA_FLUSH();
1146                         return ret;
1147                 }
1148         }
1149
1150         DMA_FLUSH();
1151
1152         if (dmabuf && cmdbuf.discard) {
1153                 drm_savage_buf_priv_t *buf_priv = dmabuf->dev_private;
1154                 uint16_t event;
1155                 event = savage_bci_emit_event(dev_priv, SAVAGE_WAIT_3D);
1156                 SET_AGE(&buf_priv->age, event, dev_priv->event_wrap);
1157                 savage_freelist_put(dev, dmabuf);
1158         }
1159
1160         return 0;
1161 }