1
2#include "util/u_format.h"
3
4#include "nvc0_context.h"
5
6#include "nv50/nv50_defs.xml.h"
7
8struct nvc0_transfer {
9   struct pipe_transfer base;
10   struct nv50_m2mf_rect rect[2];
11   uint32_t nblocksx;
12   uint16_t nblocksy;
13   uint16_t nlayers;
14};
15
16static void
17nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0,
18                        const struct nv50_m2mf_rect *dst,
19                        const struct nv50_m2mf_rect *src,
20                        uint32_t nblocksx, uint32_t nblocksy)
21{
22   struct nouveau_pushbuf *push = nvc0->base.pushbuf;
23   struct nouveau_bufctx *bctx = nvc0->bufctx;
24   const int cpp = dst->cpp;
25   uint32_t src_ofst = src->base;
26   uint32_t dst_ofst = dst->base;
27   uint32_t height = nblocksy;
28   uint32_t sy = src->y;
29   uint32_t dy = dst->y;
30   uint32_t exec = (1 << 20);
31
32   assert(dst->cpp == src->cpp);
33
34   nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
35   nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
36   nouveau_pushbuf_bufctx(push, bctx);
37   nouveau_pushbuf_validate(push);
38
39   if (nouveau_bo_memtype(src->bo)) {
40      BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5);
41      PUSH_DATA (push, src->tile_mode);
42      PUSH_DATA (push, src->width * cpp);
43      PUSH_DATA (push, src->height);
44      PUSH_DATA (push, src->depth);
45      PUSH_DATA (push, src->z);
46   } else {
47      src_ofst += src->y * src->pitch + src->x * cpp;
48
49      BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1);
50      PUSH_DATA (push, src->width * cpp);
51
52      exec |= NVC0_M2MF_EXEC_LINEAR_IN;
53   }
54
55   if (nouveau_bo_memtype(dst->bo)) {
56      BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5);
57      PUSH_DATA (push, dst->tile_mode);
58      PUSH_DATA (push, dst->width * cpp);
59      PUSH_DATA (push, dst->height);
60      PUSH_DATA (push, dst->depth);
61      PUSH_DATA (push, dst->z);
62   } else {
63      dst_ofst += dst->y * dst->pitch + dst->x * cpp;
64
65      BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1);
66      PUSH_DATA (push, dst->width * cpp);
67
68      exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
69   }
70
71   while (height) {
72      int line_count = height > 2047 ? 2047 : height;
73
74      BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
75      PUSH_DATAh(push, src->bo->offset + src_ofst);
76      PUSH_DATA (push, src->bo->offset + src_ofst);
77
78      BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
79      PUSH_DATAh(push, dst->bo->offset + dst_ofst);
80      PUSH_DATA (push, dst->bo->offset + dst_ofst);
81
82      if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
83         BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2);
84         PUSH_DATA (push, src->x * cpp);
85         PUSH_DATA (push, sy);
86      } else {
87         src_ofst += line_count * src->pitch;
88      }
89      if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
90         BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2);
91         PUSH_DATA (push, dst->x * cpp);
92         PUSH_DATA (push, dy);
93      } else {
94         dst_ofst += line_count * dst->pitch;
95      }
96
97      BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
98      PUSH_DATA (push, nblocksx * cpp);
99      PUSH_DATA (push, line_count);
100      BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
101      PUSH_DATA (push, exec);
102
103      height -= line_count;
104      sy += line_count;
105      dy += line_count;
106   }
107
108   nouveau_bufctx_reset(bctx, 0);
109}
110
111static void
112nve4_m2mf_transfer_rect(struct nvc0_context *nvc0,
113                        const struct nv50_m2mf_rect *dst,
114                        const struct nv50_m2mf_rect *src,
115                        uint32_t nblocksx, uint32_t nblocksy)
116{
117   struct nouveau_pushbuf *push = nvc0->base.pushbuf;
118   struct nouveau_bufctx *bctx = nvc0->bufctx;
119   uint32_t exec;
120   uint32_t src_base = src->base;
121   uint32_t dst_base = dst->base;
122   const int cpp = dst->cpp;
123
124   assert(dst->cpp == src->cpp);
125
126   nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
127   nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
128   nouveau_pushbuf_bufctx(push, bctx);
129   nouveau_pushbuf_validate(push);
130
131   exec = 0x200 /* 2D_ENABLE */ | 0x6 /* UNK */;
132
133   if (!nouveau_bo_memtype(dst->bo)) {
134      assert(!dst->z);
135      dst_base += dst->y * dst->pitch + dst->x * cpp;
136      exec |= 0x100; /* DST_MODE_2D_LINEAR */
137   }
138   if (!nouveau_bo_memtype(src->bo)) {
139      assert(!src->z);
140      src_base += src->y * src->pitch + src->x * cpp;
141      exec |= 0x080; /* SRC_MODE_2D_LINEAR */
142   }
143
144   BEGIN_NVC0(push, SUBC_COPY(0x070c), 6);
145   PUSH_DATA (push, 0x1000 | dst->tile_mode);
146   PUSH_DATA (push, dst->pitch);
147   PUSH_DATA (push, dst->height);
148   PUSH_DATA (push, dst->depth);
149   PUSH_DATA (push, dst->z);
150   PUSH_DATA (push, (dst->y << 16) | (dst->x * cpp));
151
152   BEGIN_NVC0(push, SUBC_COPY(0x0728), 6);
153   PUSH_DATA (push, 0x1000 | src->tile_mode);
154   PUSH_DATA (push, src->pitch);
155   PUSH_DATA (push, src->height);
156   PUSH_DATA (push, src->depth);
157   PUSH_DATA (push, src->z);
158   PUSH_DATA (push, (src->y << 16) | (src->x * cpp));
159
160   BEGIN_NVC0(push, SUBC_COPY(0x0400), 8);
161   PUSH_DATAh(push, src->bo->offset + src_base);
162   PUSH_DATA (push, src->bo->offset + src_base);
163   PUSH_DATAh(push, dst->bo->offset + dst_base);
164   PUSH_DATA (push, dst->bo->offset + dst_base);
165   PUSH_DATA (push, src->pitch);
166   PUSH_DATA (push, dst->pitch);
167   PUSH_DATA (push, nblocksx * cpp);
168   PUSH_DATA (push, nblocksy);
169
170   BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
171   PUSH_DATA (push, exec);
172
173   nouveau_bufctx_reset(bctx, 0);
174}
175
176void
177nvc0_m2mf_push_linear(struct nouveau_context *nv,
178                      struct nouveau_bo *dst, unsigned offset, unsigned domain,
179                      unsigned size, const void *data)
180{
181   struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
182   struct nouveau_pushbuf *push = nv->pushbuf;
183   uint32_t *src = (uint32_t *)data;
184   unsigned count = (size + 3) / 4;
185
186   nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
187   nouveau_pushbuf_bufctx(push, nvc0->bufctx);
188   nouveau_pushbuf_validate(push);
189
190   while (count) {
191      unsigned nr;
192
193      if (!PUSH_SPACE(push, 16))
194         break;
195      nr = PUSH_AVAIL(push);
196      assert(nr >= 16);
197      nr = MIN2(count, nr - 9);
198      nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
199
200      BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
201      PUSH_DATAh(push, dst->offset + offset);
202      PUSH_DATA (push, dst->offset + offset);
203      BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
204      PUSH_DATA (push, nr * 4);
205      PUSH_DATA (push, 1);
206      BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
207      PUSH_DATA (push, 0x100111);
208
209      /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
210      BEGIN_NIC0(push, NVC0_M2MF(DATA), nr);
211      PUSH_DATAp(push, src, nr);
212
213      count -= nr;
214      src += nr;
215      offset += nr * 4;
216   }
217
218   nouveau_bufctx_reset(nvc0->bufctx, 0);
219}
220
221void
222nve4_p2mf_push_linear(struct nouveau_context *nv,
223                      struct nouveau_bo *dst, unsigned offset, unsigned domain,
224                      unsigned size, const void *data)
225{
226   struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
227   struct nouveau_pushbuf *push = nv->pushbuf;
228   uint32_t *src = (uint32_t *)data;
229   unsigned count = (size + 3) / 4;
230
231   nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
232   nouveau_pushbuf_bufctx(push, nvc0->bufctx);
233   nouveau_pushbuf_validate(push);
234
235   while (count) {
236      unsigned nr;
237
238      if (!PUSH_SPACE(push, 16))
239         break;
240      nr = PUSH_AVAIL(push);
241      assert(nr >= 16);
242      nr = MIN2(count, nr - 8);
243      nr = MIN2(nr, (NV04_PFIFO_MAX_PACKET_LEN - 1));
244
245      BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2);
246      PUSH_DATAh(push, dst->offset + offset);
247      PUSH_DATA (push, dst->offset + offset);
248      BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2);
249      PUSH_DATA (push, nr * 4);
250      PUSH_DATA (push, 1);
251      /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
252      BEGIN_1IC0(push, NVE4_P2MF(EXEC), nr + 1);
253      PUSH_DATA (push, 0x1001);
254      PUSH_DATAp(push, src, nr);
255
256      count -= nr;
257      src += nr;
258      offset += nr * 4;
259   }
260
261   nouveau_bufctx_reset(nvc0->bufctx, 0);
262}
263
264static void
265nvc0_m2mf_copy_linear(struct nouveau_context *nv,
266                      struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
267                      struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
268                      unsigned size)
269{
270   struct nouveau_pushbuf *push = nv->pushbuf;
271   struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
272
273   nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
274   nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
275   nouveau_pushbuf_bufctx(push, bctx);
276   nouveau_pushbuf_validate(push);
277
278   while (size) {
279      unsigned bytes = MIN2(size, 1 << 17);
280
281      BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
282      PUSH_DATAh(push, dst->offset + dstoff);
283      PUSH_DATA (push, dst->offset + dstoff);
284      BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
285      PUSH_DATAh(push, src->offset + srcoff);
286      PUSH_DATA (push, src->offset + srcoff);
287      BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
288      PUSH_DATA (push, bytes);
289      PUSH_DATA (push, 1);
290      BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
291      PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
292                 NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
293
294      srcoff += bytes;
295      dstoff += bytes;
296      size -= bytes;
297   }
298
299   nouveau_bufctx_reset(bctx, 0);
300}
301
302static void
303nve4_m2mf_copy_linear(struct nouveau_context *nv,
304                      struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
305                      struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
306                      unsigned size)
307{
308   struct nouveau_pushbuf *push = nv->pushbuf;
309   struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
310
311   nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
312   nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
313   nouveau_pushbuf_bufctx(push, bctx);
314   nouveau_pushbuf_validate(push);
315
316   BEGIN_NVC0(push, SUBC_COPY(0x0400), 4);
317   PUSH_DATAh(push, src->offset + srcoff);
318   PUSH_DATA (push, src->offset + srcoff);
319   PUSH_DATAh(push, dst->offset + dstoff);
320   PUSH_DATA (push, dst->offset + dstoff);
321   BEGIN_NVC0(push, SUBC_COPY(0x0418), 1);
322   PUSH_DATA (push, size);
323   BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
324   PUSH_DATA (push, 0x186);
325
326   nouveau_bufctx_reset(bctx, 0);
327}
328
329struct pipe_transfer *
330nvc0_miptree_transfer_new(struct pipe_context *pctx,
331                          struct pipe_resource *res,
332                          unsigned level,
333                          unsigned usage,
334                          const struct pipe_box *box)
335{
336   struct nvc0_context *nvc0 = nvc0_context(pctx);
337   struct nouveau_device *dev = nvc0->screen->base.device;
338   struct nv50_miptree *mt = nv50_miptree(res);
339   struct nvc0_transfer *tx;
340   uint32_t size;
341   int ret;
342
343   if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
344      return NULL;
345
346   tx = CALLOC_STRUCT(nvc0_transfer);
347   if (!tx)
348      return NULL;
349
350   pipe_resource_reference(&tx->base.resource, res);
351
352   tx->base.level = level;
353   tx->base.usage = usage;
354   tx->base.box = *box;
355
356   if (util_format_is_plain(res->format)) {
357      tx->nblocksx = box->width << mt->ms_x;
358      tx->nblocksy = box->height << mt->ms_y;
359   } else {
360      tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
361      tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
362   }
363   tx->nlayers = box->depth;
364
365   tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
366   tx->base.layer_stride = tx->nblocksy * tx->base.stride;
367
368   nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
369
370   size = tx->base.layer_stride;
371
372   ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
373                        size * tx->nlayers, NULL, &tx->rect[1].bo);
374   if (ret) {
375      FREE(tx);
376      return NULL;
377   }
378
379   tx->rect[1].cpp = tx->rect[0].cpp;
380   tx->rect[1].width = tx->nblocksx;
381   tx->rect[1].height = tx->nblocksy;
382   tx->rect[1].depth = 1;
383   tx->rect[1].pitch = tx->base.stride;
384   tx->rect[1].domain = NOUVEAU_BO_GART;
385
386   if (usage & PIPE_TRANSFER_READ) {
387      unsigned base = tx->rect[0].base;
388      unsigned z = tx->rect[0].z;
389      unsigned i;
390      for (i = 0; i < tx->nlayers; ++i) {
391         nvc0->m2mf_copy_rect(nvc0, &tx->rect[1], &tx->rect[0],
392                              tx->nblocksx, tx->nblocksy);
393         if (mt->layout_3d)
394            tx->rect[0].z++;
395         else
396            tx->rect[0].base += mt->layer_stride;
397         tx->rect[1].base += size;
398      }
399      tx->rect[0].z = z;
400      tx->rect[0].base = base;
401      tx->rect[1].base = 0;
402   }
403
404   return &tx->base;
405}
406
407void
408nvc0_miptree_transfer_del(struct pipe_context *pctx,
409                          struct pipe_transfer *transfer)
410{
411   struct nvc0_context *nvc0 = nvc0_context(pctx);
412   struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
413   struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
414   unsigned i;
415
416   if (tx->base.usage & PIPE_TRANSFER_WRITE) {
417      for (i = 0; i < tx->nlayers; ++i) {
418         nvc0->m2mf_copy_rect(nvc0, &tx->rect[0], &tx->rect[1],
419                              tx->nblocksx, tx->nblocksy);
420         if (mt->layout_3d)
421            tx->rect[0].z++;
422         else
423            tx->rect[0].base += mt->layer_stride;
424         tx->rect[1].base += tx->nblocksy * tx->base.stride;
425      }
426   }
427
428   nouveau_bo_ref(NULL, &tx->rect[1].bo);
429   pipe_resource_reference(&transfer->resource, NULL);
430
431   FREE(tx);
432}
433
434void *
435nvc0_miptree_transfer_map(struct pipe_context *pctx,
436                          struct pipe_transfer *transfer)
437{
438   struct nvc0_context *nvc0 = nvc0_context(pctx);
439   struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
440   int ret;
441   unsigned flags = 0;
442
443   if (tx->rect[1].bo->map)
444      return tx->rect[1].bo->map;
445
446   if (transfer->usage & PIPE_TRANSFER_READ)
447      flags = NOUVEAU_BO_RD;
448   if (transfer->usage & PIPE_TRANSFER_WRITE)
449      flags |= NOUVEAU_BO_WR;
450
451   ret = nouveau_bo_map(tx->rect[1].bo, flags, nvc0->screen->base.client);
452   if (ret)
453      return NULL;
454   return tx->rect[1].bo->map;
455}
456
457void
458nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
459                            struct pipe_transfer *transfer)
460{
461}
462
463void
464nvc0_cb_push(struct nouveau_context *nv,
465             struct nouveau_bo *bo, unsigned domain,
466             unsigned base, unsigned size,
467             unsigned offset, unsigned words, const uint32_t *data)
468{
469   struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
470   struct nouveau_pushbuf *push = nv->pushbuf;
471
472   assert(!(offset & 3));
473   size = align(size, 0x100);
474
475   nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
476   nouveau_pushbuf_bufctx(push, bctx);
477   nouveau_pushbuf_validate(push);
478
479   BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
480   PUSH_DATA (push, size);
481   PUSH_DATAh(push, bo->offset + base);
482   PUSH_DATA (push, bo->offset + base);
483
484   while (words) {
485      unsigned nr = PUSH_AVAIL(push);
486      nr = MIN2(nr, words);
487      nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
488
489      BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
490      PUSH_DATA (push, offset);
491      PUSH_DATAp(push, data, nr);
492
493      words -= nr;
494      data += nr;
495      offset += nr * 4;
496   }
497
498   nouveau_bufctx_reset(bctx, 0);
499}
500
501void
502nvc0_init_transfer_functions(struct nvc0_context *nvc0)
503{
504   if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS) {
505      nvc0->m2mf_copy_rect = nve4_m2mf_transfer_rect;
506      nvc0->base.copy_data = nve4_m2mf_copy_linear;
507      nvc0->base.push_data = nve4_p2mf_push_linear;
508   } else {
509      nvc0->m2mf_copy_rect = nvc0_m2mf_transfer_rect;
510      nvc0->base.copy_data = nvc0_m2mf_copy_linear;
511      nvc0->base.push_data = nvc0_m2mf_push_linear;
512   }
513   nvc0->base.push_cb = nvc0_cb_push;
514}
515