1
2#include "util/u_format.h"
3
4#include "nv50_context.h"
5
6#include "nv50_defs.xml.h"
7
8struct nv50_transfer {
9   struct pipe_transfer base;
10   struct nv50_m2mf_rect rect[2];
11   uint32_t nblocksx;
12   uint32_t nblocksy;
13};
14
15void
16nv50_m2mf_rect_setup(struct nv50_m2mf_rect *rect,
17                     struct pipe_resource *restrict res, unsigned l,
18                     unsigned x, unsigned y, unsigned z)
19{
20   struct nv50_miptree *mt = nv50_miptree(res);
21   const unsigned w = u_minify(res->width0, l);
22   const unsigned h = u_minify(res->height0, l);
23
24   rect->bo = mt->base.bo;
25   rect->domain = mt->base.domain;
26   rect->base = mt->level[l].offset;
27   rect->pitch = mt->level[l].pitch;
28   if (util_format_is_plain(res->format)) {
29      rect->width = w << mt->ms_x;
30      rect->height = h << mt->ms_y;
31      rect->x = x << mt->ms_x;
32      rect->y = y << mt->ms_y;
33   } else {
34      rect->width = util_format_get_nblocksx(res->format, w);
35      rect->height = util_format_get_nblocksy(res->format, h);
36      rect->x = util_format_get_nblocksx(res->format, x);
37      rect->y = util_format_get_nblocksy(res->format, y);
38   }
39   rect->tile_mode = mt->level[l].tile_mode;
40   rect->cpp = util_format_get_blocksize(res->format);
41
42   if (mt->layout_3d) {
43      rect->z = z;
44      rect->depth = u_minify(res->depth0, l);
45   } else {
46      rect->base += z * mt->layer_stride;
47      rect->z = 0;
48      rect->depth = 1;
49   }
50}
51
52void
53nv50_m2mf_transfer_rect(struct nv50_context *nv50,
54                        const struct nv50_m2mf_rect *dst,
55                        const struct nv50_m2mf_rect *src,
56                        uint32_t nblocksx, uint32_t nblocksy)
57{
58   struct nouveau_pushbuf *push = nv50->base.pushbuf;
59   struct nouveau_bufctx *bctx = nv50->bufctx;
60   const int cpp = dst->cpp;
61   uint32_t src_ofst = src->base;
62   uint32_t dst_ofst = dst->base;
63   uint32_t height = nblocksy;
64   uint32_t sy = src->y;
65   uint32_t dy = dst->y;
66
67   assert(dst->cpp == src->cpp);
68
69   nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
70   nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
71   nouveau_pushbuf_bufctx(push, bctx);
72   nouveau_pushbuf_validate(push);
73
74   if (nouveau_bo_memtype(src->bo)) {
75      BEGIN_NV04(push, NV50_M2MF(LINEAR_IN), 6);
76      PUSH_DATA (push, 0);
77      PUSH_DATA (push, src->tile_mode);
78      PUSH_DATA (push, src->width * cpp);
79      PUSH_DATA (push, src->height);
80      PUSH_DATA (push, src->depth);
81      PUSH_DATA (push, src->z);
82   } else {
83      src_ofst += src->y * src->pitch + src->x * cpp;
84
85      BEGIN_NV04(push, NV50_M2MF(LINEAR_IN), 1);
86      PUSH_DATA (push, 1);
87      BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_PITCH_IN), 1);
88      PUSH_DATA (push, src->pitch);
89   }
90
91   if (nouveau_bo_memtype(dst->bo)) {
92      BEGIN_NV04(push, NV50_M2MF(LINEAR_OUT), 6);
93      PUSH_DATA (push, 0);
94      PUSH_DATA (push, dst->tile_mode);
95      PUSH_DATA (push, dst->width * cpp);
96      PUSH_DATA (push, dst->height);
97      PUSH_DATA (push, dst->depth);
98      PUSH_DATA (push, dst->z);
99   } else {
100      dst_ofst += dst->y * dst->pitch + dst->x * cpp;
101
102      BEGIN_NV04(push, NV50_M2MF(LINEAR_OUT), 1);
103      PUSH_DATA (push, 1);
104      BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_PITCH_OUT), 1);
105      PUSH_DATA (push, dst->pitch);
106   }
107
108   while (height) {
109      int line_count = height > 2047 ? 2047 : height;
110
111      BEGIN_NV04(push, NV50_M2MF(OFFSET_IN_HIGH), 2);
112      PUSH_DATAh(push, src->bo->offset + src_ofst);
113      PUSH_DATAh(push, dst->bo->offset + dst_ofst);
114
115      BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_OFFSET_IN), 2);
116      PUSH_DATA (push, src->bo->offset + src_ofst);
117      PUSH_DATA (push, dst->bo->offset + dst_ofst);
118
119      if (nouveau_bo_memtype(src->bo)) {
120         BEGIN_NV04(push, NV50_M2MF(TILING_POSITION_IN), 1);
121         PUSH_DATA (push, (sy << 16) | (src->x * cpp));
122      } else {
123         src_ofst += line_count * src->pitch;
124      }
125      if (nouveau_bo_memtype(dst->bo)) {
126         BEGIN_NV04(push, NV50_M2MF(TILING_POSITION_OUT), 1);
127         PUSH_DATA (push, (dy << 16) | (dst->x * cpp));
128      } else {
129         dst_ofst += line_count * dst->pitch;
130      }
131
132      BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_LINE_LENGTH_IN), 4);
133      PUSH_DATA (push, nblocksx * cpp);
134      PUSH_DATA (push, line_count);
135      PUSH_DATA (push, (1 << 8) | (1 << 0));
136      PUSH_DATA (push, 0);
137
138      height -= line_count;
139      sy += line_count;
140      dy += line_count;
141   }
142
143   nouveau_bufctx_reset(bctx, 0);
144}
145
146void
147nv50_sifc_linear_u8(struct nouveau_context *nv,
148                    struct nouveau_bo *dst, unsigned offset, unsigned domain,
149                    unsigned size, const void *data)
150{
151   struct nv50_context *nv50 = nv50_context(&nv->pipe);
152   struct nouveau_pushbuf *push = nv50->base.pushbuf;
153   uint32_t *src = (uint32_t *)data;
154   unsigned count = (size + 3) / 4;
155   unsigned xcoord = offset & 0xff;
156
157   nouveau_bufctx_refn(nv50->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
158   nouveau_pushbuf_bufctx(push, nv50->bufctx);
159   nouveau_pushbuf_validate(push);
160
161   offset &= ~0xff;
162
163   BEGIN_NV04(push, NV50_2D(DST_FORMAT), 2);
164   PUSH_DATA (push, NV50_SURFACE_FORMAT_R8_UNORM);
165   PUSH_DATA (push, 1);
166   BEGIN_NV04(push, NV50_2D(DST_PITCH), 5);
167   PUSH_DATA (push, 262144);
168   PUSH_DATA (push, 65536);
169   PUSH_DATA (push, 1);
170   PUSH_DATAh(push, dst->offset + offset);
171   PUSH_DATA (push, dst->offset + offset);
172   BEGIN_NV04(push, NV50_2D(SIFC_BITMAP_ENABLE), 2);
173   PUSH_DATA (push, 0);
174   PUSH_DATA (push, NV50_SURFACE_FORMAT_R8_UNORM);
175   BEGIN_NV04(push, NV50_2D(SIFC_WIDTH), 10);
176   PUSH_DATA (push, size);
177   PUSH_DATA (push, 1);
178   PUSH_DATA (push, 0);
179   PUSH_DATA (push, 1);
180   PUSH_DATA (push, 0);
181   PUSH_DATA (push, 1);
182   PUSH_DATA (push, 0);
183   PUSH_DATA (push, xcoord);
184   PUSH_DATA (push, 0);
185   PUSH_DATA (push, 0);
186
187   while (count) {
188      unsigned nr;
189
190      if (!PUSH_SPACE(push, 16))
191         break;
192      nr = PUSH_AVAIL(push);
193      assert(nr >= 16);
194      nr = MIN2(count, nr - 1);
195      nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
196
197      BEGIN_NI04(push, NV50_2D(SIFC_DATA), nr);
198      PUSH_DATAp(push, src, nr);
199
200      src += nr;
201      count -= nr;
202   }
203
204   nouveau_bufctx_reset(nv50->bufctx, 0);
205}
206
207void
208nv50_m2mf_copy_linear(struct nouveau_context *nv,
209                      struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
210                      struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
211                      unsigned size)
212{
213   struct nouveau_pushbuf *push = nv->pushbuf;
214   struct nouveau_bufctx *bctx = nv50_context(&nv->pipe)->bufctx;
215
216   nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
217   nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
218   nouveau_pushbuf_bufctx(push, bctx);
219   nouveau_pushbuf_validate(push);
220
221   BEGIN_NV04(push, NV50_M2MF(LINEAR_IN), 1);
222   PUSH_DATA (push, 1);
223   BEGIN_NV04(push, NV50_M2MF(LINEAR_OUT), 1);
224   PUSH_DATA (push, 1);
225
226   while (size) {
227      unsigned bytes = MIN2(size, 1 << 17);
228
229      BEGIN_NV04(push, NV50_M2MF(OFFSET_IN_HIGH), 2);
230      PUSH_DATAh(push, src->offset + srcoff);
231      PUSH_DATAh(push, dst->offset + dstoff);
232      BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_OFFSET_IN), 2);
233      PUSH_DATA (push, src->offset + srcoff);
234      PUSH_DATA (push, dst->offset + dstoff);
235      BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_LINE_LENGTH_IN), 4);
236      PUSH_DATA (push, bytes);
237      PUSH_DATA (push, 1);
238      PUSH_DATA (push, (1 << 8) | (1 << 0));
239      PUSH_DATA (push, 0);
240
241      srcoff += bytes;
242      dstoff += bytes;
243      size -= bytes;
244   }
245
246   nouveau_bufctx_reset(bctx, 0);
247}
248
249struct pipe_transfer *
250nv50_miptree_transfer_new(struct pipe_context *pctx,
251                          struct pipe_resource *res,
252                          unsigned level,
253                          unsigned usage,
254                          const struct pipe_box *box)
255{
256   struct nv50_context *nv50 = nv50_context(pctx);
257   struct nouveau_device *dev = nv50->screen->base.device;
258   const struct nv50_miptree *mt = nv50_miptree(res);
259   struct nv50_transfer *tx;
260   uint32_t size;
261   int ret;
262
263   if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
264      return NULL;
265
266   tx = CALLOC_STRUCT(nv50_transfer);
267   if (!tx)
268      return NULL;
269
270   pipe_resource_reference(&tx->base.resource, res);
271
272   tx->base.level = level;
273   tx->base.usage = usage;
274   tx->base.box = *box;
275
276   if (util_format_is_plain(res->format)) {
277      tx->nblocksx = box->width << mt->ms_x;
278      tx->nblocksy = box->height << mt->ms_x;
279   } else {
280      tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
281      tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
282   }
283
284   tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
285   tx->base.layer_stride = tx->nblocksy * tx->base.stride;
286
287   nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
288
289   size = tx->base.layer_stride;
290
291   ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
292                        size * tx->base.box.depth, NULL, &tx->rect[1].bo);
293   if (ret) {
294      FREE(tx);
295      return NULL;
296   }
297
298   tx->rect[1].cpp = tx->rect[0].cpp;
299   tx->rect[1].width = tx->nblocksx;
300   tx->rect[1].height = tx->nblocksy;
301   tx->rect[1].depth = 1;
302   tx->rect[1].pitch = tx->base.stride;
303   tx->rect[1].domain = NOUVEAU_BO_GART;
304
305   if (usage & PIPE_TRANSFER_READ) {
306      unsigned base = tx->rect[0].base;
307      unsigned z = tx->rect[0].z;
308      unsigned i;
309      for (i = 0; i < box->depth; ++i) {
310         nv50_m2mf_transfer_rect(nv50, &tx->rect[1], &tx->rect[0],
311                                 tx->nblocksx, tx->nblocksy);
312         if (mt->layout_3d)
313            tx->rect[0].z++;
314         else
315            tx->rect[0].base += mt->layer_stride;
316         tx->rect[1].base += size;
317      }
318      tx->rect[0].z = z;
319      tx->rect[0].base = base;
320      tx->rect[1].base = 0;
321   }
322
323   return &tx->base;
324}
325
326void
327nv50_miptree_transfer_del(struct pipe_context *pctx,
328                          struct pipe_transfer *transfer)
329{
330   struct nv50_context *nv50 = nv50_context(pctx);
331   struct nv50_transfer *tx = (struct nv50_transfer *)transfer;
332   struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
333   unsigned i;
334
335   if (tx->base.usage & PIPE_TRANSFER_WRITE) {
336      for (i = 0; i < tx->base.box.depth; ++i) {
337         nv50_m2mf_transfer_rect(nv50, &tx->rect[0], &tx->rect[1],
338                                 tx->nblocksx, tx->nblocksy);
339         if (mt->layout_3d)
340            tx->rect[0].z++;
341         else
342            tx->rect[0].base += mt->layer_stride;
343         tx->rect[1].base += tx->nblocksy * tx->base.stride;
344      }
345   }
346
347   nouveau_bo_ref(NULL, &tx->rect[1].bo);
348   pipe_resource_reference(&transfer->resource, NULL);
349
350   FREE(tx);
351}
352
353void *
354nv50_miptree_transfer_map(struct pipe_context *pctx,
355                          struct pipe_transfer *transfer)
356{
357   struct nv50_screen *screen = nv50_screen(pctx->screen);
358   struct nv50_transfer *tx = (struct nv50_transfer *)transfer;
359   int ret;
360   unsigned flags = 0;
361
362   if (tx->rect[1].bo->map)
363      return tx->rect[1].bo->map;
364
365   if (transfer->usage & PIPE_TRANSFER_READ)
366      flags = NOUVEAU_BO_RD;
367   if (transfer->usage & PIPE_TRANSFER_WRITE)
368      flags |= NOUVEAU_BO_WR;
369
370   ret = nouveau_bo_map(tx->rect[1].bo, flags, screen->base.client);
371   if (ret)
372      return NULL;
373   return tx->rect[1].bo->map;
374}
375
376void
377nv50_miptree_transfer_unmap(struct pipe_context *pctx,
378                            struct pipe_transfer *transfer)
379{
380   /* nothing to do */
381}
382
383void
384nv50_cb_push(struct nouveau_context *nv,
385             struct nouveau_bo *bo, unsigned domain,
386             unsigned base, unsigned size,
387             unsigned offset, unsigned words, const uint32_t *data)
388{
389   struct nouveau_pushbuf *push = nv->pushbuf;
390   struct nouveau_bufctx *bctx = nv50_context(&nv->pipe)->bufctx;
391
392   assert(!(offset & 3));
393   size = align(size, 0x100);
394
395   nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
396   nouveau_pushbuf_bufctx(push, bctx);
397   nouveau_pushbuf_validate(push);
398
399   while (words) {
400      unsigned nr;
401
402      nr = PUSH_AVAIL(push);
403      nr = MIN2(nr - 7, words);
404      nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
405
406      BEGIN_NV04(push, NV50_3D(CB_DEF_ADDRESS_HIGH), 3);
407      PUSH_DATAh(push, bo->offset + base);
408      PUSH_DATA (push, bo->offset + base);
409      PUSH_DATA (push, (NV50_CB_TMP << 16) | (size & 0xffff));
410      BEGIN_NV04(push, NV50_3D(CB_ADDR), 1);
411      PUSH_DATA (push, (offset << 6) | NV50_CB_TMP);
412      BEGIN_NI04(push, NV50_3D(CB_DATA(0)), nr);
413      PUSH_DATAp(push, data, nr);
414
415      words -= nr;
416      data += nr;
417      offset += nr * 4;
418   }
419
420   nouveau_bufctx_reset(bctx, 0);
421}
422