1
2#include "util/u_inlines.h"
3#include "util/u_memory.h"
4#include "util/u_math.h"
5
6#include "nouveau_screen.h"
7#include "nouveau_context.h"
8#include "nouveau_winsys.h"
9#include "nouveau_fence.h"
10#include "nouveau_buffer.h"
11#include "nouveau_mm.h"
12
13struct nouveau_transfer {
14   struct pipe_transfer base;
15};
16
17static INLINE struct nouveau_transfer *
18nouveau_transfer(struct pipe_transfer *transfer)
19{
20   return (struct nouveau_transfer *)transfer;
21}
22
23static INLINE boolean
24nouveau_buffer_allocate(struct nouveau_screen *screen,
25                        struct nv04_resource *buf, unsigned domain)
26{
27   uint32_t size = buf->base.width0;
28
29   if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
30      size = align(size, 0x100);
31
32   if (domain == NOUVEAU_BO_VRAM) {
33      buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
34                                    &buf->bo, &buf->offset);
35      if (!buf->bo)
36         return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
37   } else
38   if (domain == NOUVEAU_BO_GART) {
39      buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
40                                    &buf->bo, &buf->offset);
41      if (!buf->bo)
42         return FALSE;
43   }
44   if (domain != NOUVEAU_BO_GART) {
45      if (!buf->data) {
46         buf->data = MALLOC(buf->base.width0);
47         if (!buf->data)
48            return FALSE;
49      }
50   }
51   buf->domain = domain;
52   if (buf->bo)
53      buf->address = buf->bo->offset + buf->offset;
54
55   return TRUE;
56}
57
58static INLINE void
59release_allocation(struct nouveau_mm_allocation **mm,
60                   struct nouveau_fence *fence)
61{
62   nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
63   (*mm) = NULL;
64}
65
66INLINE void
67nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
68{
69   nouveau_bo_ref(NULL, &buf->bo);
70
71   if (buf->mm)
72      release_allocation(&buf->mm, buf->fence);
73
74   buf->domain = 0;
75}
76
77static INLINE boolean
78nouveau_buffer_reallocate(struct nouveau_screen *screen,
79                          struct nv04_resource *buf, unsigned domain)
80{
81   nouveau_buffer_release_gpu_storage(buf);
82
83   return nouveau_buffer_allocate(screen, buf, domain);
84}
85
86static void
87nouveau_buffer_destroy(struct pipe_screen *pscreen,
88                       struct pipe_resource *presource)
89{
90   struct nv04_resource *res = nv04_resource(presource);
91
92   nouveau_buffer_release_gpu_storage(res);
93
94   if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
95      FREE(res->data);
96
97   nouveau_fence_ref(NULL, &res->fence);
98   nouveau_fence_ref(NULL, &res->fence_wr);
99
100   FREE(res);
101}
102
103/* Maybe just migrate to GART right away if we actually need to do this. */
104boolean
105nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf,
106                        unsigned start, unsigned size)
107{
108   struct nouveau_mm_allocation *mm;
109   struct nouveau_bo *bounce = NULL;
110   uint32_t offset;
111
112   assert(buf->domain == NOUVEAU_BO_VRAM);
113
114   mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
115   if (!bounce)
116      return FALSE;
117
118   nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART,
119                 buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size);
120
121   if (nouveau_bo_map(bounce, NOUVEAU_BO_RD, nv->screen->client))
122      return FALSE;
123   memcpy(buf->data + start, (uint8_t *)bounce->map + offset, size);
124
125   buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
126
127   nouveau_bo_ref(NULL, &bounce);
128   if (mm)
129      nouveau_mm_free(mm);
130   return TRUE;
131}
132
133static boolean
134nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf,
135                      unsigned start, unsigned size)
136{
137   struct nouveau_mm_allocation *mm;
138   struct nouveau_bo *bounce = NULL;
139   uint32_t offset;
140
141   if (size <= 192 && (nv->push_data || nv->push_cb)) {
142      if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
143         nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
144                     start, size / 4, (const uint32_t *)(buf->data + start));
145      else
146         nv->push_data(nv, buf->bo, buf->offset + start, buf->domain,
147                       size, buf->data + start);
148      return TRUE;
149   }
150
151   mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
152   if (!bounce)
153      return FALSE;
154
155   nouveau_bo_map(bounce, 0, nv->screen->client);
156   memcpy((uint8_t *)bounce->map + offset, buf->data + start, size);
157
158   nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
159                 bounce, offset, NOUVEAU_BO_GART, size);
160
161   nouveau_bo_ref(NULL, &bounce);
162   if (mm)
163      release_allocation(&mm, nv->screen->fence.current);
164
165   if (start == 0 && size == buf->base.width0)
166      buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
167   return TRUE;
168}
169
170static struct pipe_transfer *
171nouveau_buffer_transfer_get(struct pipe_context *pipe,
172                            struct pipe_resource *resource,
173                            unsigned level, unsigned usage,
174                            const struct pipe_box *box)
175{
176   struct nv04_resource *buf = nv04_resource(resource);
177   struct nouveau_context *nv = nouveau_context(pipe);
178   struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer);
179   if (!xfr)
180      return NULL;
181
182   xfr->base.resource = resource;
183   xfr->base.box.x = box->x;
184   xfr->base.box.width = box->width;
185   xfr->base.usage = usage;
186
187   if (buf->domain == NOUVEAU_BO_VRAM) {
188      if (usage & PIPE_TRANSFER_READ) {
189         if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)
190            nouveau_buffer_download(nv, buf, 0, buf->base.width0);
191      }
192   }
193
194   return &xfr->base;
195}
196
197static void
198nouveau_buffer_transfer_destroy(struct pipe_context *pipe,
199                                struct pipe_transfer *transfer)
200{
201   struct nv04_resource *buf = nv04_resource(transfer->resource);
202   struct nouveau_transfer *xfr = nouveau_transfer(transfer);
203   struct nouveau_context *nv = nouveau_context(pipe);
204
205   if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
206      if (buf->domain == NOUVEAU_BO_VRAM) {
207         nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width);
208      }
209
210      if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
211                                                 PIPE_BIND_INDEX_BUFFER)))
212         nouveau_context(pipe)->vbo_dirty = TRUE;
213   }
214
215   FREE(xfr);
216}
217
218static INLINE boolean
219nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
220{
221   if (rw == PIPE_TRANSFER_READ) {
222      if (!buf->fence_wr)
223         return TRUE;
224      if (!nouveau_fence_wait(buf->fence_wr))
225         return FALSE;
226   } else {
227      if (!buf->fence)
228         return TRUE;
229      if (!nouveau_fence_wait(buf->fence))
230         return FALSE;
231
232      nouveau_fence_ref(NULL, &buf->fence);
233   }
234   nouveau_fence_ref(NULL, &buf->fence_wr);
235
236   return TRUE;
237}
238
239static INLINE boolean
240nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
241{
242   if (rw == PIPE_TRANSFER_READ)
243      return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
244   else
245      return (buf->fence && !nouveau_fence_signalled(buf->fence));
246}
247
248static void *
249nouveau_buffer_transfer_map(struct pipe_context *pipe,
250                            struct pipe_transfer *transfer)
251{
252   struct nouveau_context *nv = nouveau_context(pipe);
253   struct nouveau_transfer *xfr = nouveau_transfer(transfer);
254   struct nv04_resource *buf = nv04_resource(transfer->resource);
255   struct nouveau_bo *bo = buf->bo;
256   uint8_t *map;
257   int ret;
258   uint32_t offset = xfr->base.box.x;
259   uint32_t flags = 0;
260
261   if (buf->domain != NOUVEAU_BO_GART)
262      return buf->data + offset;
263
264   if (!buf->mm)
265      flags = nouveau_screen_transfer_flags(xfr->base.usage);
266
267   offset += buf->offset;
268
269   ret = nouveau_bo_map(buf->bo, flags, nv->screen->client);
270   if (ret)
271      return NULL;
272   map = (uint8_t *)bo->map + offset;
273
274   if (buf->mm) {
275      if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
276         if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
277            return NULL;
278      } else
279      if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
280         nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
281      }
282   }
283   return map;
284}
285
286
287
288static void
289nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
290                                     struct pipe_transfer *transfer,
291                                     const struct pipe_box *box)
292{
293#if 0
294   struct nv04_resource *res = nv04_resource(transfer->resource);
295   struct nouveau_bo *bo = res->bo;
296   unsigned offset = res->offset + transfer->box.x + box->x;
297
298   /* not using non-snoop system memory yet, no need for cflush */
299   if (1)
300      return;
301
302   /* XXX: maybe need to upload for VRAM buffers here */
303#endif
304}
305
306static void
307nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
308                              struct pipe_transfer *transfer)
309{
310}
311
312
313void *
314nouveau_resource_map_offset(struct nouveau_context *nv,
315                            struct nv04_resource *res, uint32_t offset,
316                            uint32_t flags)
317{
318   if ((res->domain == NOUVEAU_BO_VRAM) &&
319       (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
320      nouveau_buffer_download(nv, res, 0, res->base.width0);
321
322   if ((res->domain != NOUVEAU_BO_GART) ||
323       (res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
324      return res->data + offset;
325
326   if (res->mm) {
327      unsigned rw;
328      rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
329      nouveau_buffer_sync(res, rw);
330      if (nouveau_bo_map(res->bo, 0, NULL))
331         return NULL;
332   } else {
333      if (nouveau_bo_map(res->bo, flags, nv->screen->client))
334         return NULL;
335   }
336   return (uint8_t *)res->bo->map + res->offset + offset;
337}
338
339
340const struct u_resource_vtbl nouveau_buffer_vtbl =
341{
342   u_default_resource_get_handle,     /* get_handle */
343   nouveau_buffer_destroy,               /* resource_destroy */
344   nouveau_buffer_transfer_get,          /* get_transfer */
345   nouveau_buffer_transfer_destroy,      /* transfer_destroy */
346   nouveau_buffer_transfer_map,          /* transfer_map */
347   nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
348   nouveau_buffer_transfer_unmap,        /* transfer_unmap */
349   u_default_transfer_inline_write    /* transfer_inline_write */
350};
351
352struct pipe_resource *
353nouveau_buffer_create(struct pipe_screen *pscreen,
354                      const struct pipe_resource *templ)
355{
356   struct nouveau_screen *screen = nouveau_screen(pscreen);
357   struct nv04_resource *buffer;
358   boolean ret;
359
360   buffer = CALLOC_STRUCT(nv04_resource);
361   if (!buffer)
362      return NULL;
363
364   buffer->base = *templ;
365   buffer->vtbl = &nouveau_buffer_vtbl;
366   pipe_reference_init(&buffer->base.reference, 1);
367   buffer->base.screen = pscreen;
368
369   if (buffer->base.bind &
370       (screen->vidmem_bindings & screen->sysmem_bindings)) {
371      switch (buffer->base.usage) {
372      case PIPE_USAGE_DEFAULT:
373      case PIPE_USAGE_IMMUTABLE:
374      case PIPE_USAGE_STATIC:
375         buffer->domain = NOUVEAU_BO_VRAM;
376         break;
377      case PIPE_USAGE_DYNAMIC:
378      case PIPE_USAGE_STAGING:
379      case PIPE_USAGE_STREAM:
380         buffer->domain = NOUVEAU_BO_GART;
381         break;
382      default:
383         assert(0);
384         break;
385      }
386   } else {
387      if (buffer->base.bind & screen->vidmem_bindings)
388         buffer->domain = NOUVEAU_BO_VRAM;
389      else
390      if (buffer->base.bind & screen->sysmem_bindings)
391         buffer->domain = NOUVEAU_BO_GART;
392   }
393   ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
394
395   if (ret == FALSE)
396      goto fail;
397
398   return &buffer->base;
399
400fail:
401   FREE(buffer);
402   return NULL;
403}
404
405
406struct pipe_resource *
407nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
408                           unsigned bytes, unsigned bind)
409{
410   struct nv04_resource *buffer;
411
412   buffer = CALLOC_STRUCT(nv04_resource);
413   if (!buffer)
414      return NULL;
415
416   pipe_reference_init(&buffer->base.reference, 1);
417   buffer->vtbl = &nouveau_buffer_vtbl;
418   buffer->base.screen = pscreen;
419   buffer->base.format = PIPE_FORMAT_R8_UNORM;
420   buffer->base.usage = PIPE_USAGE_IMMUTABLE;
421   buffer->base.bind = bind;
422   buffer->base.width0 = bytes;
423   buffer->base.height0 = 1;
424   buffer->base.depth0 = 1;
425
426   buffer->data = ptr;
427   buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
428
429   return &buffer->base;
430}
431
432/* Like download, but for GART buffers. Merge ? */
433static INLINE boolean
434nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
435                          struct nouveau_bo *bo, unsigned offset, unsigned size)
436{
437   if (!buf->data) {
438      buf->data = MALLOC(size);
439      if (!buf->data)
440         return FALSE;
441   }
442   if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->screen->client))
443      return FALSE;
444   memcpy(buf->data, (uint8_t *)bo->map + offset, size);
445
446   return TRUE;
447}
448
449/* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
450boolean
451nouveau_buffer_migrate(struct nouveau_context *nv,
452                       struct nv04_resource *buf, const unsigned new_domain)
453{
454   struct nouveau_screen *screen = nv->screen;
455   struct nouveau_bo *bo;
456   const unsigned old_domain = buf->domain;
457   unsigned size = buf->base.width0;
458   unsigned offset;
459   int ret;
460
461   assert(new_domain != old_domain);
462
463   if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
464      if (!nouveau_buffer_allocate(screen, buf, new_domain))
465         return FALSE;
466      ret = nouveau_bo_map(buf->bo, 0, nv->screen->client);
467      if (ret)
468         return ret;
469      memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
470      FREE(buf->data);
471   } else
472   if (old_domain != 0 && new_domain != 0) {
473      struct nouveau_mm_allocation *mm = buf->mm;
474
475      if (new_domain == NOUVEAU_BO_VRAM) {
476         /* keep a system memory copy of our data in case we hit a fallback */
477         if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
478            return FALSE;
479         if (nouveau_mesa_debug)
480            debug_printf("migrating %u KiB to VRAM\n", size / 1024);
481      }
482
483      offset = buf->offset;
484      bo = buf->bo;
485      buf->bo = NULL;
486      buf->mm = NULL;
487      nouveau_buffer_allocate(screen, buf, new_domain);
488
489      nv->copy_data(nv, buf->bo, buf->offset, new_domain,
490                    bo, offset, old_domain, buf->base.width0);
491
492      nouveau_bo_ref(NULL, &bo);
493      if (mm)
494         release_allocation(&mm, screen->fence.current);
495   } else
496   if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
497      if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
498         return FALSE;
499      if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0))
500         return FALSE;
501   } else
502      return FALSE;
503
504   assert(buf->domain == new_domain);
505   return TRUE;
506}
507
508/* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
509 * We'd like to only allocate @size bytes here, but then we'd have to rebase
510 * the vertex indices ...
511 */
512boolean
513nouveau_user_buffer_upload(struct nouveau_context *nv,
514                           struct nv04_resource *buf,
515                           unsigned base, unsigned size)
516{
517   struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
518   int ret;
519
520   assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
521
522   buf->base.width0 = base + size;
523   if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
524      return FALSE;
525
526   ret = nouveau_bo_map(buf->bo, 0, nv->screen->client);
527   if (ret)
528      return FALSE;
529   memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
530
531   return TRUE;
532}
533
534
535/* Scratch data allocation. */
536
537static INLINE int
538nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
539                         unsigned size)
540{
541   return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
542                         4096, size, NULL, pbo);
543}
544
545void
546nouveau_scratch_runout_release(struct nouveau_context *nv)
547{
548   if (!nv->scratch.nr_runout)
549      return;
550   do {
551      --nv->scratch.nr_runout;
552      nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
553   } while (nv->scratch.nr_runout);
554
555   FREE(nv->scratch.runout);
556   nv->scratch.end = 0;
557   nv->scratch.runout = NULL;
558}
559
560/* Allocate an extra bo if we can't fit everything we need simultaneously.
561 * (Could happen for very large user arrays.)
562 */
563static INLINE boolean
564nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
565{
566   int ret;
567   const unsigned n = nv->scratch.nr_runout++;
568
569   nv->scratch.runout = REALLOC(nv->scratch.runout,
570                                (n + 0) * sizeof(*nv->scratch.runout),
571                                (n + 1) * sizeof(*nv->scratch.runout));
572   nv->scratch.runout[n] = NULL;
573
574   ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
575   if (!ret) {
576      ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
577      if (ret)
578         nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
579   }
580   if (!ret) {
581      nv->scratch.current = nv->scratch.runout[n];
582      nv->scratch.offset = 0;
583      nv->scratch.end = size;
584      nv->scratch.map = nv->scratch.current->map;
585   }
586   return !ret;
587}
588
589/* Continue to next scratch buffer, if available (no wrapping, large enough).
590 * Allocate it if it has not yet been created.
591 */
592static INLINE boolean
593nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
594{
595   struct nouveau_bo *bo;
596   int ret;
597   const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
598
599   if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
600      return FALSE;
601   nv->scratch.id = i;
602
603   bo = nv->scratch.bo[i];
604   if (!bo) {
605      ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
606      if (ret)
607         return FALSE;
608      nv->scratch.bo[i] = bo;
609   }
610   nv->scratch.current = bo;
611   nv->scratch.offset = 0;
612   nv->scratch.end = nv->scratch.bo_size;
613
614   ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->screen->client);
615   if (!ret)
616      nv->scratch.map = bo->map;
617   return !ret;
618}
619
620static boolean
621nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
622{
623   boolean ret;
624
625   ret = nouveau_scratch_next(nv, min_size);
626   if (!ret)
627      ret = nouveau_scratch_runout(nv, min_size);
628   return ret;
629}
630
631
632/* Copy data to a scratch buffer and return address & bo the data resides in. */
633uint64_t
634nouveau_scratch_data(struct nouveau_context *nv,
635                     const void *data, unsigned base, unsigned size,
636                     struct nouveau_bo **bo)
637{
638   unsigned bgn = MAX2(base, nv->scratch.offset);
639   unsigned end = bgn + size;
640
641   if (end >= nv->scratch.end) {
642      end = base + size;
643      if (!nouveau_scratch_more(nv, end))
644         return 0;
645      bgn = base;
646   }
647   nv->scratch.offset = align(end, 4);
648
649   memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
650
651   *bo = nv->scratch.current;
652   return (*bo)->offset + (bgn - base);
653}
654
655void *
656nouveau_scratch_get(struct nouveau_context *nv,
657                    unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
658{
659   unsigned bgn = nv->scratch.offset;
660   unsigned end = nv->scratch.offset + size;
661
662   if (end >= nv->scratch.end) {
663      end = size;
664      if (!nouveau_scratch_more(nv, end))
665         return NULL;
666      bgn = 0;
667   }
668   nv->scratch.offset = align(end, 4);
669
670   *pbo = nv->scratch.current;
671   *gpu_addr = nv->scratch.current->offset + bgn;
672   return nv->scratch.map + bgn;
673}
674