freedreno_resource.c revision 7105774babc4d23623c3547cd19122a55c1090db
1/* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */ 2 3/* 4 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org> 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Rob Clark <robclark@freedesktop.org> 27 */ 28 29#include "util/u_format.h" 30#include "util/u_format_rgtc.h" 31#include "util/u_format_zs.h" 32#include "util/u_inlines.h" 33#include "util/u_transfer.h" 34#include "util/u_string.h" 35#include "util/u_surface.h" 36#include "util/set.h" 37 38#include "freedreno_resource.h" 39#include "freedreno_batch_cache.h" 40#include "freedreno_screen.h" 41#include "freedreno_surface.h" 42#include "freedreno_context.h" 43#include "freedreno_query_hw.h" 44#include "freedreno_util.h" 45 46#include <errno.h> 47 48/* XXX this should go away, needed for 'struct winsys_handle' */ 49#include "state_tracker/drm_driver.h" 50 51static bool 52pending(struct fd_resource *rsc, bool write) 53{ 54 /* if we have a pending GPU write, we are busy in any case: */ 55 if (rsc->write_batch) 56 return true; 57 58 /* if CPU wants to write, but we are pending a GPU read, we are busy: */ 59 if (write && rsc->batch_mask) 60 return true; 61 62 if (rsc->stencil && pending(rsc->stencil, write)) 63 return true; 64 65 return false; 66} 67 68static void 69fd_invalidate_resource(struct fd_context *ctx, struct pipe_resource *prsc) 70{ 71 int i; 72 73 /* Go through the entire state and see if the resource is bound 74 * anywhere. If it is, mark the relevant state as dirty. This is called on 75 * realloc_bo. 76 */ 77 78 /* Constbufs */ 79 for (i = 1; i < PIPE_MAX_CONSTANT_BUFFERS && !(ctx->dirty & FD_DIRTY_CONSTBUF); i++) { 80 if (ctx->constbuf[PIPE_SHADER_VERTEX].cb[i].buffer == prsc) 81 ctx->dirty |= FD_DIRTY_CONSTBUF; 82 if (ctx->constbuf[PIPE_SHADER_FRAGMENT].cb[i].buffer == prsc) 83 ctx->dirty |= FD_DIRTY_CONSTBUF; 84 } 85 86 /* VBOs */ 87 for (i = 0; i < ctx->vtx.vertexbuf.count && !(ctx->dirty & FD_DIRTY_VTXBUF); i++) { 88 if (ctx->vtx.vertexbuf.vb[i].buffer == prsc) 89 ctx->dirty |= FD_DIRTY_VTXBUF; 90 } 91 92 /* Index buffer */ 93 if (ctx->indexbuf.buffer == prsc) 94 ctx->dirty |= FD_DIRTY_INDEXBUF; 95 96 /* Textures */ 97 for (i = 0; i < ctx->verttex.num_textures && !(ctx->dirty & FD_DIRTY_VERTTEX); i++) { 98 if (ctx->verttex.textures[i] && (ctx->verttex.textures[i]->texture == prsc)) 99 ctx->dirty |= FD_DIRTY_VERTTEX; 100 } 101 for (i = 0; i < ctx->fragtex.num_textures && !(ctx->dirty & FD_DIRTY_FRAGTEX); i++) { 102 if (ctx->fragtex.textures[i] && (ctx->fragtex.textures[i]->texture == prsc)) 103 ctx->dirty |= FD_DIRTY_FRAGTEX; 104 } 105} 106 107static void 108realloc_bo(struct fd_resource *rsc, uint32_t size) 109{ 110 struct fd_screen *screen = fd_screen(rsc->base.b.screen); 111 uint32_t flags = DRM_FREEDRENO_GEM_CACHE_WCOMBINE | 112 DRM_FREEDRENO_GEM_TYPE_KMEM; /* TODO */ 113 114 /* if we start using things other than write-combine, 115 * be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT 116 */ 117 118 if (rsc->bo) 119 fd_bo_del(rsc->bo); 120 121 rsc->bo = fd_bo_new(screen->dev, size, flags); 122 rsc->timestamp = 0; 123 util_range_set_empty(&rsc->valid_buffer_range); 124 fd_bc_invalidate_resource(rsc, true); 125} 126 127static void fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond); 128static void fd_blitter_pipe_end(struct fd_context *ctx); 129 130static void 131do_blit(struct fd_context *ctx, const struct pipe_blit_info *blit, bool fallback) 132{ 133 /* TODO size threshold too?? */ 134 if ((blit->src.resource->target != PIPE_BUFFER) && !fallback) { 135 /* do blit on gpu: */ 136 fd_blitter_pipe_begin(ctx, false); 137 util_blitter_blit(ctx->blitter, blit); 138 fd_blitter_pipe_end(ctx); 139 } else { 140 /* do blit on cpu: */ 141 util_resource_copy_region(&ctx->base, 142 blit->dst.resource, blit->dst.level, blit->dst.box.x, 143 blit->dst.box.y, blit->dst.box.z, 144 blit->src.resource, blit->src.level, &blit->src.box); 145 } 146} 147 148static bool 149fd_try_shadow_resource(struct fd_context *ctx, struct fd_resource *rsc, 150 unsigned level, unsigned usage, const struct pipe_box *box) 151{ 152 struct pipe_context *pctx = &ctx->base; 153 struct pipe_resource *prsc = &rsc->base.b; 154 bool fallback = false; 155 156 /* TODO: somehow munge dimensions and format to copy unsupported 157 * render target format to something that is supported? 158 */ 159 if (!pctx->screen->is_format_supported(pctx->screen, 160 prsc->format, prsc->target, prsc->nr_samples, 161 PIPE_BIND_RENDER_TARGET)) 162 fallback = true; 163 164 /* these cases should be handled elsewhere.. just for future 165 * reference in case this gets split into a more generic(ish) 166 * helper. 167 */ 168 debug_assert(!(usage & PIPE_TRANSFER_READ)); 169 debug_assert(!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)); 170 171 /* if we do a gpu blit to clone the whole resource, we'll just 172 * end up stalling on that.. so only allow if we can discard 173 * current range (and blit, possibly cpu or gpu, the rest) 174 */ 175 if (!(usage & PIPE_TRANSFER_DISCARD_RANGE)) 176 return false; 177 178 bool whole_level = util_texrange_covers_whole_level(prsc, level, 179 box->x, box->y, box->z, box->width, box->height, box->depth); 180 181 /* TODO need to be more clever about current level */ 182 if ((prsc->target >= PIPE_TEXTURE_2D) && !whole_level) 183 return false; 184 185 struct pipe_resource *pshadow = 186 pctx->screen->resource_create(pctx->screen, prsc); 187 188 if (!pshadow) 189 return false; 190 191 assert(!ctx->in_shadow); 192 ctx->in_shadow = true; 193 194 /* get rid of any references that batch-cache might have to us (which 195 * should empty/destroy rsc->batches hashset) 196 */ 197 fd_bc_invalidate_resource(rsc, false); 198 199 /* Swap the backing bo's, so shadow becomes the old buffer, 200 * blit from shadow to new buffer. From here on out, we 201 * cannot fail. 202 * 203 * Note that we need to do it in this order, otherwise if 204 * we go down cpu blit path, the recursive transfer_map() 205 * sees the wrong status.. 206 */ 207 struct fd_resource *shadow = fd_resource(pshadow); 208 209 DBG("shadow: %p (%d) -> %p (%d)\n", rsc, rsc->base.b.reference.count, 210 shadow, shadow->base.b.reference.count); 211 212 /* TODO valid_buffer_range?? */ 213 swap(rsc->bo, shadow->bo); 214 swap(rsc->timestamp, shadow->timestamp); 215 swap(rsc->write_batch, shadow->write_batch); 216 217 /* at this point, the newly created shadow buffer is not referenced 218 * by any batches, but the existing rsc (probably) is. We need to 219 * transfer those references over: 220 */ 221 debug_assert(shadow->batch_mask == 0); 222 struct fd_batch *batch; 223 foreach_batch(batch, &ctx->screen->batch_cache, rsc->batch_mask) { 224 struct set_entry *entry = _mesa_set_search(batch->resources, rsc); 225 _mesa_set_remove(batch->resources, entry); 226 _mesa_set_add(batch->resources, shadow); 227 } 228 swap(rsc->batch_mask, shadow->batch_mask); 229 230 struct pipe_blit_info blit = {0}; 231 blit.dst.resource = prsc; 232 blit.dst.format = prsc->format; 233 blit.src.resource = pshadow; 234 blit.src.format = pshadow->format; 235 blit.mask = util_format_get_mask(prsc->format); 236 blit.filter = PIPE_TEX_FILTER_NEAREST; 237 238#define set_box(field, val) do { \ 239 blit.dst.field = (val); \ 240 blit.src.field = (val); \ 241 } while (0) 242 243 /* blit the other levels in their entirety: */ 244 for (unsigned l = 0; l <= prsc->last_level; l++) { 245 if (l == level) 246 continue; 247 248 /* just blit whole level: */ 249 set_box(level, l); 250 set_box(box.width, u_minify(prsc->width0, l)); 251 set_box(box.height, u_minify(prsc->height0, l)); 252 set_box(box.depth, u_minify(prsc->depth0, l)); 253 254 do_blit(ctx, &blit, fallback); 255 } 256 257 /* deal w/ current level specially, since we might need to split 258 * it up into a couple blits: 259 */ 260 if (!whole_level) { 261 set_box(level, level); 262 263 switch (prsc->target) { 264 case PIPE_BUFFER: 265 case PIPE_TEXTURE_1D: 266 set_box(box.y, 0); 267 set_box(box.z, 0); 268 set_box(box.height, 1); 269 set_box(box.depth, 1); 270 271 if (box->x > 0) { 272 set_box(box.x, 0); 273 set_box(box.width, box->x); 274 275 do_blit(ctx, &blit, fallback); 276 } 277 if ((box->x + box->width) < u_minify(prsc->width0, level)) { 278 set_box(box.x, box->x + box->width); 279 set_box(box.width, u_minify(prsc->width0, level) - (box->x + box->width)); 280 281 do_blit(ctx, &blit, fallback); 282 } 283 break; 284 case PIPE_TEXTURE_2D: 285 /* TODO */ 286 default: 287 unreachable("TODO"); 288 } 289 } 290 291 ctx->in_shadow = false; 292 293 pipe_resource_reference(&pshadow, NULL); 294 295 return true; 296} 297 298static unsigned 299fd_resource_layer_offset(struct fd_resource *rsc, 300 struct fd_resource_slice *slice, 301 unsigned layer) 302{ 303 if (rsc->layer_first) 304 return layer * rsc->layer_size; 305 else 306 return layer * slice->size0; 307} 308 309static void 310fd_resource_flush_z32s8(struct fd_transfer *trans, const struct pipe_box *box) 311{ 312 struct fd_resource *rsc = fd_resource(trans->base.resource); 313 struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level); 314 struct fd_resource_slice *sslice = fd_resource_slice(rsc->stencil, trans->base.level); 315 enum pipe_format format = trans->base.resource->format; 316 317 float *depth = fd_bo_map(rsc->bo) + slice->offset + 318 fd_resource_layer_offset(rsc, slice, trans->base.box.z) + 319 (trans->base.box.y + box->y) * slice->pitch * 4 + (trans->base.box.x + box->x) * 4; 320 uint8_t *stencil = fd_bo_map(rsc->stencil->bo) + sslice->offset + 321 fd_resource_layer_offset(rsc->stencil, sslice, trans->base.box.z) + 322 (trans->base.box.y + box->y) * sslice->pitch + trans->base.box.x + box->x; 323 324 if (format != PIPE_FORMAT_X32_S8X24_UINT) 325 util_format_z32_float_s8x24_uint_unpack_z_float( 326 depth, slice->pitch * 4, 327 trans->staging, trans->base.stride, 328 box->width, box->height); 329 330 util_format_z32_float_s8x24_uint_unpack_s_8uint( 331 stencil, sslice->pitch, 332 trans->staging, trans->base.stride, 333 box->width, box->height); 334} 335 336static void 337fd_resource_flush_rgtc(struct fd_transfer *trans, const struct pipe_box *box) 338{ 339 struct fd_resource *rsc = fd_resource(trans->base.resource); 340 struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level); 341 enum pipe_format format = trans->base.resource->format; 342 343 uint8_t *data = fd_bo_map(rsc->bo) + slice->offset + 344 fd_resource_layer_offset(rsc, slice, trans->base.box.z) + 345 ((trans->base.box.y + box->y) * slice->pitch + 346 trans->base.box.x + box->x) * rsc->cpp; 347 348 uint8_t *source = trans->staging + 349 util_format_get_nblocksy(format, box->y) * trans->base.stride + 350 util_format_get_stride(format, box->x); 351 352 switch (format) { 353 case PIPE_FORMAT_RGTC1_UNORM: 354 case PIPE_FORMAT_RGTC1_SNORM: 355 case PIPE_FORMAT_LATC1_UNORM: 356 case PIPE_FORMAT_LATC1_SNORM: 357 util_format_rgtc1_unorm_unpack_rgba_8unorm( 358 data, slice->pitch * rsc->cpp, 359 source, trans->base.stride, 360 box->width, box->height); 361 break; 362 case PIPE_FORMAT_RGTC2_UNORM: 363 case PIPE_FORMAT_RGTC2_SNORM: 364 case PIPE_FORMAT_LATC2_UNORM: 365 case PIPE_FORMAT_LATC2_SNORM: 366 util_format_rgtc2_unorm_unpack_rgba_8unorm( 367 data, slice->pitch * rsc->cpp, 368 source, trans->base.stride, 369 box->width, box->height); 370 break; 371 default: 372 assert(!"Unexpected format\n"); 373 break; 374 } 375} 376 377static void 378fd_resource_flush(struct fd_transfer *trans, const struct pipe_box *box) 379{ 380 enum pipe_format format = trans->base.resource->format; 381 382 switch (format) { 383 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 384 case PIPE_FORMAT_X32_S8X24_UINT: 385 fd_resource_flush_z32s8(trans, box); 386 break; 387 case PIPE_FORMAT_RGTC1_UNORM: 388 case PIPE_FORMAT_RGTC1_SNORM: 389 case PIPE_FORMAT_RGTC2_UNORM: 390 case PIPE_FORMAT_RGTC2_SNORM: 391 case PIPE_FORMAT_LATC1_UNORM: 392 case PIPE_FORMAT_LATC1_SNORM: 393 case PIPE_FORMAT_LATC2_UNORM: 394 case PIPE_FORMAT_LATC2_SNORM: 395 fd_resource_flush_rgtc(trans, box); 396 break; 397 default: 398 assert(!"Unexpected staging transfer type"); 399 break; 400 } 401} 402 403static void fd_resource_transfer_flush_region(struct pipe_context *pctx, 404 struct pipe_transfer *ptrans, 405 const struct pipe_box *box) 406{ 407 struct fd_resource *rsc = fd_resource(ptrans->resource); 408 struct fd_transfer *trans = fd_transfer(ptrans); 409 410 if (ptrans->resource->target == PIPE_BUFFER) 411 util_range_add(&rsc->valid_buffer_range, 412 ptrans->box.x + box->x, 413 ptrans->box.x + box->x + box->width); 414 415 if (trans->staging) 416 fd_resource_flush(trans, box); 417} 418 419static void 420fd_resource_transfer_unmap(struct pipe_context *pctx, 421 struct pipe_transfer *ptrans) 422{ 423 struct fd_context *ctx = fd_context(pctx); 424 struct fd_resource *rsc = fd_resource(ptrans->resource); 425 struct fd_transfer *trans = fd_transfer(ptrans); 426 427 if (trans->staging && !(ptrans->usage & PIPE_TRANSFER_FLUSH_EXPLICIT)) { 428 struct pipe_box box; 429 u_box_2d(0, 0, ptrans->box.width, ptrans->box.height, &box); 430 fd_resource_flush(trans, &box); 431 } 432 433 if (!(ptrans->usage & PIPE_TRANSFER_UNSYNCHRONIZED)) { 434 fd_bo_cpu_fini(rsc->bo); 435 if (rsc->stencil) 436 fd_bo_cpu_fini(rsc->stencil->bo); 437 } 438 439 util_range_add(&rsc->valid_buffer_range, 440 ptrans->box.x, 441 ptrans->box.x + ptrans->box.width); 442 443 pipe_resource_reference(&ptrans->resource, NULL); 444 util_slab_free(&ctx->transfer_pool, ptrans); 445 446 free(trans->staging); 447} 448 449static void * 450fd_resource_transfer_map(struct pipe_context *pctx, 451 struct pipe_resource *prsc, 452 unsigned level, unsigned usage, 453 const struct pipe_box *box, 454 struct pipe_transfer **pptrans) 455{ 456 struct fd_context *ctx = fd_context(pctx); 457 struct fd_resource *rsc = fd_resource(prsc); 458 struct fd_resource_slice *slice = fd_resource_slice(rsc, level); 459 struct fd_transfer *trans; 460 struct pipe_transfer *ptrans; 461 enum pipe_format format = prsc->format; 462 uint32_t op = 0; 463 uint32_t offset; 464 char *buf; 465 int ret = 0; 466 467 DBG("prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d", prsc, level, usage, 468 box->width, box->height, box->x, box->y); 469 470 ptrans = util_slab_alloc(&ctx->transfer_pool); 471 if (!ptrans) 472 return NULL; 473 474 /* util_slab_alloc() doesn't zero: */ 475 trans = fd_transfer(ptrans); 476 memset(trans, 0, sizeof(*trans)); 477 478 pipe_resource_reference(&ptrans->resource, prsc); 479 ptrans->level = level; 480 ptrans->usage = usage; 481 ptrans->box = *box; 482 ptrans->stride = util_format_get_nblocksx(format, slice->pitch) * rsc->cpp; 483 ptrans->layer_stride = rsc->layer_first ? rsc->layer_size : slice->size0; 484 485 if (ctx->in_shadow && !(usage & PIPE_TRANSFER_READ)) 486 usage |= PIPE_TRANSFER_UNSYNCHRONIZED; 487 488 if (usage & PIPE_TRANSFER_READ) 489 op |= DRM_FREEDRENO_PREP_READ; 490 491 if (usage & PIPE_TRANSFER_WRITE) 492 op |= DRM_FREEDRENO_PREP_WRITE; 493 494 if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) { 495 realloc_bo(rsc, fd_bo_size(rsc->bo)); 496 if (rsc->stencil) 497 realloc_bo(rsc->stencil, fd_bo_size(rsc->stencil->bo)); 498 fd_invalidate_resource(ctx, prsc); 499 } else if ((usage & PIPE_TRANSFER_WRITE) && 500 prsc->target == PIPE_BUFFER && 501 !util_ranges_intersect(&rsc->valid_buffer_range, 502 box->x, box->x + box->width)) { 503 /* We are trying to write to a previously uninitialized range. No need 504 * to wait. 505 */ 506 } else if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) { 507 /* If the GPU is writing to the resource, or if it is reading from the 508 * resource and we're trying to write to it, flush the renders. 509 */ 510 bool needs_flush = pending(rsc, !!(usage & PIPE_TRANSFER_WRITE)); 511 bool busy = needs_flush || (0 != fd_bo_cpu_prep(rsc->bo, 512 ctx->screen->pipe, op | DRM_FREEDRENO_PREP_NOSYNC)); 513 514 /* if we need to flush/stall, see if we can make a shadow buffer 515 * to avoid this: 516 * 517 * TODO we could go down this path !reorder && !busy_for_read 518 * ie. we only *don't* want to go down this path if the blit 519 * will trigger a flush! 520 */ 521 if (ctx->screen->reorder && busy && !(usage & PIPE_TRANSFER_READ)) { 522 if (fd_try_shadow_resource(ctx, rsc, level, usage, box)) { 523 needs_flush = busy = false; 524 fd_invalidate_resource(ctx, prsc); 525 } 526 } 527 528 if (needs_flush) { 529 if (usage & PIPE_TRANSFER_WRITE) { 530 struct fd_batch *batch; 531 foreach_batch(batch, &ctx->screen->batch_cache, rsc->batch_mask) 532 fd_batch_flush(batch); 533 assert(rsc->batch_mask == 0); 534 } else { 535 fd_batch_flush(rsc->write_batch); 536 } 537 assert(!rsc->write_batch); 538 } 539 540 /* The GPU keeps track of how the various bo's are being used, and 541 * will wait if necessary for the proper operation to have 542 * completed. 543 */ 544 if (busy) { 545 ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, op); 546 if (ret) 547 goto fail; 548 } 549 } 550 551 buf = fd_bo_map(rsc->bo); 552 if (!buf) 553 goto fail; 554 555 offset = slice->offset + 556 box->y / util_format_get_blockheight(format) * ptrans->stride + 557 box->x / util_format_get_blockwidth(format) * rsc->cpp + 558 fd_resource_layer_offset(rsc, slice, box->z); 559 560 if (prsc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT || 561 prsc->format == PIPE_FORMAT_X32_S8X24_UINT) { 562 assert(trans->base.box.depth == 1); 563 564 trans->base.stride = trans->base.box.width * rsc->cpp * 2; 565 trans->staging = malloc(trans->base.stride * trans->base.box.height); 566 if (!trans->staging) 567 goto fail; 568 569 /* if we're not discarding the whole range (or resource), we must copy 570 * the real data in. 571 */ 572 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | 573 PIPE_TRANSFER_DISCARD_RANGE))) { 574 struct fd_resource_slice *sslice = 575 fd_resource_slice(rsc->stencil, level); 576 void *sbuf = fd_bo_map(rsc->stencil->bo); 577 if (!sbuf) 578 goto fail; 579 580 float *depth = (float *)(buf + slice->offset + 581 fd_resource_layer_offset(rsc, slice, box->z) + 582 box->y * slice->pitch * 4 + box->x * 4); 583 uint8_t *stencil = sbuf + sslice->offset + 584 fd_resource_layer_offset(rsc->stencil, sslice, box->z) + 585 box->y * sslice->pitch + box->x; 586 587 if (format != PIPE_FORMAT_X32_S8X24_UINT) 588 util_format_z32_float_s8x24_uint_pack_z_float( 589 trans->staging, trans->base.stride, 590 depth, slice->pitch * 4, 591 box->width, box->height); 592 593 util_format_z32_float_s8x24_uint_pack_s_8uint( 594 trans->staging, trans->base.stride, 595 stencil, sslice->pitch, 596 box->width, box->height); 597 } 598 599 buf = trans->staging; 600 offset = 0; 601 } else if (rsc->internal_format != format && 602 util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC) { 603 assert(trans->base.box.depth == 1); 604 605 trans->base.stride = util_format_get_stride( 606 format, trans->base.box.width); 607 trans->staging = malloc( 608 util_format_get_2d_size(format, trans->base.stride, 609 trans->base.box.height)); 610 if (!trans->staging) 611 goto fail; 612 613 /* if we're not discarding the whole range (or resource), we must copy 614 * the real data in. 615 */ 616 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | 617 PIPE_TRANSFER_DISCARD_RANGE))) { 618 uint8_t *rgba8 = (uint8_t *)buf + slice->offset + 619 fd_resource_layer_offset(rsc, slice, box->z) + 620 box->y * slice->pitch * rsc->cpp + box->x * rsc->cpp; 621 622 switch (format) { 623 case PIPE_FORMAT_RGTC1_UNORM: 624 case PIPE_FORMAT_RGTC1_SNORM: 625 case PIPE_FORMAT_LATC1_UNORM: 626 case PIPE_FORMAT_LATC1_SNORM: 627 util_format_rgtc1_unorm_pack_rgba_8unorm( 628 trans->staging, trans->base.stride, 629 rgba8, slice->pitch * rsc->cpp, 630 box->width, box->height); 631 break; 632 case PIPE_FORMAT_RGTC2_UNORM: 633 case PIPE_FORMAT_RGTC2_SNORM: 634 case PIPE_FORMAT_LATC2_UNORM: 635 case PIPE_FORMAT_LATC2_SNORM: 636 util_format_rgtc2_unorm_pack_rgba_8unorm( 637 trans->staging, trans->base.stride, 638 rgba8, slice->pitch * rsc->cpp, 639 box->width, box->height); 640 break; 641 default: 642 assert(!"Unexpected format"); 643 break; 644 } 645 } 646 647 buf = trans->staging; 648 offset = 0; 649 } 650 651 *pptrans = ptrans; 652 653 return buf + offset; 654 655fail: 656 fd_resource_transfer_unmap(pctx, ptrans); 657 return NULL; 658} 659 660static void 661fd_resource_destroy(struct pipe_screen *pscreen, 662 struct pipe_resource *prsc) 663{ 664 struct fd_resource *rsc = fd_resource(prsc); 665 fd_bc_invalidate_resource(rsc, true); 666 if (rsc->bo) 667 fd_bo_del(rsc->bo); 668 util_range_destroy(&rsc->valid_buffer_range); 669 FREE(rsc); 670} 671 672static boolean 673fd_resource_get_handle(struct pipe_screen *pscreen, 674 struct pipe_resource *prsc, 675 struct winsys_handle *handle) 676{ 677 struct fd_resource *rsc = fd_resource(prsc); 678 679 return fd_screen_bo_get_handle(pscreen, rsc->bo, 680 rsc->slices[0].pitch * rsc->cpp, handle); 681} 682 683 684static const struct u_resource_vtbl fd_resource_vtbl = { 685 .resource_get_handle = fd_resource_get_handle, 686 .resource_destroy = fd_resource_destroy, 687 .transfer_map = fd_resource_transfer_map, 688 .transfer_flush_region = fd_resource_transfer_flush_region, 689 .transfer_unmap = fd_resource_transfer_unmap, 690}; 691 692static uint32_t 693setup_slices(struct fd_resource *rsc, uint32_t alignment, enum pipe_format format) 694{ 695 struct pipe_resource *prsc = &rsc->base.b; 696 enum util_format_layout layout = util_format_description(format)->layout; 697 uint32_t level, size = 0; 698 uint32_t width = prsc->width0; 699 uint32_t height = prsc->height0; 700 uint32_t depth = prsc->depth0; 701 /* in layer_first layout, the level (slice) contains just one 702 * layer (since in fact the layer contains the slices) 703 */ 704 uint32_t layers_in_level = rsc->layer_first ? 1 : prsc->array_size; 705 706 for (level = 0; level <= prsc->last_level; level++) { 707 struct fd_resource_slice *slice = fd_resource_slice(rsc, level); 708 uint32_t blocks; 709 710 if (layout == UTIL_FORMAT_LAYOUT_ASTC) 711 slice->pitch = width = 712 util_align_npot(width, 32 * util_format_get_blockwidth(format)); 713 else 714 slice->pitch = width = align(width, 32); 715 slice->offset = size; 716 blocks = util_format_get_nblocks(format, width, height); 717 /* 1d array and 2d array textures must all have the same layer size 718 * for each miplevel on a3xx. 3d textures can have different layer 719 * sizes for high levels, but the hw auto-sizer is buggy (or at least 720 * different than what this code does), so as soon as the layer size 721 * range gets into range, we stop reducing it. 722 */ 723 if (prsc->target == PIPE_TEXTURE_3D && ( 724 level == 1 || 725 (level > 1 && rsc->slices[level - 1].size0 > 0xf000))) 726 slice->size0 = align(blocks * rsc->cpp, alignment); 727 else if (level == 0 || rsc->layer_first || alignment == 1) 728 slice->size0 = align(blocks * rsc->cpp, alignment); 729 else 730 slice->size0 = rsc->slices[level - 1].size0; 731 732 size += slice->size0 * depth * layers_in_level; 733 734 width = u_minify(width, 1); 735 height = u_minify(height, 1); 736 depth = u_minify(depth, 1); 737 } 738 739 return size; 740} 741 742static uint32_t 743slice_alignment(struct pipe_screen *pscreen, const struct pipe_resource *tmpl) 744{ 745 /* on a3xx, 2d array and 3d textures seem to want their 746 * layers aligned to page boundaries: 747 */ 748 switch (tmpl->target) { 749 case PIPE_TEXTURE_3D: 750 case PIPE_TEXTURE_1D_ARRAY: 751 case PIPE_TEXTURE_2D_ARRAY: 752 return 4096; 753 default: 754 return 1; 755 } 756} 757 758/** 759 * Create a new texture object, using the given template info. 760 */ 761static struct pipe_resource * 762fd_resource_create(struct pipe_screen *pscreen, 763 const struct pipe_resource *tmpl) 764{ 765 struct fd_resource *rsc = CALLOC_STRUCT(fd_resource); 766 struct pipe_resource *prsc = &rsc->base.b; 767 enum pipe_format format = tmpl->format; 768 uint32_t size, alignment; 769 770 DBG("%p: target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, " 771 "nr_samples=%u, usage=%u, bind=%x, flags=%x", prsc, 772 tmpl->target, util_format_name(format), 773 tmpl->width0, tmpl->height0, tmpl->depth0, 774 tmpl->array_size, tmpl->last_level, tmpl->nr_samples, 775 tmpl->usage, tmpl->bind, tmpl->flags); 776 777 if (!rsc) 778 return NULL; 779 780 *prsc = *tmpl; 781 782 pipe_reference_init(&prsc->reference, 1); 783 784 prsc->screen = pscreen; 785 786 util_range_init(&rsc->valid_buffer_range); 787 788 rsc->base.vtbl = &fd_resource_vtbl; 789 790 if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) 791 format = PIPE_FORMAT_Z32_FLOAT; 792 else if (fd_screen(pscreen)->gpu_id < 400 && 793 util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC) 794 format = PIPE_FORMAT_R8G8B8A8_UNORM; 795 rsc->internal_format = format; 796 rsc->cpp = util_format_get_blocksize(format); 797 798 assert(rsc->cpp); 799 800 alignment = slice_alignment(pscreen, tmpl); 801 if (is_a4xx(fd_screen(pscreen))) { 802 switch (tmpl->target) { 803 case PIPE_TEXTURE_3D: 804 rsc->layer_first = false; 805 break; 806 default: 807 rsc->layer_first = true; 808 alignment = 1; 809 break; 810 } 811 } 812 813 size = setup_slices(rsc, alignment, format); 814 815 if (rsc->layer_first) { 816 rsc->layer_size = align(size, 4096); 817 size = rsc->layer_size * prsc->array_size; 818 } 819 820 realloc_bo(rsc, size); 821 if (!rsc->bo) 822 goto fail; 823 824 /* There is no native Z32F_S8 sampling or rendering format, so this must 825 * be emulated via two separate textures. The depth texture still keeps 826 * its Z32F_S8 format though, and we also keep a reference to a separate 827 * S8 texture. 828 */ 829 if (tmpl->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) { 830 struct pipe_resource stencil = *tmpl; 831 stencil.format = PIPE_FORMAT_S8_UINT; 832 rsc->stencil = fd_resource(fd_resource_create(pscreen, &stencil)); 833 if (!rsc->stencil) 834 goto fail; 835 } 836 837 return prsc; 838fail: 839 fd_resource_destroy(pscreen, prsc); 840 return NULL; 841} 842 843/** 844 * Create a texture from a winsys_handle. The handle is often created in 845 * another process by first creating a pipe texture and then calling 846 * resource_get_handle. 847 */ 848static struct pipe_resource * 849fd_resource_from_handle(struct pipe_screen *pscreen, 850 const struct pipe_resource *tmpl, 851 struct winsys_handle *handle, unsigned usage) 852{ 853 struct fd_resource *rsc = CALLOC_STRUCT(fd_resource); 854 struct fd_resource_slice *slice = &rsc->slices[0]; 855 struct pipe_resource *prsc = &rsc->base.b; 856 857 DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, " 858 "nr_samples=%u, usage=%u, bind=%x, flags=%x", 859 tmpl->target, util_format_name(tmpl->format), 860 tmpl->width0, tmpl->height0, tmpl->depth0, 861 tmpl->array_size, tmpl->last_level, tmpl->nr_samples, 862 tmpl->usage, tmpl->bind, tmpl->flags); 863 864 if (!rsc) 865 return NULL; 866 867 *prsc = *tmpl; 868 869 pipe_reference_init(&prsc->reference, 1); 870 871 prsc->screen = pscreen; 872 873 util_range_init(&rsc->valid_buffer_range); 874 875 rsc->bo = fd_screen_bo_from_handle(pscreen, handle, &slice->pitch); 876 if (!rsc->bo) 877 goto fail; 878 879 rsc->base.vtbl = &fd_resource_vtbl; 880 rsc->cpp = util_format_get_blocksize(tmpl->format); 881 slice->pitch /= rsc->cpp; 882 slice->offset = handle->offset; 883 884 assert(rsc->cpp); 885 886 return prsc; 887 888fail: 889 fd_resource_destroy(pscreen, prsc); 890 return NULL; 891} 892 893/** 894 * _copy_region using pipe (3d engine) 895 */ 896static bool 897fd_blitter_pipe_copy_region(struct fd_context *ctx, 898 struct pipe_resource *dst, 899 unsigned dst_level, 900 unsigned dstx, unsigned dsty, unsigned dstz, 901 struct pipe_resource *src, 902 unsigned src_level, 903 const struct pipe_box *src_box) 904{ 905 /* not until we allow rendertargets to be buffers */ 906 if (dst->target == PIPE_BUFFER || src->target == PIPE_BUFFER) 907 return false; 908 909 if (!util_blitter_is_copy_supported(ctx->blitter, dst, src)) 910 return false; 911 912 fd_blitter_pipe_begin(ctx, false); 913 util_blitter_copy_texture(ctx->blitter, 914 dst, dst_level, dstx, dsty, dstz, 915 src, src_level, src_box); 916 fd_blitter_pipe_end(ctx); 917 918 return true; 919} 920 921/** 922 * Copy a block of pixels from one resource to another. 923 * The resource must be of the same format. 924 * Resources with nr_samples > 1 are not allowed. 925 */ 926static void 927fd_resource_copy_region(struct pipe_context *pctx, 928 struct pipe_resource *dst, 929 unsigned dst_level, 930 unsigned dstx, unsigned dsty, unsigned dstz, 931 struct pipe_resource *src, 932 unsigned src_level, 933 const struct pipe_box *src_box) 934{ 935 struct fd_context *ctx = fd_context(pctx); 936 937 /* TODO if we have 2d core, or other DMA engine that could be used 938 * for simple copies and reasonably easily synchronized with the 3d 939 * core, this is where we'd plug it in.. 940 */ 941 942 /* try blit on 3d pipe: */ 943 if (fd_blitter_pipe_copy_region(ctx, 944 dst, dst_level, dstx, dsty, dstz, 945 src, src_level, src_box)) 946 return; 947 948 /* else fallback to pure sw: */ 949 util_resource_copy_region(pctx, 950 dst, dst_level, dstx, dsty, dstz, 951 src, src_level, src_box); 952} 953 954bool 955fd_render_condition_check(struct pipe_context *pctx) 956{ 957 struct fd_context *ctx = fd_context(pctx); 958 959 if (!ctx->cond_query) 960 return true; 961 962 union pipe_query_result res = { 0 }; 963 bool wait = 964 ctx->cond_mode != PIPE_RENDER_COND_NO_WAIT && 965 ctx->cond_mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT; 966 967 if (pctx->get_query_result(pctx, ctx->cond_query, wait, &res)) 968 return (bool)res.u64 != ctx->cond_cond; 969 970 return true; 971} 972 973/** 974 * Optimal hardware path for blitting pixels. 975 * Scaling, format conversion, up- and downsampling (resolve) are allowed. 976 */ 977static void 978fd_blit(struct pipe_context *pctx, const struct pipe_blit_info *blit_info) 979{ 980 struct fd_context *ctx = fd_context(pctx); 981 struct pipe_blit_info info = *blit_info; 982 983 if (info.src.resource->nr_samples > 1 && 984 info.dst.resource->nr_samples <= 1 && 985 !util_format_is_depth_or_stencil(info.src.resource->format) && 986 !util_format_is_pure_integer(info.src.resource->format)) { 987 DBG("color resolve unimplemented"); 988 return; 989 } 990 991 if (info.render_condition_enable && !fd_render_condition_check(pctx)) 992 return; 993 994 if (util_try_blit_via_copy_region(pctx, &info)) { 995 return; /* done */ 996 } 997 998 if (info.mask & PIPE_MASK_S) { 999 DBG("cannot blit stencil, skipping"); 1000 info.mask &= ~PIPE_MASK_S; 1001 } 1002 1003 if (!util_blitter_is_blit_supported(ctx->blitter, &info)) { 1004 DBG("blit unsupported %s -> %s", 1005 util_format_short_name(info.src.resource->format), 1006 util_format_short_name(info.dst.resource->format)); 1007 return; 1008 } 1009 1010 fd_blitter_pipe_begin(ctx, info.render_condition_enable); 1011 util_blitter_blit(ctx->blitter, &info); 1012 fd_blitter_pipe_end(ctx); 1013} 1014 1015static void 1016fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond) 1017{ 1018 util_blitter_save_vertex_buffer_slot(ctx->blitter, ctx->vtx.vertexbuf.vb); 1019 util_blitter_save_vertex_elements(ctx->blitter, ctx->vtx.vtx); 1020 util_blitter_save_vertex_shader(ctx->blitter, ctx->prog.vp); 1021 util_blitter_save_so_targets(ctx->blitter, ctx->streamout.num_targets, 1022 ctx->streamout.targets); 1023 util_blitter_save_rasterizer(ctx->blitter, ctx->rasterizer); 1024 util_blitter_save_viewport(ctx->blitter, &ctx->viewport); 1025 util_blitter_save_scissor(ctx->blitter, &ctx->scissor); 1026 util_blitter_save_fragment_shader(ctx->blitter, ctx->prog.fp); 1027 util_blitter_save_blend(ctx->blitter, ctx->blend); 1028 util_blitter_save_depth_stencil_alpha(ctx->blitter, ctx->zsa); 1029 util_blitter_save_stencil_ref(ctx->blitter, &ctx->stencil_ref); 1030 util_blitter_save_sample_mask(ctx->blitter, ctx->sample_mask); 1031 util_blitter_save_framebuffer(ctx->blitter, 1032 ctx->batch ? &ctx->batch->framebuffer : NULL); 1033 util_blitter_save_fragment_sampler_states(ctx->blitter, 1034 ctx->fragtex.num_samplers, 1035 (void **)ctx->fragtex.samplers); 1036 util_blitter_save_fragment_sampler_views(ctx->blitter, 1037 ctx->fragtex.num_textures, ctx->fragtex.textures); 1038 if (!render_cond) 1039 util_blitter_save_render_condition(ctx->blitter, 1040 ctx->cond_query, ctx->cond_cond, ctx->cond_mode); 1041 1042 fd_hw_query_set_stage(ctx, ctx->batch->draw, FD_STAGE_BLIT); 1043} 1044 1045static void 1046fd_blitter_pipe_end(struct fd_context *ctx) 1047{ 1048 fd_hw_query_set_stage(ctx, ctx->batch->draw, FD_STAGE_NULL); 1049} 1050 1051static void 1052fd_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc) 1053{ 1054 struct fd_resource *rsc = fd_resource(prsc); 1055 1056 if (rsc->write_batch) 1057 fd_batch_flush(rsc->write_batch); 1058 1059 assert(!rsc->write_batch); 1060} 1061 1062void 1063fd_resource_screen_init(struct pipe_screen *pscreen) 1064{ 1065 pscreen->resource_create = fd_resource_create; 1066 pscreen->resource_from_handle = fd_resource_from_handle; 1067 pscreen->resource_get_handle = u_resource_get_handle_vtbl; 1068 pscreen->resource_destroy = u_resource_destroy_vtbl; 1069} 1070 1071void 1072fd_resource_context_init(struct pipe_context *pctx) 1073{ 1074 pctx->transfer_map = u_transfer_map_vtbl; 1075 pctx->transfer_flush_region = u_transfer_flush_region_vtbl; 1076 pctx->transfer_unmap = u_transfer_unmap_vtbl; 1077 pctx->buffer_subdata = u_default_buffer_subdata; 1078 pctx->texture_subdata = u_default_texture_subdata; 1079 pctx->create_surface = fd_create_surface; 1080 pctx->surface_destroy = fd_surface_destroy; 1081 pctx->resource_copy_region = fd_resource_copy_region; 1082 pctx->blit = fd_blit; 1083 pctx->flush_resource = fd_flush_resource; 1084} 1085