pb_buffer_fenced.c revision efcea15aaaaa4f1431a8c0a8521bd42a953f2e6c
1fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org/************************************************************************** 2fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * 3fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * Copyright 2007 Tungsten Graphics, Inc., Cedar Park, Texas. 4fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * All Rights Reserved. 5fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * 6fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * Permission is hereby granted, free of charge, to any person obtaining a 7fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * copy of this software and associated documentation files (the 8fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * "Software"), to deal in the Software without restriction, including 9fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * without limitation the rights to use, copy, modify, merge, publish, 10fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * distribute, sub license, and/or sell copies of the Software, and to 11fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * permit persons to whom the Software is furnished to do so, subject to 12fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * the following conditions: 13fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * 14fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * The above copyright notice and this permission notice (including the 15fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * next paragraph) shall be included in all copies or substantial portions 16fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * of the Software. 17fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * 18fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 22fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * 26fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org **************************************************************************/ 27fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org 28fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org/** 29fb732b17922ea75830be4db6b80534c4827d8a55jkummerow@chromium.org * \file 30 * Implementation of fenced buffers. 31 * 32 * \author Jose Fonseca <jrfonseca-at-tungstengraphics-dot-com> 33 * \author Thomas Hellström <thomas-at-tungstengraphics-dot-com> 34 */ 35 36 37#include "pipe/p_config.h" 38 39#if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) 40#include <unistd.h> 41#include <sched.h> 42#endif 43 44#include "pipe/p_compiler.h" 45#include "pipe/p_error.h" 46#include "util/u_debug.h" 47#include "pipe/p_thread.h" 48#include "util/u_memory.h" 49#include "util/u_double_list.h" 50 51#include "pb_buffer.h" 52#include "pb_buffer_fenced.h" 53 54 55 56/** 57 * Convenience macro (type safe). 58 */ 59#define SUPER(__derived) (&(__derived)->base) 60 61 62struct fenced_buffer_list 63{ 64 pipe_mutex mutex; 65 66 struct pb_fence_ops *ops; 67 68 size_t numDelayed; 69 struct list_head delayed; 70 71#ifdef DEBUG 72 size_t numUnfenced; 73 struct list_head unfenced; 74#endif 75}; 76 77 78/** 79 * Wrapper around a pipe buffer which adds fencing and reference counting. 80 */ 81struct fenced_buffer 82{ 83 struct pb_buffer base; 84 85 struct pb_buffer *buffer; 86 87 /* FIXME: protect access with mutex */ 88 89 /** 90 * A bitmask of PIPE_BUFFER_USAGE_CPU/GPU_READ/WRITE describing the current 91 * buffer usage. 92 */ 93 unsigned flags; 94 95 unsigned mapcount; 96 struct pb_validate *vl; 97 unsigned validation_flags; 98 struct pipe_fence_handle *fence; 99 100 struct list_head head; 101 struct fenced_buffer_list *list; 102}; 103 104 105static INLINE struct fenced_buffer * 106fenced_buffer(struct pb_buffer *buf) 107{ 108 assert(buf); 109 return (struct fenced_buffer *)buf; 110} 111 112 113static INLINE void 114_fenced_buffer_add(struct fenced_buffer *fenced_buf) 115{ 116 struct fenced_buffer_list *fenced_list = fenced_buf->list; 117 118 assert(fenced_buf->base.base.refcount); 119 assert(fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE); 120 assert(fenced_buf->fence); 121 122#ifdef DEBUG 123 LIST_DEL(&fenced_buf->head); 124 assert(fenced_list->numUnfenced); 125 --fenced_list->numUnfenced; 126#endif 127 LIST_ADDTAIL(&fenced_buf->head, &fenced_list->delayed); 128 ++fenced_list->numDelayed; 129} 130 131 132/** 133 * Actually destroy the buffer. 134 */ 135static INLINE void 136_fenced_buffer_destroy(struct fenced_buffer *fenced_buf) 137{ 138 struct fenced_buffer_list *fenced_list = fenced_buf->list; 139 140 assert(!fenced_buf->base.base.refcount); 141 assert(!fenced_buf->fence); 142#ifdef DEBUG 143 assert(fenced_buf->head.prev); 144 assert(fenced_buf->head.next); 145 LIST_DEL(&fenced_buf->head); 146 assert(fenced_list->numUnfenced); 147 --fenced_list->numUnfenced; 148#else 149 (void)fenced_list; 150#endif 151 pb_reference(&fenced_buf->buffer, NULL); 152 FREE(fenced_buf); 153} 154 155 156static INLINE void 157_fenced_buffer_remove(struct fenced_buffer_list *fenced_list, 158 struct fenced_buffer *fenced_buf) 159{ 160 struct pb_fence_ops *ops = fenced_list->ops; 161 162 assert(fenced_buf->fence); 163 assert(fenced_buf->list == fenced_list); 164 165 ops->fence_reference(ops, &fenced_buf->fence, NULL); 166 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE; 167 168 assert(fenced_buf->head.prev); 169 assert(fenced_buf->head.next); 170 171 LIST_DEL(&fenced_buf->head); 172 assert(fenced_list->numDelayed); 173 --fenced_list->numDelayed; 174 175#ifdef DEBUG 176 LIST_ADDTAIL(&fenced_buf->head, &fenced_list->unfenced); 177 ++fenced_list->numUnfenced; 178#endif 179 180 if(!fenced_buf->base.base.refcount) 181 _fenced_buffer_destroy(fenced_buf); 182} 183 184 185static INLINE enum pipe_error 186_fenced_buffer_finish(struct fenced_buffer *fenced_buf) 187{ 188 struct fenced_buffer_list *fenced_list = fenced_buf->list; 189 struct pb_fence_ops *ops = fenced_list->ops; 190 191#if 0 192 debug_warning("waiting for GPU"); 193#endif 194 195 assert(fenced_buf->fence); 196 if(fenced_buf->fence) { 197 if(ops->fence_finish(ops, fenced_buf->fence, 0) != 0) { 198 return PIPE_ERROR; 199 } 200 /* Remove from the fenced list */ 201 /* TODO: remove consequents */ 202 _fenced_buffer_remove(fenced_list, fenced_buf); 203 } 204 205 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE; 206 return PIPE_OK; 207} 208 209 210/** 211 * Free as many fenced buffers from the list head as possible. 212 */ 213static void 214_fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list, 215 int wait) 216{ 217 struct pb_fence_ops *ops = fenced_list->ops; 218 struct list_head *curr, *next; 219 struct fenced_buffer *fenced_buf; 220 struct pipe_fence_handle *prev_fence = NULL; 221 222 curr = fenced_list->delayed.next; 223 next = curr->next; 224 while(curr != &fenced_list->delayed) { 225 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 226 227 if(fenced_buf->fence != prev_fence) { 228 int signaled; 229 if (wait) 230 signaled = ops->fence_finish(ops, fenced_buf->fence, 0); 231 else 232 signaled = ops->fence_signalled(ops, fenced_buf->fence, 0); 233 if (signaled != 0) 234 break; 235 prev_fence = fenced_buf->fence; 236 } 237 else { 238 assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0); 239 } 240 241 _fenced_buffer_remove(fenced_list, fenced_buf); 242 243 curr = next; 244 next = curr->next; 245 } 246} 247 248 249static void 250fenced_buffer_destroy(struct pb_buffer *buf) 251{ 252 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 253 struct fenced_buffer_list *fenced_list = fenced_buf->list; 254 255 pipe_mutex_lock(fenced_list->mutex); 256 assert(fenced_buf->base.base.refcount == 0); 257 if (fenced_buf->fence) { 258 struct pb_fence_ops *ops = fenced_list->ops; 259 if(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0) { 260 struct list_head *curr, *prev; 261 curr = &fenced_buf->head; 262 prev = curr->prev; 263 do { 264 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 265 assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0); 266 _fenced_buffer_remove(fenced_list, fenced_buf); 267 curr = prev; 268 prev = curr->prev; 269 } while (curr != &fenced_list->delayed); 270 } 271 else { 272 /* delay destruction */ 273 } 274 } 275 else { 276 _fenced_buffer_destroy(fenced_buf); 277 } 278 pipe_mutex_unlock(fenced_list->mutex); 279} 280 281 282static void * 283fenced_buffer_map(struct pb_buffer *buf, 284 unsigned flags) 285{ 286 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 287 struct fenced_buffer_list *fenced_list = fenced_buf->list; 288 struct pb_fence_ops *ops = fenced_list->ops; 289 void *map; 290 291 assert(!(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE)); 292 293 /* Serialize writes */ 294 if((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_WRITE) || 295 ((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ) && (flags & PIPE_BUFFER_USAGE_CPU_WRITE))) { 296 if(flags & PIPE_BUFFER_USAGE_DONTBLOCK) { 297 /* Don't wait for the GPU to finish writing */ 298 if(ops->fence_finish(ops, fenced_buf->fence, 0) == 0) 299 _fenced_buffer_remove(fenced_list, fenced_buf); 300 else 301 return NULL; 302 } 303 else { 304 /* Wait for the GPU to finish writing */ 305 _fenced_buffer_finish(fenced_buf); 306 } 307 } 308 309#if 0 310 /* Check for CPU write access (read is OK) */ 311 if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) { 312 /* this is legal -- just for debugging */ 313 debug_warning("concurrent CPU writes"); 314 } 315#endif 316 317 map = pb_map(fenced_buf->buffer, flags); 318 if(map) { 319 ++fenced_buf->mapcount; 320 fenced_buf->flags |= flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE; 321 } 322 323 return map; 324} 325 326 327static void 328fenced_buffer_unmap(struct pb_buffer *buf) 329{ 330 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 331 assert(fenced_buf->mapcount); 332 if(fenced_buf->mapcount) { 333 pb_unmap(fenced_buf->buffer); 334 --fenced_buf->mapcount; 335 if(!fenced_buf->mapcount) 336 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_CPU_READ_WRITE; 337 } 338} 339 340 341static enum pipe_error 342fenced_buffer_validate(struct pb_buffer *buf, 343 struct pb_validate *vl, 344 unsigned flags) 345{ 346 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 347 enum pipe_error ret; 348 349 if(!vl) { 350 /* invalidate */ 351 fenced_buf->vl = NULL; 352 fenced_buf->validation_flags = 0; 353 return PIPE_OK; 354 } 355 356 assert(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE); 357 assert(!(flags & ~PIPE_BUFFER_USAGE_GPU_READ_WRITE)); 358 flags &= PIPE_BUFFER_USAGE_GPU_READ_WRITE; 359 360 /* Buffer cannot be validated in two different lists */ 361 if(fenced_buf->vl && fenced_buf->vl != vl) 362 return PIPE_ERROR_RETRY; 363 364 /* Do not validate if buffer is still mapped */ 365 if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) { 366 /* TODO: wait for the thread that mapped the buffer to unmap it */ 367 return PIPE_ERROR_RETRY; 368 } 369 370 if(fenced_buf->vl == vl && 371 (fenced_buf->validation_flags & flags) == flags) { 372 /* Nothing to do -- buffer already validated */ 373 return PIPE_OK; 374 } 375 376 /* Final sanity checking */ 377 assert(!(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE)); 378 assert(!fenced_buf->mapcount); 379 380 ret = pb_validate(fenced_buf->buffer, vl, flags); 381 if (ret != PIPE_OK) 382 return ret; 383 384 fenced_buf->vl = vl; 385 fenced_buf->validation_flags |= flags; 386 387 return PIPE_OK; 388} 389 390 391static void 392fenced_buffer_fence(struct pb_buffer *buf, 393 struct pipe_fence_handle *fence) 394{ 395 struct fenced_buffer *fenced_buf; 396 struct fenced_buffer_list *fenced_list; 397 struct pb_fence_ops *ops; 398 399 fenced_buf = fenced_buffer(buf); 400 fenced_list = fenced_buf->list; 401 ops = fenced_list->ops; 402 403 if(fence == fenced_buf->fence) { 404 /* Nothing to do */ 405 return; 406 } 407 408 assert(fenced_buf->vl); 409 assert(fenced_buf->validation_flags); 410 411 pipe_mutex_lock(fenced_list->mutex); 412 if (fenced_buf->fence) 413 _fenced_buffer_remove(fenced_list, fenced_buf); 414 if (fence) { 415 ops->fence_reference(ops, &fenced_buf->fence, fence); 416 fenced_buf->flags |= fenced_buf->validation_flags; 417 _fenced_buffer_add(fenced_buf); 418 } 419 pipe_mutex_unlock(fenced_list->mutex); 420 421 pb_fence(fenced_buf->buffer, fence); 422 423 fenced_buf->vl = NULL; 424 fenced_buf->validation_flags = 0; 425} 426 427 428static void 429fenced_buffer_get_base_buffer(struct pb_buffer *buf, 430 struct pb_buffer **base_buf, 431 unsigned *offset) 432{ 433 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 434 pb_get_base_buffer(fenced_buf->buffer, base_buf, offset); 435} 436 437 438static const struct pb_vtbl 439fenced_buffer_vtbl = { 440 fenced_buffer_destroy, 441 fenced_buffer_map, 442 fenced_buffer_unmap, 443 fenced_buffer_validate, 444 fenced_buffer_fence, 445 fenced_buffer_get_base_buffer 446}; 447 448 449struct pb_buffer * 450fenced_buffer_create(struct fenced_buffer_list *fenced_list, 451 struct pb_buffer *buffer) 452{ 453 struct fenced_buffer *buf; 454 455 if(!buffer) 456 return NULL; 457 458 buf = CALLOC_STRUCT(fenced_buffer); 459 if(!buf) { 460 pb_reference(&buffer, NULL); 461 return NULL; 462 } 463 464 buf->base.base.refcount = 1; 465 buf->base.base.alignment = buffer->base.alignment; 466 buf->base.base.usage = buffer->base.usage; 467 buf->base.base.size = buffer->base.size; 468 469 buf->base.vtbl = &fenced_buffer_vtbl; 470 buf->buffer = buffer; 471 buf->list = fenced_list; 472 473#ifdef DEBUG 474 pipe_mutex_lock(fenced_list->mutex); 475 LIST_ADDTAIL(&buf->head, &fenced_list->unfenced); 476 ++fenced_list->numUnfenced; 477 pipe_mutex_unlock(fenced_list->mutex); 478#endif 479 480 return &buf->base; 481} 482 483 484struct fenced_buffer_list * 485fenced_buffer_list_create(struct pb_fence_ops *ops) 486{ 487 struct fenced_buffer_list *fenced_list; 488 489 fenced_list = CALLOC_STRUCT(fenced_buffer_list); 490 if (!fenced_list) 491 return NULL; 492 493 fenced_list->ops = ops; 494 495 LIST_INITHEAD(&fenced_list->delayed); 496 fenced_list->numDelayed = 0; 497 498#ifdef DEBUG 499 LIST_INITHEAD(&fenced_list->unfenced); 500 fenced_list->numUnfenced = 0; 501#endif 502 503 pipe_mutex_init(fenced_list->mutex); 504 505 return fenced_list; 506} 507 508 509void 510fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list, 511 int wait) 512{ 513 pipe_mutex_lock(fenced_list->mutex); 514 _fenced_buffer_list_check_free(fenced_list, wait); 515 pipe_mutex_unlock(fenced_list->mutex); 516} 517 518 519#ifdef DEBUG 520void 521fenced_buffer_list_dump(struct fenced_buffer_list *fenced_list) 522{ 523 struct pb_fence_ops *ops = fenced_list->ops; 524 struct list_head *curr, *next; 525 struct fenced_buffer *fenced_buf; 526 527 pipe_mutex_lock(fenced_list->mutex); 528 529 debug_printf("%10s %7s %10s %s\n", 530 "buffer", "refcount", "fence", "signalled"); 531 532 curr = fenced_list->unfenced.next; 533 next = curr->next; 534 while(curr != &fenced_list->unfenced) { 535 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 536 assert(!fenced_buf->fence); 537 debug_printf("%10p %7u\n", 538 fenced_buf, 539 fenced_buf->base.base.refcount); 540 curr = next; 541 next = curr->next; 542 } 543 544 curr = fenced_list->delayed.next; 545 next = curr->next; 546 while(curr != &fenced_list->delayed) { 547 int signaled; 548 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 549 signaled = ops->fence_signalled(ops, fenced_buf->fence, 0); 550 debug_printf("%10p %7u %10p %s\n", 551 fenced_buf, 552 fenced_buf->base.base.refcount, 553 fenced_buf->fence, 554 signaled == 0 ? "y" : "n"); 555 curr = next; 556 next = curr->next; 557 } 558 559 pipe_mutex_unlock(fenced_list->mutex); 560} 561#endif 562 563 564void 565fenced_buffer_list_destroy(struct fenced_buffer_list *fenced_list) 566{ 567 pipe_mutex_lock(fenced_list->mutex); 568 569 /* Wait on outstanding fences */ 570 while (fenced_list->numDelayed) { 571 pipe_mutex_unlock(fenced_list->mutex); 572#if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) 573 sched_yield(); 574#endif 575 _fenced_buffer_list_check_free(fenced_list, 1); 576 pipe_mutex_lock(fenced_list->mutex); 577 } 578 579#ifdef DEBUG 580 //assert(!fenced_list->numUnfenced); 581#endif 582 583 pipe_mutex_unlock(fenced_list->mutex); 584 585 fenced_list->ops->destroy(fenced_list->ops); 586 587 FREE(fenced_list); 588} 589 590 591