pb_buffer_fenced.c revision 55770d09c18c4d33403abb97dfef4f897efbbe2a
1/************************************************************************** 2 * 3 * Copyright 2007 Tungsten Graphics, Inc., Cedar Park, Texas. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28/** 29 * \file 30 * Implementation of fenced buffers. 31 * 32 * \author Jose Fonseca <jrfonseca-at-tungstengraphics-dot-com> 33 * \author Thomas Hellström <thomas-at-tungstengraphics-dot-com> 34 */ 35 36 37#include "pipe/p_config.h" 38 39#if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS) 40#include <unistd.h> 41#include <sched.h> 42#endif 43 44#include "pipe/p_compiler.h" 45#include "pipe/p_error.h" 46#include "util/u_debug.h" 47#include "pipe/p_thread.h" 48#include "util/u_memory.h" 49#include "util/u_double_list.h" 50 51#include "pb_buffer.h" 52#include "pb_buffer_fenced.h" 53 54 55 56/** 57 * Convenience macro (type safe). 58 */ 59#define SUPER(__derived) (&(__derived)->base) 60 61 62struct fenced_buffer_list 63{ 64 pipe_mutex mutex; 65 66 struct pb_fence_ops *ops; 67 68 pb_size numDelayed; 69 struct list_head delayed; 70 71#ifdef DEBUG 72 pb_size numUnfenced; 73 struct list_head unfenced; 74#endif 75}; 76 77 78/** 79 * Wrapper around a pipe buffer which adds fencing and reference counting. 80 */ 81struct fenced_buffer 82{ 83 struct pb_buffer base; 84 85 struct pb_buffer *buffer; 86 87 /* FIXME: protect access with mutex */ 88 89 /** 90 * A bitmask of PIPE_BUFFER_USAGE_CPU/GPU_READ/WRITE describing the current 91 * buffer usage. 92 */ 93 unsigned flags; 94 95 unsigned mapcount; 96 struct pb_validate *vl; 97 unsigned validation_flags; 98 struct pipe_fence_handle *fence; 99 100 struct list_head head; 101 struct fenced_buffer_list *list; 102}; 103 104 105static INLINE struct fenced_buffer * 106fenced_buffer(struct pb_buffer *buf) 107{ 108 assert(buf); 109 return (struct fenced_buffer *)buf; 110} 111 112 113static INLINE void 114_fenced_buffer_add(struct fenced_buffer *fenced_buf) 115{ 116 struct fenced_buffer_list *fenced_list = fenced_buf->list; 117 118 assert(pipe_is_referenced(&fenced_buf->base.base.reference)); 119 assert(fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE); 120 assert(fenced_buf->fence); 121 122#ifdef DEBUG 123 LIST_DEL(&fenced_buf->head); 124 assert(fenced_list->numUnfenced); 125 --fenced_list->numUnfenced; 126#endif 127 LIST_ADDTAIL(&fenced_buf->head, &fenced_list->delayed); 128 ++fenced_list->numDelayed; 129} 130 131 132/** 133 * Actually destroy the buffer. 134 */ 135static INLINE void 136_fenced_buffer_destroy(struct fenced_buffer *fenced_buf) 137{ 138 struct fenced_buffer_list *fenced_list = fenced_buf->list; 139 140 assert(!pipe_is_referenced(&fenced_buf->base.base.reference)); 141 assert(!fenced_buf->fence); 142#ifdef DEBUG 143 assert(fenced_buf->head.prev); 144 assert(fenced_buf->head.next); 145 LIST_DEL(&fenced_buf->head); 146 assert(fenced_list->numUnfenced); 147 --fenced_list->numUnfenced; 148#else 149 (void)fenced_list; 150#endif 151 pb_reference(&fenced_buf->buffer, NULL); 152 FREE(fenced_buf); 153} 154 155 156static INLINE void 157_fenced_buffer_remove(struct fenced_buffer_list *fenced_list, 158 struct fenced_buffer *fenced_buf) 159{ 160 struct pb_fence_ops *ops = fenced_list->ops; 161 162 assert(fenced_buf->fence); 163 assert(fenced_buf->list == fenced_list); 164 165 ops->fence_reference(ops, &fenced_buf->fence, NULL); 166 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE; 167 168 assert(fenced_buf->head.prev); 169 assert(fenced_buf->head.next); 170 171 LIST_DEL(&fenced_buf->head); 172 assert(fenced_list->numDelayed); 173 --fenced_list->numDelayed; 174 175#ifdef DEBUG 176 LIST_ADDTAIL(&fenced_buf->head, &fenced_list->unfenced); 177 ++fenced_list->numUnfenced; 178#endif 179 180 /** 181 * FIXME!!! 182 */ 183 184 if(!pipe_is_referenced(&fenced_buf->base.base.reference)) 185 _fenced_buffer_destroy(fenced_buf); 186} 187 188 189static INLINE enum pipe_error 190_fenced_buffer_finish(struct fenced_buffer *fenced_buf) 191{ 192 struct fenced_buffer_list *fenced_list = fenced_buf->list; 193 struct pb_fence_ops *ops = fenced_list->ops; 194 195#if 0 196 debug_warning("waiting for GPU"); 197#endif 198 199 assert(fenced_buf->fence); 200 if(fenced_buf->fence) { 201 if(ops->fence_finish(ops, fenced_buf->fence, 0) != 0) { 202 return PIPE_ERROR; 203 } 204 /* Remove from the fenced list */ 205 /* TODO: remove consequents */ 206 _fenced_buffer_remove(fenced_list, fenced_buf); 207 } 208 209 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE; 210 return PIPE_OK; 211} 212 213 214/** 215 * Free as many fenced buffers from the list head as possible. 216 */ 217static void 218_fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list, 219 int wait) 220{ 221 struct pb_fence_ops *ops = fenced_list->ops; 222 struct list_head *curr, *next; 223 struct fenced_buffer *fenced_buf; 224 struct pipe_fence_handle *prev_fence = NULL; 225 226 curr = fenced_list->delayed.next; 227 next = curr->next; 228 while(curr != &fenced_list->delayed) { 229 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 230 231 if(fenced_buf->fence != prev_fence) { 232 int signaled; 233 if (wait) 234 signaled = ops->fence_finish(ops, fenced_buf->fence, 0); 235 else 236 signaled = ops->fence_signalled(ops, fenced_buf->fence, 0); 237 if (signaled != 0) 238 break; 239 prev_fence = fenced_buf->fence; 240 } 241 else { 242 assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0); 243 } 244 245 _fenced_buffer_remove(fenced_list, fenced_buf); 246 247 curr = next; 248 next = curr->next; 249 } 250} 251 252 253static void 254fenced_buffer_destroy(struct pb_buffer *buf) 255{ 256 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 257 struct fenced_buffer_list *fenced_list = fenced_buf->list; 258 259 pipe_mutex_lock(fenced_list->mutex); 260 assert(!pipe_is_referenced(&fenced_buf->base.base.reference)); 261 if (fenced_buf->fence) { 262 struct pb_fence_ops *ops = fenced_list->ops; 263 if(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0) { 264 struct list_head *curr, *prev; 265 curr = &fenced_buf->head; 266 prev = curr->prev; 267 do { 268 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 269 assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0); 270 _fenced_buffer_remove(fenced_list, fenced_buf); 271 curr = prev; 272 prev = curr->prev; 273 } while (curr != &fenced_list->delayed); 274 } 275 else { 276 /* delay destruction */ 277 } 278 } 279 else { 280 _fenced_buffer_destroy(fenced_buf); 281 } 282 pipe_mutex_unlock(fenced_list->mutex); 283} 284 285 286static void * 287fenced_buffer_map(struct pb_buffer *buf, 288 unsigned flags) 289{ 290 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 291 struct fenced_buffer_list *fenced_list = fenced_buf->list; 292 struct pb_fence_ops *ops = fenced_list->ops; 293 void *map; 294 295 assert(!(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE)); 296 297 /* Serialize writes */ 298 if((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_WRITE) || 299 ((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ) && (flags & PIPE_BUFFER_USAGE_CPU_WRITE))) { 300 if(flags & PIPE_BUFFER_USAGE_DONTBLOCK) { 301 /* Don't wait for the GPU to finish writing */ 302 if(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0) 303 _fenced_buffer_remove(fenced_list, fenced_buf); 304 else 305 return NULL; 306 } 307 else { 308 /* Wait for the GPU to finish writing */ 309 _fenced_buffer_finish(fenced_buf); 310 } 311 } 312 313#if 0 314 /* Check for CPU write access (read is OK) */ 315 if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) { 316 /* this is legal -- just for debugging */ 317 debug_warning("concurrent CPU writes"); 318 } 319#endif 320 321 map = pb_map(fenced_buf->buffer, flags); 322 if(map) { 323 ++fenced_buf->mapcount; 324 fenced_buf->flags |= flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE; 325 } 326 327 return map; 328} 329 330 331static void 332fenced_buffer_unmap(struct pb_buffer *buf) 333{ 334 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 335 assert(fenced_buf->mapcount); 336 if(fenced_buf->mapcount) { 337 pb_unmap(fenced_buf->buffer); 338 --fenced_buf->mapcount; 339 if(!fenced_buf->mapcount) 340 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_CPU_READ_WRITE; 341 } 342} 343 344 345static enum pipe_error 346fenced_buffer_validate(struct pb_buffer *buf, 347 struct pb_validate *vl, 348 unsigned flags) 349{ 350 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 351 enum pipe_error ret; 352 353 if(!vl) { 354 /* invalidate */ 355 fenced_buf->vl = NULL; 356 fenced_buf->validation_flags = 0; 357 return PIPE_OK; 358 } 359 360 assert(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE); 361 assert(!(flags & ~PIPE_BUFFER_USAGE_GPU_READ_WRITE)); 362 flags &= PIPE_BUFFER_USAGE_GPU_READ_WRITE; 363 364 /* Buffer cannot be validated in two different lists */ 365 if(fenced_buf->vl && fenced_buf->vl != vl) 366 return PIPE_ERROR_RETRY; 367 368#if 0 369 /* Do not validate if buffer is still mapped */ 370 if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) { 371 /* TODO: wait for the thread that mapped the buffer to unmap it */ 372 return PIPE_ERROR_RETRY; 373 } 374 /* Final sanity checking */ 375 assert(!(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE)); 376 assert(!fenced_buf->mapcount); 377#endif 378 379 if(fenced_buf->vl == vl && 380 (fenced_buf->validation_flags & flags) == flags) { 381 /* Nothing to do -- buffer already validated */ 382 return PIPE_OK; 383 } 384 385 ret = pb_validate(fenced_buf->buffer, vl, flags); 386 if (ret != PIPE_OK) 387 return ret; 388 389 fenced_buf->vl = vl; 390 fenced_buf->validation_flags |= flags; 391 392 return PIPE_OK; 393} 394 395 396static void 397fenced_buffer_fence(struct pb_buffer *buf, 398 struct pipe_fence_handle *fence) 399{ 400 struct fenced_buffer *fenced_buf; 401 struct fenced_buffer_list *fenced_list; 402 struct pb_fence_ops *ops; 403 404 fenced_buf = fenced_buffer(buf); 405 fenced_list = fenced_buf->list; 406 ops = fenced_list->ops; 407 408 if(fence == fenced_buf->fence) { 409 /* Nothing to do */ 410 return; 411 } 412 413 assert(fenced_buf->vl); 414 assert(fenced_buf->validation_flags); 415 416 pipe_mutex_lock(fenced_list->mutex); 417 if (fenced_buf->fence) 418 _fenced_buffer_remove(fenced_list, fenced_buf); 419 if (fence) { 420 ops->fence_reference(ops, &fenced_buf->fence, fence); 421 fenced_buf->flags |= fenced_buf->validation_flags; 422 _fenced_buffer_add(fenced_buf); 423 } 424 pipe_mutex_unlock(fenced_list->mutex); 425 426 pb_fence(fenced_buf->buffer, fence); 427 428 fenced_buf->vl = NULL; 429 fenced_buf->validation_flags = 0; 430} 431 432 433static void 434fenced_buffer_get_base_buffer(struct pb_buffer *buf, 435 struct pb_buffer **base_buf, 436 pb_size *offset) 437{ 438 struct fenced_buffer *fenced_buf = fenced_buffer(buf); 439 pb_get_base_buffer(fenced_buf->buffer, base_buf, offset); 440} 441 442 443static const struct pb_vtbl 444fenced_buffer_vtbl = { 445 fenced_buffer_destroy, 446 fenced_buffer_map, 447 fenced_buffer_unmap, 448 fenced_buffer_validate, 449 fenced_buffer_fence, 450 fenced_buffer_get_base_buffer 451}; 452 453 454struct pb_buffer * 455fenced_buffer_create(struct fenced_buffer_list *fenced_list, 456 struct pb_buffer *buffer) 457{ 458 struct fenced_buffer *buf; 459 460 if(!buffer) 461 return NULL; 462 463 buf = CALLOC_STRUCT(fenced_buffer); 464 if(!buf) { 465 pb_reference(&buffer, NULL); 466 return NULL; 467 } 468 469 pipe_reference_init(&buf->base.base.reference, 1); 470 buf->base.base.alignment = buffer->base.alignment; 471 buf->base.base.usage = buffer->base.usage; 472 buf->base.base.size = buffer->base.size; 473 474 buf->base.vtbl = &fenced_buffer_vtbl; 475 buf->buffer = buffer; 476 buf->list = fenced_list; 477 478#ifdef DEBUG 479 pipe_mutex_lock(fenced_list->mutex); 480 LIST_ADDTAIL(&buf->head, &fenced_list->unfenced); 481 ++fenced_list->numUnfenced; 482 pipe_mutex_unlock(fenced_list->mutex); 483#endif 484 485 return &buf->base; 486} 487 488 489struct fenced_buffer_list * 490fenced_buffer_list_create(struct pb_fence_ops *ops) 491{ 492 struct fenced_buffer_list *fenced_list; 493 494 fenced_list = CALLOC_STRUCT(fenced_buffer_list); 495 if (!fenced_list) 496 return NULL; 497 498 fenced_list->ops = ops; 499 500 LIST_INITHEAD(&fenced_list->delayed); 501 fenced_list->numDelayed = 0; 502 503#ifdef DEBUG 504 LIST_INITHEAD(&fenced_list->unfenced); 505 fenced_list->numUnfenced = 0; 506#endif 507 508 pipe_mutex_init(fenced_list->mutex); 509 510 return fenced_list; 511} 512 513 514void 515fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list, 516 int wait) 517{ 518 pipe_mutex_lock(fenced_list->mutex); 519 _fenced_buffer_list_check_free(fenced_list, wait); 520 pipe_mutex_unlock(fenced_list->mutex); 521} 522 523 524#ifdef DEBUG 525void 526fenced_buffer_list_dump(struct fenced_buffer_list *fenced_list) 527{ 528 struct pb_fence_ops *ops = fenced_list->ops; 529 struct list_head *curr, *next; 530 struct fenced_buffer *fenced_buf; 531 532 pipe_mutex_lock(fenced_list->mutex); 533 534 debug_printf("%10s %7s %7s %10s %s\n", 535 "buffer", "size", "refcount", "fence", "signalled"); 536 537 curr = fenced_list->unfenced.next; 538 next = curr->next; 539 while(curr != &fenced_list->unfenced) { 540 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 541 assert(!fenced_buf->fence); 542 debug_printf("%10p %7u %7u\n", 543 fenced_buf, 544 fenced_buf->base.base.size, 545 p_atomic_read(&fenced_buf->base.base.reference.count)); 546 curr = next; 547 next = curr->next; 548 } 549 550 curr = fenced_list->delayed.next; 551 next = curr->next; 552 while(curr != &fenced_list->delayed) { 553 int signaled; 554 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 555 signaled = ops->fence_signalled(ops, fenced_buf->fence, 0); 556 debug_printf("%10p %7u %7u %10p %s\n", 557 fenced_buf, 558 fenced_buf->base.base.size, 559 p_atomic_read(&fenced_buf->base.base.reference.count), 560 fenced_buf->fence, 561 signaled == 0 ? "y" : "n"); 562 curr = next; 563 next = curr->next; 564 } 565 566 pipe_mutex_unlock(fenced_list->mutex); 567} 568#endif 569 570 571void 572fenced_buffer_list_destroy(struct fenced_buffer_list *fenced_list) 573{ 574 pipe_mutex_lock(fenced_list->mutex); 575 576 /* Wait on outstanding fences */ 577 while (fenced_list->numDelayed) { 578 pipe_mutex_unlock(fenced_list->mutex); 579#if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS) 580 sched_yield(); 581#endif 582 _fenced_buffer_list_check_free(fenced_list, 1); 583 pipe_mutex_lock(fenced_list->mutex); 584 } 585 586#ifdef DEBUG 587 /*assert(!fenced_list->numUnfenced);*/ 588#endif 589 590 pipe_mutex_unlock(fenced_list->mutex); 591 592 fenced_list->ops->destroy(fenced_list->ops); 593 594 FREE(fenced_list); 595} 596 597 598