1/************************************************************************** 2 * 3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA 4 * All Rights Reserved. 5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 26 * USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 **************************************************************************/ 29/* 30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 31 */ 32 33#ifdef HAVE_CONFIG_H 34#include "config.h" 35#endif 36 37#include <drm/psb_ttm_placement_user.h> 38#include <stdint.h> 39#include <sys/time.h> 40#include <errno.h> 41#include <unistd.h> 42#include <assert.h> 43#include <string.h> 44#include <sys/mman.h> 45#include <xf86drm.h> 46#include "wsbm_pool.h" 47#include "wsbm_fencemgr.h" 48#include "wsbm_priv.h" 49#include "wsbm_manager.h" 50 51#define WSBM_SLABPOOL_ALLOC_RETRIES 100 52#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ 53 do { \ 54 (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ 55 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ 56 57#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ 58 do { \ 59 (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ 60 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ 61 62 63#ifdef DEBUG_FENCESIGNALED 64static int createbuffer = 0; 65static int fencesignaled = 0; 66#endif 67 68struct _WsbmSlab; 69 70struct _WsbmSlabBuffer 71{ 72 struct _WsbmKernelBuf kBuf; 73 struct _WsbmBufStorage storage; 74 struct _WsbmCond event; 75 76 /* 77 * Remains constant after creation. 78 */ 79 80 int isSlabBuffer; 81 struct _WsbmSlab *parent; 82 uint32_t start; 83 void *virtual; 84 unsigned long requestedSize; 85 uint64_t mapHandle; 86 87 /* 88 * Protected by struct _WsbmSlabSizeHeader::mutex 89 */ 90 91 struct _WsbmListHead head; 92 93 /* 94 * Protected by this::mutex 95 */ 96 97 struct _WsbmFenceObject *fence; 98 uint32_t fenceType; 99 struct _WsbmAtomic writers; /* (Only upping) */ 100 int unFenced; 101}; 102 103struct _WsbmSlabPool; 104struct _WsbmSlabKernelBO 105{ 106 107 /* 108 * Constant at creation 109 */ 110 111 struct _WsbmKernelBuf kBuf; 112 uint32_t pageAlignment; 113 void *virtual; 114 unsigned long actualSize; 115 uint64_t mapHandle; 116 117 /* 118 * Protected by struct _WsbmSlabCache::mutex 119 */ 120 121 struct _WsbmSlabPool *slabPool; 122 uint32_t proposedPlacement; 123 struct _WsbmListHead timeoutHead; 124 struct _WsbmListHead head; 125 struct timeval timeFreed; 126}; 127 128struct _WsbmSlab 129{ 130 struct _WsbmListHead head; 131 struct _WsbmListHead freeBuffers; 132 uint32_t numBuffers; 133 uint32_t numFree; 134 struct _WsbmSlabBuffer *buffers; 135 struct _WsbmSlabSizeHeader *header; 136 struct _WsbmSlabKernelBO *kbo; 137}; 138 139struct _WsbmSlabSizeHeader 140{ 141 /* 142 * Constant at creation. 143 */ 144 struct _WsbmSlabPool *slabPool; 145 uint32_t bufSize; 146 147 /* 148 * Protected by this::mutex 149 */ 150 151 struct _WsbmListHead slabs; 152 struct _WsbmListHead freeSlabs; 153 struct _WsbmListHead delayedBuffers; 154 uint32_t numDelayed; 155 struct _WsbmMutex mutex; 156}; 157 158struct _WsbmSlabCache 159{ 160 struct timeval slabTimeout; 161 struct timeval checkInterval; 162 struct timeval nextCheck; 163 struct _WsbmListHead timeoutList; 164 struct _WsbmListHead unCached; 165 struct _WsbmListHead cached; 166 struct _WsbmMutex mutex; 167}; 168 169struct _WsbmSlabPool 170{ 171 struct _WsbmBufferPool pool; 172 173 /* 174 * The data of this structure remains constant after 175 * initialization and thus needs no mutex protection. 176 */ 177 178 unsigned int devOffset; 179 struct _WsbmSlabCache *cache; 180 uint32_t proposedPlacement; 181 uint32_t validMask; 182 uint32_t *bucketSizes; 183 uint32_t numBuckets; 184 uint32_t pageSize; 185 int pageAlignment; 186 int maxSlabSize; 187 int desiredNumBuffers; 188 struct _WsbmSlabSizeHeader *headers; 189}; 190 191static inline struct _WsbmSlabPool * 192slabPoolFromPool(struct _WsbmBufferPool *pool) 193{ 194 return containerOf(pool, struct _WsbmSlabPool, pool); 195} 196 197static inline struct _WsbmSlabPool * 198slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf) 199{ 200 return slabPoolFromPool(sBuf->storage.pool); 201} 202 203static inline struct _WsbmSlabBuffer * 204slabBuffer(struct _WsbmBufStorage *buf) 205{ 206 return containerOf(buf, struct _WsbmSlabBuffer, storage); 207} 208 209/* 210 * FIXME: Perhaps arrange timeout slabs in size buckets for fast 211 * retreival?? 212 */ 213 214static inline int 215wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2) 216{ 217 return ((arg1->tv_sec > arg2->tv_sec) || 218 ((arg1->tv_sec == arg2->tv_sec) && 219 (arg1->tv_usec > arg2->tv_usec))); 220} 221 222static inline void 223wsbmTimeAdd(struct timeval *arg, struct timeval *add) 224{ 225 unsigned int sec; 226 227 arg->tv_sec += add->tv_sec; 228 arg->tv_usec += add->tv_usec; 229 sec = arg->tv_usec / 1000000; 230 arg->tv_sec += sec; 231 arg->tv_usec -= sec * 1000000; 232} 233 234static void 235wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo) 236{ 237 struct ttm_pl_reference_req arg; 238 struct _WsbmSlabPool *slabPool; 239 240 if (!kbo) 241 return; 242 243 slabPool = kbo->slabPool; 244 arg.handle = kbo->kBuf.handle; 245 (void)munmap(kbo->virtual, kbo->actualSize); 246 (void)drmCommandWrite(slabPool->pool.fd, 247 slabPool->devOffset + TTM_PL_UNREF, &arg, 248 sizeof(arg)); 249 free(kbo); 250} 251 252static void 253wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time) 254{ 255 struct _WsbmListHead *list, *next; 256 struct _WsbmSlabKernelBO *kbo; 257 258 if (!wsbmTimeAfterEq(time, &cache->nextCheck)) 259 return; 260 261 WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) { 262 kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead); 263 264 if (!wsbmTimeAfterEq(time, &kbo->timeFreed)) 265 break; 266 267 WSBMLISTDELINIT(&kbo->timeoutHead); 268 WSBMLISTDELINIT(&kbo->head); 269 wsbmFreeKernelBO(kbo); 270 } 271 272 cache->nextCheck = *time; 273 wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval); 274} 275 276/* 277 * Add a _SlabKernelBO to the free slab manager. 278 * This means that it is available for reuse, but if it's not 279 * reused in a while, it will be freed. 280 */ 281 282static void 283wsbmSetKernelBOFree(struct _WsbmSlabCache *cache, 284 struct _WsbmSlabKernelBO *kbo) 285{ 286 struct timeval time; 287 struct timeval timeFreed; 288 289 gettimeofday(&time, NULL); 290 timeFreed = time; 291 WSBM_MUTEX_LOCK(&cache->mutex); 292 wsbmTimeAdd(&timeFreed, &cache->slabTimeout); 293 kbo->timeFreed = timeFreed; 294 295 if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED) 296 WSBMLISTADD(&kbo->head, &cache->cached); 297 else 298 WSBMLISTADD(&kbo->head, &cache->unCached); 299 300 WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList); 301 wsbmFreeTimeoutKBOsLocked(cache, &time); 302 303 WSBM_MUTEX_UNLOCK(&cache->mutex); 304} 305 306/* 307 * Get a _SlabKernelBO for us to use as storage for a slab. 308 */ 309 310static struct _WsbmSlabKernelBO * 311wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) 312{ 313 struct _WsbmSlabPool *slabPool = header->slabPool; 314 struct _WsbmSlabCache *cache = slabPool->cache; 315 struct _WsbmListHead *list, *head; 316 uint32_t size = header->bufSize * slabPool->desiredNumBuffers; 317 struct _WsbmSlabKernelBO *kbo; 318 struct _WsbmSlabKernelBO *kboTmp; 319 int ret; 320 321 /* 322 * FIXME: We should perhaps allow some variation in slabsize in order 323 * to efficiently reuse slabs. 324 */ 325 326 size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize; 327 if (size < header->bufSize) 328 size = header->bufSize; 329 size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1); 330 WSBM_MUTEX_LOCK(&cache->mutex); 331 332 kbo = NULL; 333 334 retry: 335 head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ? 336 &cache->cached : &cache->unCached; 337 338 WSBMLISTFOREACH(list, head) { 339 kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head); 340 341 if ((kboTmp->actualSize == size) && 342 (slabPool->pageAlignment == 0 || 343 (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) { 344 345 if (!kbo) 346 kbo = kboTmp; 347 348 if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0) 349 break; 350 351 } 352 } 353 354 if (kbo) { 355 WSBMLISTDELINIT(&kbo->head); 356 WSBMLISTDELINIT(&kbo->timeoutHead); 357 } 358 359 WSBM_MUTEX_UNLOCK(&cache->mutex); 360 361 if (kbo) { 362 uint32_t new_mask = 363 kbo->proposedPlacement ^ slabPool->proposedPlacement; 364 365 ret = 0; 366 if (new_mask) { 367 union ttm_pl_setstatus_arg arg; 368 struct ttm_pl_setstatus_req *req = &arg.req; 369 struct ttm_pl_rep *rep = &arg.rep; 370 371 req->handle = kbo->kBuf.handle; 372 req->set_placement = slabPool->proposedPlacement & new_mask; 373 req->clr_placement = ~slabPool->proposedPlacement & new_mask; 374 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, 375 slabPool->devOffset + TTM_PL_SETSTATUS, 376 arg, ret); 377 if (ret == 0) { 378 kbo->kBuf.gpuOffset = rep->gpu_offset; 379 kbo->kBuf.placement = rep->placement; 380 } 381 kbo->proposedPlacement = slabPool->proposedPlacement; 382 } 383 384 if (ret == 0) 385 return kbo; 386 387 wsbmFreeKernelBO(kbo); 388 kbo = NULL; 389 goto retry; 390 } 391 392 kbo = calloc(1, sizeof(*kbo)); 393 if (!kbo) 394 return NULL; 395 396 { 397 union ttm_pl_create_arg arg; 398 399 kbo->slabPool = slabPool; 400 WSBMINITLISTHEAD(&kbo->head); 401 WSBMINITLISTHEAD(&kbo->timeoutHead); 402 403 arg.req.size = size; 404 arg.req.placement = slabPool->proposedPlacement; 405 arg.req.page_alignment = slabPool->pageAlignment; 406 407 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, 408 slabPool->devOffset + TTM_PL_CREATE, 409 arg, ret); 410 if (ret) 411 goto out_err0; 412 413 kbo->kBuf.gpuOffset = arg.rep.gpu_offset; 414 kbo->kBuf.placement = arg.rep.placement; 415 kbo->kBuf.handle = arg.rep.handle; 416 417 kbo->actualSize = arg.rep.bo_size; 418 kbo->mapHandle = arg.rep.map_handle; 419 kbo->proposedPlacement = slabPool->proposedPlacement; 420 } 421 422 kbo->virtual = mmap(0, kbo->actualSize, 423 PROT_READ | PROT_WRITE, MAP_SHARED, 424 slabPool->pool.fd, kbo->mapHandle); 425 426 if (kbo->virtual == MAP_FAILED) { 427 ret = -errno; 428 goto out_err1; 429 } 430 431 return kbo; 432 433 out_err1: 434 { 435 struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle }; 436 437 (void)drmCommandWrite(slabPool->pool.fd, 438 slabPool->devOffset + TTM_PL_UNREF, 439 &arg, sizeof(arg)); 440 } 441 out_err0: 442 free(kbo); 443 return NULL; 444} 445 446static int 447wsbmAllocSlab(struct _WsbmSlabSizeHeader *header) 448{ 449 struct _WsbmSlab *slab; 450 struct _WsbmSlabBuffer *sBuf; 451 uint32_t numBuffers; 452 uint32_t ret; 453 uint32_t i; 454 455 slab = calloc(1, sizeof(*slab)); 456 if (!slab) 457 return -ENOMEM; 458 459 slab->kbo = wsbmAllocKernelBO(header); 460 if (!slab->kbo) { 461 ret = -ENOMEM; 462 goto out_err0; 463 } 464 465 numBuffers = slab->kbo->actualSize / header->bufSize; 466 if (!numBuffers) { 467 ret = -ENOMEM; 468 goto out_err1; 469 } 470 471 slab->buffers = calloc(numBuffers, sizeof(*slab->buffers)); 472 if (!slab->buffers) { 473 ret = -ENOMEM; 474 goto out_err1; 475 } 476 477 WSBMINITLISTHEAD(&slab->head); 478 WSBMINITLISTHEAD(&slab->freeBuffers); 479 slab->numBuffers = numBuffers; 480 slab->numFree = 0; 481 slab->header = header; 482 483 sBuf = slab->buffers; 484 for (i = 0; i < numBuffers; ++i) { 485 ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool); 486 if (ret) 487 goto out_err2; 488 sBuf->parent = slab; 489 sBuf->start = i * header->bufSize; 490 sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual + 491 sBuf->start); 492 wsbmAtomicSet(&sBuf->writers, 0); 493 sBuf->isSlabBuffer = 1; 494 WSBM_COND_INIT(&sBuf->event); 495 WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers); 496 slab->numFree++; 497 sBuf++; 498 } 499 500 WSBMLISTADDTAIL(&slab->head, &header->slabs); 501 502 return 0; 503 504 out_err2: 505 sBuf = slab->buffers; 506 for (i = 0; i < numBuffers; ++i) { 507 if (sBuf->parent == slab) { 508 WSBM_COND_FREE(&sBuf->event); 509 wsbmBufStorageTakedown(&sBuf->storage); 510 } 511 sBuf++; 512 } 513 free(slab->buffers); 514 out_err1: 515 wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo); 516 out_err0: 517 free(slab); 518 return ret; 519} 520 521/* 522 * Delete a buffer from the slab header delayed list and put 523 * it on the slab free list. 524 */ 525 526static void 527wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf) 528{ 529 struct _WsbmSlab *slab = buf->parent; 530 struct _WsbmSlabSizeHeader *header = slab->header; 531 struct _WsbmListHead *list = &buf->head; 532 533 WSBMLISTDEL(list); 534 WSBMLISTADDTAIL(list, &slab->freeBuffers); 535 slab->numFree++; 536 537 if (slab->head.next == &slab->head) 538 WSBMLISTADDTAIL(&slab->head, &header->slabs); 539 540 if (slab->numFree == slab->numBuffers) { 541 list = &slab->head; 542 WSBMLISTDEL(list); 543 WSBMLISTADDTAIL(list, &header->freeSlabs); 544 } 545 546 if (header->slabs.next == &header->slabs || 547 slab->numFree != slab->numBuffers) { 548 549 struct _WsbmListHead *next; 550 struct _WsbmSlabCache *cache = header->slabPool->cache; 551 552 WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) { 553 uint32_t i; 554 struct _WsbmSlabBuffer *sBuf; 555 556 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); 557 558 WSBMLISTDELINIT(list); 559 560 sBuf = slab->buffers; 561 for (i = 0; i < slab->numBuffers; ++i) { 562 if (sBuf->parent == slab) { 563 WSBM_COND_FREE(&sBuf->event); 564 wsbmBufStorageTakedown(&sBuf->storage); 565 } 566 sBuf++; 567 } 568 wsbmSetKernelBOFree(cache, slab->kbo); 569 free(slab->buffers); 570 free(slab); 571 } 572 } 573} 574 575static void 576wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait) 577{ 578 struct _WsbmListHead *list, *prev, *first, *head; 579 struct _WsbmSlabBuffer *sBuf; 580 struct _WsbmSlab *slab; 581 int firstWasSignaled = 1; 582 int signaled; 583 uint32_t i; 584 int ret; 585 586 /* 587 * Rerun the freeing test if the youngest tested buffer 588 * was signaled, since there might be more idle buffers 589 * in the delay list. 590 */ 591 592 while (firstWasSignaled) { 593 firstWasSignaled = 0; 594 signaled = 0; 595 first = header->delayedBuffers.next; 596 597 /* Only examine the oldest 1/3 of delayed buffers: 598 */ 599 if (header->numDelayed > 3) { 600 for (i = 0; i < header->numDelayed; i += 3) { 601 first = first->next; 602 } 603 } 604 605 /* 606 * No need to take the buffer mutex for each buffer we loop 607 * through since we're currently the only user. 608 */ 609 610 head = first->next; 611 WSBMLISTFOREACHPREVSAFE(list, prev, head) { 612 613 if (list == &header->delayedBuffers) 614 break; 615 616 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); 617 618 slab = sBuf->parent; 619 620 if (!signaled) { 621 if (wait) { 622 ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); 623 if (ret) 624 break; 625 signaled = 1; 626 wait = 0; 627 } else { 628 signaled = 629 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); 630#ifdef DEBUG_FENCESIGNALED 631 fencesignaled++; 632#endif 633 } 634 if (signaled) { 635 if (list == first) 636 firstWasSignaled = 1; 637 wsbmFenceUnreference(&sBuf->fence); 638 header->numDelayed--; 639 wsbmSlabFreeBufferLocked(sBuf); 640 } else 641 break; 642 } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { 643 wsbmFenceUnreference(&sBuf->fence); 644 header->numDelayed--; 645 wsbmSlabFreeBufferLocked(sBuf); 646 } 647 } 648 } 649} 650 651static struct _WsbmSlabBuffer * 652wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header) 653{ 654 static struct _WsbmSlabBuffer *buf; 655 struct _WsbmSlab *slab; 656 struct _WsbmListHead *list; 657 int count = WSBM_SLABPOOL_ALLOC_RETRIES; 658 659 WSBM_MUTEX_LOCK(&header->mutex); 660 while (header->slabs.next == &header->slabs && count > 0) { 661 wsbmSlabCheckFreeLocked(header, 0); 662 if (header->slabs.next != &header->slabs) 663 break; 664 665 WSBM_MUTEX_UNLOCK(&header->mutex); 666 if (count != WSBM_SLABPOOL_ALLOC_RETRIES) 667 usleep(1000); 668 WSBM_MUTEX_LOCK(&header->mutex); 669 (void)wsbmAllocSlab(header); 670 count--; 671 } 672 673 list = header->slabs.next; 674 if (list == &header->slabs) { 675 WSBM_MUTEX_UNLOCK(&header->mutex); 676 return NULL; 677 } 678 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); 679 if (--slab->numFree == 0) 680 WSBMLISTDELINIT(list); 681 682 list = slab->freeBuffers.next; 683 WSBMLISTDELINIT(list); 684 685 WSBM_MUTEX_UNLOCK(&header->mutex); 686 buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); 687 688 buf->storage.destroyContainer = NULL; 689 690#ifdef DEBUG_FENCESIGNALED 691 createbuffer++; 692#endif 693 return buf; 694} 695 696static struct _WsbmBufStorage * 697pool_create(struct _WsbmBufferPool *pool, unsigned long size, 698 uint32_t placement, unsigned alignment) 699{ 700 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); 701 struct _WsbmSlabSizeHeader *header; 702 struct _WsbmSlabBuffer *sBuf; 703 uint32_t i; 704 int ret; 705 706 /* 707 * FIXME: Check for compatibility. 708 */ 709 710 header = slabPool->headers; 711 for (i = 0; i < slabPool->numBuckets; ++i) { 712 if (header->bufSize >= size) 713 break; 714 header++; 715 } 716 717 if (i < slabPool->numBuckets) { 718 sBuf = wsbmSlabAllocBuffer(header); 719 return ((sBuf) ? &sBuf->storage : NULL); 720 } 721 722 /* 723 * Fall back to allocate a buffer object directly from DRM. 724 * and wrap it in a wsbmBO structure. 725 */ 726 727 sBuf = calloc(1, sizeof(*sBuf)); 728 729 if (!sBuf) 730 return NULL; 731 732 if (alignment) { 733 if ((alignment < slabPool->pageSize) 734 && (slabPool->pageSize % alignment)) 735 goto out_err0; 736 if ((alignment > slabPool->pageSize) 737 && (alignment % slabPool->pageSize)) 738 goto out_err0; 739 } 740 741 ret = wsbmBufStorageInit(&sBuf->storage, pool); 742 if (ret) 743 goto out_err0; 744 745 ret = WSBM_COND_INIT(&sBuf->event); 746 if (ret) 747 goto out_err1; 748 749 { 750 union ttm_pl_create_arg arg; 751 752 arg.req.size = size; 753 arg.req.placement = placement; 754 arg.req.page_alignment = alignment / slabPool->pageSize; 755 756 DRMRESTARTCOMMANDWRITEREAD(pool->fd, 757 slabPool->devOffset + TTM_PL_CREATE, 758 arg, ret); 759 760 if (ret) 761 goto out_err2; 762 763 sBuf->kBuf.gpuOffset = arg.rep.gpu_offset; 764 sBuf->kBuf.placement = arg.rep.placement; 765 sBuf->kBuf.handle = arg.rep.handle; 766 sBuf->mapHandle = arg.rep.map_handle; 767 sBuf->requestedSize = size; 768 769 sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, 770 pool->fd, sBuf->mapHandle); 771 772 if (sBuf->virtual == MAP_FAILED) 773 goto out_err3; 774 } 775 776 wsbmAtomicSet(&sBuf->writers, 0); 777 return &sBuf->storage; 778 out_err3: 779 { 780 struct ttm_pl_reference_req arg; 781 782 arg.handle = sBuf->kBuf.handle; 783 (void)drmCommandWriteRead(pool->fd, 784 slabPool->devOffset + TTM_PL_UNREF, 785 &arg, sizeof(arg)); 786 } 787 out_err2: 788 WSBM_COND_FREE(&sBuf->event); 789 out_err1: 790 wsbmBufStorageTakedown(&sBuf->storage); 791 out_err0: 792 free(sBuf); 793 return NULL; 794} 795 796static void 797pool_destroy(struct _WsbmBufStorage **p_buf) 798{ 799 struct _WsbmBufStorage *buf = *p_buf; 800 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 801 struct _WsbmSlab *slab; 802 struct _WsbmSlabSizeHeader *header; 803 804 *p_buf = NULL; 805 806 if (!sBuf->isSlabBuffer) { 807 struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf); 808 struct ttm_pl_reference_req arg; 809 810 if (sBuf->virtual != NULL) { 811 (void)munmap(sBuf->virtual, sBuf->requestedSize); 812 sBuf->virtual = NULL; 813 } 814 815 arg.handle = sBuf->kBuf.handle; 816 (void)drmCommandWrite(slabPool->pool.fd, 817 slabPool->devOffset + TTM_PL_UNREF, 818 &arg, sizeof(arg)); 819 820 WSBM_COND_FREE(&sBuf->event); 821 wsbmBufStorageTakedown(&sBuf->storage); 822 free(sBuf); 823 return; 824 } 825 826 slab = sBuf->parent; 827 header = slab->header; 828 829 /* 830 * No need to take the buffer mutex below since we're the only user. 831 */ 832 833 WSBM_MUTEX_LOCK(&header->mutex); 834 sBuf->unFenced = 0; 835 wsbmAtomicSet(&sBuf->writers, 0); 836 wsbmAtomicSet(&sBuf->storage.refCount, 1); 837 838 if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { 839 WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers); 840 header->numDelayed++; 841 } else { 842 if (sBuf->fence) 843 wsbmFenceUnreference(&sBuf->fence); 844 wsbmSlabFreeBufferLocked(sBuf); 845 } 846 847 WSBM_MUTEX_UNLOCK(&header->mutex); 848} 849 850static void 851waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy) 852{ 853 struct _WsbmBufStorage *storage = &sBuf->storage; 854 855 while (sBuf->unFenced || sBuf->fence != NULL) { 856 857 if (sBuf->unFenced) 858 WSBM_COND_WAIT(&sBuf->event, &storage->mutex); 859 860 if (sBuf->fence != NULL) { 861 if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) { 862 struct _WsbmFenceObject *fence = 863 wsbmFenceReference(sBuf->fence); 864 865 WSBM_MUTEX_UNLOCK(&storage->mutex); 866 (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy); 867 WSBM_MUTEX_LOCK(&storage->mutex); 868 if (sBuf->fence == fence) 869 wsbmFenceUnreference(&sBuf->fence); 870 871 wsbmFenceUnreference(&fence); 872 } else { 873 wsbmFenceUnreference(&sBuf->fence); 874 } 875 } 876 } 877} 878 879static int 880pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) 881{ 882 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 883 884 WSBM_MUTEX_LOCK(&buf->mutex); 885 waitIdleLocked(sBuf, lazy); 886 WSBM_MUTEX_UNLOCK(&buf->mutex); 887 888 return 0; 889} 890 891static int 892pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual) 893{ 894 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 895 896 *virtual = sBuf->virtual; 897 898 return 0; 899} 900 901static void 902pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused))) 903{ 904 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 905 906 if (wsbmAtomicDecZero(&sBuf->writers)) 907 WSBM_COND_BROADCAST(&sBuf->event); 908} 909 910static int 911pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) 912{ 913 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 914 int ret = 0; 915 916 WSBM_MUTEX_LOCK(&buf->mutex); 917 if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { 918 int signaled; 919 920 if (sBuf->unFenced) { 921 ret = -EBUSY; 922 goto out_unlock; 923 } 924 925 if (sBuf->isSlabBuffer) 926 signaled = (sBuf->fence == NULL) || 927 wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType); 928 else 929 signaled = (sBuf->fence == NULL) || 930 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); 931 932 ret = 0; 933 if (signaled) { 934 wsbmFenceUnreference(&sBuf->fence); 935 wsbmAtomicInc(&sBuf->writers); 936 } else 937 ret = -EBUSY; 938 goto out_unlock; 939 } 940 waitIdleLocked(sBuf, 0); 941 wsbmAtomicInc(&sBuf->writers); 942 out_unlock: 943 WSBM_MUTEX_UNLOCK(&buf->mutex); 944 return ret; 945} 946 947static void 948pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused))) 949{ 950 ; 951} 952 953static unsigned long 954pool_poolOffset(struct _WsbmBufStorage *buf) 955{ 956 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 957 958 return sBuf->start; 959} 960 961static unsigned long 962pool_size(struct _WsbmBufStorage *buf) 963{ 964 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 965 966 if (!sBuf->isSlabBuffer) 967 return sBuf->requestedSize; 968 969 return sBuf->parent->header->bufSize; 970} 971 972static struct _WsbmKernelBuf * 973pool_kernel(struct _WsbmBufStorage *buf) 974{ 975 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 976 977 return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf; 978} 979 980static unsigned long 981pool_offset(struct _WsbmBufStorage *buf) 982{ 983 return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf); 984} 985 986static void 987pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) 988{ 989 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 990 struct _WsbmKernelBuf *kBuf; 991 992 WSBM_MUTEX_LOCK(&buf->mutex); 993 if (sBuf->fence) 994 wsbmFenceUnreference(&sBuf->fence); 995 996 kBuf = pool_kernel(buf); 997 sBuf->fenceType = kBuf->fence_type_mask; 998 if (!wsbmFenceSignaledCached(fence, sBuf->fenceType)) 999 sBuf->fence = wsbmFenceReference(fence); 1000 1001 sBuf->unFenced = 0; 1002 WSBM_COND_BROADCAST(&sBuf->event); 1003 WSBM_MUTEX_UNLOCK(&buf->mutex); 1004} 1005 1006static int 1007pool_validate(struct _WsbmBufStorage *buf, 1008 uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused))) 1009{ 1010 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 1011 1012 WSBM_MUTEX_LOCK(&buf->mutex); 1013 while (wsbmAtomicRead(&sBuf->writers) != 0) { 1014 WSBM_COND_WAIT(&sBuf->event, &buf->mutex); 1015 } 1016 1017 sBuf->unFenced = 1; 1018 WSBM_MUTEX_UNLOCK(&buf->mutex); 1019 return 0; 1020} 1021 1022static void 1023pool_unvalidate(struct _WsbmBufStorage *buf) 1024{ 1025 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 1026 1027 WSBM_MUTEX_LOCK(&buf->mutex); 1028 if (sBuf->unFenced) { 1029 sBuf->unFenced = 0; 1030 WSBM_COND_BROADCAST(&sBuf->event); 1031 } 1032 WSBM_MUTEX_UNLOCK(&buf->mutex); 1033} 1034 1035struct _WsbmSlabCache * 1036wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec) 1037{ 1038 struct _WsbmSlabCache *tmp; 1039 1040 tmp = calloc(1, sizeof(*tmp)); 1041 if (!tmp) 1042 return NULL; 1043 1044 WSBM_MUTEX_INIT(&tmp->mutex); 1045 WSBM_MUTEX_LOCK(&tmp->mutex); 1046 tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000; 1047 tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000; 1048 tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000; 1049 1050 tmp->checkInterval.tv_usec = checkIntervalMsec * 1000; 1051 tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000; 1052 tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000; 1053 1054 gettimeofday(&tmp->nextCheck, NULL); 1055 wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval); 1056 WSBMINITLISTHEAD(&tmp->timeoutList); 1057 WSBMINITLISTHEAD(&tmp->unCached); 1058 WSBMINITLISTHEAD(&tmp->cached); 1059 WSBM_MUTEX_UNLOCK(&tmp->mutex); 1060 1061 return tmp; 1062} 1063 1064void 1065wsbmSlabCacheFinish(struct _WsbmSlabCache *cache) 1066{ 1067 struct timeval time; 1068 1069 time = cache->nextCheck; 1070 WSBM_MUTEX_LOCK(&cache->mutex); 1071 wsbmTimeAdd(&time, &cache->checkInterval); 1072 wsbmFreeTimeoutKBOsLocked(cache, &time); 1073 WSBM_MUTEX_UNLOCK(&cache->mutex); 1074 1075 assert(cache->timeoutList.next == &cache->timeoutList); 1076 assert(cache->unCached.next == &cache->unCached); 1077 assert(cache->cached.next == &cache->cached); 1078 1079 WSBM_MUTEX_FREE(&cache->mutex); 1080 free(cache); 1081} 1082 1083static void 1084wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size, 1085 struct _WsbmSlabSizeHeader *header) 1086{ 1087 WSBM_MUTEX_INIT(&header->mutex); 1088 WSBM_MUTEX_LOCK(&header->mutex); 1089 1090 WSBMINITLISTHEAD(&header->slabs); 1091 WSBMINITLISTHEAD(&header->freeSlabs); 1092 WSBMINITLISTHEAD(&header->delayedBuffers); 1093 1094 header->numDelayed = 0; 1095 header->slabPool = slabPool; 1096 header->bufSize = size; 1097 1098 WSBM_MUTEX_UNLOCK(&header->mutex); 1099} 1100 1101static void 1102wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header) 1103{ 1104 struct _WsbmListHead *list, *next; 1105 struct _WsbmSlabBuffer *sBuf; 1106 1107 WSBM_MUTEX_LOCK(&header->mutex); 1108 WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) { 1109 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); 1110 1111 if (sBuf->fence) { 1112 (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); 1113 wsbmFenceUnreference(&sBuf->fence); 1114 } 1115 header->numDelayed--; 1116 wsbmSlabFreeBufferLocked(sBuf); 1117 } 1118 WSBM_MUTEX_UNLOCK(&header->mutex); 1119 WSBM_MUTEX_FREE(&header->mutex); 1120} 1121 1122static void 1123pool_takedown(struct _WsbmBufferPool *pool) 1124{ 1125 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); 1126 unsigned int i; 1127 1128 for (i = 0; i < slabPool->numBuckets; ++i) { 1129 wsbmFinishSizeHeader(&slabPool->headers[i]); 1130 } 1131 1132 free(slabPool->headers); 1133 free(slabPool->bucketSizes); 1134 free(slabPool); 1135} 1136 1137struct _WsbmBufferPool * 1138wsbmSlabPoolInit(int fd, 1139 uint32_t devOffset, 1140 uint32_t placement, 1141 uint32_t validMask, 1142 uint32_t smallestSize, 1143 uint32_t numSizes, 1144 uint32_t desiredNumBuffers, 1145 uint32_t maxSlabSize, 1146 uint32_t pageAlignment, struct _WsbmSlabCache *cache) 1147{ 1148 struct _WsbmBufferPool *pool; 1149 struct _WsbmSlabPool *slabPool; 1150 uint32_t i; 1151 1152 slabPool = calloc(1, sizeof(*slabPool)); 1153 if (!slabPool) 1154 return NULL; 1155 1156 pool = &slabPool->pool; 1157 1158 slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes)); 1159 if (!slabPool->bucketSizes) 1160 goto out_err0; 1161 1162 slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers)); 1163 if (!slabPool->headers) 1164 goto out_err1; 1165 1166 slabPool->devOffset = devOffset; 1167 slabPool->cache = cache; 1168 slabPool->proposedPlacement = placement; 1169 slabPool->validMask = validMask; 1170 slabPool->numBuckets = numSizes; 1171 slabPool->pageSize = getpagesize(); 1172 slabPool->pageAlignment = pageAlignment; 1173 slabPool->maxSlabSize = maxSlabSize; 1174 slabPool->desiredNumBuffers = desiredNumBuffers; 1175 1176 for (i = 0; i < slabPool->numBuckets; ++i) { 1177 slabPool->bucketSizes[i] = (smallestSize << i); 1178 wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i], 1179 &slabPool->headers[i]); 1180 } 1181 1182 pool->fd = fd; 1183 pool->map = &pool_map; 1184 pool->unmap = &pool_unmap; 1185 pool->destroy = &pool_destroy; 1186 pool->offset = &pool_offset; 1187 pool->poolOffset = &pool_poolOffset; 1188 pool->size = &pool_size; 1189 pool->create = &pool_create; 1190 pool->fence = &pool_fence; 1191 pool->kernel = &pool_kernel; 1192 pool->validate = &pool_validate; 1193 pool->unvalidate = &pool_unvalidate; 1194 pool->waitIdle = &pool_waitIdle; 1195 pool->takeDown = &pool_takedown; 1196 pool->releasefromcpu = &pool_releaseFromCpu; 1197 pool->syncforcpu = &pool_syncForCpu; 1198 1199 return pool; 1200 1201 out_err1: 1202 free(slabPool->bucketSizes); 1203 out_err0: 1204 free(slabPool); 1205 1206 return NULL; 1207} 1208