183827f40a2d97261528087331b0bee6ce2cf27c5root/************************************************************************** 283827f40a2d97261528087331b0bee6ce2cf27c5root * 383827f40a2d97261528087331b0bee6ce2cf27c5root * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA 483827f40a2d97261528087331b0bee6ce2cf27c5root * All Rights Reserved. 583827f40a2d97261528087331b0bee6ce2cf27c5root * Copyright 2009 VMware, Inc., Palo Alto, CA., USA 683827f40a2d97261528087331b0bee6ce2cf27c5root * All Rights Reserved. 783827f40a2d97261528087331b0bee6ce2cf27c5root * 883827f40a2d97261528087331b0bee6ce2cf27c5root * Permission is hereby granted, free of charge, to any person obtaining a 983827f40a2d97261528087331b0bee6ce2cf27c5root * copy of this software and associated documentation files (the 1083827f40a2d97261528087331b0bee6ce2cf27c5root * "Software"), to deal in the Software without restriction, including 1183827f40a2d97261528087331b0bee6ce2cf27c5root * without limitation the rights to use, copy, modify, merge, publish, 1283827f40a2d97261528087331b0bee6ce2cf27c5root * distribute, sub license, and/or sell copies of the Software, and to 1383827f40a2d97261528087331b0bee6ce2cf27c5root * permit persons to whom the Software is furnished to do so, subject to 1483827f40a2d97261528087331b0bee6ce2cf27c5root * the following conditions: 1583827f40a2d97261528087331b0bee6ce2cf27c5root * 1683827f40a2d97261528087331b0bee6ce2cf27c5root * The above copyright notice and this permission notice (including the 1783827f40a2d97261528087331b0bee6ce2cf27c5root * next paragraph) shall be included in all copies or substantial portions 1883827f40a2d97261528087331b0bee6ce2cf27c5root * of the Software. 1983827f40a2d97261528087331b0bee6ce2cf27c5root * 2083827f40a2d97261528087331b0bee6ce2cf27c5root * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 2183827f40a2d97261528087331b0bee6ce2cf27c5root * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 2283827f40a2d97261528087331b0bee6ce2cf27c5root * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 2383827f40a2d97261528087331b0bee6ce2cf27c5root * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 2483827f40a2d97261528087331b0bee6ce2cf27c5root * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 2583827f40a2d97261528087331b0bee6ce2cf27c5root * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 2683827f40a2d97261528087331b0bee6ce2cf27c5root * USE OR OTHER DEALINGS IN THE SOFTWARE. 2783827f40a2d97261528087331b0bee6ce2cf27c5root * 2883827f40a2d97261528087331b0bee6ce2cf27c5root **************************************************************************/ 2983827f40a2d97261528087331b0bee6ce2cf27c5root/* 3083827f40a2d97261528087331b0bee6ce2cf27c5root * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 3183827f40a2d97261528087331b0bee6ce2cf27c5root */ 3283827f40a2d97261528087331b0bee6ce2cf27c5root 3383827f40a2d97261528087331b0bee6ce2cf27c5root#ifdef HAVE_CONFIG_H 3483827f40a2d97261528087331b0bee6ce2cf27c5root#include "config.h" 3583827f40a2d97261528087331b0bee6ce2cf27c5root#endif 3683827f40a2d97261528087331b0bee6ce2cf27c5root 3783827f40a2d97261528087331b0bee6ce2cf27c5root#include <stdint.h> 3883827f40a2d97261528087331b0bee6ce2cf27c5root#include <errno.h> 3983827f40a2d97261528087331b0bee6ce2cf27c5root#include <unistd.h> 4083827f40a2d97261528087331b0bee6ce2cf27c5root#include <assert.h> 4183827f40a2d97261528087331b0bee6ce2cf27c5root#include <stdio.h> 4283827f40a2d97261528087331b0bee6ce2cf27c5root#include <string.h> 4383827f40a2d97261528087331b0bee6ce2cf27c5root#include "wsbm_pool.h" 4483827f40a2d97261528087331b0bee6ce2cf27c5root#include "wsbm_fencemgr.h" 4583827f40a2d97261528087331b0bee6ce2cf27c5root#include "wsbm_manager.h" 4683827f40a2d97261528087331b0bee6ce2cf27c5root#include "wsbm_mm.h" 4783827f40a2d97261528087331b0bee6ce2cf27c5root#include "wsbm_priv.h" 4883827f40a2d97261528087331b0bee6ce2cf27c5root 4983827f40a2d97261528087331b0bee6ce2cf27c5root/* 5083827f40a2d97261528087331b0bee6ce2cf27c5root * Malloced memory must be aligned to 16 bytes, since that's what 5183827f40a2d97261528087331b0bee6ce2cf27c5root * the DMA bitblt requires. 5283827f40a2d97261528087331b0bee6ce2cf27c5root */ 5383827f40a2d97261528087331b0bee6ce2cf27c5root 5483827f40a2d97261528087331b0bee6ce2cf27c5root#define WSBM_USER_ALIGN_ADD 16 5583827f40a2d97261528087331b0bee6ce2cf27c5root#define WSBM_USER_ALIGN_SYSMEM(_val) \ 5683827f40a2d97261528087331b0bee6ce2cf27c5root ((void *)(((unsigned long) (_val) + 15) & ~15)) 5783827f40a2d97261528087331b0bee6ce2cf27c5root 5883827f40a2d97261528087331b0bee6ce2cf27c5rootstruct _WsbmUserBuffer 5983827f40a2d97261528087331b0bee6ce2cf27c5root{ 6083827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmBufStorage buf; 6183827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmKernelBuf kBuf; 6283827f40a2d97261528087331b0bee6ce2cf27c5root 6383827f40a2d97261528087331b0bee6ce2cf27c5root /* Protected by the pool mutex */ 6483827f40a2d97261528087331b0bee6ce2cf27c5root 6583827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead lru; 6683827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead delayed; 6783827f40a2d97261528087331b0bee6ce2cf27c5root 6883827f40a2d97261528087331b0bee6ce2cf27c5root /* Protected by the buffer mutex */ 6983827f40a2d97261528087331b0bee6ce2cf27c5root 7083827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long size; 7183827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long alignment; 7283827f40a2d97261528087331b0bee6ce2cf27c5root 7383827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmCond event; 7483827f40a2d97261528087331b0bee6ce2cf27c5root uint32_t proposedPlacement; 7583827f40a2d97261528087331b0bee6ce2cf27c5root uint32_t newFenceType; 7683827f40a2d97261528087331b0bee6ce2cf27c5root 7783827f40a2d97261528087331b0bee6ce2cf27c5root void *map; 7883827f40a2d97261528087331b0bee6ce2cf27c5root void *sysmem; 7983827f40a2d97261528087331b0bee6ce2cf27c5root int unFenced; 8083827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmFenceObject *fence; 8183827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmMMNode *node; 8283827f40a2d97261528087331b0bee6ce2cf27c5root 8383827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmAtomic writers; 8483827f40a2d97261528087331b0bee6ce2cf27c5root}; 8583827f40a2d97261528087331b0bee6ce2cf27c5root 8683827f40a2d97261528087331b0bee6ce2cf27c5rootstruct _WsbmUserPool 8783827f40a2d97261528087331b0bee6ce2cf27c5root{ 8883827f40a2d97261528087331b0bee6ce2cf27c5root /* 8983827f40a2d97261528087331b0bee6ce2cf27c5root * Constant after initialization. 9083827f40a2d97261528087331b0bee6ce2cf27c5root */ 9183827f40a2d97261528087331b0bee6ce2cf27c5root 9283827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmBufferPool pool; 9383827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long agpOffset; 9483827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long agpMap; 9583827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long agpSize; 9683827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long vramOffset; 9783827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long vramMap; 9883827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long vramSize; 9983827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmMutex mutex; 10083827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead delayed; 10183827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead vramLRU; 10283827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead agpLRU; 10383827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmMM vramMM; 10483827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmMM agpMM; 10583827f40a2d97261528087331b0bee6ce2cf27c5root uint32_t(*fenceTypes) (uint64_t); 10683827f40a2d97261528087331b0bee6ce2cf27c5root}; 10783827f40a2d97261528087331b0bee6ce2cf27c5root 10883827f40a2d97261528087331b0bee6ce2cf27c5rootstatic inline struct _WsbmUserPool * 10983827f40a2d97261528087331b0bee6ce2cf27c5rootuserPool(struct _WsbmUserBuffer *buf) 11083827f40a2d97261528087331b0bee6ce2cf27c5root{ 11183827f40a2d97261528087331b0bee6ce2cf27c5root return containerOf(buf->buf.pool, struct _WsbmUserPool, pool); 11283827f40a2d97261528087331b0bee6ce2cf27c5root} 11383827f40a2d97261528087331b0bee6ce2cf27c5root 11483827f40a2d97261528087331b0bee6ce2cf27c5rootstatic inline struct _WsbmUserBuffer * 11583827f40a2d97261528087331b0bee6ce2cf27c5rootuserBuf(struct _WsbmBufStorage *buf) 11683827f40a2d97261528087331b0bee6ce2cf27c5root{ 11783827f40a2d97261528087331b0bee6ce2cf27c5root return containerOf(buf, struct _WsbmUserBuffer, buf); 11883827f40a2d97261528087331b0bee6ce2cf27c5root} 11983827f40a2d97261528087331b0bee6ce2cf27c5root 12083827f40a2d97261528087331b0bee6ce2cf27c5rootstatic void 12183827f40a2d97261528087331b0bee6ce2cf27c5rootwaitIdleLocked(struct _WsbmBufStorage *buf, int lazy) 12283827f40a2d97261528087331b0bee6ce2cf27c5root{ 12383827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 12483827f40a2d97261528087331b0bee6ce2cf27c5root 12583827f40a2d97261528087331b0bee6ce2cf27c5root while (vBuf->unFenced || vBuf->fence != NULL) { 12683827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->unFenced) 12783827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_COND_WAIT(&vBuf->event, &buf->mutex); 12883827f40a2d97261528087331b0bee6ce2cf27c5root 12983827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->fence != NULL) { 13083827f40a2d97261528087331b0bee6ce2cf27c5root if (!wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { 13183827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmFenceObject *fence = 13283827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceReference(vBuf->fence); 13383827f40a2d97261528087331b0bee6ce2cf27c5root 13483827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&buf->mutex); 13583827f40a2d97261528087331b0bee6ce2cf27c5root (void)wsbmFenceFinish(fence, vBuf->kBuf.fence_type_mask, 13683827f40a2d97261528087331b0bee6ce2cf27c5root lazy); 13783827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&buf->mutex); 13883827f40a2d97261528087331b0bee6ce2cf27c5root 13983827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->fence == fence) 14083827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&vBuf->fence); 14183827f40a2d97261528087331b0bee6ce2cf27c5root 14283827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&fence); 14383827f40a2d97261528087331b0bee6ce2cf27c5root } else { 14483827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&vBuf->fence); 14583827f40a2d97261528087331b0bee6ce2cf27c5root } 14683827f40a2d97261528087331b0bee6ce2cf27c5root } 14783827f40a2d97261528087331b0bee6ce2cf27c5root } 14883827f40a2d97261528087331b0bee6ce2cf27c5root} 14983827f40a2d97261528087331b0bee6ce2cf27c5root 15083827f40a2d97261528087331b0bee6ce2cf27c5rootstatic int 15183827f40a2d97261528087331b0bee6ce2cf27c5rootpool_waitIdle(struct _WsbmBufStorage *buf, int lazy) 15283827f40a2d97261528087331b0bee6ce2cf27c5root{ 15383827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&buf->mutex); 15483827f40a2d97261528087331b0bee6ce2cf27c5root waitIdleLocked(buf, lazy); 15583827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&buf->mutex); 15683827f40a2d97261528087331b0bee6ce2cf27c5root 15783827f40a2d97261528087331b0bee6ce2cf27c5root return 0; 15883827f40a2d97261528087331b0bee6ce2cf27c5root} 15983827f40a2d97261528087331b0bee6ce2cf27c5root 16083827f40a2d97261528087331b0bee6ce2cf27c5rootstatic int 16183827f40a2d97261528087331b0bee6ce2cf27c5rootevict_lru(struct _WsbmListHead *lru) 16283827f40a2d97261528087331b0bee6ce2cf27c5root{ 16383827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf; 16483827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p; 16583827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead *list = lru->next; 16683827f40a2d97261528087331b0bee6ce2cf27c5root int err; 16783827f40a2d97261528087331b0bee6ce2cf27c5root 16883827f40a2d97261528087331b0bee6ce2cf27c5root if (list == lru) { 16983827f40a2d97261528087331b0bee6ce2cf27c5root return -ENOMEM; 17083827f40a2d97261528087331b0bee6ce2cf27c5root } 17183827f40a2d97261528087331b0bee6ce2cf27c5root 17283827f40a2d97261528087331b0bee6ce2cf27c5root vBuf = WSBMLISTENTRY(list, struct _WsbmUserBuffer, lru); 17383827f40a2d97261528087331b0bee6ce2cf27c5root p = userPool(vBuf); 17483827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 17583827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&vBuf->buf.mutex); 17683827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 17783827f40a2d97261528087331b0bee6ce2cf27c5root 17883827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->sysmem = malloc(vBuf->size + WSBM_USER_ALIGN_ADD); 17983827f40a2d97261528087331b0bee6ce2cf27c5root 18083827f40a2d97261528087331b0bee6ce2cf27c5root if (!vBuf->sysmem) { 18183827f40a2d97261528087331b0bee6ce2cf27c5root err = -ENOMEM; 18283827f40a2d97261528087331b0bee6ce2cf27c5root goto out_unlock; 18383827f40a2d97261528087331b0bee6ce2cf27c5root } 18483827f40a2d97261528087331b0bee6ce2cf27c5root 18583827f40a2d97261528087331b0bee6ce2cf27c5root (void)wsbmFenceFinish(vBuf->fence, vBuf->kBuf.fence_type_mask, 0); 18683827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&vBuf->fence); 18783827f40a2d97261528087331b0bee6ce2cf27c5root 18883827f40a2d97261528087331b0bee6ce2cf27c5root memcpy(WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), vBuf->map, vBuf->size); 18983827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTDELINIT(&vBuf->lru); 19083827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; 19183827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); 19283827f40a2d97261528087331b0bee6ce2cf27c5root 19383827f40a2d97261528087331b0bee6ce2cf27c5root /* 19483827f40a2d97261528087331b0bee6ce2cf27c5root * FIXME: Free memory. 19583827f40a2d97261528087331b0bee6ce2cf27c5root */ 19683827f40a2d97261528087331b0bee6ce2cf27c5root 19783827f40a2d97261528087331b0bee6ce2cf27c5root err = 0; 19883827f40a2d97261528087331b0bee6ce2cf27c5root out_unlock: 19983827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&vBuf->buf.mutex); 20083827f40a2d97261528087331b0bee6ce2cf27c5root return err; 20183827f40a2d97261528087331b0bee6ce2cf27c5root} 20283827f40a2d97261528087331b0bee6ce2cf27c5root 20383827f40a2d97261528087331b0bee6ce2cf27c5rootstatic struct _WsbmBufStorage * 20483827f40a2d97261528087331b0bee6ce2cf27c5rootpool_create(struct _WsbmBufferPool *pool, 20583827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long size, uint32_t placement, unsigned alignment) 20683827f40a2d97261528087331b0bee6ce2cf27c5root{ 20783827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); 20883827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = calloc(1, sizeof(*vBuf)); 20983827f40a2d97261528087331b0bee6ce2cf27c5root 21083827f40a2d97261528087331b0bee6ce2cf27c5root if (!vBuf) 21183827f40a2d97261528087331b0bee6ce2cf27c5root return NULL; 21283827f40a2d97261528087331b0bee6ce2cf27c5root 21383827f40a2d97261528087331b0bee6ce2cf27c5root wsbmBufStorageInit(&vBuf->buf, pool); 21483827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->sysmem = NULL; 21583827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->proposedPlacement = placement; 21683827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->size = size; 21783827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->alignment = alignment; 21883827f40a2d97261528087331b0bee6ce2cf27c5root 21983827f40a2d97261528087331b0bee6ce2cf27c5root WSBMINITLISTHEAD(&vBuf->lru); 22083827f40a2d97261528087331b0bee6ce2cf27c5root WSBMINITLISTHEAD(&vBuf->delayed); 22183827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 22283827f40a2d97261528087331b0bee6ce2cf27c5root 22383827f40a2d97261528087331b0bee6ce2cf27c5root if (placement & WSBM_PL_FLAG_TT) { 22483827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = wsbmMMSearchFree(&p->agpMM, size, alignment, 1); 22583827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->node) 22683827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); 22783827f40a2d97261528087331b0bee6ce2cf27c5root 22883827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->node) { 22983827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.placement = WSBM_PL_FLAG_TT; 23083827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; 23183827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->map = (void *)(p->agpMap + vBuf->node->start); 23283827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); 23383827f40a2d97261528087331b0bee6ce2cf27c5root goto have_mem; 23483827f40a2d97261528087331b0bee6ce2cf27c5root } 23583827f40a2d97261528087331b0bee6ce2cf27c5root } 23683827f40a2d97261528087331b0bee6ce2cf27c5root 23783827f40a2d97261528087331b0bee6ce2cf27c5root if (placement & WSBM_PL_FLAG_VRAM) { 23883827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = wsbmMMSearchFree(&p->vramMM, size, alignment, 1); 23983827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->node) 24083827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); 24183827f40a2d97261528087331b0bee6ce2cf27c5root 24283827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->node) { 24383827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; 24483827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; 24583827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->map = (void *)(p->vramMap + vBuf->node->start); 24683827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); 24783827f40a2d97261528087331b0bee6ce2cf27c5root goto have_mem; 24883827f40a2d97261528087331b0bee6ce2cf27c5root } 24983827f40a2d97261528087331b0bee6ce2cf27c5root } 25083827f40a2d97261528087331b0bee6ce2cf27c5root 25183827f40a2d97261528087331b0bee6ce2cf27c5root if ((placement & WSBM_PL_FLAG_NO_EVICT) 25283827f40a2d97261528087331b0bee6ce2cf27c5root && !(placement & WSBM_PL_FLAG_SYSTEM)) { 25383827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 25483827f40a2d97261528087331b0bee6ce2cf27c5root goto out_err; 25583827f40a2d97261528087331b0bee6ce2cf27c5root } 25683827f40a2d97261528087331b0bee6ce2cf27c5root 25783827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->sysmem = malloc(size + WSBM_USER_ALIGN_ADD); 25883827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; 25983827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); 26083827f40a2d97261528087331b0bee6ce2cf27c5root 26183827f40a2d97261528087331b0bee6ce2cf27c5root have_mem: 26283827f40a2d97261528087331b0bee6ce2cf27c5root 26383827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 26483827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->sysmem != NULL 26583827f40a2d97261528087331b0bee6ce2cf27c5root || (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM))) 26683827f40a2d97261528087331b0bee6ce2cf27c5root return &vBuf->buf; 26783827f40a2d97261528087331b0bee6ce2cf27c5root out_err: 26883827f40a2d97261528087331b0bee6ce2cf27c5root free(vBuf); 26983827f40a2d97261528087331b0bee6ce2cf27c5root return NULL; 27083827f40a2d97261528087331b0bee6ce2cf27c5root} 27183827f40a2d97261528087331b0bee6ce2cf27c5root 27283827f40a2d97261528087331b0bee6ce2cf27c5rootstatic int 27383827f40a2d97261528087331b0bee6ce2cf27c5rootpool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, 27483827f40a2d97261528087331b0bee6ce2cf27c5root uint64_t clr_flags) 27583827f40a2d97261528087331b0bee6ce2cf27c5root{ 27683827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 27783827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p = userPool(vBuf); 27883827f40a2d97261528087331b0bee6ce2cf27c5root int err = -ENOMEM; 27983827f40a2d97261528087331b0bee6ce2cf27c5root 28083827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&buf->mutex); 28183827f40a2d97261528087331b0bee6ce2cf27c5root 28283827f40a2d97261528087331b0bee6ce2cf27c5root while (wsbmAtomicRead(&vBuf->writers) != 0) 28383827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_COND_WAIT(&vBuf->event, &buf->mutex); 28483827f40a2d97261528087331b0bee6ce2cf27c5root 28583827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->unFenced = 1; 28683827f40a2d97261528087331b0bee6ce2cf27c5root 28783827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 28883827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTDELINIT(&vBuf->lru); 28983827f40a2d97261528087331b0bee6ce2cf27c5root 29083827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->proposedPlacement = 29183827f40a2d97261528087331b0bee6ce2cf27c5root (vBuf->proposedPlacement | set_flags) & ~clr_flags; 29283827f40a2d97261528087331b0bee6ce2cf27c5root 29383827f40a2d97261528087331b0bee6ce2cf27c5root if ((vBuf->proposedPlacement & vBuf->kBuf.placement & WSBM_PL_MASK_MEM) == 29483827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.placement) { 29583827f40a2d97261528087331b0bee6ce2cf27c5root err = 0; 29683827f40a2d97261528087331b0bee6ce2cf27c5root goto have_mem; 29783827f40a2d97261528087331b0bee6ce2cf27c5root } 29883827f40a2d97261528087331b0bee6ce2cf27c5root 29983827f40a2d97261528087331b0bee6ce2cf27c5root /* 30083827f40a2d97261528087331b0bee6ce2cf27c5root * We're moving to another memory region, so evict first and we'll 30183827f40a2d97261528087331b0bee6ce2cf27c5root * do a sw copy to the other region. 30283827f40a2d97261528087331b0bee6ce2cf27c5root */ 30383827f40a2d97261528087331b0bee6ce2cf27c5root 30483827f40a2d97261528087331b0bee6ce2cf27c5root if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) { 30583827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead tmpLRU; 30683827f40a2d97261528087331b0bee6ce2cf27c5root 30783827f40a2d97261528087331b0bee6ce2cf27c5root WSBMINITLISTHEAD(&tmpLRU); 30883827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&tmpLRU, &vBuf->lru); 30983827f40a2d97261528087331b0bee6ce2cf27c5root err = evict_lru(&tmpLRU); 31083827f40a2d97261528087331b0bee6ce2cf27c5root if (err) 31183827f40a2d97261528087331b0bee6ce2cf27c5root goto have_mem; 31283827f40a2d97261528087331b0bee6ce2cf27c5root } 31383827f40a2d97261528087331b0bee6ce2cf27c5root 31483827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->proposedPlacement & WSBM_PL_FLAG_TT) { 31583827f40a2d97261528087331b0bee6ce2cf27c5root do { 31683827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = 31783827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMSearchFree(&p->agpMM, vBuf->size, vBuf->alignment, 1); 31883827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->node) 31983827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = 32083827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); 32183827f40a2d97261528087331b0bee6ce2cf27c5root 32283827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->node) { 32383827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.placement = WSBM_PL_FLAG_TT; 32483827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; 32583827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->map = (void *)(p->agpMap + vBuf->node->start); 32683827f40a2d97261528087331b0bee6ce2cf27c5root memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), 32783827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->size); 32883827f40a2d97261528087331b0bee6ce2cf27c5root free(vBuf->sysmem); 32983827f40a2d97261528087331b0bee6ce2cf27c5root goto have_mem; 33083827f40a2d97261528087331b0bee6ce2cf27c5root } 33183827f40a2d97261528087331b0bee6ce2cf27c5root } while (evict_lru(&p->agpLRU) == 0); 33283827f40a2d97261528087331b0bee6ce2cf27c5root } 33383827f40a2d97261528087331b0bee6ce2cf27c5root 33483827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->proposedPlacement & WSBM_PL_FLAG_VRAM) { 33583827f40a2d97261528087331b0bee6ce2cf27c5root do { 33683827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = 33783827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMSearchFree(&p->vramMM, vBuf->size, vBuf->alignment, 1); 33883827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->node) 33983827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->node = 34083827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); 34183827f40a2d97261528087331b0bee6ce2cf27c5root 3426f2c71fc4f82da090495374fd3386f237192743cElaine Wang if (!err && vBuf->node) { 34383827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; 34483827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; 34583827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->map = (void *)(p->vramMap + vBuf->node->start); 34683827f40a2d97261528087331b0bee6ce2cf27c5root memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), 34783827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->size); 34883827f40a2d97261528087331b0bee6ce2cf27c5root free(vBuf->sysmem); 34983827f40a2d97261528087331b0bee6ce2cf27c5root goto have_mem; 35083827f40a2d97261528087331b0bee6ce2cf27c5root } 35183827f40a2d97261528087331b0bee6ce2cf27c5root } while (evict_lru(&p->vramLRU) == 0); 35283827f40a2d97261528087331b0bee6ce2cf27c5root } 35383827f40a2d97261528087331b0bee6ce2cf27c5root 35483827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->proposedPlacement & WSBM_PL_FLAG_SYSTEM) 35583827f40a2d97261528087331b0bee6ce2cf27c5root goto have_mem; 35683827f40a2d97261528087331b0bee6ce2cf27c5root 35783827f40a2d97261528087331b0bee6ce2cf27c5root err = -ENOMEM; 35883827f40a2d97261528087331b0bee6ce2cf27c5root 35983827f40a2d97261528087331b0bee6ce2cf27c5root have_mem: 36083827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->newFenceType = p->fenceTypes(set_flags); 36183827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 36283827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&buf->mutex); 36383827f40a2d97261528087331b0bee6ce2cf27c5root return err; 36483827f40a2d97261528087331b0bee6ce2cf27c5root} 36583827f40a2d97261528087331b0bee6ce2cf27c5root 36683827f40a2d97261528087331b0bee6ce2cf27c5rootstatic int 36783827f40a2d97261528087331b0bee6ce2cf27c5rootpool_setStatus(struct _WsbmBufStorage *buf, 36883827f40a2d97261528087331b0bee6ce2cf27c5root uint32_t set_placement, uint32_t clr_placement) 36983827f40a2d97261528087331b0bee6ce2cf27c5root{ 37083827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 37183827f40a2d97261528087331b0bee6ce2cf27c5root int ret; 37283827f40a2d97261528087331b0bee6ce2cf27c5root 37383827f40a2d97261528087331b0bee6ce2cf27c5root ret = pool_validate(buf, set_placement, clr_placement); 37483827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->unFenced = 0; 37583827f40a2d97261528087331b0bee6ce2cf27c5root return ret; 37683827f40a2d97261528087331b0bee6ce2cf27c5root} 37783827f40a2d97261528087331b0bee6ce2cf27c5root 37883827f40a2d97261528087331b0bee6ce2cf27c5rootvoid 37983827f40a2d97261528087331b0bee6ce2cf27c5rootrelease_delayed_buffers(struct _WsbmUserPool *p) 38083827f40a2d97261528087331b0bee6ce2cf27c5root{ 38183827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf; 38283827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmListHead *list, *next; 38383827f40a2d97261528087331b0bee6ce2cf27c5root 38483827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 38583827f40a2d97261528087331b0bee6ce2cf27c5root 38683827f40a2d97261528087331b0bee6ce2cf27c5root /* 38783827f40a2d97261528087331b0bee6ce2cf27c5root * We don't need to take the buffer mutexes in this loop, since 38883827f40a2d97261528087331b0bee6ce2cf27c5root * the only other user is the evict_lru function, which has the 38983827f40a2d97261528087331b0bee6ce2cf27c5root * pool mutex held when accessing the buffer fence member. 39083827f40a2d97261528087331b0bee6ce2cf27c5root */ 39183827f40a2d97261528087331b0bee6ce2cf27c5root 39283827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTFOREACHSAFE(list, next, &p->delayed) { 39383827f40a2d97261528087331b0bee6ce2cf27c5root vBuf = WSBMLISTENTRY(list, struct _WsbmUserBuffer, delayed); 39483827f40a2d97261528087331b0bee6ce2cf27c5root 39583827f40a2d97261528087331b0bee6ce2cf27c5root if (!vBuf->fence 39683827f40a2d97261528087331b0bee6ce2cf27c5root || wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { 39783827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->fence) 39883827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&vBuf->fence); 39983827f40a2d97261528087331b0bee6ce2cf27c5root 40083827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTDEL(&vBuf->delayed); 40183827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTDEL(&vBuf->lru); 40283827f40a2d97261528087331b0bee6ce2cf27c5root 40383827f40a2d97261528087331b0bee6ce2cf27c5root if ((vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM) == 0) 40483827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMPutBlock(vBuf->node); 40583827f40a2d97261528087331b0bee6ce2cf27c5root else 40683827f40a2d97261528087331b0bee6ce2cf27c5root free(vBuf->sysmem); 40783827f40a2d97261528087331b0bee6ce2cf27c5root 40883827f40a2d97261528087331b0bee6ce2cf27c5root free(vBuf); 40983827f40a2d97261528087331b0bee6ce2cf27c5root } else 41083827f40a2d97261528087331b0bee6ce2cf27c5root break; 41183827f40a2d97261528087331b0bee6ce2cf27c5root 41283827f40a2d97261528087331b0bee6ce2cf27c5root } 41383827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 41483827f40a2d97261528087331b0bee6ce2cf27c5root} 41583827f40a2d97261528087331b0bee6ce2cf27c5root 41683827f40a2d97261528087331b0bee6ce2cf27c5rootstatic void 41783827f40a2d97261528087331b0bee6ce2cf27c5rootpool_destroy(struct _WsbmBufStorage **buf) 41883827f40a2d97261528087331b0bee6ce2cf27c5root{ 41983827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(*buf); 42083827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p = userPool(vBuf); 42183827f40a2d97261528087331b0bee6ce2cf27c5root 42283827f40a2d97261528087331b0bee6ce2cf27c5root *buf = NULL; 42383827f40a2d97261528087331b0bee6ce2cf27c5root 42483827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&vBuf->buf.mutex); 42583827f40a2d97261528087331b0bee6ce2cf27c5root if ((vBuf->fence 42683827f40a2d97261528087331b0bee6ce2cf27c5root && !wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask))) { 42783827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 42883827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&vBuf->delayed, &p->delayed); 42983827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 43083827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&vBuf->buf.mutex); 43183827f40a2d97261528087331b0bee6ce2cf27c5root return; 43283827f40a2d97261528087331b0bee6ce2cf27c5root } 43383827f40a2d97261528087331b0bee6ce2cf27c5root 43483827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->fence) 43583827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&vBuf->fence); 43683827f40a2d97261528087331b0bee6ce2cf27c5root 43783827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 43883827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTDEL(&vBuf->lru); 43983827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 44083827f40a2d97261528087331b0bee6ce2cf27c5root 44183827f40a2d97261528087331b0bee6ce2cf27c5root if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) 44283827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMPutBlock(vBuf->node); 44383827f40a2d97261528087331b0bee6ce2cf27c5root else 44483827f40a2d97261528087331b0bee6ce2cf27c5root free(vBuf->sysmem); 44583827f40a2d97261528087331b0bee6ce2cf27c5root 44683827f40a2d97261528087331b0bee6ce2cf27c5root free(vBuf); 44783827f40a2d97261528087331b0bee6ce2cf27c5root return; 44883827f40a2d97261528087331b0bee6ce2cf27c5root} 44983827f40a2d97261528087331b0bee6ce2cf27c5root 45083827f40a2d97261528087331b0bee6ce2cf27c5rootstatic int 4513795b36d0c937666a5bddc352c781d5e0beec26bThierry Strudelpool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual) 45283827f40a2d97261528087331b0bee6ce2cf27c5root{ 45383827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 45483827f40a2d97261528087331b0bee6ce2cf27c5root 45583827f40a2d97261528087331b0bee6ce2cf27c5root *virtual = vBuf->map; 45683827f40a2d97261528087331b0bee6ce2cf27c5root return 0; 45783827f40a2d97261528087331b0bee6ce2cf27c5root} 45883827f40a2d97261528087331b0bee6ce2cf27c5root 45983827f40a2d97261528087331b0bee6ce2cf27c5rootstatic void 4603795b36d0c937666a5bddc352c781d5e0beec26bThierry Strudelpool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused))) 46183827f40a2d97261528087331b0bee6ce2cf27c5root{ 46283827f40a2d97261528087331b0bee6ce2cf27c5root ; 46383827f40a2d97261528087331b0bee6ce2cf27c5root} 46483827f40a2d97261528087331b0bee6ce2cf27c5root 46583827f40a2d97261528087331b0bee6ce2cf27c5rootstatic void 4663795b36d0c937666a5bddc352c781d5e0beec26bThierry Strudelpool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused))) 46783827f40a2d97261528087331b0bee6ce2cf27c5root{ 46883827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 46983827f40a2d97261528087331b0bee6ce2cf27c5root 47083827f40a2d97261528087331b0bee6ce2cf27c5root if (wsbmAtomicDecZero(&vBuf->writers)) 47183827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_COND_BROADCAST(&vBuf->event); 47283827f40a2d97261528087331b0bee6ce2cf27c5root 47383827f40a2d97261528087331b0bee6ce2cf27c5root} 47483827f40a2d97261528087331b0bee6ce2cf27c5root 47583827f40a2d97261528087331b0bee6ce2cf27c5rootstatic int 47683827f40a2d97261528087331b0bee6ce2cf27c5rootpool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) 47783827f40a2d97261528087331b0bee6ce2cf27c5root{ 47883827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 47983827f40a2d97261528087331b0bee6ce2cf27c5root int ret = 0; 48083827f40a2d97261528087331b0bee6ce2cf27c5root 48183827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&buf->mutex); 48283827f40a2d97261528087331b0bee6ce2cf27c5root if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { 48383827f40a2d97261528087331b0bee6ce2cf27c5root 48483827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->unFenced) { 48583827f40a2d97261528087331b0bee6ce2cf27c5root ret = -EBUSY; 48683827f40a2d97261528087331b0bee6ce2cf27c5root goto out_unlock; 48783827f40a2d97261528087331b0bee6ce2cf27c5root } 48883827f40a2d97261528087331b0bee6ce2cf27c5root 48983827f40a2d97261528087331b0bee6ce2cf27c5root ret = 0; 49083827f40a2d97261528087331b0bee6ce2cf27c5root if ((vBuf->fence == NULL) || 49183827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { 49283827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&vBuf->fence); 49383827f40a2d97261528087331b0bee6ce2cf27c5root wsbmAtomicInc(&vBuf->writers); 49483827f40a2d97261528087331b0bee6ce2cf27c5root } else 49583827f40a2d97261528087331b0bee6ce2cf27c5root ret = -EBUSY; 49683827f40a2d97261528087331b0bee6ce2cf27c5root 49783827f40a2d97261528087331b0bee6ce2cf27c5root goto out_unlock; 49883827f40a2d97261528087331b0bee6ce2cf27c5root } 49983827f40a2d97261528087331b0bee6ce2cf27c5root waitIdleLocked(buf, 0); 50083827f40a2d97261528087331b0bee6ce2cf27c5root wsbmAtomicInc(&vBuf->writers); 50183827f40a2d97261528087331b0bee6ce2cf27c5root out_unlock: 50283827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&buf->mutex); 50383827f40a2d97261528087331b0bee6ce2cf27c5root return ret; 50483827f40a2d97261528087331b0bee6ce2cf27c5root} 50583827f40a2d97261528087331b0bee6ce2cf27c5root 50683827f40a2d97261528087331b0bee6ce2cf27c5rootstatic unsigned long 50783827f40a2d97261528087331b0bee6ce2cf27c5rootpool_offset(struct _WsbmBufStorage *buf) 50883827f40a2d97261528087331b0bee6ce2cf27c5root{ 50983827f40a2d97261528087331b0bee6ce2cf27c5root return userBuf(buf)->kBuf.gpuOffset; 51083827f40a2d97261528087331b0bee6ce2cf27c5root} 51183827f40a2d97261528087331b0bee6ce2cf27c5root 51283827f40a2d97261528087331b0bee6ce2cf27c5rootstatic unsigned long 5133795b36d0c937666a5bddc352c781d5e0beec26bThierry Strudelpool_poolOffset(struct _WsbmBufStorage *buf __attribute__ ((unused))) 51483827f40a2d97261528087331b0bee6ce2cf27c5root{ 51583827f40a2d97261528087331b0bee6ce2cf27c5root return 0UL; 51683827f40a2d97261528087331b0bee6ce2cf27c5root} 51783827f40a2d97261528087331b0bee6ce2cf27c5root 51883827f40a2d97261528087331b0bee6ce2cf27c5rootstatic unsigned long 51983827f40a2d97261528087331b0bee6ce2cf27c5rootpool_size(struct _WsbmBufStorage *buf) 52083827f40a2d97261528087331b0bee6ce2cf27c5root{ 52183827f40a2d97261528087331b0bee6ce2cf27c5root return userBuf(buf)->size; 52283827f40a2d97261528087331b0bee6ce2cf27c5root} 52383827f40a2d97261528087331b0bee6ce2cf27c5root 52483827f40a2d97261528087331b0bee6ce2cf27c5rootstatic void 52583827f40a2d97261528087331b0bee6ce2cf27c5rootpool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) 52683827f40a2d97261528087331b0bee6ce2cf27c5root{ 52783827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 52883827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p = userPool(vBuf); 52983827f40a2d97261528087331b0bee6ce2cf27c5root 53083827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&buf->mutex); 53183827f40a2d97261528087331b0bee6ce2cf27c5root 53283827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->fence) 53383827f40a2d97261528087331b0bee6ce2cf27c5root wsbmFenceUnreference(&vBuf->fence); 53483827f40a2d97261528087331b0bee6ce2cf27c5root 53583827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->fence = wsbmFenceReference(fence); 53683827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->unFenced = 0; 53783827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->kBuf.fence_type_mask = vBuf->newFenceType; 53883827f40a2d97261528087331b0bee6ce2cf27c5root 53983827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_COND_BROADCAST(&vBuf->event); 54083827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 54183827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) 54283827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); 54383827f40a2d97261528087331b0bee6ce2cf27c5root else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) 54483827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); 54583827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 54683827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&buf->mutex); 54783827f40a2d97261528087331b0bee6ce2cf27c5root} 54883827f40a2d97261528087331b0bee6ce2cf27c5root 54983827f40a2d97261528087331b0bee6ce2cf27c5rootstatic void 55083827f40a2d97261528087331b0bee6ce2cf27c5rootpool_unvalidate(struct _WsbmBufStorage *buf) 55183827f40a2d97261528087331b0bee6ce2cf27c5root{ 55283827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 55383827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p = userPool(vBuf); 55483827f40a2d97261528087331b0bee6ce2cf27c5root 55583827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&buf->mutex); 55683827f40a2d97261528087331b0bee6ce2cf27c5root 55783827f40a2d97261528087331b0bee6ce2cf27c5root if (!vBuf->unFenced) 55883827f40a2d97261528087331b0bee6ce2cf27c5root goto out_unlock; 55983827f40a2d97261528087331b0bee6ce2cf27c5root 56083827f40a2d97261528087331b0bee6ce2cf27c5root vBuf->unFenced = 0; 56183827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_COND_BROADCAST(&vBuf->event); 56283827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 56383827f40a2d97261528087331b0bee6ce2cf27c5root if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) 56483827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); 56583827f40a2d97261528087331b0bee6ce2cf27c5root else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) 56683827f40a2d97261528087331b0bee6ce2cf27c5root WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); 56783827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 56883827f40a2d97261528087331b0bee6ce2cf27c5root 56983827f40a2d97261528087331b0bee6ce2cf27c5root out_unlock: 57083827f40a2d97261528087331b0bee6ce2cf27c5root 57183827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&buf->mutex); 57283827f40a2d97261528087331b0bee6ce2cf27c5root} 57383827f40a2d97261528087331b0bee6ce2cf27c5root 57483827f40a2d97261528087331b0bee6ce2cf27c5rootstatic struct _WsbmKernelBuf * 57583827f40a2d97261528087331b0bee6ce2cf27c5rootpool_kernel(struct _WsbmBufStorage *buf) 57683827f40a2d97261528087331b0bee6ce2cf27c5root{ 57783827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserBuffer *vBuf = userBuf(buf); 57883827f40a2d97261528087331b0bee6ce2cf27c5root 57983827f40a2d97261528087331b0bee6ce2cf27c5root return &vBuf->kBuf; 58083827f40a2d97261528087331b0bee6ce2cf27c5root} 58183827f40a2d97261528087331b0bee6ce2cf27c5root 58283827f40a2d97261528087331b0bee6ce2cf27c5rootstatic void 58383827f40a2d97261528087331b0bee6ce2cf27c5rootpool_takedown(struct _WsbmBufferPool *pool) 58483827f40a2d97261528087331b0bee6ce2cf27c5root{ 58583827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); 58683827f40a2d97261528087331b0bee6ce2cf27c5root int empty; 58783827f40a2d97261528087331b0bee6ce2cf27c5root 58883827f40a2d97261528087331b0bee6ce2cf27c5root do { 58983827f40a2d97261528087331b0bee6ce2cf27c5root release_delayed_buffers(p); 59083827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 59183827f40a2d97261528087331b0bee6ce2cf27c5root empty = (p->delayed.next == &p->delayed); 59283827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 59383827f40a2d97261528087331b0bee6ce2cf27c5root 59483827f40a2d97261528087331b0bee6ce2cf27c5root if (!empty) 59583827f40a2d97261528087331b0bee6ce2cf27c5root usleep(1000); 59683827f40a2d97261528087331b0bee6ce2cf27c5root 59783827f40a2d97261528087331b0bee6ce2cf27c5root } while (!empty); 59883827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 59983827f40a2d97261528087331b0bee6ce2cf27c5root 60083827f40a2d97261528087331b0bee6ce2cf27c5root while (evict_lru(&p->vramLRU) == 0) ; 60183827f40a2d97261528087331b0bee6ce2cf27c5root while (evict_lru(&p->agpLRU) == 0) ; 60283827f40a2d97261528087331b0bee6ce2cf27c5root 60383827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 60483827f40a2d97261528087331b0bee6ce2cf27c5root 60583827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMtakedown(&p->agpMM); 60683827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMtakedown(&p->vramMM); 60783827f40a2d97261528087331b0bee6ce2cf27c5root 60883827f40a2d97261528087331b0bee6ce2cf27c5root free(p); 60983827f40a2d97261528087331b0bee6ce2cf27c5root} 61083827f40a2d97261528087331b0bee6ce2cf27c5root 61183827f40a2d97261528087331b0bee6ce2cf27c5rootvoid 61283827f40a2d97261528087331b0bee6ce2cf27c5rootwsbmUserPoolClean(struct _WsbmBufferPool *pool, int cleanVram, int cleanAgp) 61383827f40a2d97261528087331b0bee6ce2cf27c5root{ 61483827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); 61583827f40a2d97261528087331b0bee6ce2cf27c5root 61683827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_LOCK(&p->mutex); 61783827f40a2d97261528087331b0bee6ce2cf27c5root if (cleanVram) 61883827f40a2d97261528087331b0bee6ce2cf27c5root while (evict_lru(&p->vramLRU) == 0) ; 61983827f40a2d97261528087331b0bee6ce2cf27c5root if (cleanAgp) 62083827f40a2d97261528087331b0bee6ce2cf27c5root while (evict_lru(&p->agpLRU) == 0) ; 62183827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_UNLOCK(&p->mutex); 62283827f40a2d97261528087331b0bee6ce2cf27c5root} 62383827f40a2d97261528087331b0bee6ce2cf27c5root 62483827f40a2d97261528087331b0bee6ce2cf27c5rootstruct _WsbmBufferPool * 62583827f40a2d97261528087331b0bee6ce2cf27c5rootwsbmUserPoolInit(void *vramAddr, 62683827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long vramStart, unsigned long vramSize, 62783827f40a2d97261528087331b0bee6ce2cf27c5root void *agpAddr, unsigned long agpStart, 62883827f40a2d97261528087331b0bee6ce2cf27c5root unsigned long agpSize, 62983827f40a2d97261528087331b0bee6ce2cf27c5root uint32_t(*fenceTypes) (uint64_t set_flags)) 63083827f40a2d97261528087331b0bee6ce2cf27c5root{ 63183827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmBufferPool *pool; 63283827f40a2d97261528087331b0bee6ce2cf27c5root struct _WsbmUserPool *uPool; 63383827f40a2d97261528087331b0bee6ce2cf27c5root int ret; 63483827f40a2d97261528087331b0bee6ce2cf27c5root 63583827f40a2d97261528087331b0bee6ce2cf27c5root uPool = calloc(1, sizeof(*uPool)); 63683827f40a2d97261528087331b0bee6ce2cf27c5root if (!uPool) 63783827f40a2d97261528087331b0bee6ce2cf27c5root goto out_err0; 63883827f40a2d97261528087331b0bee6ce2cf27c5root 63983827f40a2d97261528087331b0bee6ce2cf27c5root ret = WSBM_MUTEX_INIT(&uPool->mutex); 64083827f40a2d97261528087331b0bee6ce2cf27c5root if (ret) 64183827f40a2d97261528087331b0bee6ce2cf27c5root goto out_err0; 64283827f40a2d97261528087331b0bee6ce2cf27c5root 64383827f40a2d97261528087331b0bee6ce2cf27c5root ret = wsbmMMinit(&uPool->vramMM, 0, vramSize); 64483827f40a2d97261528087331b0bee6ce2cf27c5root if (ret) 64583827f40a2d97261528087331b0bee6ce2cf27c5root goto out_err1; 64683827f40a2d97261528087331b0bee6ce2cf27c5root 64783827f40a2d97261528087331b0bee6ce2cf27c5root ret = wsbmMMinit(&uPool->agpMM, 0, agpSize); 64883827f40a2d97261528087331b0bee6ce2cf27c5root if (ret) 64983827f40a2d97261528087331b0bee6ce2cf27c5root goto out_err2; 65083827f40a2d97261528087331b0bee6ce2cf27c5root 65183827f40a2d97261528087331b0bee6ce2cf27c5root WSBMINITLISTHEAD(&uPool->delayed); 65283827f40a2d97261528087331b0bee6ce2cf27c5root WSBMINITLISTHEAD(&uPool->vramLRU); 65383827f40a2d97261528087331b0bee6ce2cf27c5root WSBMINITLISTHEAD(&uPool->agpLRU); 65483827f40a2d97261528087331b0bee6ce2cf27c5root 65583827f40a2d97261528087331b0bee6ce2cf27c5root uPool->agpOffset = agpStart; 65683827f40a2d97261528087331b0bee6ce2cf27c5root uPool->agpMap = (unsigned long)agpAddr; 65783827f40a2d97261528087331b0bee6ce2cf27c5root uPool->vramOffset = vramStart; 65883827f40a2d97261528087331b0bee6ce2cf27c5root uPool->vramMap = (unsigned long)vramAddr; 65983827f40a2d97261528087331b0bee6ce2cf27c5root uPool->fenceTypes = fenceTypes; 66083827f40a2d97261528087331b0bee6ce2cf27c5root 66183827f40a2d97261528087331b0bee6ce2cf27c5root pool = &uPool->pool; 66283827f40a2d97261528087331b0bee6ce2cf27c5root pool->map = &pool_map; 66383827f40a2d97261528087331b0bee6ce2cf27c5root pool->unmap = &pool_unmap; 66483827f40a2d97261528087331b0bee6ce2cf27c5root pool->destroy = &pool_destroy; 66583827f40a2d97261528087331b0bee6ce2cf27c5root pool->offset = &pool_offset; 66683827f40a2d97261528087331b0bee6ce2cf27c5root pool->poolOffset = &pool_poolOffset; 66783827f40a2d97261528087331b0bee6ce2cf27c5root pool->size = &pool_size; 66883827f40a2d97261528087331b0bee6ce2cf27c5root pool->create = &pool_create; 66983827f40a2d97261528087331b0bee6ce2cf27c5root pool->fence = &pool_fence; 67083827f40a2d97261528087331b0bee6ce2cf27c5root pool->unvalidate = &pool_unvalidate; 67183827f40a2d97261528087331b0bee6ce2cf27c5root pool->kernel = &pool_kernel; 67283827f40a2d97261528087331b0bee6ce2cf27c5root pool->validate = &pool_validate; 67383827f40a2d97261528087331b0bee6ce2cf27c5root pool->waitIdle = &pool_waitIdle; 67483827f40a2d97261528087331b0bee6ce2cf27c5root pool->takeDown = &pool_takedown; 67583827f40a2d97261528087331b0bee6ce2cf27c5root pool->setStatus = &pool_setStatus; 67683827f40a2d97261528087331b0bee6ce2cf27c5root pool->syncforcpu = &pool_syncForCpu; 67783827f40a2d97261528087331b0bee6ce2cf27c5root pool->releasefromcpu = &pool_releaseFromCpu; 67883827f40a2d97261528087331b0bee6ce2cf27c5root 67983827f40a2d97261528087331b0bee6ce2cf27c5root return pool; 68083827f40a2d97261528087331b0bee6ce2cf27c5root 68183827f40a2d97261528087331b0bee6ce2cf27c5root out_err2: 68283827f40a2d97261528087331b0bee6ce2cf27c5root wsbmMMtakedown(&uPool->vramMM); 68383827f40a2d97261528087331b0bee6ce2cf27c5root out_err1: 68483827f40a2d97261528087331b0bee6ce2cf27c5root WSBM_MUTEX_FREE(&uPool->mutex); 68583827f40a2d97261528087331b0bee6ce2cf27c5root out_err0: 68683827f40a2d97261528087331b0bee6ce2cf27c5root free(uPool); 68783827f40a2d97261528087331b0bee6ce2cf27c5root 68883827f40a2d97261528087331b0bee6ce2cf27c5root return NULL; 68983827f40a2d97261528087331b0bee6ce2cf27c5root} 690