1f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/************************************************************************** 2f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 3f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Copyright 2007-2010 VMware, Inc. 4f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * All Rights Reserved. 5f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 6f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Permission is hereby granted, free of charge, to any person obtaining a 7f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * copy of this software and associated documentation files (the 8f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * "Software"), to deal in the Software without restriction, including 9f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * without limitation the rights to use, copy, modify, merge, publish, 10f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * distribute, sub license, and/or sell copies of the Software, and to 11f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * permit persons to whom the Software is furnished to do so, subject to 12f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * the following conditions: 13f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 14f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * The above copyright notice and this permission notice (including the 15f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * next paragraph) shall be included in all copies or substantial portions 16f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * of the Software. 17f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 18f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 26f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org **************************************************************************/ 27f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 28f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 29f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * \file 30f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Implementation of fenced buffers. 31f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 32f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * \author Jose Fonseca <jfonseca-at-vmware-dot-com> 33f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * \author Thomas Hellström <thellstrom-at-vmware-dot-com> 34f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 35f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 36f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 37f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "pipe/p_config.h" 38f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 39f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS) 40f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include <unistd.h> 41f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include <sched.h> 42f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#endif 43f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 44f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "pipe/p_compiler.h" 45f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "pipe/p_defines.h" 46f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "util/u_debug.h" 47f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "os/os_thread.h" 48f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "util/u_memory.h" 49f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "util/u_double_list.h" 50f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 51f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "pb_buffer.h" 52f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "pb_buffer_fenced.h" 53f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#include "pb_bufmgr.h" 54f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 55f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 56f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 57f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 58f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Convenience macro (type safe). 59f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 60f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#define SUPER(__derived) (&(__derived)->base) 61f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 62f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 63f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstruct fenced_manager 64f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 65f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_manager base; 66f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_manager *provider; 67f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops; 68f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 69f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 70f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Maximum buffer size that can be safely allocated. 71f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 72f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size max_buffer_size; 73f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 74f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 75f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Maximum cpu memory we can allocate before we start waiting for the 76f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * GPU to idle. 77f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 78f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size max_cpu_total_size; 79f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 80f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 81f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Following members are mutable and protected by this mutex. 82f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 83f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex mutex; 84f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 85f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 86f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Fenced buffer list. 87f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 88f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * All fenced buffers are placed in this listed, ordered from the oldest 89f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * fence to the newest fence. 90f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 91f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct list_head fenced; 92f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size num_fenced; 93f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 94f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct list_head unfenced; 95f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size num_unfenced; 96f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 97f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 98f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * How much temporary CPU memory is being used to hold unvalidated buffers. 99f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 100f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size cpu_total_size; 101f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org}; 102f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 103f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 104f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 105f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Fenced buffer. 106f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 107f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Wrapper around a pipe buffer which adds fencing and reference counting. 108f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 109f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstruct fenced_buffer 110f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 111f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 112f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Immutable members. 113f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 114f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 115f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_buffer base; 116f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *mgr; 117f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 118f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 119f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Following members are mutable and protected by fenced_manager::mutex. 120f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 121f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 122f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct list_head head; 123f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 124f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 125f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Buffer with storage. 126f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 127f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_buffer *buffer; 128f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size size; 129f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_desc desc; 130f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 131f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 132f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Temporary CPU storage data. Used when there isn't enough GPU memory to 133f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * store the buffer. 134f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 135f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org void *data; 136f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 137f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /** 138f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * A bitmask of PB_USAGE_CPU/GPU_READ/WRITE describing the current 139f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * buffer usage. 140f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 141f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org unsigned flags; 142f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 143f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org unsigned mapcount; 144f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 145f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_validate *vl; 146f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org unsigned validation_flags; 147f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 148f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pipe_fence_handle *fence; 149f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org}; 150f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 151f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 152f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic INLINE struct fenced_manager * 153f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_manager(struct pb_manager *mgr) 154f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 155f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(mgr); 156f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return (struct fenced_manager *)mgr; 157f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 158f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 159f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 160f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic INLINE struct fenced_buffer * 161f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer(struct pb_buffer *buf) 162f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 163f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(buf); 164f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return (struct fenced_buffer *)buf; 165f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 166f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 167f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 168f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 169f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_destroy_cpu_storage_locked(struct fenced_buffer *fenced_buf); 170f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 171f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 172f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_create_cpu_storage_locked(struct fenced_manager *fenced_mgr, 173f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf); 174f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 175f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 176f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer *fenced_buf); 177f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 178f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 179f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_create_gpu_storage_locked(struct fenced_manager *fenced_mgr, 180f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf, 181f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org boolean wait); 182f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 183f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 184f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_copy_storage_to_gpu_locked(struct fenced_buffer *fenced_buf); 185f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 186f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 187f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_copy_storage_to_cpu_locked(struct fenced_buffer *fenced_buf); 188f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 189f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 190f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 191f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Dump the fenced buffer list. 192f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 193f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Useful to understand failures to allocate buffers. 194f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 195f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 196f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_manager_dump_locked(struct fenced_manager *fenced_mgr) 197f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 198f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#ifdef DEBUG 199f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops = fenced_mgr->ops; 200f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct list_head *curr, *next; 201f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf; 202f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 203f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org debug_printf("%10s %7s %8s %7s %10s %s\n", 204f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org "buffer", "size", "refcount", "storage", "fence", "signalled"); 205f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 206f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = fenced_mgr->unfenced.next; 207f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 208f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(curr != &fenced_mgr->unfenced) { 209f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 210f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!fenced_buf->fence); 211f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org debug_printf("%10p %7u %8u %7s\n", 212f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (void *) fenced_buf, 213f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->base.size, 214f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org p_atomic_read(&fenced_buf->base.reference.count), 215f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->buffer ? "gpu" : (fenced_buf->data ? "cpu" : "none")); 216f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = next; 217f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 218f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 219f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 220f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = fenced_mgr->fenced.next; 221f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 222f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(curr != &fenced_mgr->fenced) { 223f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org int signaled; 224f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 225f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->buffer); 226f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org signaled = ops->fence_signalled(ops, fenced_buf->fence, 0); 227f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org debug_printf("%10p %7u %8u %7s %10p %s\n", 228f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (void *) fenced_buf, 229f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->base.size, 230f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org p_atomic_read(&fenced_buf->base.reference.count), 231f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org "gpu", 232f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (void *) fenced_buf->fence, 233f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org signaled == 0 ? "y" : "n"); 234f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = next; 235f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 236f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 237f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#else 238f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (void)fenced_mgr; 239f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#endif 240f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 241f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 242f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 243f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic INLINE void 244f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_destroy_locked(struct fenced_manager *fenced_mgr, 245f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf) 246f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 247f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!pipe_is_referenced(&fenced_buf->base.reference)); 248f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 249f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!fenced_buf->fence); 250f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->head.prev); 251f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->head.next); 252f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_DEL(&fenced_buf->head); 253f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_mgr->num_unfenced); 254f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org --fenced_mgr->num_unfenced; 255f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 256f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_gpu_storage_locked(fenced_buf); 257f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_cpu_storage_locked(fenced_buf); 258f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 259f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org FREE(fenced_buf); 260f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 261f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 262f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 263f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 264f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Add the buffer to the fenced list. 265f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 266f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Reference count should be incremented before calling this function. 267f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 268f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic INLINE void 269f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_add_locked(struct fenced_manager *fenced_mgr, 270f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf) 271f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 272f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(pipe_is_referenced(&fenced_buf->base.reference)); 273f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->flags & PB_USAGE_GPU_READ_WRITE); 274f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->fence); 275f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 276f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org p_atomic_inc(&fenced_buf->base.reference.count); 277f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 278f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_DEL(&fenced_buf->head); 279f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_mgr->num_unfenced); 280f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org --fenced_mgr->num_unfenced; 281f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_ADDTAIL(&fenced_buf->head, &fenced_mgr->fenced); 282f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ++fenced_mgr->num_fenced; 283f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 284f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 285f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 286f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 287f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Remove the buffer from the fenced list, and potentially destroy the buffer 288f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * if the reference count reaches zero. 289f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 290f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Returns TRUE if the buffer was detroyed. 291f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 292f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic INLINE boolean 293f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_remove_locked(struct fenced_manager *fenced_mgr, 294f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf) 295f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 296f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops = fenced_mgr->ops; 297f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 298f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->fence); 299f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->mgr == fenced_mgr); 300f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 301f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ops->fence_reference(ops, &fenced_buf->fence, NULL); 302f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->flags &= ~PB_USAGE_GPU_READ_WRITE; 303f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 304f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->head.prev); 305f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->head.next); 306f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 307f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_DEL(&fenced_buf->head); 308f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_mgr->num_fenced); 309f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org --fenced_mgr->num_fenced; 310f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 311f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_ADDTAIL(&fenced_buf->head, &fenced_mgr->unfenced); 312f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ++fenced_mgr->num_unfenced; 313f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 314f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (p_atomic_dec_zero(&fenced_buf->base.reference.count)) { 315f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_locked(fenced_mgr, fenced_buf); 316f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return TRUE; 317f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 318f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 319f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return FALSE; 320f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 321f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 322f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 323f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 324f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Wait for the fence to expire, and remove it from the fenced list. 325f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 326f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * This function will release and re-aquire the mutex, so any copy of mutable 327f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * state must be discarded after calling it. 328f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 329f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic INLINE enum pipe_error 330f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_finish_locked(struct fenced_manager *fenced_mgr, 331f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf) 332f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 333f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops = fenced_mgr->ops; 334f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org enum pipe_error ret = PIPE_ERROR; 335f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 336f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#if 0 337f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org debug_warning("waiting for GPU"); 338f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#endif 339f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 340f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(pipe_is_referenced(&fenced_buf->base.reference)); 341f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->fence); 342f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 343f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->fence) { 344f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pipe_fence_handle *fence = NULL; 345f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org int finished; 346f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org boolean proceed; 347f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 348f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ops->fence_reference(ops, &fence, fenced_buf->fence); 349f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 350f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 351f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 352f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org finished = ops->fence_finish(ops, fenced_buf->fence, 0); 353f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 354f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 355f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 356f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(pipe_is_referenced(&fenced_buf->base.reference)); 357f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 358f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 359f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Only proceed if the fence object didn't change in the meanwhile. 360f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Otherwise assume the work has been already carried out by another 361f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * thread that re-aquired the lock before us. 362f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 363f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org proceed = fence == fenced_buf->fence ? TRUE : FALSE; 364f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 365f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ops->fence_reference(ops, &fence, NULL); 366f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 367f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(proceed && finished == 0) { 368f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 369f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Remove from the fenced list 370f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 371f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 372f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org boolean destroyed; 373f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 374f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org destroyed = fenced_buffer_remove_locked(fenced_mgr, fenced_buf); 375f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 376f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* TODO: remove consequents buffers with the same fence? */ 377f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 378f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!destroyed); 379f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 380f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->flags &= ~PB_USAGE_GPU_READ_WRITE; 381f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 382f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = PIPE_OK; 383f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 384f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 385f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 386f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return ret; 387f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 388f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 389f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 390f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 391f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Remove as many fenced buffers from the fenced list as possible. 392f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 393f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Returns TRUE if at least one buffer was removed. 394f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 395f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic boolean 396f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_manager_check_signalled_locked(struct fenced_manager *fenced_mgr, 397f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org boolean wait) 398f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 399f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops = fenced_mgr->ops; 400f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct list_head *curr, *next; 401f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf; 402f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pipe_fence_handle *prev_fence = NULL; 403f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org boolean ret = FALSE; 404f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 405f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = fenced_mgr->fenced.next; 406f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 407f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(curr != &fenced_mgr->fenced) { 408f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 409f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 410f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->fence != prev_fence) { 411f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org int signaled; 412f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 413f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (wait) { 414f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org signaled = ops->fence_finish(ops, fenced_buf->fence, 0); 415f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 416f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 417f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Don't return just now. Instead preemptively check if the 418f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * following buffers' fences already expired, without further waits. 419f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 420f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org wait = FALSE; 421f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 422f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org else { 423f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org signaled = ops->fence_signalled(ops, fenced_buf->fence, 0); 424f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 425f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 426f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (signaled != 0) { 427f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return ret; 428f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 429f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 430f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org prev_fence = fenced_buf->fence; 431f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 432f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org else { 433f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* This buffer's fence object is identical to the previous buffer's 434f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * fence object, so no need to check the fence again. 435f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 436f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0); 437f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 438f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 439f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_remove_locked(fenced_mgr, fenced_buf); 440f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 441f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = TRUE; 442f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 443f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = next; 444f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 445f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 446f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 447f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return ret; 448f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 449f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 450f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 451f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 452f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Try to free some GPU memory by backing it up into CPU memory. 453f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 454f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Returns TRUE if at least one buffer was freed. 455f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 456f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic boolean 457f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_manager_free_gpu_storage_locked(struct fenced_manager *fenced_mgr) 458f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 459f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct list_head *curr, *next; 460f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf; 461f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 462f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = fenced_mgr->unfenced.next; 463f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 464f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(curr != &fenced_mgr->unfenced) { 465f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head); 466f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 467f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 468f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * We can only move storage if the buffer is not mapped and not 469f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * validated. 470f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 471f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->buffer && 472f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org !fenced_buf->mapcount && 473f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org !fenced_buf->vl) { 474f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org enum pipe_error ret; 475f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 476f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = fenced_buffer_create_cpu_storage_locked(fenced_mgr, fenced_buf); 477f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(ret == PIPE_OK) { 478f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = fenced_buffer_copy_storage_to_cpu_locked(fenced_buf); 479f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(ret == PIPE_OK) { 480f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_gpu_storage_locked(fenced_buf); 481f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return TRUE; 482f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 483f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_cpu_storage_locked(fenced_buf); 484f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 485f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 486f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 487f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org curr = next; 488f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org next = curr->next; 489f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 490f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 491f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return FALSE; 492f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 493f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 494f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 495f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 496f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Destroy CPU storage for this buffer. 497f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 498f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 499f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_destroy_cpu_storage_locked(struct fenced_buffer *fenced_buf) 500f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 501f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->data) { 502f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org align_free(fenced_buf->data); 503f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->data = NULL; 504f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->mgr->cpu_total_size >= fenced_buf->size); 505f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->mgr->cpu_total_size -= fenced_buf->size; 506f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 507f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 508f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 509f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 510f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 511f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Create CPU storage for this buffer. 512f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 513f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 514f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_create_cpu_storage_locked(struct fenced_manager *fenced_mgr, 515f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf) 516f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 517f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!fenced_buf->data); 518f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->data) 519f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_OK; 520f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 521f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (fenced_mgr->cpu_total_size + fenced_buf->size > fenced_mgr->max_cpu_total_size) 522f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_ERROR_OUT_OF_MEMORY; 523f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 524f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->data = align_malloc(fenced_buf->size, fenced_buf->desc.alignment); 525f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!fenced_buf->data) 526f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_ERROR_OUT_OF_MEMORY; 527f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 528f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->cpu_total_size += fenced_buf->size; 529f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 530f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_OK; 531f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 532f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 533f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 534f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 535f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Destroy the GPU storage. 536f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 537f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 538f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer *fenced_buf) 539f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 540f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->buffer) { 541f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_reference(&fenced_buf->buffer, NULL); 542f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 543f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 544f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 545f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 546f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 547f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Try to create GPU storage for this buffer. 548f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * 549f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * This function is a shorthand around pb_manager::create_buffer for 550f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * fenced_buffer_create_gpu_storage_locked()'s benefit. 551f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 552f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic INLINE boolean 553f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_try_create_gpu_storage_locked(struct fenced_manager *fenced_mgr, 554f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf) 555f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 556f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_manager *provider = fenced_mgr->provider; 557f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 558f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!fenced_buf->buffer); 559f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 560f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->buffer = provider->create_buffer(fenced_mgr->provider, 561f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->size, 562f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org &fenced_buf->desc); 563f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return fenced_buf->buffer ? TRUE : FALSE; 564f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 565f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 566f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 567f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 568f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Create GPU storage for this buffer. 569f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 570f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 571f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_create_gpu_storage_locked(struct fenced_manager *fenced_mgr, 572f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf, 573f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org boolean wait) 574f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 575f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!fenced_buf->buffer); 576f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 577f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 578f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Check for signaled buffers before trying to allocate. 579f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 580f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_manager_check_signalled_locked(fenced_mgr, FALSE); 581f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 582f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_try_create_gpu_storage_locked(fenced_mgr, fenced_buf); 583f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 584f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 585f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Keep trying while there is some sort of progress: 586f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * - fences are expiring, 587f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * - or buffers are being being swapped out from GPU memory into CPU memory. 588f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 589f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(!fenced_buf->buffer && 590f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (fenced_manager_check_signalled_locked(fenced_mgr, FALSE) || 591f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_manager_free_gpu_storage_locked(fenced_mgr))) { 592f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_try_create_gpu_storage_locked(fenced_mgr, fenced_buf); 593f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 594f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 595f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!fenced_buf->buffer && wait) { 596f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 597f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Same as before, but this time around, wait to free buffers if 598f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * necessary. 599f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 600f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(!fenced_buf->buffer && 601f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (fenced_manager_check_signalled_locked(fenced_mgr, TRUE) || 602f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_manager_free_gpu_storage_locked(fenced_mgr))) { 603f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_try_create_gpu_storage_locked(fenced_mgr, fenced_buf); 604f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 605f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 606f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 607f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!fenced_buf->buffer) { 608f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(0) 609f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_manager_dump_locked(fenced_mgr); 610f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 611f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* give up */ 612f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_ERROR_OUT_OF_MEMORY; 613f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 614f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 615f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_OK; 616f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 617f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 618f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 619f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 620f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_copy_storage_to_gpu_locked(struct fenced_buffer *fenced_buf) 621f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 622f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org uint8_t *map; 623f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 624f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->data); 625f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->buffer); 626f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 627f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org map = pb_map(fenced_buf->buffer, PB_USAGE_CPU_WRITE, NULL); 628f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!map) 629f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_ERROR; 630f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 631f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org memcpy(map, fenced_buf->data, fenced_buf->size); 632f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 633f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_unmap(fenced_buf->buffer); 634f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 635f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_OK; 636f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 637f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 638f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 639f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 640f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_copy_storage_to_cpu_locked(struct fenced_buffer *fenced_buf) 641f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 642f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org const uint8_t *map; 643f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 644f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->data); 645f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->buffer); 646f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 647f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org map = pb_map(fenced_buf->buffer, PB_USAGE_CPU_READ, NULL); 648f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!map) 649f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_ERROR; 650f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 651f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org memcpy(fenced_buf->data, map, fenced_buf->size); 652f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 653f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_unmap(fenced_buf->buffer); 654f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 655f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return PIPE_OK; 656f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 657f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 658f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 659f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 660f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_destroy(struct pb_buffer *buf) 661f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 662f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf = fenced_buffer(buf); 663f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_buf->mgr; 664f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 665f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!pipe_is_referenced(&fenced_buf->base.reference)); 666f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 667f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 668f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 669f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_locked(fenced_mgr, fenced_buf); 670f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 671f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 672f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 673f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 674f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 675f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void * 676f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_map(struct pb_buffer *buf, 677f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org unsigned flags, void *flush_ctx) 678f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 679f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf = fenced_buffer(buf); 680f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_buf->mgr; 681f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops = fenced_mgr->ops; 682f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org void *map = NULL; 683f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 684f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 685f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 686f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!(flags & PB_USAGE_GPU_READ_WRITE)); 687f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 688f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 689f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Serialize writes. 690f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 691f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while((fenced_buf->flags & PB_USAGE_GPU_WRITE) || 692f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ((fenced_buf->flags & PB_USAGE_GPU_READ) && 693f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (flags & PB_USAGE_CPU_WRITE))) { 694f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 695f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 696f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Don't wait for the GPU to finish accessing it, if blocking is forbidden. 697f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 698f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if((flags & PB_USAGE_DONTBLOCK) && 699f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ops->fence_signalled(ops, fenced_buf->fence, 0) != 0) { 700f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto done; 701f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 702f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 703f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (flags & PB_USAGE_UNSYNCHRONIZED) { 704f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org break; 705f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 706f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 707f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 708f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Wait for the GPU to finish accessing. This will release and re-acquire 709f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * the mutex, so all copies of mutable state must be discarded. 710f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 711f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_finish_locked(fenced_mgr, fenced_buf); 712f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 713f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 714f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->buffer) { 715f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org map = pb_map(fenced_buf->buffer, flags, flush_ctx); 716f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 717f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org else { 718f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->data); 719f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org map = fenced_buf->data; 720f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 721f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 722f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(map) { 723f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ++fenced_buf->mapcount; 724f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->flags |= flags & PB_USAGE_CPU_READ_WRITE; 725f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 726f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 727f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgdone: 728f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 729f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 730f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return map; 731f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 732f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 733f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 734f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 735f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_unmap(struct pb_buffer *buf) 736f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 737f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf = fenced_buffer(buf); 738f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_buf->mgr; 739f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 740f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 741f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 742f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->mapcount); 743f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->mapcount) { 744f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (fenced_buf->buffer) 745f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_unmap(fenced_buf->buffer); 746f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org --fenced_buf->mapcount; 747f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!fenced_buf->mapcount) 748f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->flags &= ~PB_USAGE_CPU_READ_WRITE; 749f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 750f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 751f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 752f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 753f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 754f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 755f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic enum pipe_error 756f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_validate(struct pb_buffer *buf, 757f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_validate *vl, 758f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org unsigned flags) 759f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 760f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf = fenced_buffer(buf); 761f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_buf->mgr; 762f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org enum pipe_error ret; 763f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 764f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 765f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 766f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!vl) { 767f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* invalidate */ 768f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->vl = NULL; 769f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->validation_flags = 0; 770f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = PIPE_OK; 771f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto done; 772f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 773f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 774f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(flags & PB_USAGE_GPU_READ_WRITE); 775f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!(flags & ~PB_USAGE_GPU_READ_WRITE)); 776f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org flags &= PB_USAGE_GPU_READ_WRITE; 777f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 778f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* Buffer cannot be validated in two different lists */ 779f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->vl && fenced_buf->vl != vl) { 780f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = PIPE_ERROR_RETRY; 781f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto done; 782f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 783f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 784f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->vl == vl && 785f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org (fenced_buf->validation_flags & flags) == flags) { 786f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* Nothing to do -- buffer already validated */ 787f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = PIPE_OK; 788f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto done; 789f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 790f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 791f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 792f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Create and update GPU storage. 793f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 794f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!fenced_buf->buffer) { 795f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!fenced_buf->mapcount); 796f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 797f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = fenced_buffer_create_gpu_storage_locked(fenced_mgr, fenced_buf, TRUE); 798f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(ret != PIPE_OK) { 799f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto done; 800f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 801f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 802f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = fenced_buffer_copy_storage_to_gpu_locked(fenced_buf); 803f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(ret != PIPE_OK) { 804f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_gpu_storage_locked(fenced_buf); 805f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto done; 806f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 807f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 808f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->mapcount) { 809f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org debug_printf("warning: validating a buffer while it is still mapped\n"); 810f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 811f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org else { 812f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy_cpu_storage_locked(fenced_buf); 813f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 814f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 815f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 816f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = pb_validate(fenced_buf->buffer, vl, flags); 817f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (ret != PIPE_OK) 818f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto done; 819f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 820f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->vl = vl; 821f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->validation_flags |= flags; 822f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 823f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgdone: 824f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 825f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 826f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return ret; 827f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 828f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 829f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 830f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 831f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_fence(struct pb_buffer *buf, 832f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pipe_fence_handle *fence) 833f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 834f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf = fenced_buffer(buf); 835f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_buf->mgr; 836f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops = fenced_mgr->ops; 837f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 838f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 839f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 840f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(pipe_is_referenced(&fenced_buf->base.reference)); 841f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->buffer); 842f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 843f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fence != fenced_buf->fence) { 844f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->vl); 845f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->validation_flags); 846f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 847f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (fenced_buf->fence) { 848f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org boolean destroyed; 849f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org destroyed = fenced_buffer_remove_locked(fenced_mgr, fenced_buf); 850f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(!destroyed); 851f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 852f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (fence) { 853f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ops->fence_reference(ops, &fenced_buf->fence, fence); 854f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->flags |= fenced_buf->validation_flags; 855f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_add_locked(fenced_mgr, fenced_buf); 856f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 857f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 858f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_fence(fenced_buf->buffer, fence); 859f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 860f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->vl = NULL; 861f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->validation_flags = 0; 862f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 863f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 864f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 865f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 866f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 867f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 868f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 869f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_get_base_buffer(struct pb_buffer *buf, 870f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_buffer **base_buf, 871f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size *offset) 872f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 873f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf = fenced_buffer(buf); 874f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_buf->mgr; 875f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 876f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 877f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 878f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 879f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * This should only be called when the buffer is validated. Typically 880f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * when processing relocations. 881f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 882f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->vl); 883f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->buffer); 884f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 885f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_buf->buffer) 886f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_get_base_buffer(fenced_buf->buffer, base_buf, offset); 887f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org else { 888f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org *base_buf = buf; 889f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org *offset = 0; 890f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 891f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 892f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 893f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 894f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 895f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 896f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic const struct pb_vtbl 897f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_buffer_vtbl = { 898f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_destroy, 899f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_map, 900f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_unmap, 901f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_validate, 902f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_fence, 903f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buffer_get_base_buffer 904f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org}; 905f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 906f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 907f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org/** 908f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Wrap a buffer in a fenced buffer. 909f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 910f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic struct pb_buffer * 911f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_bufmgr_create_buffer(struct pb_manager *mgr, 912f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size size, 913f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org const struct pb_desc *desc) 914f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 915f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_manager(mgr); 916f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_buffer *fenced_buf; 917f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org enum pipe_error ret; 918f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 919f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 920f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Don't stall the GPU, waste time evicting buffers, or waste memory 921f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * trying to create a buffer that will most likely never fit into the 922f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * graphics aperture. 923f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 924f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(size > fenced_mgr->max_buffer_size) { 925f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto no_buffer; 926f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 927f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 928f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf = CALLOC_STRUCT(fenced_buffer); 929f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!fenced_buf) 930f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto no_buffer; 931f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 932f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_reference_init(&fenced_buf->base.reference, 1); 933f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->base.alignment = desc->alignment; 934f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->base.usage = desc->usage; 935f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->base.size = size; 936f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->size = size; 937f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->desc = *desc; 938f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 939f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->base.vtbl = &fenced_buffer_vtbl; 940f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_buf->mgr = fenced_mgr; 941f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 942f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 943f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 944f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 945f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Try to create GPU storage without stalling, 946f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 947f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = fenced_buffer_create_gpu_storage_locked(fenced_mgr, fenced_buf, FALSE); 948f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 949f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 950f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Attempt to use CPU memory to avoid stalling the GPU. 951f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 952f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(ret != PIPE_OK) { 953f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = fenced_buffer_create_cpu_storage_locked(fenced_mgr, fenced_buf); 954f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 955f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 956f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 957f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Create GPU storage, waiting for some to be available. 958f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 959f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(ret != PIPE_OK) { 960f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ret = fenced_buffer_create_gpu_storage_locked(fenced_mgr, fenced_buf, TRUE); 961f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 962f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 963f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* 964f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org * Give up. 965f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org */ 966f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(ret != PIPE_OK) { 967f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org goto no_storage; 968f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 969f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 970f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_buf->buffer || fenced_buf->data); 971f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 972f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_ADDTAIL(&fenced_buf->head, &fenced_mgr->unfenced); 973f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ++fenced_mgr->num_unfenced; 974f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 975f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 976f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return &fenced_buf->base; 977f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 978f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgno_storage: 979f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 980f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org FREE(fenced_buf); 981f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgno_buffer: 982f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return NULL; 983f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 984f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 985f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 986f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 987f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_bufmgr_flush(struct pb_manager *mgr) 988f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 989f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_manager(mgr); 990f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 991f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 992f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(fenced_manager_check_signalled_locked(fenced_mgr, TRUE)) 993f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ; 994f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 995f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 996f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org assert(fenced_mgr->provider->flush); 997f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_mgr->provider->flush) 998f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->provider->flush(fenced_mgr->provider); 999f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 1000f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1001f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1002f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstatic void 1003f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_bufmgr_destroy(struct pb_manager *mgr) 1004f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 1005f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr = fenced_manager(mgr); 1006f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1007f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 1008f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1009f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /* Wait on outstanding fences */ 1010f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while (fenced_mgr->num_fenced) { 1011f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 1012f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS) 1013f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org sched_yield(); 1014f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#endif 1015f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_lock(fenced_mgr->mutex); 1016f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org while(fenced_manager_check_signalled_locked(fenced_mgr, TRUE)) 1017f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org ; 1018f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org } 1019f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1020f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#ifdef DEBUG 1021f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org /*assert(!fenced_mgr->num_unfenced);*/ 1022f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org#endif 1023f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1024f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_unlock(fenced_mgr->mutex); 1025f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_destroy(fenced_mgr->mutex); 1026f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1027f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(fenced_mgr->provider) 1028f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->provider->destroy(fenced_mgr->provider); 1029f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1030f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->ops->destroy(fenced_mgr->ops); 1031f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1032f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org FREE(fenced_mgr); 1033f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 1034f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1035f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1036f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgstruct pb_manager * 1037f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.orgfenced_bufmgr_create(struct pb_manager *provider, 1038f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct pb_fence_ops *ops, 1039f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size max_buffer_size, 1040f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pb_size max_cpu_total_size) 1041f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org{ 1042f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org struct fenced_manager *fenced_mgr; 1043f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1044f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if(!provider) 1045f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return NULL; 1046f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1047f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr = CALLOC_STRUCT(fenced_manager); 1048f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org if (!fenced_mgr) 1049f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return NULL; 1050f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1051f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->base.destroy = fenced_bufmgr_destroy; 1052f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->base.create_buffer = fenced_bufmgr_create_buffer; 1053f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->base.flush = fenced_bufmgr_flush; 1054f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1055f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->provider = provider; 1056f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->ops = ops; 1057f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->max_buffer_size = max_buffer_size; 1058f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->max_cpu_total_size = max_cpu_total_size; 1059f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1060f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_INITHEAD(&fenced_mgr->fenced); 1061f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->num_fenced = 0; 1062f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1063f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org LIST_INITHEAD(&fenced_mgr->unfenced); 1064f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org fenced_mgr->num_unfenced = 0; 1065f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1066f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org pipe_mutex_init(fenced_mgr->mutex); 1067f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org 1068f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org return &fenced_mgr->base; 1069f2ba7591b1407a7ee9209f842c50696914dc2dedkbr@chromium.org} 1070