/* * Copyright (C) 2011 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package androidx.media.filterfw; import android.annotation.TargetApi; import android.graphics.Bitmap; import android.os.Build; import android.renderscript.Allocation; import android.renderscript.Element; import android.renderscript.RenderScript; import android.renderscript.Type; import android.util.Log; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; import java.util.Vector; final class BackingStore { /** Access mode None: Frame data will not be accessed at all. */ static final int ACCESS_NONE = 0x00; /** Access mode Bytes: Frame data will be accessed as a ByteBuffer. */ static final int ACCESS_BYTES = 0x01; /** Access mode Texture: Frame data will be accessed as a TextureSource. */ static final int ACCESS_TEXTURE = 0x02; /** Access mode RenderTarget: Frame data will be accessed as a RenderTarget. */ static final int ACCESS_RENDERTARGET = 0x04; /** Access mode Object: Frame data will be accessed as a generic Object. */ static final int ACCESS_OBJECT = 0x08; /** Access mode Bitmap: Frame data will be accessed as a Bitmap. */ static final int ACCESS_BITMAP = 0x10; /** Access mode Allocation: Frame data will be accessed as a RenderScript Allocation. */ static final int ACCESS_ALLOCATION = 0x20; private static final int BACKING_BYTEBUFFER = 1; private static final int BACKING_TEXTURE = 2; private static final int BACKING_OBJECT = 3; private static final int BACKING_BITMAP = 4; private static final int BACKING_ALLOCATION = 5; private final FrameType mType; private int[] mDimensions; private long mTimestamp = Frame.TIMESTAMP_NOT_SET; private final FrameManager mFrameManager; private Vector mBackings = new Vector(); private boolean mWriteLocked = false; private int mReadLocks = 0; private int mRefCount = 1; /** The most up-to-date data backing */ private Backing mCurrentBacking = null; /** The currently locked backing */ private Backing mLockedBacking = null; // Public Methods ////////////////////////////////////////////////////////////////////////////// public BackingStore(FrameType type, int[] dimensions, FrameManager frameManager) { mType = type; mDimensions = dimensions != null ? Arrays.copyOf(dimensions, dimensions.length) : null; mFrameManager = frameManager; } public FrameType getFrameType() { return mType; } public Object lockData(int mode, int accessFormat) { return lockBacking(mode, accessFormat).lock(accessFormat); } public Backing lockBacking(int mode, int access) { Backing backing = fetchBacking(mode, access); if (backing == null) { throw new RuntimeException("Could not fetch frame data!"); } lock(backing, mode); return backing; } public boolean unlock() { if (mWriteLocked) { mWriteLocked = false; } else if (mReadLocks > 0) { --mReadLocks; } else { return false; } mLockedBacking.unlock(); mLockedBacking = null; return true; } public BackingStore retain() { if (mRefCount >= 10) { Log.w("BackingStore", "High ref-count of " + mRefCount + " on " + this + "!"); } if (mRefCount <= 0) { throw new RuntimeException("RETAINING RELEASED"); } ++mRefCount; return this; } public BackingStore release() { if (mRefCount <= 0) { throw new RuntimeException("DOUBLE-RELEASE"); } --mRefCount; if (mRefCount == 0) { releaseBackings(); return null; } return this; } /** * Resizes the backing store. This invalidates all data in the store. */ public void resize(int[] newDimensions) { Vector resized = new Vector(); for (Backing backing : mBackings) { if (backing.resize(newDimensions)) { resized.add(backing); } else { releaseBacking(backing); } } mBackings = resized; mDimensions = newDimensions; } public int[] getDimensions() { return mDimensions; } public int getElementCount() { int result = 1; if (mDimensions != null) { for (int dim : mDimensions) { result *= dim; } } return result; } public void importStore(BackingStore store) { // TODO: Better backing selection? if (store.mBackings.size() > 0) { importBacking(store.mBackings.firstElement()); } mTimestamp = store.mTimestamp; } /** * @return the timestamp */ public long getTimestamp() { return mTimestamp; } /** * @param timestamp the timestamp to set */ public void setTimestamp(long timestamp) { mTimestamp = timestamp; } // Internal Methods //////////////////////////////////////////////////////////////////////////// private Backing fetchBacking(int mode, int access) { Backing backing = getBacking(mode, access); if (backing == null) { backing = attachNewBacking(mode, access); } syncBacking(backing); return backing; } private void syncBacking(Backing backing) { if (backing != null && backing.isDirty() && mCurrentBacking != null) { backing.syncTo(mCurrentBacking); } } private Backing getBacking(int mode, int access) { // [Non-iterator looping] for (int i = 0; i < mBackings.size(); ++i) { final Backing backing = mBackings.get(i); int backingAccess = (mode == Frame.MODE_WRITE) ? backing.writeAccess() : backing.readAccess(); if ((backingAccess & access) == access) { return backing; } } return null; } private Backing attachNewBacking(int mode, int access) { Backing backing = createBacking(mode, access); if (mBackings.size() > 0) { backing.markDirty(); } mBackings.add(backing); return backing; } private Backing createBacking(int mode, int access) { // TODO: If the read/write access flags indicate, make/fetch a GraphicBuffer backing. Backing backing = null; int elemSize = mType.getElementSize(); if (shouldFetchCached(access)) { backing = mFrameManager.fetchBacking(mode, access, mDimensions, elemSize); } if (backing == null) { switch (access) { case ACCESS_BYTES: backing = new ByteBufferBacking(); break; case ACCESS_TEXTURE: case ACCESS_RENDERTARGET: backing = new TextureBacking(); break; case ACCESS_OBJECT: backing = new ObjectBacking(); break; case ACCESS_BITMAP: backing = new BitmapBacking(); break; case ACCESS_ALLOCATION: if (!AllocationBacking.isSupported()) { throw new RuntimeException( "Attempted to create an AllocationBacking in context that does " + "not support RenderScript!"); } backing = new AllocationBacking(mFrameManager.getContext().getRenderScript()); break; } if (backing == null) { throw new RuntimeException( "Could not create backing for access type " + access + "!"); } if (backing.requiresGpu() && !mFrameManager.getRunner().isOpenGLSupported()) { throw new RuntimeException( "Cannot create backing that requires GPU in a runner that does not " + "support OpenGL!"); } backing.setDimensions(mDimensions); backing.setElementSize(elemSize); backing.setElementId(mType.getElementId()); backing.allocate(mType); mFrameManager.onBackingCreated(backing); } return backing; } private void importBacking(Backing backing) { // TODO: This actually needs synchronization between the two BackingStore threads for the // general case int access = backing.requiresGpu() ? ACCESS_BYTES : backing.readAccess(); Backing newBacking = createBacking(Frame.MODE_READ, access); newBacking.syncTo(backing); mBackings.add(newBacking); mCurrentBacking = newBacking; } private void releaseBackings() { // [Non-iterator looping] for (int i = 0; i < mBackings.size(); ++i) { releaseBacking(mBackings.get(i)); } mBackings.clear(); mCurrentBacking = null; } private void releaseBacking(Backing backing) { mFrameManager.onBackingAvailable(backing); } private void lock(Backing backingToLock, int mode) { if (mode == Frame.MODE_WRITE) { // Make sure frame is not read-locked if (mReadLocks > 0) { throw new RuntimeException( "Attempting to write-lock the read-locked frame " + this + "!"); } else if (mWriteLocked) { throw new RuntimeException( "Attempting to write-lock the write-locked frame " + this + "!"); } // Mark all other backings dirty // [Non-iterator looping] for (int i = 0; i < mBackings.size(); ++i) { final Backing backing = mBackings.get(i); if (backing != backingToLock) { backing.markDirty(); } } mWriteLocked = true; mCurrentBacking = backingToLock; } else { if (mWriteLocked) { throw new RuntimeException("Attempting to read-lock locked frame " + this + "!"); } ++mReadLocks; } mLockedBacking = backingToLock; } private static boolean shouldFetchCached(int access) { return access != ACCESS_OBJECT; } // Backings //////////////////////////////////////////////////////////////////////////////////// static abstract class Backing { protected int[] mDimensions = null; private int mElementSize; private int mElementID; protected boolean mIsDirty = false; int cachePriority = 0; public abstract void allocate(FrameType frameType); public abstract int readAccess(); public abstract int writeAccess(); public abstract void syncTo(Backing backing); public abstract Object lock(int accessType); public abstract int getType(); public abstract boolean shouldCache(); public abstract boolean requiresGpu(); public abstract void destroy(); public abstract int getSize(); public void unlock() { // Default implementation does nothing. } public void setData(Object data) { throw new RuntimeException("Internal error: Setting data on frame backing " + this + ", which does not support setting data directly!"); } public void setDimensions(int[] dimensions) { mDimensions = dimensions; } public void setElementSize(int elemSize) { mElementSize = elemSize; } public void setElementId(int elemId) { mElementID = elemId; } public int[] getDimensions() { return mDimensions; } public int getElementSize() { return mElementSize; } public int getElementId() { return mElementID; } public boolean resize(int[] newDimensions) { return false; } public void markDirty() { mIsDirty = true; } public boolean isDirty() { return mIsDirty; } protected void assertImageCompatible(FrameType type) { if (type.getElementId() != FrameType.ELEMENT_RGBA8888) { throw new RuntimeException("Cannot allocate texture with non-RGBA data type!"); } else if (mDimensions == null || mDimensions.length != 2) { throw new RuntimeException("Cannot allocate non 2-dimensional texture!"); } } } static class ObjectBacking extends Backing { private Object mObject = null; @Override public void allocate(FrameType frameType) { mObject = null; } @Override public int readAccess() { return ACCESS_OBJECT; } @Override public int writeAccess() { return ACCESS_OBJECT; } @Override public void syncTo(Backing backing) { switch (backing.getType()) { case BACKING_OBJECT: mObject = backing.lock(ACCESS_OBJECT); backing.unlock(); break; case BACKING_BITMAP: mObject = backing.lock(ACCESS_BITMAP); backing.unlock(); break; default: mObject = null; } mIsDirty = false; } @Override public Object lock(int accessType) { return mObject; } @Override public int getType() { return BACKING_OBJECT; } @Override public boolean shouldCache() { return false; } @Override public boolean requiresGpu() { return false; } @Override public void destroy() { mObject = null; } @Override public int getSize() { return 0; } @Override public void setData(Object data) { mObject = data; } } static class BitmapBacking extends Backing { private Bitmap mBitmap = null; @Override public void allocate(FrameType frameType) { assertImageCompatible(frameType); } @Override public int readAccess() { return ACCESS_BITMAP; } @Override public int writeAccess() { return ACCESS_BITMAP; } @Override public void syncTo(Backing backing) { int access = backing.readAccess(); if ((access & ACCESS_BITMAP) != 0) { mBitmap = (Bitmap) backing.lock(ACCESS_BITMAP); } else if ((access & ACCESS_BYTES) != 0) { createBitmap(); ByteBuffer buffer = (ByteBuffer) backing.lock(ACCESS_BYTES); mBitmap.copyPixelsFromBuffer(buffer); buffer.rewind(); } else if ((access & ACCESS_TEXTURE) != 0) { createBitmap(); RenderTarget renderTarget = (RenderTarget) backing.lock(ACCESS_RENDERTARGET); mBitmap.copyPixelsFromBuffer( renderTarget.getPixelData(mDimensions[0], mDimensions[1])); } else if ((access & ACCESS_ALLOCATION) != 0 && AllocationBacking.isSupported()) { createBitmap(); syncToAllocationBacking(backing); } else { throw new RuntimeException("Cannot sync bytebuffer backing!"); } backing.unlock(); mIsDirty = false; } @TargetApi(11) private void syncToAllocationBacking(Backing backing) { Allocation allocation = (Allocation) backing.lock(ACCESS_ALLOCATION); allocation.copyTo(mBitmap); } @Override public Object lock(int accessType) { return mBitmap; } @Override public int getType() { return BACKING_BITMAP; } @Override public boolean shouldCache() { return false; } @Override public boolean requiresGpu() { return false; } @Override public void destroy() { // As we share the bitmap with other backings (such as object backings), we must not // recycle it here. mBitmap = null; } @Override public int getSize() { return 4 * mDimensions[0] * mDimensions[1]; } @Override public void setData(Object data) { // We can assume that data will always be a Bitmap instance. mBitmap = (Bitmap) data; } private void createBitmap() { mBitmap = Bitmap.createBitmap(mDimensions[0], mDimensions[1], Bitmap.Config.ARGB_8888); } } static class TextureBacking extends Backing { private RenderTarget mRenderTarget = null; private TextureSource mTexture = null; @Override public void allocate(FrameType frameType) { assertImageCompatible(frameType); mTexture = TextureSource.newTexture(); } @Override public int readAccess() { return ACCESS_TEXTURE; } @Override public int writeAccess() { return ACCESS_RENDERTARGET; } @Override public void syncTo(Backing backing) { int access = backing.readAccess(); if ((access & ACCESS_BYTES) != 0) { ByteBuffer pixels = (ByteBuffer) backing.lock(ACCESS_BYTES); mTexture.allocateWithPixels(pixels, mDimensions[0], mDimensions[1]); } else if ((access & ACCESS_BITMAP) != 0) { Bitmap bitmap = (Bitmap) backing.lock(ACCESS_BITMAP); mTexture.allocateWithBitmapPixels(bitmap); } else if ((access & ACCESS_TEXTURE) != 0) { TextureSource texture = (TextureSource) backing.lock(ACCESS_TEXTURE); int w = mDimensions[0]; int h = mDimensions[1]; ImageShader.renderTextureToTarget(texture, getRenderTarget(), w, h); } else if ((access & ACCESS_ALLOCATION) != 0 && AllocationBacking.isSupported()) { syncToAllocationBacking(backing); } else { throw new RuntimeException("Cannot sync bytebuffer backing!"); } backing.unlock(); mIsDirty = false; } @TargetApi(11) private void syncToAllocationBacking(Backing backing) { Allocation allocation = (Allocation) backing.lock(ACCESS_ALLOCATION); ByteBuffer pixels = ByteBuffer.allocateDirect(getSize()); allocation.copyTo(pixels.array()); mTexture.allocateWithPixels(pixels, mDimensions[0], mDimensions[1]); } @Override public Object lock(int accessType) { switch (accessType) { case ACCESS_TEXTURE: return getTexture(); case ACCESS_RENDERTARGET: return getRenderTarget(); default: throw new RuntimeException("Illegal access to texture!"); } } @Override public int getType() { return BACKING_TEXTURE; } @Override public boolean shouldCache() { return true; } @Override public boolean requiresGpu() { return true; } @Override public void destroy() { if (mRenderTarget != null) { mRenderTarget.release(); } if (mTexture.isAllocated()) { mTexture.release(); } } @Override public int getSize() { return 4 * mDimensions[0] * mDimensions[1]; } private TextureSource getTexture() { if (!mTexture.isAllocated()) { mTexture.allocate(mDimensions[0], mDimensions[1]); } return mTexture; } private RenderTarget getRenderTarget() { if (mRenderTarget == null) { int w = mDimensions[0]; int h = mDimensions[1]; mRenderTarget = RenderTarget.currentTarget().forTexture(getTexture(), w, h); } return mRenderTarget; } } static class ByteBufferBacking extends Backing { ByteBuffer mBuffer = null; @Override public void allocate(FrameType frameType) { int size = frameType.getElementSize(); for (int dim : mDimensions) { size *= dim; } mBuffer = ByteBuffer.allocateDirect(size); } @Override public int readAccess() { return ACCESS_BYTES; } @Override public int writeAccess() { return ACCESS_BYTES; } @Override public boolean requiresGpu() { return false; } @Override public void syncTo(Backing backing) { int access = backing.readAccess(); if ((access & ACCESS_TEXTURE) != 0) { RenderTarget target = (RenderTarget) backing.lock(ACCESS_RENDERTARGET); GLToolbox.readTarget(target, mBuffer, mDimensions[0], mDimensions[1]); } else if ((access & ACCESS_BITMAP) != 0) { Bitmap bitmap = (Bitmap) backing.lock(ACCESS_BITMAP); bitmap.copyPixelsToBuffer(mBuffer); mBuffer.rewind(); } else if ((access & ACCESS_BYTES) != 0) { ByteBuffer otherBuffer = (ByteBuffer) backing.lock(ACCESS_BYTES); mBuffer.put(otherBuffer); otherBuffer.rewind(); } else if ((access & ACCESS_ALLOCATION) != 0 && AllocationBacking.isSupported()) { syncToAllocationBacking(backing); } else { throw new RuntimeException("Cannot sync bytebuffer backing!"); } backing.unlock(); mBuffer.rewind(); mIsDirty = false; } @TargetApi(11) private void syncToAllocationBacking(Backing backing) { Allocation allocation = (Allocation) backing.lock(ACCESS_ALLOCATION); if (getElementId() == FrameType.ELEMENT_RGBA8888) { byte[] bytes = mBuffer.array(); allocation.copyTo(bytes); } else if (getElementId() == FrameType.ELEMENT_FLOAT32) { float[] floats = new float[getSize() / 4]; allocation.copyTo(floats); mBuffer.asFloatBuffer().put(floats); } else { throw new RuntimeException( "Trying to sync to an allocation with an unsupported element id: " + getElementId()); } } @Override public Object lock(int accessType) { return mBuffer.rewind(); } @Override public void unlock() { mBuffer.rewind(); } @Override public int getType() { return BACKING_BYTEBUFFER; } @Override public boolean shouldCache() { return true; } @Override public void destroy() { mBuffer = null; } @Override public int getSize() { return mBuffer.remaining(); } } @TargetApi(11) static class AllocationBacking extends Backing { private final RenderScript mRenderScript; private Allocation mAllocation = null; public AllocationBacking(RenderScript renderScript) { mRenderScript = renderScript; } @Override public void allocate(FrameType frameType) { assertCompatible(frameType); Element element = null; switch (frameType.getElementId()) { case FrameType.ELEMENT_RGBA8888: element = Element.RGBA_8888(mRenderScript); break; case FrameType.ELEMENT_FLOAT32: element = Element.F32(mRenderScript); break; } Type.Builder imageTypeBuilder = new Type.Builder(mRenderScript, element); imageTypeBuilder.setX(mDimensions.length >= 1 ? mDimensions[0] : 1); imageTypeBuilder.setY(mDimensions.length == 2 ? mDimensions[1] : 1); Type imageType = imageTypeBuilder.create(); mAllocation = Allocation.createTyped(mRenderScript, imageType); } @Override public int readAccess() { return ACCESS_ALLOCATION; } @Override public int writeAccess() { return ACCESS_ALLOCATION; } @Override public boolean requiresGpu() { return false; } @Override public void syncTo(Backing backing) { int access = backing.readAccess(); if ((access & ACCESS_TEXTURE) != 0) { RenderTarget target = (RenderTarget) backing.lock(ACCESS_RENDERTARGET); ByteBuffer pixels = ByteBuffer.allocateDirect(getSize()); GLToolbox.readTarget(target, pixels, mDimensions[0], mDimensions[1]); mAllocation.copyFrom(pixels.array()); } else if ((access & ACCESS_BITMAP) != 0) { Bitmap bitmap = (Bitmap) backing.lock(ACCESS_BITMAP); mAllocation.copyFrom(bitmap); } else if ((access & ACCESS_BYTES) != 0) { ByteBuffer buffer = (ByteBuffer) backing.lock(ACCESS_BYTES); if (buffer.order() != ByteOrder.nativeOrder()) { throw new RuntimeException( "Trying to sync to the ByteBufferBacking with non-native byte order!"); } byte[] bytes; if (buffer.hasArray()) { bytes = buffer.array(); } else { bytes = new byte[getSize()]; buffer.get(bytes); buffer.rewind(); } mAllocation.copyFromUnchecked(bytes); } else { throw new RuntimeException("Cannot sync allocation backing!"); } backing.unlock(); mIsDirty = false; } @Override public Object lock(int accessType) { return mAllocation; } @Override public void unlock() { } @Override public int getType() { return BACKING_ALLOCATION; } @Override public boolean shouldCache() { return true; } @Override public void destroy() { if (mAllocation != null) { mAllocation.destroy(); mAllocation = null; } } @Override public int getSize() { int elementCount = 1; for (int dim : mDimensions) { elementCount *= dim; } return getElementSize() * elementCount; } public static boolean isSupported() { return Build.VERSION.SDK_INT >= 11; } private void assertCompatible(FrameType type) { // TODO: consider adding support for other data types. if (type.getElementId() != FrameType.ELEMENT_RGBA8888 && type.getElementId() != FrameType.ELEMENT_FLOAT32) { throw new RuntimeException( "Cannot allocate allocation with a non-RGBA or non-float data type!"); } if (mDimensions == null || mDimensions.length > 2) { throw new RuntimeException( "Cannot create an allocation with more than 2 dimensions!"); } } } }