GraphicBufferSource.h revision 00beaa2776382e8fa1844cd85e00813c3f226970
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef GRAPHIC_BUFFER_SOURCE_H_ 18 19#define GRAPHIC_BUFFER_SOURCE_H_ 20 21#include <gui/IGraphicBufferProducer.h> 22#include <gui/BufferQueue.h> 23#include <utils/RefBase.h> 24 25#include <OMX_Core.h> 26#include <VideoAPI.h> 27#include "../include/OMXNodeInstance.h" 28#include <media/stagefright/foundation/ABase.h> 29#include <media/stagefright/foundation/AHandlerReflector.h> 30#include <media/stagefright/foundation/ALooper.h> 31 32namespace android { 33 34struct FrameDropper; 35 36/* 37 * This class is used to feed OMX codecs from a Surface via BufferQueue. 38 * 39 * Instances of the class don't run on a dedicated thread. Instead, 40 * various events trigger data movement: 41 * 42 * - Availability of a new frame of data from the BufferQueue (notified 43 * via the onFrameAvailable callback). 44 * - The return of a codec buffer (via OnEmptyBufferDone). 45 * - Application signaling end-of-stream. 46 * - Transition to or from "executing" state. 47 * 48 * Frames of data (and, perhaps, the end-of-stream indication) can arrive 49 * before the codec is in the "executing" state, so we need to queue 50 * things up until we're ready to go. 51 */ 52class GraphicBufferSource : public BufferQueue::ConsumerListener { 53public: 54 GraphicBufferSource( 55 OMXNodeInstance* nodeInstance, 56 uint32_t bufferWidth, 57 uint32_t bufferHeight, 58 uint32_t bufferCount, 59 uint32_t consumerUsage, 60 const sp<IGraphicBufferConsumer> &consumer = NULL 61 ); 62 63 virtual ~GraphicBufferSource(); 64 65 // We can't throw an exception if the constructor fails, so we just set 66 // this and require that the caller test the value. 67 status_t initCheck() const { 68 return mInitCheck; 69 } 70 71 // Returns the handle to the producer side of the BufferQueue. Buffers 72 // queued on this will be received by GraphicBufferSource. 73 sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { 74 return mProducer; 75 } 76 77 // Sets the default buffer data space 78 void setDefaultDataSpace(android_dataspace dataSpace); 79 80 // This is called when OMX transitions to OMX_StateExecuting, which means 81 // we can start handing it buffers. If we already have buffers of data 82 // sitting in the BufferQueue, this will send them to the codec. 83 void omxExecuting(); 84 85 // This is called when OMX transitions to OMX_StateIdle, indicating that 86 // the codec is meant to return all buffers back to the client for them 87 // to be freed. Do NOT submit any more buffers to the component. 88 void omxIdle(); 89 90 // This is called when OMX transitions to OMX_StateLoaded, indicating that 91 // we are shutting down. 92 void omxLoaded(); 93 94 // A "codec buffer", i.e. a buffer that can be used to pass data into 95 // the encoder, has been allocated. (This call does not call back into 96 // OMXNodeInstance.) 97 void addCodecBuffer(OMX_BUFFERHEADERTYPE* header); 98 99 // Called from OnEmptyBufferDone. If we have a BQ buffer available, 100 // fill it with a new frame of data; otherwise, just mark it as available. 101 void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd); 102 103 // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the 104 // buffer source will fix timestamp in the header if needed.) 105 void codecBufferFilled(OMX_BUFFERHEADERTYPE* header); 106 107 // This is called after the last input frame has been submitted. We 108 // need to submit an empty buffer with the EOS flag set. If we don't 109 // have a codec buffer ready, we just set the mEndOfStream flag. 110 status_t signalEndOfInputStream(); 111 112 // If suspend is true, all incoming buffers (including those currently 113 // in the BufferQueue) will be discarded until the suspension is lifted. 114 void suspend(bool suspend); 115 116 // Specifies the interval after which we requeue the buffer previously 117 // queued to the encoder. This is useful in the case of surface flinger 118 // providing the input surface if the resulting encoded stream is to 119 // be displayed "live". If we were not to push through the extra frame 120 // the decoder on the remote end would be unable to decode the latest frame. 121 // This API must be called before transitioning the encoder to "executing" 122 // state and once this behaviour is specified it cannot be reset. 123 status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); 124 125 // When set, the timestamp fed to the encoder will be modified such that 126 // the gap between two adjacent frames is capped at maxGapUs. Timestamp 127 // will be restored to the original when the encoded frame is returned to 128 // the client. 129 // This is to solve a problem in certain real-time streaming case, where 130 // encoder's rate control logic produces huge frames after a long period 131 // of suspension on input. 132 status_t setMaxTimestampGapUs(int64_t maxGapUs); 133 134 // When set, the max frame rate fed to the encoder will be capped at maxFps. 135 status_t setMaxFps(float maxFps); 136 137 struct TimeLapseConfig { 138 int64_t mTimePerFrameUs; // the time (us) between two frames for playback 139 int64_t mTimePerCaptureUs; // the time (us) between two frames for capture 140 }; 141 142 // Sets the time lapse (or slow motion) parameters. 143 // When set, the sample's timestamp will be modified to playback framerate, 144 // and capture timestamp will be modified to capture rate. 145 status_t setTimeLapseConfig(const TimeLapseConfig &config); 146 147 // Sets the start time us (in system time), samples before which should 148 // be dropped and not submitted to encoder 149 void setSkipFramesBeforeUs(int64_t startTimeUs); 150 151 // Sets the desired color aspects, e.g. to be used when producer does not specify a dataspace. 152 void setColorAspects(const ColorAspects &aspects); 153 154protected: 155 // BufferQueue::ConsumerListener interface, called when a new frame of 156 // data is available. If we're executing and a codec buffer is 157 // available, we acquire the buffer, copy the GraphicBuffer reference 158 // into the codec buffer, and call Empty[This]Buffer. If we're not yet 159 // executing or there's no codec buffer available, we just increment 160 // mNumFramesAvailable and return. 161 virtual void onFrameAvailable(const BufferItem& item); 162 163 // BufferQueue::ConsumerListener interface, called when the client has 164 // released one or more GraphicBuffers. We clear out the appropriate 165 // set of mBufferSlot entries. 166 virtual void onBuffersReleased(); 167 168 // BufferQueue::ConsumerListener interface, called when the client has 169 // changed the sideband stream. GraphicBufferSource doesn't handle sideband 170 // streams so this is a no-op (and should never be called). 171 virtual void onSidebandStreamChanged(); 172 173private: 174 // PersistentProxyListener is similar to BufferQueue::ProxyConsumerListener 175 // except that it returns (acquire/detach/re-attache/release) buffers 176 // in onFrameAvailable() if the actual consumer object is no longer valid. 177 // 178 // This class is used in persistent input surface case to prevent buffer 179 // loss when onFrameAvailable() is received while we don't have a valid 180 // consumer around. 181 class PersistentProxyListener : public BnConsumerListener { 182 public: 183 PersistentProxyListener( 184 const wp<IGraphicBufferConsumer> &consumer, 185 const wp<ConsumerListener>& consumerListener); 186 virtual ~PersistentProxyListener(); 187 virtual void onFrameAvailable(const BufferItem& item) override; 188 virtual void onFrameReplaced(const BufferItem& item) override; 189 virtual void onBuffersReleased() override; 190 virtual void onSidebandStreamChanged() override; 191 private: 192 // mConsumerListener is a weak reference to the IConsumerListener. 193 wp<ConsumerListener> mConsumerListener; 194 // mConsumer is a weak reference to the IGraphicBufferConsumer, use 195 // a weak ref to avoid circular ref between mConsumer and this class 196 wp<IGraphicBufferConsumer> mConsumer; 197 }; 198 199 // Keep track of codec input buffers. They may either be available 200 // (mGraphicBuffer == NULL) or in use by the codec. 201 struct CodecBuffer { 202 OMX_BUFFERHEADERTYPE* mHeader; 203 204 // buffer producer's frame-number for buffer 205 uint64_t mFrameNumber; 206 207 // buffer producer's buffer slot for buffer 208 int mSlot; 209 210 sp<GraphicBuffer> mGraphicBuffer; 211 }; 212 213 // Returns the index of an available codec buffer. If none are 214 // available, returns -1. Mutex must be held by caller. 215 int findAvailableCodecBuffer_l(); 216 217 // Returns true if a codec buffer is available. 218 bool isCodecBufferAvailable_l() { 219 return findAvailableCodecBuffer_l() >= 0; 220 } 221 222 // Finds the mCodecBuffers entry that matches. Returns -1 if not found. 223 int findMatchingCodecBuffer_l(const OMX_BUFFERHEADERTYPE* header); 224 225 // Fills a codec buffer with a frame from the BufferQueue. This must 226 // only be called when we know that a frame of data is ready (i.e. we're 227 // in the onFrameAvailable callback, or if we're in codecBufferEmptied 228 // and mNumFramesAvailable is nonzero). Returns without doing anything if 229 // we don't have a codec buffer available. 230 // 231 // Returns true if we successfully filled a codec buffer with a BQ buffer. 232 bool fillCodecBuffer_l(); 233 234 // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer 235 // reference into the codec buffer, and submits the data to the codec. 236 status_t submitBuffer_l(const BufferItem &item, int cbi); 237 238 // Submits an empty buffer, with the EOS flag set. Returns without 239 // doing anything if we don't have a codec buffer available. 240 void submitEndOfInputStream_l(); 241 242 // Release buffer to the consumer 243 void releaseBuffer( 244 int &id, uint64_t frameNum, 245 const sp<GraphicBuffer> buffer, const sp<Fence> &fence); 246 247 void setLatestBuffer_l(const BufferItem &item, bool dropped); 248 bool repeatLatestBuffer_l(); 249 int64_t getTimestamp(const BufferItem &item); 250 251 // called when the data space of the input buffer changes 252 void onDataSpaceChanged_l(android_dataspace dataSpace, android_pixel_format pixelFormat); 253 254 // Lock, covers all member variables. 255 mutable Mutex mMutex; 256 257 // Used to report constructor failure. 258 status_t mInitCheck; 259 260 // Pointer back to the object that contains us. We send buffers here. 261 OMXNodeInstance* mNodeInstance; 262 263 // Set by omxExecuting() / omxIdling(). 264 bool mExecuting; 265 266 bool mSuspended; 267 268 // Last dataspace seen 269 android_dataspace mLastDataSpace; 270 271 // Our BufferQueue interfaces. mProducer is passed to the producer through 272 // getIGraphicBufferProducer, and mConsumer is used internally to retrieve 273 // the buffers queued by the producer. 274 bool mIsPersistent; 275 sp<IGraphicBufferProducer> mProducer; 276 sp<IGraphicBufferConsumer> mConsumer; 277 278 // Number of frames pending in BufferQueue that haven't yet been 279 // forwarded to the codec. 280 size_t mNumFramesAvailable; 281 282 // Number of frames acquired from consumer (debug only) 283 int32_t mNumBufferAcquired; 284 285 // Set to true if we want to send end-of-stream after we run out of 286 // frames in BufferQueue. 287 bool mEndOfStream; 288 bool mEndOfStreamSent; 289 290 // Cache of GraphicBuffers from the buffer queue. When the codec 291 // is done processing a GraphicBuffer, we can use this to map back 292 // to a slot number. 293 sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS]; 294 295 // Tracks codec buffers. 296 Vector<CodecBuffer> mCodecBuffers; 297 298 //// 299 friend struct AHandlerReflector<GraphicBufferSource>; 300 301 enum { 302 kWhatRepeatLastFrame, 303 }; 304 enum { 305 kRepeatLastFrameCount = 10, 306 }; 307 308 KeyedVector<int64_t, int64_t> mOriginalTimeUs; 309 int64_t mMaxTimestampGapUs; 310 int64_t mPrevOriginalTimeUs; 311 int64_t mPrevModifiedTimeUs; 312 int64_t mSkipFramesBeforeNs; 313 314 sp<FrameDropper> mFrameDropper; 315 316 sp<ALooper> mLooper; 317 sp<AHandlerReflector<GraphicBufferSource> > mReflector; 318 319 int64_t mRepeatAfterUs; 320 int32_t mRepeatLastFrameGeneration; 321 int64_t mRepeatLastFrameTimestamp; 322 int32_t mRepeatLastFrameCount; 323 324 int mLatestBufferId; 325 uint64_t mLatestBufferFrameNum; 326 int32_t mLatestBufferUseCount; 327 sp<Fence> mLatestBufferFence; 328 329 // The previous buffer should've been repeated but 330 // no codec buffer was available at the time. 331 bool mRepeatBufferDeferred; 332 333 // Time lapse / slow motion configuration 334 int64_t mTimePerCaptureUs; 335 int64_t mTimePerFrameUs; 336 int64_t mPrevCaptureUs; 337 int64_t mPrevFrameUs; 338 339 MetadataBufferType mMetadataBufferType; 340 ColorAspects mColorAspects; 341 342 void onMessageReceived(const sp<AMessage> &msg); 343 344 DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource); 345}; 346 347} // namespace android 348 349#endif // GRAPHIC_BUFFER_SOURCE_H_ 350