1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef GRAPHIC_BUFFER_SOURCE_H_ 18 19#define GRAPHIC_BUFFER_SOURCE_H_ 20 21#include <gui/IGraphicBufferProducer.h> 22#include <gui/BufferQueue.h> 23#include <utils/RefBase.h> 24 25#include <OMX_Core.h> 26#include "../include/OMXNodeInstance.h" 27#include <media/stagefright/foundation/ABase.h> 28#include <media/stagefright/foundation/AHandlerReflector.h> 29#include <media/stagefright/foundation/ALooper.h> 30 31namespace android { 32 33struct FrameDropper; 34 35/* 36 * This class is used to feed OMX codecs from a Surface via BufferQueue. 37 * 38 * Instances of the class don't run on a dedicated thread. Instead, 39 * various events trigger data movement: 40 * 41 * - Availability of a new frame of data from the BufferQueue (notified 42 * via the onFrameAvailable callback). 43 * - The return of a codec buffer (via OnEmptyBufferDone). 44 * - Application signaling end-of-stream. 45 * - Transition to or from "executing" state. 46 * 47 * Frames of data (and, perhaps, the end-of-stream indication) can arrive 48 * before the codec is in the "executing" state, so we need to queue 49 * things up until we're ready to go. 50 */ 51class GraphicBufferSource : public BufferQueue::ConsumerListener { 52public: 53 GraphicBufferSource( 54 OMXNodeInstance* nodeInstance, 55 uint32_t bufferWidth, 56 uint32_t bufferHeight, 57 uint32_t bufferCount, 58 uint32_t consumerUsage, 59 const sp<IGraphicBufferConsumer> &consumer = NULL 60 ); 61 62 virtual ~GraphicBufferSource(); 63 64 // We can't throw an exception if the constructor fails, so we just set 65 // this and require that the caller test the value. 66 status_t initCheck() const { 67 return mInitCheck; 68 } 69 70 // Returns the handle to the producer side of the BufferQueue. Buffers 71 // queued on this will be received by GraphicBufferSource. 72 sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { 73 return mProducer; 74 } 75 76 // This is called when OMX transitions to OMX_StateExecuting, which means 77 // we can start handing it buffers. If we already have buffers of data 78 // sitting in the BufferQueue, this will send them to the codec. 79 void omxExecuting(); 80 81 // This is called when OMX transitions to OMX_StateIdle, indicating that 82 // the codec is meant to return all buffers back to the client for them 83 // to be freed. Do NOT submit any more buffers to the component. 84 void omxIdle(); 85 86 // This is called when OMX transitions to OMX_StateLoaded, indicating that 87 // we are shutting down. 88 void omxLoaded(); 89 90 // A "codec buffer", i.e. a buffer that can be used to pass data into 91 // the encoder, has been allocated. (This call does not call back into 92 // OMXNodeInstance.) 93 void addCodecBuffer(OMX_BUFFERHEADERTYPE* header); 94 95 // Called from OnEmptyBufferDone. If we have a BQ buffer available, 96 // fill it with a new frame of data; otherwise, just mark it as available. 97 void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header, int fenceFd); 98 99 // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the 100 // buffer source will fix timestamp in the header if needed.) 101 void codecBufferFilled(OMX_BUFFERHEADERTYPE* header); 102 103 // This is called after the last input frame has been submitted. We 104 // need to submit an empty buffer with the EOS flag set. If we don't 105 // have a codec buffer ready, we just set the mEndOfStream flag. 106 status_t signalEndOfInputStream(); 107 108 // If suspend is true, all incoming buffers (including those currently 109 // in the BufferQueue) will be discarded until the suspension is lifted. 110 void suspend(bool suspend); 111 112 // Specifies the interval after which we requeue the buffer previously 113 // queued to the encoder. This is useful in the case of surface flinger 114 // providing the input surface if the resulting encoded stream is to 115 // be displayed "live". If we were not to push through the extra frame 116 // the decoder on the remote end would be unable to decode the latest frame. 117 // This API must be called before transitioning the encoder to "executing" 118 // state and once this behaviour is specified it cannot be reset. 119 status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); 120 121 // When set, the timestamp fed to the encoder will be modified such that 122 // the gap between two adjacent frames is capped at maxGapUs. Timestamp 123 // will be restored to the original when the encoded frame is returned to 124 // the client. 125 // This is to solve a problem in certain real-time streaming case, where 126 // encoder's rate control logic produces huge frames after a long period 127 // of suspension on input. 128 status_t setMaxTimestampGapUs(int64_t maxGapUs); 129 130 // When set, the max frame rate fed to the encoder will be capped at maxFps. 131 status_t setMaxFps(float maxFps); 132 133 // Sets the time lapse (or slow motion) parameters. 134 // data[0] is the time (us) between two frames for playback 135 // data[1] is the time (us) between two frames for capture 136 // When set, the sample's timestamp will be modified to playback framerate, 137 // and capture timestamp will be modified to capture rate. 138 status_t setTimeLapseUs(int64_t* data); 139 140 // Sets the start time us (in system time), samples before which should 141 // be dropped and not submitted to encoder 142 void setSkipFramesBeforeUs(int64_t startTimeUs); 143 144protected: 145 // BufferQueue::ConsumerListener interface, called when a new frame of 146 // data is available. If we're executing and a codec buffer is 147 // available, we acquire the buffer, copy the GraphicBuffer reference 148 // into the codec buffer, and call Empty[This]Buffer. If we're not yet 149 // executing or there's no codec buffer available, we just increment 150 // mNumFramesAvailable and return. 151 virtual void onFrameAvailable(const BufferItem& item); 152 153 // BufferQueue::ConsumerListener interface, called when the client has 154 // released one or more GraphicBuffers. We clear out the appropriate 155 // set of mBufferSlot entries. 156 virtual void onBuffersReleased(); 157 158 // BufferQueue::ConsumerListener interface, called when the client has 159 // changed the sideband stream. GraphicBufferSource doesn't handle sideband 160 // streams so this is a no-op (and should never be called). 161 virtual void onSidebandStreamChanged(); 162 163private: 164 // PersistentProxyListener is similar to BufferQueue::ProxyConsumerListener 165 // except that it returns (acquire/detach/re-attache/release) buffers 166 // in onFrameAvailable() if the actual consumer object is no longer valid. 167 // 168 // This class is used in persistent input surface case to prevent buffer 169 // loss when onFrameAvailable() is received while we don't have a valid 170 // consumer around. 171 class PersistentProxyListener : public BnConsumerListener { 172 public: 173 PersistentProxyListener( 174 const wp<IGraphicBufferConsumer> &consumer, 175 const wp<ConsumerListener>& consumerListener); 176 virtual ~PersistentProxyListener(); 177 virtual void onFrameAvailable(const BufferItem& item) override; 178 virtual void onFrameReplaced(const BufferItem& item) override; 179 virtual void onBuffersReleased() override; 180 virtual void onSidebandStreamChanged() override; 181 private: 182 // mConsumerListener is a weak reference to the IConsumerListener. 183 wp<ConsumerListener> mConsumerListener; 184 // mConsumer is a weak reference to the IGraphicBufferConsumer, use 185 // a weak ref to avoid circular ref between mConsumer and this class 186 wp<IGraphicBufferConsumer> mConsumer; 187 }; 188 189 // Keep track of codec input buffers. They may either be available 190 // (mGraphicBuffer == NULL) or in use by the codec. 191 struct CodecBuffer { 192 OMX_BUFFERHEADERTYPE* mHeader; 193 194 // buffer producer's frame-number for buffer 195 uint64_t mFrameNumber; 196 197 // buffer producer's buffer slot for buffer 198 int mBuf; 199 200 sp<GraphicBuffer> mGraphicBuffer; 201 }; 202 203 // Returns the index of an available codec buffer. If none are 204 // available, returns -1. Mutex must be held by caller. 205 int findAvailableCodecBuffer_l(); 206 207 // Returns true if a codec buffer is available. 208 bool isCodecBufferAvailable_l() { 209 return findAvailableCodecBuffer_l() >= 0; 210 } 211 212 // Finds the mCodecBuffers entry that matches. Returns -1 if not found. 213 int findMatchingCodecBuffer_l(const OMX_BUFFERHEADERTYPE* header); 214 215 // Fills a codec buffer with a frame from the BufferQueue. This must 216 // only be called when we know that a frame of data is ready (i.e. we're 217 // in the onFrameAvailable callback, or if we're in codecBufferEmptied 218 // and mNumFramesAvailable is nonzero). Returns without doing anything if 219 // we don't have a codec buffer available. 220 // 221 // Returns true if we successfully filled a codec buffer with a BQ buffer. 222 bool fillCodecBuffer_l(); 223 224 // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer 225 // reference into the codec buffer, and submits the data to the codec. 226 status_t submitBuffer_l(const BufferItem &item, int cbi); 227 228 // Submits an empty buffer, with the EOS flag set. Returns without 229 // doing anything if we don't have a codec buffer available. 230 void submitEndOfInputStream_l(); 231 232 // Release buffer to the consumer 233 void releaseBuffer( 234 int &id, uint64_t frameNum, 235 const sp<GraphicBuffer> buffer, const sp<Fence> &fence); 236 237 void setLatestBuffer_l(const BufferItem &item, bool dropped); 238 bool repeatLatestBuffer_l(); 239 int64_t getTimestamp(const BufferItem &item); 240 241 // Lock, covers all member variables. 242 mutable Mutex mMutex; 243 244 // Used to report constructor failure. 245 status_t mInitCheck; 246 247 // Pointer back to the object that contains us. We send buffers here. 248 OMXNodeInstance* mNodeInstance; 249 250 // Set by omxExecuting() / omxIdling(). 251 bool mExecuting; 252 253 bool mSuspended; 254 255 // Our BufferQueue interfaces. mProducer is passed to the producer through 256 // getIGraphicBufferProducer, and mConsumer is used internally to retrieve 257 // the buffers queued by the producer. 258 bool mIsPersistent; 259 sp<IGraphicBufferProducer> mProducer; 260 sp<IGraphicBufferConsumer> mConsumer; 261 262 // Number of frames pending in BufferQueue that haven't yet been 263 // forwarded to the codec. 264 size_t mNumFramesAvailable; 265 266 // Number of frames acquired from consumer (debug only) 267 int32_t mNumBufferAcquired; 268 269 // Set to true if we want to send end-of-stream after we run out of 270 // frames in BufferQueue. 271 bool mEndOfStream; 272 bool mEndOfStreamSent; 273 274 // Cache of GraphicBuffers from the buffer queue. When the codec 275 // is done processing a GraphicBuffer, we can use this to map back 276 // to a slot number. 277 sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS]; 278 279 // Tracks codec buffers. 280 Vector<CodecBuffer> mCodecBuffers; 281 282 //// 283 friend struct AHandlerReflector<GraphicBufferSource>; 284 285 enum { 286 kWhatRepeatLastFrame, 287 }; 288 enum { 289 kRepeatLastFrameCount = 10, 290 }; 291 292 KeyedVector<int64_t, int64_t> mOriginalTimeUs; 293 int64_t mMaxTimestampGapUs; 294 int64_t mPrevOriginalTimeUs; 295 int64_t mPrevModifiedTimeUs; 296 int64_t mSkipFramesBeforeNs; 297 298 sp<FrameDropper> mFrameDropper; 299 300 sp<ALooper> mLooper; 301 sp<AHandlerReflector<GraphicBufferSource> > mReflector; 302 303 int64_t mRepeatAfterUs; 304 int32_t mRepeatLastFrameGeneration; 305 int64_t mRepeatLastFrameTimestamp; 306 int32_t mRepeatLastFrameCount; 307 308 int mLatestBufferId; 309 uint64_t mLatestBufferFrameNum; 310 int32_t mLatestBufferUseCount; 311 sp<Fence> mLatestBufferFence; 312 313 // The previous buffer should've been repeated but 314 // no codec buffer was available at the time. 315 bool mRepeatBufferDeferred; 316 317 // Time lapse / slow motion configuration 318 int64_t mTimePerCaptureUs; 319 int64_t mTimePerFrameUs; 320 int64_t mPrevCaptureUs; 321 int64_t mPrevFrameUs; 322 323 MetadataBufferType mMetadataBufferType; 324 325 void onMessageReceived(const sp<AMessage> &msg); 326 327 DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource); 328}; 329 330} // namespace android 331 332#endif // GRAPHIC_BUFFER_SOURCE_H_ 333