1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef MEDIA_CODEC_H_
18
19#define MEDIA_CODEC_H_
20
21#include <memory>
22#include <vector>
23
24#include <gui/IGraphicBufferProducer.h>
25#include <media/hardware/CryptoAPI.h>
26#include <media/MediaCodecInfo.h>
27#include <media/MediaResource.h>
28#include <media/MediaAnalyticsItem.h>
29#include <media/stagefright/foundation/AHandler.h>
30#include <media/stagefright/FrameRenderTracker.h>
31#include <utils/Vector.h>
32
33namespace android {
34
35struct ABuffer;
36struct AMessage;
37struct AReplyToken;
38struct AString;
39class BufferChannelBase;
40struct CodecBase;
41class IBatteryStats;
42struct ICrypto;
43class MediaCodecBuffer;
44class IMemory;
45class IResourceManagerClient;
46class IResourceManagerService;
47struct PersistentSurface;
48class SoftwareRenderer;
49class Surface;
50namespace hardware {
51namespace cas {
52namespace native {
53namespace V1_0 {
54struct IDescrambler;
55}}}}
56using hardware::cas::native::V1_0::IDescrambler;
57
58struct MediaCodec : public AHandler {
59    enum ConfigureFlags {
60        CONFIGURE_FLAG_ENCODE   = 1,
61    };
62
63    enum BufferFlags {
64        BUFFER_FLAG_SYNCFRAME     = 1,
65        BUFFER_FLAG_CODECCONFIG   = 2,
66        BUFFER_FLAG_EOS           = 4,
67        BUFFER_FLAG_PARTIAL_FRAME = 8,
68        BUFFER_FLAG_MUXER_DATA    = 16,
69    };
70
71    enum {
72        CB_INPUT_AVAILABLE = 1,
73        CB_OUTPUT_AVAILABLE = 2,
74        CB_ERROR = 3,
75        CB_OUTPUT_FORMAT_CHANGED = 4,
76        CB_RESOURCE_RECLAIMED = 5,
77    };
78
79    static const pid_t kNoPid = -1;
80    static const uid_t kNoUid = -1;
81
82    static sp<MediaCodec> CreateByType(
83            const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
84            pid_t pid = kNoPid, uid_t uid = kNoUid);
85
86    static sp<MediaCodec> CreateByComponentName(
87            const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
88            pid_t pid = kNoPid, uid_t uid = kNoUid);
89
90    static sp<PersistentSurface> CreatePersistentInputSurface();
91
92    status_t configure(
93            const sp<AMessage> &format,
94            const sp<Surface> &nativeWindow,
95            const sp<ICrypto> &crypto,
96            uint32_t flags);
97
98    status_t configure(
99            const sp<AMessage> &format,
100            const sp<Surface> &nativeWindow,
101            const sp<ICrypto> &crypto,
102            const sp<IDescrambler> &descrambler,
103            uint32_t flags);
104
105    status_t releaseCrypto();
106
107    status_t setCallback(const sp<AMessage> &callback);
108
109    status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
110
111    status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
112
113    status_t setInputSurface(const sp<PersistentSurface> &surface);
114
115    status_t start();
116
117    // Returns to a state in which the component remains allocated but
118    // unconfigured.
119    status_t stop();
120
121    // Resets the codec to the INITIALIZED state.  Can be called after an error
122    // has occured to make the codec usable.
123    status_t reset();
124
125    // Client MUST call release before releasing final reference to this
126    // object.
127    status_t release();
128
129    status_t flush();
130
131    status_t queueInputBuffer(
132            size_t index,
133            size_t offset,
134            size_t size,
135            int64_t presentationTimeUs,
136            uint32_t flags,
137            AString *errorDetailMsg = NULL);
138
139    status_t queueSecureInputBuffer(
140            size_t index,
141            size_t offset,
142            const CryptoPlugin::SubSample *subSamples,
143            size_t numSubSamples,
144            const uint8_t key[16],
145            const uint8_t iv[16],
146            CryptoPlugin::Mode mode,
147            const CryptoPlugin::Pattern &pattern,
148            int64_t presentationTimeUs,
149            uint32_t flags,
150            AString *errorDetailMsg = NULL);
151
152    status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
153
154    status_t dequeueOutputBuffer(
155            size_t *index,
156            size_t *offset,
157            size_t *size,
158            int64_t *presentationTimeUs,
159            uint32_t *flags,
160            int64_t timeoutUs = 0ll);
161
162    status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
163    status_t renderOutputBufferAndRelease(size_t index);
164    status_t releaseOutputBuffer(size_t index);
165
166    status_t signalEndOfInputStream();
167
168    status_t getOutputFormat(sp<AMessage> *format) const;
169    status_t getInputFormat(sp<AMessage> *format) const;
170
171    status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
172    status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
173
174    status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
175    status_t getOutputFormat(size_t index, sp<AMessage> *format);
176    status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
177
178    status_t setSurface(const sp<Surface> &nativeWindow);
179
180    status_t requestIDRFrame();
181
182    // Notification will be posted once there "is something to do", i.e.
183    // an input/output buffer has become available, a format change is
184    // pending, an error is pending.
185    void requestActivityNotification(const sp<AMessage> &notify);
186
187    status_t getName(AString *componentName) const;
188
189    status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const;
190
191    status_t getMetrics(MediaAnalyticsItem * &reply);
192
193    status_t setParameters(const sp<AMessage> &params);
194
195    // Create a MediaCodec notification message from a list of rendered or dropped render infos
196    // by adding rendered frame information to a base notification message. Returns the number
197    // of frames that were rendered.
198    static size_t CreateFramesRenderedMessage(
199            const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
200
201protected:
202    virtual ~MediaCodec();
203    virtual void onMessageReceived(const sp<AMessage> &msg);
204
205private:
206    // used by ResourceManagerClient
207    status_t reclaim(bool force = false);
208    friend struct ResourceManagerClient;
209
210private:
211    enum State {
212        UNINITIALIZED,
213        INITIALIZING,
214        INITIALIZED,
215        CONFIGURING,
216        CONFIGURED,
217        STARTING,
218        STARTED,
219        FLUSHING,
220        FLUSHED,
221        STOPPING,
222        RELEASING,
223    };
224    std::string stateString(State state);
225
226    enum {
227        kPortIndexInput         = 0,
228        kPortIndexOutput        = 1,
229    };
230
231    enum {
232        kWhatInit                           = 'init',
233        kWhatConfigure                      = 'conf',
234        kWhatSetSurface                     = 'sSur',
235        kWhatCreateInputSurface             = 'cisf',
236        kWhatSetInputSurface                = 'sisf',
237        kWhatStart                          = 'strt',
238        kWhatStop                           = 'stop',
239        kWhatRelease                        = 'rele',
240        kWhatDequeueInputBuffer             = 'deqI',
241        kWhatQueueInputBuffer               = 'queI',
242        kWhatDequeueOutputBuffer            = 'deqO',
243        kWhatReleaseOutputBuffer            = 'relO',
244        kWhatSignalEndOfInputStream         = 'eois',
245        kWhatGetBuffers                     = 'getB',
246        kWhatFlush                          = 'flus',
247        kWhatGetOutputFormat                = 'getO',
248        kWhatGetInputFormat                 = 'getI',
249        kWhatDequeueInputTimedOut           = 'dITO',
250        kWhatDequeueOutputTimedOut          = 'dOTO',
251        kWhatCodecNotify                    = 'codc',
252        kWhatRequestIDRFrame                = 'ridr',
253        kWhatRequestActivityNotification    = 'racN',
254        kWhatGetName                        = 'getN',
255        kWhatGetCodecInfo                   = 'gCoI',
256        kWhatSetParameters                  = 'setP',
257        kWhatSetCallback                    = 'setC',
258        kWhatSetNotification                = 'setN',
259        kWhatDrmReleaseCrypto               = 'rDrm',
260    };
261
262    enum {
263        kFlagUsesSoftwareRenderer       = 1,
264        kFlagOutputFormatChanged        = 2,
265        kFlagOutputBuffersChanged       = 4,
266        kFlagStickyError                = 8,
267        kFlagDequeueInputPending        = 16,
268        kFlagDequeueOutputPending       = 32,
269        kFlagIsSecure                   = 64,
270        kFlagSawMediaServerDie          = 128,
271        kFlagIsEncoder                  = 256,
272        // 512 skipped
273        kFlagIsAsync                    = 1024,
274        kFlagIsComponentAllocated       = 2048,
275        kFlagPushBlankBuffersOnShutdown = 4096,
276    };
277
278    struct BufferInfo {
279        BufferInfo();
280
281        sp<MediaCodecBuffer> mData;
282        bool mOwnedByClient;
283    };
284
285    struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
286        ResourceManagerServiceProxy(pid_t pid);
287        ~ResourceManagerServiceProxy();
288
289        void init();
290
291        // implements DeathRecipient
292        virtual void binderDied(const wp<IBinder>& /*who*/);
293
294        void addResource(
295                int64_t clientId,
296                const sp<IResourceManagerClient> &client,
297                const Vector<MediaResource> &resources);
298
299        void removeResource(int64_t clientId);
300
301        bool reclaimResource(const Vector<MediaResource> &resources);
302
303    private:
304        Mutex mLock;
305        sp<IResourceManagerService> mService;
306        pid_t mPid;
307    };
308
309    State mState;
310    uid_t mUid;
311    bool mReleasedByResourceManager;
312    sp<ALooper> mLooper;
313    sp<ALooper> mCodecLooper;
314    sp<CodecBase> mCodec;
315    AString mComponentName;
316    sp<MediaCodecInfo> mCodecInfo;
317    sp<AReplyToken> mReplyID;
318    uint32_t mFlags;
319    status_t mStickyError;
320    sp<Surface> mSurface;
321    SoftwareRenderer *mSoftRenderer;
322
323    MediaAnalyticsItem *mAnalyticsItem;
324    void initAnalyticsItem();
325    void updateAnalyticsItem();
326    void flushAnalyticsItem();
327    void updateEphemeralAnalytics(MediaAnalyticsItem *item);
328
329    sp<AMessage> mOutputFormat;
330    sp<AMessage> mInputFormat;
331    sp<AMessage> mCallback;
332    sp<AMessage> mOnFrameRenderedNotification;
333
334    sp<IResourceManagerClient> mResourceManagerClient;
335    sp<ResourceManagerServiceProxy> mResourceManagerService;
336
337    bool mBatteryStatNotified;
338    bool mIsVideo;
339    int32_t mVideoWidth;
340    int32_t mVideoHeight;
341    int32_t mRotationDegrees;
342
343    // initial create parameters
344    AString mInitName;
345
346    // configure parameter
347    sp<AMessage> mConfigureMsg;
348
349    // Used only to synchronize asynchronous getBufferAndFormat
350    // across all the other (synchronous) buffer state change
351    // operations, such as de/queueIn/OutputBuffer, start and
352    // stop/flush/reset/release.
353    Mutex mBufferLock;
354
355    List<size_t> mAvailPortBuffers[2];
356    std::vector<BufferInfo> mPortBuffers[2];
357
358    int32_t mDequeueInputTimeoutGeneration;
359    sp<AReplyToken> mDequeueInputReplyID;
360
361    int32_t mDequeueOutputTimeoutGeneration;
362    sp<AReplyToken> mDequeueOutputReplyID;
363
364    sp<ICrypto> mCrypto;
365
366    sp<IDescrambler> mDescrambler;
367
368    List<sp<ABuffer> > mCSD;
369
370    sp<AMessage> mActivityNotify;
371
372    bool mHaveInputSurface;
373    bool mHavePendingInputBuffers;
374    bool mCpuBoostRequested;
375
376    std::shared_ptr<BufferChannelBase> mBufferChannel;
377
378    MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid);
379
380    static sp<CodecBase> GetCodecBase(const AString &name);
381
382    static status_t PostAndAwaitResponse(
383            const sp<AMessage> &msg, sp<AMessage> *response);
384
385    void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
386
387    status_t init(const AString &name);
388
389    void setState(State newState);
390    void returnBuffersToCodec(bool isReclaim = false);
391    void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
392    size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
393    status_t onQueueInputBuffer(const sp<AMessage> &msg);
394    status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
395    ssize_t dequeuePortBuffer(int32_t portIndex);
396
397    status_t getBufferAndFormat(
398            size_t portIndex, size_t index,
399            sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
400
401    bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
402    bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
403    void cancelPendingDequeueOperations();
404
405    void extractCSD(const sp<AMessage> &format);
406    status_t queueCSDInputBuffer(size_t bufferIndex);
407
408    status_t handleSetSurface(const sp<Surface> &surface);
409    status_t connectToSurface(const sp<Surface> &surface);
410    status_t disconnectFromSurface();
411
412    bool hasCryptoOrDescrambler() {
413        return mCrypto != NULL || mDescrambler != NULL;
414    }
415
416    void postActivityNotificationIfPossible();
417
418    void onInputBufferAvailable();
419    void onOutputBufferAvailable();
420    void onError(status_t err, int32_t actionCode, const char *detail = NULL);
421    void onOutputFormatChanged();
422
423    status_t onSetParameters(const sp<AMessage> &params);
424
425    status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
426    void updateBatteryStat();
427    bool isExecuting() const;
428
429    uint64_t getGraphicBufferSize();
430    void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
431    void requestCpuBoostIfNeeded();
432
433    bool hasPendingBuffer(int portIndex);
434    bool hasPendingBuffer();
435
436    /* called to get the last codec error when the sticky flag is set.
437     * if no such codec error is found, returns UNKNOWN_ERROR.
438     */
439    inline status_t getStickyError() const {
440        return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
441    }
442
443    inline void setStickyError(status_t err) {
444        mFlags |= kFlagStickyError;
445        mStickyError = err;
446    }
447
448    void onReleaseCrypto(const sp<AMessage>& msg);
449
450    // managing time-of-flight aka latency
451    typedef struct {
452            int64_t presentationUs;
453            int64_t startedNs;
454    } BufferFlightTiming_t;
455    std::deque<BufferFlightTiming_t> mBuffersInFlight;
456    Mutex mLatencyLock;
457    int64_t mLatencyUnknown;    // buffers for which we couldn't calculate latency
458
459    void statsBufferSent(int64_t presentationUs);
460    void statsBufferReceived(int64_t presentationUs);
461
462    enum {
463        // the default shape of our latency histogram buckets
464        // XXX: should these be configurable in some way?
465        kLatencyHistBuckets = 20,
466        kLatencyHistWidth = 2000,
467        kLatencyHistFloor = 2000,
468
469        // how many samples are in the 'recent latency' histogram
470        // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps
471        kRecentLatencyFrames = 300,
472
473        // how we initialize mRecentSamples
474        kRecentSampleInvalid = -1,
475    };
476
477    int64_t mRecentSamples[kRecentLatencyFrames];
478    int mRecentHead;
479    Mutex mRecentLock;
480
481    class Histogram {
482      public:
483        Histogram() : mFloor(0), mWidth(0), mBelow(0), mAbove(0),
484                      mMin(INT64_MAX), mMax(INT64_MIN), mSum(0), mCount(0),
485                      mBucketCount(0), mBuckets(NULL) {};
486        ~Histogram() { clear(); };
487        void clear() { if (mBuckets != NULL) free(mBuckets); mBuckets = NULL; };
488        bool setup(int nbuckets, int64_t width, int64_t floor = 0);
489        void insert(int64_t sample);
490        int64_t getMin() const { return mMin; }
491        int64_t getMax() const { return mMax; }
492        int64_t getCount() const { return mCount; }
493        int64_t getSum() const { return mSum; }
494        int64_t getAvg() const { return mSum / (mCount == 0 ? 1 : mCount); }
495        std::string emit();
496      private:
497        int64_t mFloor, mCeiling, mWidth;
498        int64_t mBelow, mAbove;
499        int64_t mMin, mMax, mSum, mCount;
500
501        int mBucketCount;
502        int64_t *mBuckets;
503    };
504
505    Histogram mLatencyHist;
506
507    DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
508};
509
510}  // namespace android
511
512#endif  // MEDIA_CODEC_H_
513