1/*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef MEDIA_CODEC_H_
18
19#define MEDIA_CODEC_H_
20
21#include <gui/IGraphicBufferProducer.h>
22#include <media/hardware/CryptoAPI.h>
23#include <media/MediaResource.h>
24#include <media/stagefright/foundation/AHandler.h>
25#include <media/stagefright/FrameRenderTracker.h>
26#include <utils/Vector.h>
27
28namespace android {
29
30struct ABuffer;
31struct AMessage;
32struct AReplyToken;
33struct AString;
34struct CodecBase;
35struct IBatteryStats;
36struct ICrypto;
37class IMemory;
38struct MemoryDealer;
39class IResourceManagerClient;
40class IResourceManagerService;
41struct PersistentSurface;
42struct SoftwareRenderer;
43struct Surface;
44
45struct MediaCodec : public AHandler {
46    enum ConfigureFlags {
47        CONFIGURE_FLAG_ENCODE   = 1,
48    };
49
50    enum BufferFlags {
51        BUFFER_FLAG_SYNCFRAME   = 1,
52        BUFFER_FLAG_CODECCONFIG = 2,
53        BUFFER_FLAG_EOS         = 4,
54    };
55
56    enum {
57        CB_INPUT_AVAILABLE = 1,
58        CB_OUTPUT_AVAILABLE = 2,
59        CB_ERROR = 3,
60        CB_OUTPUT_FORMAT_CHANGED = 4,
61        CB_RESOURCE_RECLAIMED = 5,
62    };
63
64    static const pid_t kNoPid = -1;
65
66    static sp<MediaCodec> CreateByType(
67            const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err = NULL,
68            pid_t pid = kNoPid);
69
70    static sp<MediaCodec> CreateByComponentName(
71            const sp<ALooper> &looper, const char *name, status_t *err = NULL,
72            pid_t pid = kNoPid);
73
74    static sp<PersistentSurface> CreatePersistentInputSurface();
75
76    status_t configure(
77            const sp<AMessage> &format,
78            const sp<Surface> &nativeWindow,
79            const sp<ICrypto> &crypto,
80            uint32_t flags);
81
82    status_t setCallback(const sp<AMessage> &callback);
83
84    status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
85
86    status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
87
88    status_t setInputSurface(const sp<PersistentSurface> &surface);
89
90    status_t start();
91
92    // Returns to a state in which the component remains allocated but
93    // unconfigured.
94    status_t stop();
95
96    // Resets the codec to the INITIALIZED state.  Can be called after an error
97    // has occured to make the codec usable.
98    status_t reset();
99
100    // Client MUST call release before releasing final reference to this
101    // object.
102    status_t release();
103
104    status_t flush();
105
106    status_t queueInputBuffer(
107            size_t index,
108            size_t offset,
109            size_t size,
110            int64_t presentationTimeUs,
111            uint32_t flags,
112            AString *errorDetailMsg = NULL);
113
114    status_t queueSecureInputBuffer(
115            size_t index,
116            size_t offset,
117            const CryptoPlugin::SubSample *subSamples,
118            size_t numSubSamples,
119            const uint8_t key[16],
120            const uint8_t iv[16],
121            CryptoPlugin::Mode mode,
122            int64_t presentationTimeUs,
123            uint32_t flags,
124            AString *errorDetailMsg = NULL);
125
126    status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
127
128    status_t dequeueOutputBuffer(
129            size_t *index,
130            size_t *offset,
131            size_t *size,
132            int64_t *presentationTimeUs,
133            uint32_t *flags,
134            int64_t timeoutUs = 0ll);
135
136    status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
137    status_t renderOutputBufferAndRelease(size_t index);
138    status_t releaseOutputBuffer(size_t index);
139
140    status_t signalEndOfInputStream();
141
142    status_t getOutputFormat(sp<AMessage> *format) const;
143    status_t getInputFormat(sp<AMessage> *format) const;
144
145    status_t getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const;
146
147    status_t getInputBuffers(Vector<sp<ABuffer> > *buffers) const;
148    status_t getOutputBuffers(Vector<sp<ABuffer> > *buffers) const;
149
150    status_t getOutputBuffer(size_t index, sp<ABuffer> *buffer);
151    status_t getOutputFormat(size_t index, sp<AMessage> *format);
152    status_t getInputBuffer(size_t index, sp<ABuffer> *buffer);
153
154    status_t setSurface(const sp<Surface> &nativeWindow);
155
156    status_t requestIDRFrame();
157
158    // Notification will be posted once there "is something to do", i.e.
159    // an input/output buffer has become available, a format change is
160    // pending, an error is pending.
161    void requestActivityNotification(const sp<AMessage> &notify);
162
163    status_t getName(AString *componentName) const;
164
165    status_t setParameters(const sp<AMessage> &params);
166
167    // Create a MediaCodec notification message from a list of rendered or dropped render infos
168    // by adding rendered frame information to a base notification message. Returns the number
169    // of frames that were rendered.
170    static size_t CreateFramesRenderedMessage(
171            std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);
172
173protected:
174    virtual ~MediaCodec();
175    virtual void onMessageReceived(const sp<AMessage> &msg);
176
177private:
178    // used by ResourceManagerClient
179    status_t reclaim();
180    friend struct ResourceManagerClient;
181
182private:
183    enum State {
184        UNINITIALIZED,
185        INITIALIZING,
186        INITIALIZED,
187        CONFIGURING,
188        CONFIGURED,
189        STARTING,
190        STARTED,
191        FLUSHING,
192        FLUSHED,
193        STOPPING,
194        RELEASING,
195    };
196
197    enum {
198        kPortIndexInput         = 0,
199        kPortIndexOutput        = 1,
200    };
201
202    enum {
203        kWhatInit                           = 'init',
204        kWhatConfigure                      = 'conf',
205        kWhatSetSurface                     = 'sSur',
206        kWhatCreateInputSurface             = 'cisf',
207        kWhatSetInputSurface                = 'sisf',
208        kWhatStart                          = 'strt',
209        kWhatStop                           = 'stop',
210        kWhatRelease                        = 'rele',
211        kWhatDequeueInputBuffer             = 'deqI',
212        kWhatQueueInputBuffer               = 'queI',
213        kWhatDequeueOutputBuffer            = 'deqO',
214        kWhatReleaseOutputBuffer            = 'relO',
215        kWhatSignalEndOfInputStream         = 'eois',
216        kWhatGetBuffers                     = 'getB',
217        kWhatFlush                          = 'flus',
218        kWhatGetOutputFormat                = 'getO',
219        kWhatGetInputFormat                 = 'getI',
220        kWhatDequeueInputTimedOut           = 'dITO',
221        kWhatDequeueOutputTimedOut          = 'dOTO',
222        kWhatCodecNotify                    = 'codc',
223        kWhatRequestIDRFrame                = 'ridr',
224        kWhatRequestActivityNotification    = 'racN',
225        kWhatGetName                        = 'getN',
226        kWhatSetParameters                  = 'setP',
227        kWhatSetCallback                    = 'setC',
228        kWhatSetNotification                = 'setN',
229    };
230
231    enum {
232        kFlagUsesSoftwareRenderer       = 1,
233        kFlagOutputFormatChanged        = 2,
234        kFlagOutputBuffersChanged       = 4,
235        kFlagStickyError                = 8,
236        kFlagDequeueInputPending        = 16,
237        kFlagDequeueOutputPending       = 32,
238        kFlagIsSecure                   = 64,
239        kFlagSawMediaServerDie          = 128,
240        kFlagIsEncoder                  = 256,
241        kFlagGatherCodecSpecificData    = 512,
242        kFlagIsAsync                    = 1024,
243        kFlagIsComponentAllocated       = 2048,
244        kFlagPushBlankBuffersOnShutdown = 4096,
245    };
246
247    struct BufferInfo {
248        uint32_t mBufferID;
249        sp<ABuffer> mData;
250        sp<ABuffer> mEncryptedData;
251        sp<IMemory> mSharedEncryptedBuffer;
252        sp<AMessage> mNotify;
253        sp<AMessage> mFormat;
254        bool mOwnedByClient;
255    };
256
257    struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
258        ResourceManagerServiceProxy(pid_t pid);
259        ~ResourceManagerServiceProxy();
260
261        void init();
262
263        // implements DeathRecipient
264        virtual void binderDied(const wp<IBinder>& /*who*/);
265
266        void addResource(
267                int64_t clientId,
268                const sp<IResourceManagerClient> client,
269                const Vector<MediaResource> &resources);
270
271        void removeResource(int64_t clientId);
272
273        bool reclaimResource(const Vector<MediaResource> &resources);
274
275    private:
276        Mutex mLock;
277        sp<IResourceManagerService> mService;
278        pid_t mPid;
279    };
280
281    State mState;
282    bool mReleasedByResourceManager;
283    sp<ALooper> mLooper;
284    sp<ALooper> mCodecLooper;
285    sp<CodecBase> mCodec;
286    AString mComponentName;
287    sp<AReplyToken> mReplyID;
288    uint32_t mFlags;
289    status_t mStickyError;
290    sp<Surface> mSurface;
291    SoftwareRenderer *mSoftRenderer;
292
293    sp<AMessage> mOutputFormat;
294    sp<AMessage> mInputFormat;
295    sp<AMessage> mCallback;
296    sp<AMessage> mOnFrameRenderedNotification;
297    sp<MemoryDealer> mDealer;
298
299    sp<IResourceManagerClient> mResourceManagerClient;
300    sp<ResourceManagerServiceProxy> mResourceManagerService;
301
302    bool mBatteryStatNotified;
303    bool mIsVideo;
304    int32_t mVideoWidth;
305    int32_t mVideoHeight;
306    int32_t mRotationDegrees;
307
308    // initial create parameters
309    AString mInitName;
310    bool mInitNameIsType;
311    bool mInitIsEncoder;
312
313    // configure parameter
314    sp<AMessage> mConfigureMsg;
315
316    // Used only to synchronize asynchronous getBufferAndFormat
317    // across all the other (synchronous) buffer state change
318    // operations, such as de/queueIn/OutputBuffer, start and
319    // stop/flush/reset/release.
320    Mutex mBufferLock;
321
322    List<size_t> mAvailPortBuffers[2];
323    Vector<BufferInfo> mPortBuffers[2];
324
325    int32_t mDequeueInputTimeoutGeneration;
326    sp<AReplyToken> mDequeueInputReplyID;
327
328    int32_t mDequeueOutputTimeoutGeneration;
329    sp<AReplyToken> mDequeueOutputReplyID;
330
331    sp<ICrypto> mCrypto;
332
333    List<sp<ABuffer> > mCSD;
334
335    sp<AMessage> mActivityNotify;
336
337    bool mHaveInputSurface;
338    bool mHavePendingInputBuffers;
339
340    MediaCodec(const sp<ALooper> &looper, pid_t pid);
341
342    static status_t PostAndAwaitResponse(
343            const sp<AMessage> &msg, sp<AMessage> *response);
344
345    void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
346
347    status_t init(const AString &name, bool nameIsType, bool encoder);
348
349    void setState(State newState);
350    void returnBuffersToCodec();
351    void returnBuffersToCodecOnPort(int32_t portIndex);
352    size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
353    status_t onQueueInputBuffer(const sp<AMessage> &msg);
354    status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
355    ssize_t dequeuePortBuffer(int32_t portIndex);
356
357    status_t getBufferAndFormat(
358            size_t portIndex, size_t index,
359            sp<ABuffer> *buffer, sp<AMessage> *format);
360
361    bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
362    bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
363    void cancelPendingDequeueOperations();
364
365    void extractCSD(const sp<AMessage> &format);
366    status_t queueCSDInputBuffer(size_t bufferIndex);
367
368    status_t handleSetSurface(const sp<Surface> &surface);
369    status_t connectToSurface(const sp<Surface> &surface);
370    status_t disconnectFromSurface();
371
372    void postActivityNotificationIfPossible();
373
374    void onInputBufferAvailable();
375    void onOutputBufferAvailable();
376    void onError(status_t err, int32_t actionCode, const char *detail = NULL);
377    void onOutputFormatChanged();
378
379    status_t onSetParameters(const sp<AMessage> &params);
380
381    status_t amendOutputFormatWithCodecSpecificData(const sp<ABuffer> &buffer);
382    void updateBatteryStat();
383    bool isExecuting() const;
384
385    uint64_t getGraphicBufferSize();
386    void addResource(const String8 &type, const String8 &subtype, uint64_t value);
387
388    /* called to get the last codec error when the sticky flag is set.
389     * if no such codec error is found, returns UNKNOWN_ERROR.
390     */
391    inline status_t getStickyError() const {
392        return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
393    }
394
395    inline void setStickyError(status_t err) {
396        mFlags |= kFlagStickyError;
397        mStickyError = err;
398    }
399
400    DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
401};
402
403}  // namespace android
404
405#endif  // MEDIA_CODEC_H_
406