1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef A_CODEC_H_
18
19#define A_CODEC_H_
20
21#include <stdint.h>
22#include <android/native_window.h>
23#include <media/hardware/MetadataBufferType.h>
24#include <media/IOMX.h>
25#include <media/stagefright/foundation/AHierarchicalStateMachine.h>
26#include <media/stagefright/CodecBase.h>
27#include <media/stagefright/FrameRenderTracker.h>
28#include <media/stagefright/MediaDefs.h>
29#include <media/stagefright/SkipCutBuffer.h>
30#include <utils/NativeHandle.h>
31#include <OMX_Audio.h>
32
33#define TRACK_BUFFER_TIMING     0
34
35namespace android {
36
37struct ABuffer;
38struct MemoryDealer;
39struct DescribeColorFormat2Params;
40struct DataConverter;
41
42struct ACodec : public AHierarchicalStateMachine, public CodecBase {
43    ACodec();
44
45    virtual void setNotificationMessage(const sp<AMessage> &msg);
46
47    void initiateSetup(const sp<AMessage> &msg);
48
49    virtual void initiateAllocateComponent(const sp<AMessage> &msg);
50    virtual void initiateConfigureComponent(const sp<AMessage> &msg);
51    virtual void initiateCreateInputSurface();
52    virtual void initiateSetInputSurface(const sp<PersistentSurface> &surface);
53    virtual void initiateStart();
54    virtual void initiateShutdown(bool keepComponentAllocated = false);
55
56    virtual status_t queryCapabilities(
57            const AString &name, const AString &mime, bool isEncoder,
58            sp<MediaCodecInfo::Capabilities> *caps);
59
60    virtual status_t setSurface(const sp<Surface> &surface);
61
62    virtual void signalFlush();
63    virtual void signalResume();
64
65    virtual void signalSetParameters(const sp<AMessage> &msg);
66    virtual void signalEndOfInputStream();
67    virtual void signalRequestIDRFrame();
68
69    // AHierarchicalStateMachine implements the message handling
70    virtual void onMessageReceived(const sp<AMessage> &msg) {
71        handleMessage(msg);
72    }
73
74    struct PortDescription : public CodecBase::PortDescription {
75        size_t countBuffers();
76        IOMX::buffer_id bufferIDAt(size_t index) const;
77        sp<ABuffer> bufferAt(size_t index) const;
78        sp<NativeHandle> handleAt(size_t index) const;
79        sp<RefBase> memRefAt(size_t index) const;
80
81    private:
82        friend struct ACodec;
83
84        Vector<IOMX::buffer_id> mBufferIDs;
85        Vector<sp<ABuffer> > mBuffers;
86        Vector<sp<NativeHandle> > mHandles;
87        Vector<sp<RefBase> > mMemRefs;
88
89        PortDescription();
90        void addBuffer(
91                IOMX::buffer_id id, const sp<ABuffer> &buffer,
92                const sp<NativeHandle> &handle, const sp<RefBase> &memRef);
93
94        DISALLOW_EVIL_CONSTRUCTORS(PortDescription);
95    };
96
97    static bool isFlexibleColorFormat(
98            const sp<IOMX> &omx, IOMX::node_id node,
99            uint32_t colorFormat, bool usingNativeBuffers, OMX_U32 *flexibleEquivalent);
100
101    // Returns 0 if configuration is not supported.  NOTE: this is treated by
102    // some OMX components as auto level, and by others as invalid level.
103    static int /* OMX_VIDEO_AVCLEVELTYPE */ getAVCLevelFor(
104            int width, int height, int rate, int bitrate,
105            OMX_VIDEO_AVCPROFILETYPE profile = OMX_VIDEO_AVCProfileBaseline);
106
107    // Quirk still supported, even though deprecated
108    enum Quirks {
109        kRequiresAllocateBufferOnInputPorts   = 1,
110        kRequiresAllocateBufferOnOutputPorts  = 2,
111    };
112
113    static status_t getOMXChannelMapping(size_t numChannels, OMX_AUDIO_CHANNELTYPE map[]);
114
115protected:
116    virtual ~ACodec();
117
118private:
119    struct BaseState;
120    struct UninitializedState;
121    struct LoadedState;
122    struct LoadedToIdleState;
123    struct IdleToExecutingState;
124    struct ExecutingState;
125    struct OutputPortSettingsChangedState;
126    struct ExecutingToIdleState;
127    struct IdleToLoadedState;
128    struct FlushingState;
129    struct DeathNotifier;
130
131    enum {
132        kWhatSetup                   = 'setu',
133        kWhatOMXMessage              = 'omx ',
134        // same as kWhatOMXMessage - but only used with
135        // handleMessage during OMX message-list handling
136        kWhatOMXMessageItem          = 'omxI',
137        kWhatOMXMessageList          = 'omxL',
138        kWhatInputBufferFilled       = 'inpF',
139        kWhatOutputBufferDrained     = 'outD',
140        kWhatShutdown                = 'shut',
141        kWhatFlush                   = 'flus',
142        kWhatResume                  = 'resm',
143        kWhatDrainDeferredMessages   = 'drai',
144        kWhatAllocateComponent       = 'allo',
145        kWhatConfigureComponent      = 'conf',
146        kWhatSetSurface              = 'setS',
147        kWhatCreateInputSurface      = 'cisf',
148        kWhatSetInputSurface         = 'sisf',
149        kWhatSignalEndOfInputStream  = 'eois',
150        kWhatStart                   = 'star',
151        kWhatRequestIDRFrame         = 'ridr',
152        kWhatSetParameters           = 'setP',
153        kWhatSubmitOutputMetadataBufferIfEOS = 'subm',
154        kWhatOMXDied                 = 'OMXd',
155        kWhatReleaseCodecInstance    = 'relC',
156    };
157
158    enum {
159        kPortIndexInput  = 0,
160        kPortIndexOutput = 1
161    };
162
163    enum {
164        kFlagIsSecure                                 = 1,
165        kFlagPushBlankBuffersToNativeWindowOnShutdown = 2,
166        kFlagIsGrallocUsageProtected                  = 4,
167    };
168
169    enum {
170        kVideoGrallocUsage = (GRALLOC_USAGE_HW_TEXTURE
171                            | GRALLOC_USAGE_HW_COMPOSER
172                            | GRALLOC_USAGE_EXTERNAL_DISP),
173    };
174
175    struct BufferInfo {
176        enum Status {
177            OWNED_BY_US,
178            OWNED_BY_COMPONENT,
179            OWNED_BY_UPSTREAM,
180            OWNED_BY_DOWNSTREAM,
181            OWNED_BY_NATIVE_WINDOW,
182            UNRECOGNIZED,            // not a tracked buffer
183        };
184
185        static inline Status getSafeStatus(BufferInfo *info) {
186            return info == NULL ? UNRECOGNIZED : info->mStatus;
187        }
188
189        IOMX::buffer_id mBufferID;
190        Status mStatus;
191        unsigned mDequeuedAt;
192
193        sp<ABuffer> mData;      // the client's buffer; if not using data conversion, this is the
194                                // codec buffer; otherwise, it is allocated separately
195        sp<RefBase> mMemRef;    // and a reference to the IMemory, so it does not go away
196        sp<ABuffer> mCodecData; // the codec's buffer
197        sp<RefBase> mCodecRef;  // and a reference to the IMemory
198        sp<GraphicBuffer> mGraphicBuffer;
199        sp<NativeHandle> mNativeHandle;
200        int mFenceFd;
201        FrameRenderTracker::Info *mRenderInfo;
202
203        // The following field and 4 methods are used for debugging only
204        bool mIsReadFence;
205        // Store |fenceFd| and set read/write flag. Log error, if there is already a fence stored.
206        void setReadFence(int fenceFd, const char *dbg);
207        void setWriteFence(int fenceFd, const char *dbg);
208        // Log error, if the current fence is not a read/write fence.
209        void checkReadFence(const char *dbg);
210        void checkWriteFence(const char *dbg);
211    };
212
213    static const char *_asString(BufferInfo::Status s);
214    void dumpBuffers(OMX_U32 portIndex);
215
216    // If |fd| is non-negative, waits for fence with |fd| and logs an error if it fails. Returns
217    // the error code or OK on success. If |fd| is negative, it returns OK
218    status_t waitForFence(int fd, const char *dbg);
219
220#if TRACK_BUFFER_TIMING
221    struct BufferStats {
222        int64_t mEmptyBufferTimeUs;
223        int64_t mFillBufferDoneTimeUs;
224    };
225
226    KeyedVector<int64_t, BufferStats> mBufferStats;
227#endif
228
229    sp<AMessage> mNotify;
230
231    sp<UninitializedState> mUninitializedState;
232    sp<LoadedState> mLoadedState;
233    sp<LoadedToIdleState> mLoadedToIdleState;
234    sp<IdleToExecutingState> mIdleToExecutingState;
235    sp<ExecutingState> mExecutingState;
236    sp<OutputPortSettingsChangedState> mOutputPortSettingsChangedState;
237    sp<ExecutingToIdleState> mExecutingToIdleState;
238    sp<IdleToLoadedState> mIdleToLoadedState;
239    sp<FlushingState> mFlushingState;
240    sp<SkipCutBuffer> mSkipCutBuffer;
241
242    AString mComponentName;
243    uint32_t mFlags;
244    uint32_t mQuirks;
245    sp<IOMX> mOMX;
246    sp<IBinder> mNodeBinder;
247    IOMX::node_id mNode;
248    sp<MemoryDealer> mDealer[2];
249
250    bool mUsingNativeWindow;
251    sp<ANativeWindow> mNativeWindow;
252    int mNativeWindowUsageBits;
253    android_native_rect_t mLastNativeWindowCrop;
254    int32_t mLastNativeWindowDataSpace;
255    sp<AMessage> mConfigFormat;
256    sp<AMessage> mInputFormat;
257    sp<AMessage> mOutputFormat;
258
259    // Initial output format + configuration params that is reused as the base for all subsequent
260    // format updates. This will equal to mOutputFormat until the first actual frame is received.
261    sp<AMessage> mBaseOutputFormat;
262
263    FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec
264    Vector<BufferInfo> mBuffers[2];
265    bool mPortEOS[2];
266    status_t mInputEOSResult;
267
268    List<sp<AMessage> > mDeferredQueue;
269
270    sp<AMessage> mLastOutputFormat;
271    bool mIsVideo;
272    bool mIsEncoder;
273    bool mFatalError;
274    bool mShutdownInProgress;
275    bool mExplicitShutdown;
276    bool mIsLegacyVP9Decoder;
277
278    // If "mKeepComponentAllocated" we only transition back to Loaded state
279    // and do not release the component instance.
280    bool mKeepComponentAllocated;
281
282    int32_t mEncoderDelay;
283    int32_t mEncoderPadding;
284    int32_t mRotationDegrees;
285
286    bool mChannelMaskPresent;
287    int32_t mChannelMask;
288    unsigned mDequeueCounter;
289    MetadataBufferType mInputMetadataType;
290    MetadataBufferType mOutputMetadataType;
291    bool mLegacyAdaptiveExperiment;
292    int32_t mMetadataBuffersToSubmit;
293    size_t mNumUndequeuedBuffers;
294    sp<DataConverter> mConverter[2];
295
296    int64_t mRepeatFrameDelayUs;
297    int64_t mMaxPtsGapUs;
298    float mMaxFps;
299
300    int64_t mTimePerFrameUs;
301    int64_t mTimePerCaptureUs;
302
303    bool mCreateInputBuffersSuspended;
304
305    bool mTunneled;
306
307    OMX_INDEXTYPE mDescribeColorAspectsIndex;
308    OMX_INDEXTYPE mDescribeHDRStaticInfoIndex;
309
310    status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode);
311    status_t allocateBuffersOnPort(OMX_U32 portIndex);
312    status_t freeBuffersOnPort(OMX_U32 portIndex);
313    status_t freeBuffer(OMX_U32 portIndex, size_t i);
314
315    status_t handleSetSurface(const sp<Surface> &surface);
316    status_t setupNativeWindowSizeFormatAndUsage(
317            ANativeWindow *nativeWindow /* nonnull */, int *finalUsage /* nonnull */,
318            bool reconnect);
319
320    status_t configureOutputBuffersFromNativeWindow(
321            OMX_U32 *nBufferCount, OMX_U32 *nBufferSize,
322            OMX_U32 *nMinUndequeuedBuffers, bool preregister);
323    status_t allocateOutputMetadataBuffers();
324    status_t submitOutputMetadataBuffer();
325    void signalSubmitOutputMetadataBufferIfEOS_workaround();
326    status_t allocateOutputBuffersFromNativeWindow();
327    status_t cancelBufferToNativeWindow(BufferInfo *info);
328    status_t freeOutputBuffersNotOwnedByComponent();
329    BufferInfo *dequeueBufferFromNativeWindow();
330
331    inline bool storingMetadataInDecodedBuffers() {
332        return mOutputMetadataType >= 0 && !mIsEncoder;
333    }
334
335    inline bool usingMetadataOnEncoderOutput() {
336        return mOutputMetadataType >= 0 && mIsEncoder;
337    }
338
339    BufferInfo *findBufferByID(
340            uint32_t portIndex, IOMX::buffer_id bufferID,
341            ssize_t *index = NULL);
342
343    status_t setComponentRole(bool isEncoder, const char *mime);
344    static const char *getComponentRole(bool isEncoder, const char *mime);
345    static status_t setComponentRole(
346            const sp<IOMX> &omx, IOMX::node_id node, const char *role);
347
348    status_t configureCodec(const char *mime, const sp<AMessage> &msg);
349
350    status_t configureTunneledVideoPlayback(int32_t audioHwSync,
351            const sp<ANativeWindow> &nativeWindow);
352
353    status_t setVideoPortFormatType(
354            OMX_U32 portIndex,
355            OMX_VIDEO_CODINGTYPE compressionFormat,
356            OMX_COLOR_FORMATTYPE colorFormat,
357            bool usingNativeBuffers = false);
358
359    status_t setSupportedOutputFormat(bool getLegacyFlexibleFormat);
360
361    status_t setupVideoDecoder(
362            const char *mime, const sp<AMessage> &msg, bool usingNativeBuffers, bool haveSwRenderer,
363            sp<AMessage> &outputformat);
364
365    status_t setupVideoEncoder(
366            const char *mime, const sp<AMessage> &msg,
367            sp<AMessage> &outputformat, sp<AMessage> &inputformat);
368
369    status_t setVideoFormatOnPort(
370            OMX_U32 portIndex,
371            int32_t width, int32_t height,
372            OMX_VIDEO_CODINGTYPE compressionFormat, float frameRate = -1.0);
373
374    // gets index or sets it to 0 on error. Returns error from codec.
375    status_t initDescribeColorAspectsIndex();
376
377    // sets |params|. If |readBack| is true, it re-gets them afterwards if set succeeded.
378    // returns the codec error.
379    status_t setCodecColorAspects(DescribeColorAspectsParams &params, bool readBack = false);
380
381    // gets |params|; returns the codec error. |param| should not change on error.
382    status_t getCodecColorAspects(DescribeColorAspectsParams &params);
383
384    // gets dataspace guidance from codec and platform. |params| should be set up with the color
385    // aspects to use. If |tryCodec| is true, the codec is queried first. If it succeeds, we
386    // return OK. Otherwise, we fall back to the platform guidance and return the codec error;
387    // though, we return OK if the codec failed with UNSUPPORTED, as codec guidance is optional.
388    status_t getDataSpace(
389            DescribeColorAspectsParams &params, android_dataspace *dataSpace /* nonnull */,
390            bool tryCodec);
391
392    // sets color aspects for the encoder for certain |width/height| based on |configFormat|, and
393    // set resulting color config into |outputFormat|. If |usingNativeWindow| is true, we use
394    // video defaults if config is unspecified. Returns error from the codec.
395    status_t setColorAspectsForVideoDecoder(
396            int32_t width, int32_t height, bool usingNativeWindow,
397            const sp<AMessage> &configFormat, sp<AMessage> &outputFormat);
398
399    // gets color aspects for the encoder for certain |width/height| based on |configFormat|, and
400    // set resulting color config into |outputFormat|. If |dataSpace| is non-null, it requests
401    // dataspace guidance from the codec and platform and sets it into |dataSpace|. Returns the
402    // error from the codec.
403    status_t getColorAspectsAndDataSpaceForVideoDecoder(
404            int32_t width, int32_t height, const sp<AMessage> &configFormat,
405            sp<AMessage> &outputFormat, android_dataspace *dataSpace);
406
407    // sets color aspects for the video encoder assuming bytebuffer mode for certain |configFormat|
408    // and sets resulting color config into |outputFormat|. For mediarecorder, also set dataspace
409    // into |inputFormat|. Returns the error from the codec.
410    status_t setColorAspectsForVideoEncoder(
411            const sp<AMessage> &configFormat,
412            sp<AMessage> &outputFormat, sp<AMessage> &inputFormat);
413
414    // sets color aspects for the video encoder in surface mode. This basically sets the default
415    // video values for unspecified aspects and sets the dataspace to use in the input format.
416    // Also sets the dataspace into |dataSpace|.
417    // Returns any codec errors during this configuration, except for optional steps.
418    status_t setInitialColorAspectsForVideoEncoderSurfaceAndGetDataSpace(
419            android_dataspace *dataSpace /* nonnull */);
420
421    // gets color aspects for the video encoder input port and sets them into the |format|.
422    // Returns any codec errors.
423    status_t getInputColorAspectsForVideoEncoder(sp<AMessage> &format);
424
425    // updates the encoder output format with |aspects| defaulting to |dataSpace| for
426    // unspecified values.
427    void onDataSpaceChanged(android_dataspace dataSpace, const ColorAspects &aspects);
428
429    // gets index or sets it to 0 on error. Returns error from codec.
430    status_t initDescribeHDRStaticInfoIndex();
431
432    // sets HDR static metadata for the video encoder/decoder based on |configFormat|, and
433    // sets resulting HDRStaticInfo config into |outputFormat|. Returns error from the codec.
434    status_t setHDRStaticInfoForVideoCodec(
435            OMX_U32 portIndex, const sp<AMessage> &configFormat, sp<AMessage> &outputFormat);
436
437    // sets |params|. Returns the codec error.
438    status_t setHDRStaticInfo(const DescribeHDRStaticInfoParams &params);
439
440    // gets |params|. Returns the codec error.
441    status_t getHDRStaticInfo(DescribeHDRStaticInfoParams &params);
442
443    // gets HDR static information for the video encoder/decoder port and sets them into |format|.
444    status_t getHDRStaticInfoForVideoCodec(OMX_U32 portIndex, sp<AMessage> &format);
445
446    typedef struct drcParams {
447        int32_t drcCut;
448        int32_t drcBoost;
449        int32_t heavyCompression;
450        int32_t targetRefLevel;
451        int32_t encodedTargetLevel;
452    } drcParams_t;
453
454    status_t setupAACCodec(
455            bool encoder,
456            int32_t numChannels, int32_t sampleRate, int32_t bitRate,
457            int32_t aacProfile, bool isADTS, int32_t sbrMode,
458            int32_t maxOutputChannelCount, const drcParams_t& drc,
459            int32_t pcmLimiterEnable);
460
461    status_t setupAC3Codec(bool encoder, int32_t numChannels, int32_t sampleRate);
462
463    status_t setupEAC3Codec(bool encoder, int32_t numChannels, int32_t sampleRate);
464
465    status_t selectAudioPortFormat(
466            OMX_U32 portIndex, OMX_AUDIO_CODINGTYPE desiredFormat);
467
468    status_t setupAMRCodec(bool encoder, bool isWAMR, int32_t bitRate);
469    status_t setupG711Codec(bool encoder, int32_t sampleRate, int32_t numChannels);
470
471    status_t setupFlacCodec(
472            bool encoder, int32_t numChannels, int32_t sampleRate, int32_t compressionLevel);
473
474    status_t setupRawAudioFormat(
475            OMX_U32 portIndex, int32_t sampleRate, int32_t numChannels,
476            AudioEncoding encoding = kAudioEncodingPcm16bit);
477
478    status_t setPriority(int32_t priority);
479    status_t setOperatingRate(float rateFloat, bool isVideo);
480    status_t getIntraRefreshPeriod(uint32_t *intraRefreshPeriod);
481    status_t setIntraRefreshPeriod(uint32_t intraRefreshPeriod, bool inConfigure);
482
483    status_t setMinBufferSize(OMX_U32 portIndex, size_t size);
484
485    status_t setupMPEG4EncoderParameters(const sp<AMessage> &msg);
486    status_t setupH263EncoderParameters(const sp<AMessage> &msg);
487    status_t setupAVCEncoderParameters(const sp<AMessage> &msg);
488    status_t setupHEVCEncoderParameters(const sp<AMessage> &msg);
489    status_t setupVPXEncoderParameters(const sp<AMessage> &msg);
490
491    status_t verifySupportForProfileAndLevel(int32_t profile, int32_t level);
492
493    status_t configureBitrate(
494            int32_t bitrate, OMX_VIDEO_CONTROLRATETYPE bitrateMode);
495
496    status_t setupErrorCorrectionParameters();
497
498    status_t initNativeWindow();
499
500    // Returns true iff all buffers on the given port have status
501    // OWNED_BY_US or OWNED_BY_NATIVE_WINDOW.
502    bool allYourBuffersAreBelongToUs(OMX_U32 portIndex);
503
504    bool allYourBuffersAreBelongToUs();
505
506    void waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();
507
508    size_t countBuffersOwnedByComponent(OMX_U32 portIndex) const;
509    size_t countBuffersOwnedByNativeWindow() const;
510
511    void deferMessage(const sp<AMessage> &msg);
512    void processDeferredMessages();
513
514    void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
515    // called when we have dequeued a buffer |buf| from the native window to track render info.
516    // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is
517    // stored.
518    void updateRenderInfoForDequeuedBuffer(
519            ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info);
520
521    // Checks to see if any frames have rendered up until |until|, and to notify client
522    // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first
523    // unrendered frame. These frames are removed from the render queue.
524    // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the
525    // queue, allowing all rendered framed up till then to be notified of.
526    // (This will effectively clear the render queue up-until (and including) |until|.)
527    // If |until| is NULL, or is not in the rendered queue, this method will check all frames.
528    void notifyOfRenderedFrames(
529            bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL);
530
531    // Pass |expectedFormat| to print a warning if the format differs from it.
532    // Using sp<> instead of const sp<>& because expectedFormat is likely the current mOutputFormat
533    // which will get updated inside.
534    void onOutputFormatChanged(sp<const AMessage> expectedFormat = NULL);
535    void addKeyFormatChangesToRenderBufferNotification(sp<AMessage> &notify);
536    void sendFormatChange();
537
538    status_t getPortFormat(OMX_U32 portIndex, sp<AMessage> &notify);
539
540    void signalError(
541            OMX_ERRORTYPE error = OMX_ErrorUndefined,
542            status_t internalError = UNKNOWN_ERROR);
543
544    static bool describeDefaultColorFormat(DescribeColorFormat2Params &describeParams);
545    static bool describeColorFormat(
546        const sp<IOMX> &omx, IOMX::node_id node,
547        DescribeColorFormat2Params &describeParams);
548
549    status_t requestIDRFrame();
550    status_t setParameters(const sp<AMessage> &params);
551
552    // Send EOS on input stream.
553    void onSignalEndOfInputStream();
554
555    DISALLOW_EVIL_CONSTRUCTORS(ACodec);
556};
557
558}  // namespace android
559
560#endif  // A_CODEC_H_
561