1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17
18package android.filterpacks.videosink;
19
20import android.filterfw.core.Filter;
21import android.filterfw.core.FilterContext;
22import android.filterfw.core.Frame;
23import android.filterfw.core.FrameFormat;
24import android.filterfw.core.GenerateFieldPort;
25import android.filterfw.core.GLFrame;
26import android.filterfw.core.MutableFrameFormat;
27import android.filterfw.core.ShaderProgram;
28import android.filterfw.format.ImageFormat;
29import android.filterfw.geometry.Point;
30import android.filterfw.geometry.Quad;
31import android.media.MediaRecorder;
32import android.media.CamcorderProfile;
33import android.filterfw.core.GLEnvironment;
34
35import java.io.IOException;
36import java.io.FileDescriptor;
37
38import android.util.Log;
39
40/** @hide */
41public class MediaEncoderFilter extends Filter {
42
43    /** User-visible parameters */
44
45    /** Recording state. When set to false, recording will stop, or will not
46     * start if not yet running the graph. Instead, frames are simply ignored.
47     * When switched back to true, recording will restart. This allows a single
48     * graph to both provide preview and to record video. If this is false,
49     * recording settings can be updated while the graph is running.
50     */
51    @GenerateFieldPort(name = "recording", hasDefault = true)
52    private boolean mRecording = true;
53
54    /** Filename to save the output. */
55    @GenerateFieldPort(name = "outputFile", hasDefault = true)
56    private String mOutputFile = new String("/sdcard/MediaEncoderOut.mp4");
57
58    /** File Descriptor to save the output. */
59    @GenerateFieldPort(name = "outputFileDescriptor", hasDefault = true)
60    private FileDescriptor mFd = null;
61
62    /** Input audio source. If not set, no audio will be recorded.
63     * Select from the values in MediaRecorder.AudioSource
64     */
65    @GenerateFieldPort(name = "audioSource", hasDefault = true)
66    private int mAudioSource = NO_AUDIO_SOURCE;
67
68    /** Media recorder info listener, which needs to implement
69     * MediaRecorder.OnInfoListener. Set this to receive notifications about
70     * recording events.
71     */
72    @GenerateFieldPort(name = "infoListener", hasDefault = true)
73    private MediaRecorder.OnInfoListener mInfoListener = null;
74
75    /** Media recorder error listener, which needs to implement
76     * MediaRecorder.OnErrorListener. Set this to receive notifications about
77     * recording errors.
78     */
79    @GenerateFieldPort(name = "errorListener", hasDefault = true)
80    private MediaRecorder.OnErrorListener mErrorListener = null;
81
82    /** Media recording done callback, which needs to implement OnRecordingDoneListener.
83     * Set this to finalize media upon completion of media recording.
84     */
85    @GenerateFieldPort(name = "recordingDoneListener", hasDefault = true)
86    private OnRecordingDoneListener mRecordingDoneListener = null;
87
88    /** Orientation hint. Used for indicating proper video playback orientation.
89     * Units are in degrees of clockwise rotation, valid values are (0, 90, 180,
90     * 270).
91     */
92    @GenerateFieldPort(name = "orientationHint", hasDefault = true)
93    private int mOrientationHint = 0;
94
95    /** Camcorder profile to use. Select from the profiles available in
96     * android.media.CamcorderProfile. If this field is set, it overrides
97     * settings to width, height, framerate, outputFormat, and videoEncoder.
98     */
99    @GenerateFieldPort(name = "recordingProfile", hasDefault = true)
100    private CamcorderProfile mProfile = null;
101
102    /** Frame width to be encoded, defaults to 320.
103     * Actual received frame size has to match this */
104    @GenerateFieldPort(name = "width", hasDefault = true)
105    private int mWidth = 0;
106
107    /** Frame height to to be encoded, defaults to 240.
108     * Actual received frame size has to match */
109    @GenerateFieldPort(name = "height", hasDefault = true)
110    private int mHeight = 0;
111
112    /** Stream framerate to encode the frames at.
113     * By default, frames are encoded at 30 FPS*/
114    @GenerateFieldPort(name = "framerate", hasDefault = true)
115    private int mFps = 30;
116
117    /** The output format to encode the frames in.
118     * Choose an output format from the options in
119     * android.media.MediaRecorder.OutputFormat */
120    @GenerateFieldPort(name = "outputFormat", hasDefault = true)
121    private int mOutputFormat = MediaRecorder.OutputFormat.MPEG_4;
122
123    /** The videoencoder to encode the frames with.
124     * Choose a videoencoder from the options in
125     * android.media.MediaRecorder.VideoEncoder */
126    @GenerateFieldPort(name = "videoEncoder", hasDefault = true)
127    private int mVideoEncoder = MediaRecorder.VideoEncoder.H264;
128
129    /** The input region to read from the frame. The corners of this quad are
130     * mapped to the output rectangle. The input frame ranges from (0,0)-(1,1),
131     * top-left to bottom-right. The corners of the quad are specified in the
132     * order bottom-left, bottom-right, top-left, top-right.
133     */
134    @GenerateFieldPort(name = "inputRegion", hasDefault = true)
135    private Quad mSourceRegion;
136
137    /** The maximum filesize (in bytes) of the recording session.
138     * By default, it will be 0 and will be passed on to the MediaRecorder.
139     * If the limit is zero or negative, MediaRecorder will disable the limit*/
140    @GenerateFieldPort(name = "maxFileSize", hasDefault = true)
141    private long mMaxFileSize = 0;
142
143    /** The maximum duration (in milliseconds) of the recording session.
144     * By default, it will be 0 and will be passed on to the MediaRecorder.
145     * If the limit is zero or negative, MediaRecorder will record indefinitely*/
146    @GenerateFieldPort(name = "maxDurationMs", hasDefault = true)
147    private int mMaxDurationMs = 0;
148
149    /** TimeLapse Interval between frames.
150     * By default, it will be 0. Whether the recording is timelapsed
151     * is inferred based on its value being greater than 0 */
152    @GenerateFieldPort(name = "timelapseRecordingIntervalUs", hasDefault = true)
153    private long mTimeBetweenTimeLapseFrameCaptureUs = 0;
154
155    // End of user visible parameters
156
157    private static final int NO_AUDIO_SOURCE = -1;
158
159    private int mSurfaceId;
160    private ShaderProgram mProgram;
161    private GLFrame mScreen;
162
163    private boolean mRecordingActive = false;
164    private long mTimestampNs = 0;
165    private long mLastTimeLapseFrameRealTimestampNs = 0;
166    private int mNumFramesEncoded = 0;
167    // Used to indicate whether recording is timelapsed.
168    // Inferred based on (mTimeBetweenTimeLapseFrameCaptureUs > 0)
169    private boolean mCaptureTimeLapse = false;
170
171    private boolean mLogVerbose;
172    private static final String TAG = "MediaEncoderFilter";
173
174    // Our hook to the encoder
175    private MediaRecorder mMediaRecorder;
176
177    /** Callback to be called when media recording completes. */
178
179    public interface OnRecordingDoneListener {
180        public void onRecordingDone();
181    }
182
183    public MediaEncoderFilter(String name) {
184        super(name);
185        Point bl = new Point(0, 0);
186        Point br = new Point(1, 0);
187        Point tl = new Point(0, 1);
188        Point tr = new Point(1, 1);
189        mSourceRegion = new Quad(bl, br, tl, tr);
190        mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
191    }
192
193    @Override
194    public void setupPorts() {
195        // Add input port- will accept RGBA GLFrames
196        addMaskedInputPort("videoframe", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
197                                                      FrameFormat.TARGET_GPU));
198    }
199
200    @Override
201    public void fieldPortValueUpdated(String name, FilterContext context) {
202        if (mLogVerbose) Log.v(TAG, "Port " + name + " has been updated");
203        if (name.equals("recording")) return;
204        if (name.equals("inputRegion")) {
205            if (isOpen()) updateSourceRegion();
206            return;
207        }
208        // TODO: Not sure if it is possible to update the maxFileSize
209        // when the recording is going on. For now, not doing that.
210        if (isOpen() && mRecordingActive) {
211            throw new RuntimeException("Cannot change recording parameters"
212                                       + " when the filter is recording!");
213        }
214    }
215
216    private void updateSourceRegion() {
217        // Flip source quad to map to OpenGL origin
218        Quad flippedRegion = new Quad();
219        flippedRegion.p0 = mSourceRegion.p2;
220        flippedRegion.p1 = mSourceRegion.p3;
221        flippedRegion.p2 = mSourceRegion.p0;
222        flippedRegion.p3 = mSourceRegion.p1;
223        mProgram.setSourceRegion(flippedRegion);
224    }
225
226    // update the MediaRecorderParams based on the variables.
227    // These have to be in certain order as per the MediaRecorder
228    // documentation
229    private void updateMediaRecorderParams() {
230        mCaptureTimeLapse = mTimeBetweenTimeLapseFrameCaptureUs > 0;
231        final int GRALLOC_BUFFER = 2;
232        mMediaRecorder.setVideoSource(GRALLOC_BUFFER);
233        if (!mCaptureTimeLapse && (mAudioSource != NO_AUDIO_SOURCE)) {
234            mMediaRecorder.setAudioSource(mAudioSource);
235        }
236        if (mProfile != null) {
237            mMediaRecorder.setProfile(mProfile);
238            mFps = mProfile.videoFrameRate;
239            // If width and height are set larger than 0, then those
240            // overwrite the ones in the profile.
241            if (mWidth > 0 && mHeight > 0) {
242                mMediaRecorder.setVideoSize(mWidth, mHeight);
243            }
244        } else {
245            mMediaRecorder.setOutputFormat(mOutputFormat);
246            mMediaRecorder.setVideoEncoder(mVideoEncoder);
247            mMediaRecorder.setVideoSize(mWidth, mHeight);
248            mMediaRecorder.setVideoFrameRate(mFps);
249        }
250        mMediaRecorder.setOrientationHint(mOrientationHint);
251        mMediaRecorder.setOnInfoListener(mInfoListener);
252        mMediaRecorder.setOnErrorListener(mErrorListener);
253        if (mFd != null) {
254            mMediaRecorder.setOutputFile(mFd);
255        } else {
256            mMediaRecorder.setOutputFile(mOutputFile);
257        }
258        try {
259            mMediaRecorder.setMaxFileSize(mMaxFileSize);
260        } catch (Exception e) {
261            // Following the logic in  VideoCamera.java (in Camera app)
262            // We are going to ignore failure of setMaxFileSize here, as
263            // a) The composer selected may simply not support it, or
264            // b) The underlying media framework may not handle 64-bit range
265            // on the size restriction.
266            Log.w(TAG, "Setting maxFileSize on MediaRecorder unsuccessful! "
267                    + e.getMessage());
268        }
269        mMediaRecorder.setMaxDuration(mMaxDurationMs);
270    }
271
272    @Override
273    public void prepare(FilterContext context) {
274        if (mLogVerbose) Log.v(TAG, "Preparing");
275
276        mProgram = ShaderProgram.createIdentity(context);
277
278        mRecordingActive = false;
279    }
280
281    @Override
282    public void open(FilterContext context) {
283        if (mLogVerbose) Log.v(TAG, "Opening");
284        updateSourceRegion();
285        if (mRecording) startRecording(context);
286    }
287
288    private void startRecording(FilterContext context) {
289        if (mLogVerbose) Log.v(TAG, "Starting recording");
290
291        // Create a frame representing the screen
292        MutableFrameFormat screenFormat = new MutableFrameFormat(
293                              FrameFormat.TYPE_BYTE, FrameFormat.TARGET_GPU);
294        screenFormat.setBytesPerSample(4);
295
296        int width, height;
297        boolean widthHeightSpecified = mWidth > 0 && mHeight > 0;
298        // If width and height are specified, then use those instead
299        // of that in the profile.
300        if (mProfile != null && !widthHeightSpecified) {
301            width = mProfile.videoFrameWidth;
302            height = mProfile.videoFrameHeight;
303        } else {
304            width = mWidth;
305            height = mHeight;
306        }
307        screenFormat.setDimensions(width, height);
308        mScreen = (GLFrame)context.getFrameManager().newBoundFrame(
309                           screenFormat, GLFrame.EXISTING_FBO_BINDING, 0);
310
311        // Initialize the media recorder
312
313        mMediaRecorder = new MediaRecorder();
314        updateMediaRecorderParams();
315
316        try {
317            mMediaRecorder.prepare();
318        } catch (IllegalStateException e) {
319            throw e;
320        } catch (IOException e) {
321            throw new RuntimeException("IOException in"
322                    + "MediaRecorder.prepare()!", e);
323        } catch (Exception e) {
324            throw new RuntimeException("Unknown Exception in"
325                    + "MediaRecorder.prepare()!", e);
326        }
327        // Make sure start() is called before trying to
328        // register the surface. The native window handle needed to create
329        // the surface is initiated in start()
330        mMediaRecorder.start();
331        if (mLogVerbose) Log.v(TAG, "Open: registering surface from Mediarecorder");
332        mSurfaceId = context.getGLEnvironment().
333                registerSurfaceFromMediaRecorder(mMediaRecorder);
334        mNumFramesEncoded = 0;
335        mRecordingActive = true;
336    }
337
338    public boolean skipFrameAndModifyTimestamp(long timestampNs) {
339        // first frame- encode. Don't skip
340        if (mNumFramesEncoded == 0) {
341            mLastTimeLapseFrameRealTimestampNs = timestampNs;
342            mTimestampNs = timestampNs;
343            if (mLogVerbose) Log.v(TAG, "timelapse: FIRST frame, last real t= "
344                    + mLastTimeLapseFrameRealTimestampNs +
345                    ", setting t = " + mTimestampNs );
346            return false;
347        }
348
349        // Workaround to bypass the first 2 input frames for skipping.
350        // The first 2 output frames from the encoder are: decoder specific info and
351        // the compressed video frame data for the first input video frame.
352        if (mNumFramesEncoded >= 2 && timestampNs <
353            (mLastTimeLapseFrameRealTimestampNs +  1000L * mTimeBetweenTimeLapseFrameCaptureUs)) {
354            // If 2 frames have been already encoded,
355            // Skip all frames from last encoded frame until
356            // sufficient time (mTimeBetweenTimeLapseFrameCaptureUs) has passed.
357            if (mLogVerbose) Log.v(TAG, "timelapse: skipping intermediate frame");
358            return true;
359        } else {
360            // Desired frame has arrived after mTimeBetweenTimeLapseFrameCaptureUs time:
361            // - Reset mLastTimeLapseFrameRealTimestampNs to current time.
362            // - Artificially modify timestampNs to be one frame time (1/framerate) ahead
363            // of the last encoded frame's time stamp.
364            if (mLogVerbose) Log.v(TAG, "timelapse: encoding frame, Timestamp t = " + timestampNs +
365                    ", last real t= " + mLastTimeLapseFrameRealTimestampNs +
366                    ", interval = " + mTimeBetweenTimeLapseFrameCaptureUs);
367            mLastTimeLapseFrameRealTimestampNs = timestampNs;
368            mTimestampNs = mTimestampNs + (1000000000L / (long)mFps);
369            if (mLogVerbose) Log.v(TAG, "timelapse: encoding frame, setting t = "
370                    + mTimestampNs + ", delta t = " + (1000000000L / (long)mFps) +
371                    ", fps = " + mFps );
372            return false;
373        }
374    }
375
376    @Override
377    public void process(FilterContext context) {
378        GLEnvironment glEnv = context.getGLEnvironment();
379        // Get input frame
380        Frame input = pullInput("videoframe");
381
382        // Check if recording needs to start
383        if (!mRecordingActive && mRecording) {
384            startRecording(context);
385        }
386        // Check if recording needs to stop
387        if (mRecordingActive && !mRecording) {
388            stopRecording(context);
389        }
390
391        if (!mRecordingActive) return;
392
393        if (mCaptureTimeLapse) {
394            if (skipFrameAndModifyTimestamp(input.getTimestamp())) {
395                return;
396            }
397        } else {
398            mTimestampNs = input.getTimestamp();
399        }
400
401        // Activate our surface
402        glEnv.activateSurfaceWithId(mSurfaceId);
403
404        // Process
405        mProgram.process(input, mScreen);
406
407        // Set timestamp from input
408        glEnv.setSurfaceTimestamp(mTimestampNs);
409        // And swap buffers
410        glEnv.swapBuffers();
411        mNumFramesEncoded++;
412    }
413
414    private void stopRecording(FilterContext context) {
415        if (mLogVerbose) Log.v(TAG, "Stopping recording");
416
417        mRecordingActive = false;
418        mNumFramesEncoded = 0;
419        GLEnvironment glEnv = context.getGLEnvironment();
420        // The following call will switch the surface_id to 0
421        // (thus, calling eglMakeCurrent on surface with id 0) and
422        // then call eglDestroy on the surface. Hence, this will
423        // call disconnect the SurfaceMediaSource, which is needed to
424        // be called before calling Stop on the mediarecorder
425        if (mLogVerbose) Log.v(TAG, String.format("Unregistering surface %d", mSurfaceId));
426        glEnv.unregisterSurfaceId(mSurfaceId);
427        try {
428            mMediaRecorder.stop();
429        } catch (RuntimeException e) {
430            throw new MediaRecorderStopException("MediaRecorder.stop() failed!", e);
431        }
432        mMediaRecorder.release();
433        mMediaRecorder = null;
434
435        mScreen.release();
436        mScreen = null;
437
438        // Use an EffectsRecorder callback to forward a media finalization
439        // call so that it creates the video thumbnail, and whatever else needs
440        // to be done to finalize media.
441        if (mRecordingDoneListener != null) {
442            mRecordingDoneListener.onRecordingDone();
443        }
444    }
445
446    @Override
447    public void close(FilterContext context) {
448        if (mLogVerbose) Log.v(TAG, "Closing");
449        if (mRecordingActive) stopRecording(context);
450    }
451
452    @Override
453    public void tearDown(FilterContext context) {
454        // Release all the resources associated with the MediaRecorder
455        // and GLFrame members
456        if (mMediaRecorder != null) {
457            mMediaRecorder.release();
458        }
459        if (mScreen != null) {
460            mScreen.release();
461        }
462
463    }
464
465}
466