EffectsRecorder.java revision 44d4035047c92c151ea96764d16562d7d5e3a8a0
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 * use this file except in compliance with the License. You may obtain a copy of 6 * the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 * License for the specific language governing permissions and limitations under 14 * the License. 15 */ 16 17package com.android.camera; 18 19import android.content.Context; 20import android.filterfw.GraphEnvironment; 21import android.filterfw.core.Filter; 22import android.filterfw.core.GLEnvironment; 23import android.filterfw.core.GraphRunner; 24import android.filterfw.core.GraphRunner.OnRunnerDoneListener; 25import android.filterpacks.videosrc.SurfaceTextureSource.SurfaceTextureSourceListener; 26import android.filterpacks.videoproc.BackDropperFilter; 27import android.filterpacks.videoproc.BackDropperFilter.LearningDoneListener; 28 29import android.graphics.SurfaceTexture; 30import android.hardware.Camera; 31import android.media.MediaRecorder; 32import android.media.CamcorderProfile; 33import android.os.ConditionVariable; 34import android.os.Handler; 35import android.os.Looper; 36import android.os.ParcelFileDescriptor; 37import android.util.Log; 38import android.view.Surface; 39import android.view.SurfaceHolder; 40 41import java.io.IOException; 42import java.lang.Runnable; 43 44/** 45 * Encapsulates the mobile filter framework components needed to record video with 46 * effects applied. Modeled after MediaRecorder. 47 */ 48public class EffectsRecorder { 49 50 public static final int EFFECT_NONE = 0; 51 public static final int EFFECT_GOOFY_FACE = 1; 52 public static final int EFFECT_BACKDROPPER = 2; 53 54 public static final int EFFECT_GF_SQUEEZE = 0; 55 public static final int EFFECT_GF_BIG_EYES = 1; 56 public static final int EFFECT_GF_BIG_MOUTH = 2; 57 public static final int EFFECT_GF_SMALL_MOUTH = 3; 58 public static final int EFFECT_GF_BIG_NOSE = 4; 59 public static final int EFFECT_GF_SMALL_EYES = 5; 60 public static final int NUM_OF_GF_EFFECTS = EFFECT_GF_SMALL_EYES + 1; 61 62 public static final int EFFECT_MSG_STARTED_LEARNING = 0; 63 public static final int EFFECT_MSG_DONE_LEARNING = 1; 64 public static final int EFFECT_MSG_SWITCHING_EFFECT = 2; 65 public static final int EFFECT_MSG_EFFECTS_STOPPED = 3; 66 67 private Context mContext; 68 private Handler mHandler; 69 private boolean mReleased; 70 71 private Camera mCameraDevice; 72 private CamcorderProfile mProfile; 73 private SurfaceHolder mPreviewSurfaceHolder; 74 private int mPreviewWidth; 75 private int mPreviewHeight; 76 private MediaRecorder.OnInfoListener mInfoListener; 77 private MediaRecorder.OnErrorListener mErrorListener; 78 79 private String mOutputFile; 80 private int mOrientationHint = 0; 81 82 private int mEffect = EFFECT_NONE; 83 private int mCurrentEffect = EFFECT_NONE; 84 private EffectsListener mEffectsListener; 85 86 private Object mEffectParameter; 87 88 private GraphEnvironment mGraphEnv; 89 private int mGraphId; 90 private GraphRunner mRunner = null; 91 private GraphRunner mOldRunner = null; 92 93 private SurfaceTexture mTextureSource; 94 95 private static final int STATE_CONFIGURE = 0; 96 private static final int STATE_WAITING_FOR_SURFACE = 1; 97 private static final int STATE_PREVIEW = 2; 98 private static final int STATE_RECORD = 3; 99 private static final int STATE_RELEASED = 4; 100 private int mState = STATE_CONFIGURE; 101 102 private boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); 103 private static final String TAG = "effectsrecorder"; 104 105 /** Determine if a given effect is supported at runtime 106 * Some effects require libraries not available on all devices 107 */ 108 public static boolean isEffectSupported(int effectId) { 109 switch (effectId) { 110 case EFFECT_GOOFY_FACE: 111 return Filter.isAvailable("com.google.android.filterpacks.facedetect.GoofyRenderFilter"); 112 case EFFECT_BACKDROPPER: 113 return Filter.isAvailable("android.filterpacks.videoproc.BackDropperFilter"); 114 default: 115 return false; 116 } 117 } 118 119 public EffectsRecorder(Context context) { 120 if (mLogVerbose) Log.v(TAG, "EffectsRecorder created (" + this + ")"); 121 mContext = context; 122 mHandler = new Handler(Looper.getMainLooper()); 123 } 124 125 public void setCamera(Camera cameraDevice) { 126 switch (mState) { 127 case STATE_PREVIEW: 128 throw new RuntimeException("setCamera cannot be called while previewing!"); 129 case STATE_RECORD: 130 throw new RuntimeException("setCamera cannot be called while recording!"); 131 case STATE_RELEASED: 132 throw new RuntimeException("setCamera called on an already released recorder!"); 133 default: 134 break; 135 } 136 137 mCameraDevice = cameraDevice; 138 } 139 140 public void setProfile(CamcorderProfile profile) { 141 switch (mState) { 142 case STATE_RECORD: 143 throw new RuntimeException("setProfile cannot be called while recording!"); 144 case STATE_RELEASED: 145 throw new RuntimeException("setProfile called on an already released recorder!"); 146 default: 147 break; 148 } 149 mProfile = profile; 150 } 151 152 public void setOutputFile(String outputFile) { 153 switch (mState) { 154 case STATE_RECORD: 155 throw new RuntimeException("setOutputFile cannot be called while recording!"); 156 case STATE_RELEASED: 157 throw new RuntimeException("setOutputFile called on an already released recorder!"); 158 default: 159 break; 160 } 161 162 mOutputFile = outputFile; 163 } 164 165 public void setPreviewDisplay(SurfaceHolder previewSurfaceHolder, 166 int previewWidth, 167 int previewHeight) { 168 if (mLogVerbose) Log.v(TAG, "setPreviewDisplay (" + this + ")"); 169 switch (mState) { 170 case STATE_RECORD: 171 throw new RuntimeException("setPreviewDisplay cannot be called while recording!"); 172 case STATE_RELEASED: 173 throw new RuntimeException("setPreviewDisplay called on an already released recorder!"); 174 default: 175 break; 176 } 177 178 mPreviewSurfaceHolder = previewSurfaceHolder; 179 mPreviewWidth = previewWidth; 180 mPreviewHeight = previewHeight; 181 182 switch (mState) { 183 case STATE_WAITING_FOR_SURFACE: 184 startPreview(); 185 break; 186 case STATE_PREVIEW: 187 initializeEffect(true); 188 break; 189 } 190 } 191 192 public void setEffect(int effect, Object effectParameter) { 193 if (mLogVerbose) Log.v(TAG, 194 "setEffect: effect ID " + effect + 195 ", parameter " + effectParameter.toString() ); 196 switch (mState) { 197 case STATE_RECORD: 198 throw new RuntimeException("setEffect cannot be called while recording!"); 199 case STATE_RELEASED: 200 throw new RuntimeException("setEffect called on an already released recorder!"); 201 default: 202 break; 203 } 204 205 mEffect = effect; 206 mEffectParameter = effectParameter; 207 208 if (mState == STATE_PREVIEW) { 209 initializeEffect(false); 210 } 211 } 212 213 public interface EffectsListener { 214 public void onEffectsUpdate(int effectId, int effectMsg); 215 public void onEffectsError(Exception exception, String filePath); 216 } 217 218 public void setEffectsListener(EffectsListener listener) { 219 mEffectsListener = listener; 220 } 221 222 private void setFaceDetectOrientation(int degrees) { 223 if (mCurrentEffect == EFFECT_GOOFY_FACE) { 224 Filter rotateFilter = mRunner.getGraph().getFilter("rotate"); 225 Filter metaRotateFilter = mRunner.getGraph().getFilter("metarotate"); 226 rotateFilter.setInputValue("rotation", degrees); 227 int reverseDegrees = (360 - degrees) % 360; 228 metaRotateFilter.setInputValue("rotation", reverseDegrees); 229 } 230 } 231 232 public void setOrientationHint(int degrees) { 233 switch (mState) { 234 case STATE_RELEASED: 235 throw new RuntimeException( 236 "setOrientationHint called on an already released recorder!"); 237 default: 238 break; 239 } 240 if (mLogVerbose) Log.v(TAG, "Setting orientation hint to: " + degrees); 241 242 mOrientationHint = degrees; 243 setFaceDetectOrientation(degrees); 244 } 245 246 public void setOnInfoListener(MediaRecorder.OnInfoListener infoListener) { 247 switch (mState) { 248 case STATE_RECORD: 249 throw new RuntimeException("setInfoListener cannot be called while recording!"); 250 case STATE_RELEASED: 251 throw new RuntimeException("setInfoListener called on an already released recorder!"); 252 default: 253 break; 254 } 255 mInfoListener = infoListener; 256 } 257 258 public void setOnErrorListener(MediaRecorder.OnErrorListener errorListener) { 259 switch (mState) { 260 case STATE_RECORD: 261 throw new RuntimeException("setErrorListener cannot be called while recording!"); 262 case STATE_RELEASED: 263 throw new RuntimeException("setErrorListener called on an already released recorder!"); 264 default: 265 break; 266 } 267 mErrorListener = errorListener; 268 } 269 270 private void initializeFilterFramework() { 271 mGraphEnv = new GraphEnvironment(); 272 mGraphEnv.createGLEnvironment(); 273 274 if (mLogVerbose) { 275 Log.v(TAG, "Effects framework initializing. Recording size " 276 + mProfile.videoFrameWidth + ", " + mProfile.videoFrameHeight); 277 } 278 279 mGraphEnv.addReferences( 280 "textureSourceCallback", mSourceReadyCallback, 281 "recordingWidth", mProfile.videoFrameWidth, 282 "recordingHeight", mProfile.videoFrameHeight, 283 "recordingProfile", mProfile, 284 "audioSource", MediaRecorder.AudioSource.CAMCORDER, 285 "learningDoneListener", mLearningDoneListener); 286 287 mRunner = null; 288 mGraphId = -1; 289 mCurrentEffect = EFFECT_NONE; 290 } 291 292 private synchronized void initializeEffect(boolean forceReset) { 293 if (forceReset || 294 mCurrentEffect != mEffect || 295 mCurrentEffect == EFFECT_BACKDROPPER) { 296 if (mLogVerbose) { 297 Log.v(TAG, "Effect initializing. Preview size " 298 + mPreviewWidth + ", " + mPreviewHeight); 299 } 300 mGraphEnv.addReferences( 301 "previewSurface", mPreviewSurfaceHolder.getSurface(), 302 "previewWidth", mPreviewWidth, 303 "previewHeight", mPreviewHeight); 304 if (mState == STATE_PREVIEW) { 305 // Switching effects while running. Inform video camera. 306 sendMessage(mCurrentEffect, EFFECT_MSG_SWITCHING_EFFECT); 307 } 308 309 switch (mEffect) { 310 case EFFECT_GOOFY_FACE: 311 mGraphId = mGraphEnv.loadGraph(mContext, R.raw.goofy_face); 312 break; 313 case EFFECT_BACKDROPPER: 314 sendMessage(EFFECT_BACKDROPPER, EFFECT_MSG_STARTED_LEARNING); 315 mGraphId = mGraphEnv.loadGraph(mContext, R.raw.backdropper); 316 break; 317 default: 318 throw new RuntimeException("Unknown effect ID" + mEffect + "!"); 319 } 320 mCurrentEffect = mEffect; 321 322 mOldRunner = mRunner; 323 mRunner = mGraphEnv.getRunner(mGraphId, GraphEnvironment.MODE_ASYNCHRONOUS); 324 mRunner.setDoneCallback(mRunnerDoneCallback); 325 if (mLogVerbose) { 326 Log.v(TAG, "New runner: " + mRunner 327 + ". Old runner: " + mOldRunner); 328 } 329 if (mState == STATE_PREVIEW) { 330 // Switching effects while running. Stop existing runner. 331 // The stop callback will take care of starting new runner. 332 mCameraDevice.stopPreview(); 333 try { 334 mCameraDevice.setPreviewTexture(null); 335 } catch(IOException e) { 336 throw new RuntimeException("Unable to connect camera to effect input", e); 337 } 338 mOldRunner.stop(); 339 } 340 } 341 342 switch (mCurrentEffect) { 343 case EFFECT_GOOFY_FACE: 344 Filter goofyFilter = mRunner.getGraph().getFilter("goofyrenderer"); 345 goofyFilter.setInputValue("currentEffect", 346 ((Integer)mEffectParameter).intValue()); 347 break; 348 case EFFECT_BACKDROPPER: 349 Filter backgroundSrc = mRunner.getGraph().getFilter("background"); 350 backgroundSrc.setInputValue("sourceUrl", 351 (String)mEffectParameter); 352 break; 353 default: 354 break; 355 } 356 setFaceDetectOrientation(mOrientationHint); 357 } 358 359 public synchronized void startPreview() { 360 if (mLogVerbose) Log.v(TAG, "Starting preview (" + this + ")"); 361 362 switch (mState) { 363 case STATE_PREVIEW: 364 // Already running preview 365 Log.w(TAG, "startPreview called when already running preview"); 366 return; 367 case STATE_RECORD: 368 throw new RuntimeException("Cannot start preview when already recording!"); 369 case STATE_RELEASED: 370 throw new RuntimeException("setEffect called on an already released recorder!"); 371 default: 372 break; 373 } 374 375 if (mEffect == EFFECT_NONE) { 376 throw new RuntimeException("No effect selected!"); 377 } 378 if (mEffectParameter == null) { 379 throw new RuntimeException("No effect parameter provided!"); 380 } 381 if (mProfile == null) { 382 throw new RuntimeException("No recording profile provided!"); 383 } 384 if (mPreviewSurfaceHolder == null) { 385 if (mLogVerbose) Log.v(TAG, "Passed a null surface holder; waiting for valid one"); 386 mState = STATE_WAITING_FOR_SURFACE; 387 return; 388 } 389 if (mCameraDevice == null) { 390 throw new RuntimeException("No camera to record from!"); 391 } 392 393 if (mLogVerbose) Log.v(TAG, "Initializing filter graph"); 394 395 initializeFilterFramework(); 396 397 initializeEffect(true); 398 399 if (mLogVerbose) Log.v(TAG, "Starting filter graph"); 400 401 mRunner.run(); 402 // Rest of preview startup handled in mSourceReadyCallback 403 } 404 405 private SurfaceTextureSourceListener mSourceReadyCallback = 406 new SurfaceTextureSourceListener() { 407 public void onSurfaceTextureSourceReady(SurfaceTexture source) { 408 if (mLogVerbose) Log.v(TAG, "SurfaceTexture ready callback received"); 409 synchronized(EffectsRecorder.this) { 410 mTextureSource = source; 411 412 // When shutting down a graph, we receive a null SurfaceTexture to 413 // indicate that. Don't want to connect up the camera in that case. 414 if (source == null) return; 415 416 if (mState == STATE_RELEASED) return; 417 418 mCameraDevice.stopPreview(); 419 if (mLogVerbose) Log.v(TAG, "Runner active, connecting effects preview"); 420 try { 421 mCameraDevice.setPreviewTexture(mTextureSource); 422 } catch(IOException e) { 423 throw new RuntimeException("Unable to connect camera to effect input", e); 424 } 425 426 // Lock AE/AWB to reduce transition flicker 427 tryEnable3ALocks(true); 428 mCameraDevice.startPreview(); 429 430 // Unlock AE/AWB after preview started 431 tryEnable3ALocks(false); 432 433 mState = STATE_PREVIEW; 434 435 if (mLogVerbose) Log.v(TAG, "Start preview/effect switch complete"); 436 } 437 } 438 }; 439 440 private LearningDoneListener mLearningDoneListener = 441 new LearningDoneListener() { 442 public void onLearningDone(BackDropperFilter filter) { 443 if (mLogVerbose) Log.v(TAG, "Learning done callback triggered"); 444 // Called in a processing thread, so have to post message back to UI 445 // thread 446 sendMessage(EFFECT_BACKDROPPER, EFFECT_MSG_DONE_LEARNING); 447 enable3ALocks(true); 448 } 449 }; 450 451 public synchronized void startRecording() { 452 if (mLogVerbose) Log.v(TAG, "Starting recording (" + this + ")"); 453 454 switch (mState) { 455 case STATE_RECORD: 456 throw new RuntimeException("Already recording, cannot begin anew!"); 457 case STATE_RELEASED: 458 throw new RuntimeException("startRecording called on an already released recorder!"); 459 default: 460 break; 461 } 462 463 if (mOutputFile == null) { 464 throw new RuntimeException("No output file name provided!"); 465 } 466 467 if (mState == STATE_CONFIGURE) { 468 startPreview(); 469 } 470 Filter recorder = mRunner.getGraph().getFilter("recorder"); 471 recorder.setInputValue("outputFile", mOutputFile); 472 recorder.setInputValue("orientationHint", mOrientationHint); 473 if (mInfoListener != null) { 474 recorder.setInputValue("infoListener", mInfoListener); 475 } 476 if (mErrorListener != null) { 477 recorder.setInputValue("errorListener", mErrorListener); 478 } 479 recorder.setInputValue("recording", true); 480 mState = STATE_RECORD; 481 } 482 483 public synchronized void stopRecording() { 484 if (mLogVerbose) Log.v(TAG, "Stop recording (" + this + ")"); 485 486 switch (mState) { 487 case STATE_CONFIGURE: 488 case STATE_PREVIEW: 489 Log.w(TAG, "StopRecording called when recording not active!"); 490 return; 491 case STATE_RELEASED: 492 throw new RuntimeException("stopRecording called on released EffectsRecorder!"); 493 default: 494 break; 495 } 496 Filter recorder = mRunner.getGraph().getFilter("recorder"); 497 recorder.setInputValue("recording", false); 498 mState = STATE_PREVIEW; 499 } 500 501 // Stop and release effect resources 502 public synchronized void stopPreview() { 503 if (mLogVerbose) Log.v(TAG, "Stopping preview (" + this + ")"); 504 505 switch (mState) { 506 case STATE_CONFIGURE: 507 Log.w(TAG, "StopPreview called when preview not active!"); 508 return; 509 case STATE_RELEASED: 510 throw new RuntimeException("stopPreview called on released EffectsRecorder!"); 511 default: 512 break; 513 } 514 515 if (mState == STATE_RECORD) { 516 stopRecording(); 517 } 518 519 mCurrentEffect = EFFECT_NONE; 520 521 mCameraDevice.stopPreview(); 522 try { 523 mCameraDevice.setPreviewTexture(null); 524 } catch(IOException e) { 525 throw new RuntimeException("Unable to connect camera to effect input", e); 526 } 527 528 mState = STATE_CONFIGURE; 529 mOldRunner = mRunner; 530 mRunner.stop(); 531 mRunner = null; 532 // Rest of stop and release handled in mRunnerDoneCallback 533 } 534 535 // Try to enable/disable 3A locks if supported; otherwise return false 536 boolean tryEnable3ALocks(boolean toggle) { 537 Camera.Parameters params = mCameraDevice.getParameters(); 538 if (params.isAutoExposureLockSupported() && 539 params.isAutoWhiteBalanceLockSupported() ) { 540 params.setAutoExposureLock(toggle); 541 params.setAutoWhiteBalanceLock(toggle); 542 mCameraDevice.setParameters(params); 543 return true; 544 } 545 return false; 546 } 547 548 // Try to enable/disable 3A locks if supported; otherwise, throw error 549 // Use this when locks are essential to success 550 void enable3ALocks(boolean toggle) { 551 Camera.Parameters params = mCameraDevice.getParameters(); 552 if (!tryEnable3ALocks(toggle)) { 553 throw new RuntimeException("Attempt to lock 3A on camera with no locking support!"); 554 } 555 } 556 557 private OnRunnerDoneListener mRunnerDoneCallback = 558 new OnRunnerDoneListener() { 559 public void onRunnerDone(int result) { 560 synchronized(EffectsRecorder.this) { 561 if (mLogVerbose) { 562 Log.v(TAG, 563 "Graph runner done (" + EffectsRecorder.this 564 + ", mRunner " + mRunner 565 + ", mOldRunner " + mOldRunner + ")"); 566 } 567 if (result == GraphRunner.RESULT_ERROR) { 568 // Handle error case 569 Log.e(TAG, "Error running filter graph!"); 570 raiseError(mRunner == null ? null : mRunner.getError()); 571 } 572 if (mOldRunner != null) { 573 // Tear down old graph if available 574 if (mLogVerbose) Log.v(TAG, "Tearing down old graph."); 575 GLEnvironment glEnv = mGraphEnv.getContext().getGLEnvironment(); 576 if (glEnv != null && !glEnv.isActive()) { 577 glEnv.activate(); 578 } 579 mOldRunner.getGraph().tearDown(mGraphEnv.getContext()); 580 if (glEnv != null && glEnv.isActive()) { 581 glEnv.deactivate(); 582 } 583 mOldRunner = null; 584 } 585 if (mState == STATE_PREVIEW) { 586 // Switching effects, start up the new runner 587 if (mLogVerbose) Log.v(TAG, "Previous effect halted, starting new effect."); 588 tryEnable3ALocks(false); 589 mRunner.run(); 590 } else if (mState != STATE_RELEASED) { 591 // Shutting down effects 592 if (mLogVerbose) Log.v(TAG, "Runner halted, restoring direct preview"); 593 sendMessage(EFFECT_NONE, EFFECT_MSG_EFFECTS_STOPPED); 594 } else { 595 // STATE_RELEASED - camera will be/has been released as well, do nothing. 596 } 597 } 598 } 599 }; 600 601 // Indicates that all camera/recording activity needs to halt 602 public synchronized void release() { 603 if (mLogVerbose) Log.v(TAG, "Releasing (" + this + ")"); 604 605 switch (mState) { 606 case STATE_RECORD: 607 case STATE_PREVIEW: 608 stopPreview(); 609 // Fall-through 610 default: 611 mState = STATE_RELEASED; 612 break; 613 } 614 } 615 616 private void sendMessage(final int effect, final int msg) { 617 if (mEffectsListener != null) { 618 mHandler.post(new Runnable() { 619 public void run() { 620 mEffectsListener.onEffectsUpdate(effect, msg); 621 } 622 }); 623 } 624 } 625 626 private void raiseError(final Exception exception) { 627 if (mEffectsListener != null) { 628 mHandler.post(new Runnable() { 629 public void run() { 630 mEffectsListener.onEffectsError(exception, mOutputFile); 631 } 632 }); 633 } 634 } 635} 636