1/* 2 * Copyright 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.mediaframeworktest.stress; 18 19import com.android.ex.camera2.blocking.BlockingSessionCallback; 20import com.android.mediaframeworktest.Camera2SurfaceViewTestCase; 21import com.android.mediaframeworktest.helpers.CameraTestUtils; 22 23import junit.framework.AssertionFailedError; 24 25import android.graphics.ImageFormat; 26import android.hardware.camera2.CameraCaptureSession; 27import android.hardware.camera2.CameraCharacteristics; 28import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession; 29import android.hardware.camera2.CameraDevice; 30import android.hardware.camera2.CaptureRequest; 31import android.hardware.camera2.CaptureResult; 32import android.hardware.camera2.params.StreamConfigurationMap; 33import android.media.CamcorderProfile; 34import android.media.Image; 35import android.media.ImageReader; 36import android.media.MediaExtractor; 37import android.media.MediaFormat; 38import android.media.MediaRecorder; 39import android.os.Environment; 40import android.os.SystemClock; 41import android.test.suitebuilder.annotation.LargeTest; 42import android.util.Log; 43import android.util.Range; 44import android.util.Size; 45import android.view.Surface; 46 47import java.io.File; 48import java.util.ArrayList; 49import java.util.Arrays; 50import java.util.HashMap; 51import java.util.List; 52 53import static com.android.ex.camera2.blocking.BlockingSessionCallback.SESSION_CLOSED; 54import static com.android.mediaframeworktest.helpers.CameraTestUtils.CAPTURE_IMAGE_TIMEOUT_MS; 55import static com.android.mediaframeworktest.helpers.CameraTestUtils.SESSION_CLOSE_TIMEOUT_MS; 56import static com.android.mediaframeworktest.helpers.CameraTestUtils.SIZE_BOUND_1080P; 57import static com.android.mediaframeworktest.helpers.CameraTestUtils.SIZE_BOUND_2160P; 58import static com.android.mediaframeworktest.helpers.CameraTestUtils.SimpleCaptureCallback; 59import static com.android.mediaframeworktest.helpers.CameraTestUtils.SimpleImageReaderListener; 60import static com.android.mediaframeworktest.helpers.CameraTestUtils.configureCameraSession; 61import static com.android.mediaframeworktest.helpers.CameraTestUtils.getSupportedVideoSizes; 62import static com.android.mediaframeworktest.helpers.CameraTestUtils.getValueNotNull; 63 64/** 65 * CameraDevice video recording use case tests by using MediaRecorder and 66 * MediaCodec. 67 * 68 * adb shell am instrument \ 69 * -e class com.android.mediaframeworktest.stress.Camera2RecordingTest#testBasicRecording \ 70 * -e iterations 10 \ 71 * -e waitIntervalMs 1000 \ 72 * -e resultToFile false \ 73 * -r -w com.android.mediaframeworktest/.Camera2InstrumentationTestRunner 74 */ 75@LargeTest 76public class Camera2RecordingTest extends Camera2SurfaceViewTestCase { 77 private static final String TAG = "RecordingTest"; 78 private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE); 79 private static final boolean DEBUG_DUMP = Log.isLoggable(TAG, Log.DEBUG); 80 private static final int RECORDING_DURATION_MS = 3000; 81 private static final float DURATION_MARGIN = 0.2f; 82 private static final double FRAME_DURATION_ERROR_TOLERANCE_MS = 3.0; 83 private static final int BIT_RATE_1080P = 16000000; 84 private static final int BIT_RATE_MIN = 64000; 85 private static final int BIT_RATE_MAX = 40000000; 86 private static final int VIDEO_FRAME_RATE = 30; 87 private final String VIDEO_FILE_PATH = Environment.getExternalStorageDirectory().getPath(); 88 private static final int[] mCamcorderProfileList = { 89 CamcorderProfile.QUALITY_HIGH, 90 CamcorderProfile.QUALITY_2160P, 91 CamcorderProfile.QUALITY_1080P, 92 CamcorderProfile.QUALITY_720P, 93 CamcorderProfile.QUALITY_480P, 94 CamcorderProfile.QUALITY_CIF, 95 CamcorderProfile.QUALITY_QCIF, 96 CamcorderProfile.QUALITY_QVGA, 97 CamcorderProfile.QUALITY_LOW, 98 }; 99 private static final int MAX_VIDEO_SNAPSHOT_IMAGES = 5; 100 private static final int BURST_VIDEO_SNAPSHOT_NUM = 3; 101 private static final int SLOWMO_SLOW_FACTOR = 4; 102 private static final int MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED = 4; 103 private List<Size> mSupportedVideoSizes; 104 private Surface mRecordingSurface; 105 private Surface mPersistentSurface; 106 private MediaRecorder mMediaRecorder; 107 private String mOutMediaFileName; 108 private int mVideoFrameRate; 109 private Size mVideoSize; 110 private long mRecordingStartTime; 111 112 @Override 113 protected void setUp() throws Exception { 114 super.setUp(); 115 } 116 117 @Override 118 protected void tearDown() throws Exception { 119 super.tearDown(); 120 } 121 122 private void doBasicRecording(boolean useVideoStab) throws Exception { 123 for (int i = 0; i < mCameraIds.length; i++) { 124 try { 125 Log.i(TAG, "Testing basic recording for camera " + mCameraIds[i]); 126 // Re-use the MediaRecorder object for the same camera device. 127 mMediaRecorder = new MediaRecorder(); 128 openDevice(mCameraIds[i]); 129 if (!mStaticInfo.isColorOutputSupported()) { 130 Log.i(TAG, "Camera " + mCameraIds[i] + 131 " does not support color outputs, skipping"); 132 continue; 133 } 134 135 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 136 Log.i(TAG, "Camera " + mCameraIds[i] + 137 " does not support video stabilization, skipping the stabilization" 138 + " test"); 139 continue; 140 } 141 142 initSupportedVideoSize(mCameraIds[i]); 143 144 // Test iteration starts... 145 for (int iteration = 0; iteration < getIterationCount(); ++iteration) { 146 Log.v(TAG, String.format("Recording video: %d/%d", iteration + 1, 147 getIterationCount())); 148 basicRecordingTestByCamera(mCamcorderProfileList, useVideoStab); 149 getResultPrinter().printStatus(getIterationCount(), iteration + 1, 150 mCameraIds[i]); 151 Thread.sleep(getTestWaitIntervalMs()); 152 } 153 } finally { 154 closeDevice(); 155 releaseRecorder(); 156 } 157 } 158 } 159 160 /** 161 * <p> 162 * Test basic camera recording. 163 * </p> 164 * <p> 165 * This test covers the typical basic use case of camera recording. 166 * MediaRecorder is used to record the audio and video, CamcorderProfile is 167 * used to configure the MediaRecorder. It goes through the pre-defined 168 * CamcorderProfile list, test each profile configuration and validate the 169 * recorded video. Preview is set to the video size. 170 * </p> 171 */ 172 public void testBasicRecording() throws Exception { 173 doBasicRecording(/*useVideoStab*/false); 174 } 175 176 /** 177 * <p> 178 * Test video snapshot for each camera. 179 * </p> 180 * <p> 181 * This test covers video snapshot typical use case. The MediaRecorder is used to record the 182 * video for each available video size. The largest still capture size is selected to 183 * capture the JPEG image. The still capture images are validated according to the capture 184 * configuration. The timestamp of capture result before and after video snapshot is also 185 * checked to make sure no frame drop caused by video snapshot. 186 * </p> 187 */ 188 public void testVideoSnapshot() throws Exception { 189 videoSnapshotHelper(/*burstTest*/false); 190 } 191 192 public void testConstrainedHighSpeedRecording() throws Exception { 193 constrainedHighSpeedRecording(); 194 } 195 196 private void constrainedHighSpeedRecording() throws Exception { 197 for (String id : mCameraIds) { 198 try { 199 Log.i(TAG, "Testing constrained high speed recording for camera " + id); 200 // Re-use the MediaRecorder object for the same camera device. 201 mMediaRecorder = new MediaRecorder(); 202 openDevice(id); 203 204 if (!mStaticInfo.isConstrainedHighSpeedVideoSupported()) { 205 Log.i(TAG, "Camera " + id + " doesn't support high speed recording, skipping."); 206 continue; 207 } 208 209 // Test iteration starts... 210 for (int iteration = 0; iteration < getIterationCount(); ++iteration) { 211 Log.v(TAG, String.format("Constrained high speed recording: %d/%d", 212 iteration + 1, getIterationCount())); 213 214 StreamConfigurationMap config = 215 mStaticInfo.getValueFromKeyNonNull( 216 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 217 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 218 for (Size size : highSpeedVideoSizes) { 219 List<Range<Integer>> fixedFpsRanges = 220 getHighSpeedFixedFpsRangeForSize(config, size); 221 mCollector.expectTrue("Unable to find the fixed frame rate fps range for " + 222 "size " + size, fixedFpsRanges.size() > 0); 223 // Test recording for each FPS range 224 for (Range<Integer> fpsRange : fixedFpsRanges) { 225 int captureRate = fpsRange.getLower(); 226 final int VIDEO_FRAME_RATE = 30; 227 // Skip the test if the highest recording FPS supported by CamcorderProfile 228 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 229 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 230 + " is not supported by CamcorderProfile"); 231 continue; 232 } 233 234 mOutMediaFileName = VIDEO_FILE_PATH + "/test_cslowMo_video_" + captureRate + 235 "fps_" + id + "_" + size.toString() + ".mp4"; 236 237 prepareRecording(size, VIDEO_FRAME_RATE, captureRate); 238 239 // prepare preview surface by using video size. 240 updatePreviewSurfaceWithVideo(size, captureRate); 241 242 // Start recording 243 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 244 startSlowMotionRecording(/*useMediaRecorder*/true, VIDEO_FRAME_RATE, 245 captureRate, fpsRange, resultListener, 246 /*useHighSpeedSession*/true); 247 248 // Record certain duration. 249 SystemClock.sleep(RECORDING_DURATION_MS); 250 251 // Stop recording and preview 252 stopRecording(/*useMediaRecorder*/true); 253 // Convert number of frames camera produced into the duration in unit of ms. 254 int durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 255 VIDEO_FRAME_RATE); 256 257 // Validation. 258 validateRecording(size, durationMs); 259 } 260 261 getResultPrinter().printStatus(getIterationCount(), iteration + 1, id); 262 Thread.sleep(getTestWaitIntervalMs()); 263 } 264 } 265 266 } finally { 267 closeDevice(); 268 releaseRecorder(); 269 } 270 } 271 } 272 273 /** 274 * Get high speed FPS from CamcorderProfiles for a given size. 275 * 276 * @param size The size used to search the CamcorderProfiles for the FPS. 277 * @return high speed video FPS, 0 if the given size is not supported by the CamcorderProfiles. 278 */ 279 private int getFpsFromHighSpeedProfileForSize(Size size) { 280 for (int quality = CamcorderProfile.QUALITY_HIGH_SPEED_480P; 281 quality <= CamcorderProfile.QUALITY_HIGH_SPEED_2160P; quality++) { 282 if (CamcorderProfile.hasProfile(quality)) { 283 CamcorderProfile profile = CamcorderProfile.get(quality); 284 if (size.equals(new Size(profile.videoFrameWidth, profile.videoFrameHeight))){ 285 return profile.videoFrameRate; 286 } 287 } 288 } 289 290 return 0; 291 } 292 293 private List<Range<Integer>> getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 294 Size size) { 295 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 296 List<Range<Integer>> fixedRanges = new ArrayList<Range<Integer>>(); 297 for (Range<Integer> range : availableFpsRanges) { 298 if (range.getLower().equals(range.getUpper())) { 299 fixedRanges.add(range); 300 } 301 } 302 return fixedRanges; 303 } 304 305 private void startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, 306 int captureRate, Range<Integer> fpsRange, 307 CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession) throws Exception { 308 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 309 assertTrue("Both preview and recording surfaces should be valid", 310 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 311 outputSurfaces.add(mPreviewSurface); 312 outputSurfaces.add(mRecordingSurface); 313 // Video snapshot surface 314 if (mReaderSurface != null) { 315 outputSurfaces.add(mReaderSurface); 316 } 317 mSessionListener = new BlockingSessionCallback(); 318 mSession = configureCameraSession(mCamera, outputSurfaces, useHighSpeedSession, 319 mSessionListener, mHandler); 320 321 // Create slow motion request list 322 List<CaptureRequest> slowMoRequests = null; 323 if (useHighSpeedSession) { 324 CaptureRequest.Builder requestBuilder = 325 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 326 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 327 requestBuilder.addTarget(mPreviewSurface); 328 requestBuilder.addTarget(mRecordingSurface); 329 slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession). 330 createHighSpeedRequestList(requestBuilder.build()); 331 } else { 332 CaptureRequest.Builder recordingRequestBuilder = 333 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 334 recordingRequestBuilder.set(CaptureRequest.CONTROL_MODE, 335 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 336 recordingRequestBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 337 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 338 339 CaptureRequest.Builder recordingOnlyBuilder = 340 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 341 recordingOnlyBuilder.set(CaptureRequest.CONTROL_MODE, 342 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 343 recordingOnlyBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 344 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 345 int slowMotionFactor = captureRate / videoFrameRate; 346 347 // Make sure camera output frame rate is set to correct value. 348 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 349 recordingRequestBuilder.addTarget(mRecordingSurface); 350 recordingRequestBuilder.addTarget(mPreviewSurface); 351 recordingOnlyBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 352 recordingOnlyBuilder.addTarget(mRecordingSurface); 353 354 slowMoRequests = new ArrayList<CaptureRequest>(); 355 slowMoRequests.add(recordingRequestBuilder.build());// Preview + recording. 356 357 for (int i = 0; i < slowMotionFactor - 1; i++) { 358 slowMoRequests.add(recordingOnlyBuilder.build()); // Recording only. 359 } 360 } 361 362 mSession.setRepeatingBurst(slowMoRequests, listener, mHandler); 363 364 if (useMediaRecorder) { 365 mMediaRecorder.start(); 366 } else { 367 // TODO: need implement MediaCodec path. 368 } 369 370 } 371 372 /** 373 * Test camera recording by using each available CamcorderProfile for a 374 * given camera. preview size is set to the video size. 375 */ 376 private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab) 377 throws Exception { 378 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 379 List<Range<Integer> > fpsRanges = Arrays.asList( 380 mStaticInfo.getAeAvailableTargetFpsRangesChecked()); 381 int cameraId = Integer.parseInt(mCamera.getId()); 382 int maxVideoFrameRate = -1; 383 for (int profileId : camcorderProfileList) { 384 if (!CamcorderProfile.hasProfile(cameraId, profileId) || 385 allowedUnsupported(cameraId, profileId)) { 386 continue; 387 } 388 389 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 390 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 391 Range<Integer> fpsRange = new Range(profile.videoFrameRate, profile.videoFrameRate); 392 if (maxVideoFrameRate < profile.videoFrameRate) { 393 maxVideoFrameRate = profile.videoFrameRate; 394 } 395 if (mStaticInfo.isHardwareLevelLegacy() && 396 (videoSz.getWidth() > maxPreviewSize.getWidth() || 397 videoSz.getHeight() > maxPreviewSize.getHeight())) { 398 // Skip. Legacy mode can only do recording up to max preview size 399 continue; 400 } 401 assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId + 402 " must be one of the camera device supported video size!", 403 mSupportedVideoSizes.contains(videoSz)); 404 assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId + 405 ") must be one of the camera device available FPS range!", 406 fpsRanges.contains(fpsRange)); 407 408 if (VERBOSE) { 409 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 410 } 411 412 // Configure preview and recording surfaces. 413 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 414 if (DEBUG_DUMP) { 415 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 416 + videoSz.toString() + ".mp4"; 417 } 418 419 prepareRecordingWithProfile(profile); 420 421 // prepare preview surface by using video size. 422 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate); 423 424 // Start recording 425 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 426 startRecording(/* useMediaRecorder */true, resultListener, useVideoStab); 427 428 // Record certain duration. 429 SystemClock.sleep(RECORDING_DURATION_MS); 430 431 // Stop recording and preview 432 stopRecording(/* useMediaRecorder */true); 433 // Convert number of frames camera produced into the duration in unit of ms. 434 int durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 435 profile.videoFrameRate); 436 437 if (VERBOSE) { 438 Log.v(TAG, "video frame rate: " + profile.videoFrameRate + 439 ", num of frames produced: " + resultListener.getTotalNumFrames()); 440 } 441 442 // Validation. 443 validateRecording(videoSz, durationMs); 444 } 445 if (maxVideoFrameRate != -1) { 446 // At least one CamcorderProfile is present, check FPS 447 assertTrue("At least one CamcorderProfile must support >= 24 FPS", 448 maxVideoFrameRate >= 24); 449 } 450 } 451 452 /** 453 * Initialize the supported video sizes. 454 */ 455 private void initSupportedVideoSize(String cameraId) throws Exception { 456 Size maxVideoSize = SIZE_BOUND_1080P; 457 if (CamcorderProfile.hasProfile(CamcorderProfile.QUALITY_2160P)) { 458 maxVideoSize = SIZE_BOUND_2160P; 459 } 460 mSupportedVideoSizes = 461 getSupportedVideoSizes(cameraId, mCameraManager, maxVideoSize); 462 } 463 464 /** 465 * Simple wrapper to wrap normal/burst video snapshot tests 466 */ 467 private void videoSnapshotHelper(boolean burstTest) throws Exception { 468 for (String id : mCameraIds) { 469 try { 470 Log.i(TAG, "Testing video snapshot for camera " + id); 471 // Re-use the MediaRecorder object for the same camera device. 472 mMediaRecorder = new MediaRecorder(); 473 474 openDevice(id); 475 476 if (!mStaticInfo.isColorOutputSupported()) { 477 Log.i(TAG, "Camera " + id + 478 " does not support color outputs, skipping"); 479 continue; 480 } 481 482 initSupportedVideoSize(id); 483 484 // Test iteration starts... 485 for (int iteration = 0; iteration < getIterationCount(); ++iteration) { 486 Log.v(TAG, String.format("Video snapshot: %d/%d", iteration + 1, 487 getIterationCount())); 488 videoSnapshotTestByCamera(burstTest); 489 getResultPrinter().printStatus(getIterationCount(), iteration + 1, id); 490 Thread.sleep(getTestWaitIntervalMs()); 491 } 492 } finally { 493 closeDevice(); 494 releaseRecorder(); 495 } 496 } 497 } 498 499 /** 500 * Returns {@code true} if the {@link CamcorderProfile} ID is allowed to be unsupported. 501 * 502 * <p>This only allows unsupported profiles when using the LEGACY mode of the Camera API.</p> 503 * 504 * @param profileId a {@link CamcorderProfile} ID to check. 505 * @return {@code true} if supported. 506 */ 507 private boolean allowedUnsupported(int cameraId, int profileId) { 508 if (!mStaticInfo.isHardwareLevelLegacy()) { 509 return false; 510 } 511 512 switch(profileId) { 513 case CamcorderProfile.QUALITY_2160P: 514 case CamcorderProfile.QUALITY_1080P: 515 case CamcorderProfile.QUALITY_HIGH: 516 return !CamcorderProfile.hasProfile(cameraId, profileId) || 517 CamcorderProfile.get(cameraId, profileId).videoFrameWidth >= 1080; 518 } 519 return false; 520 } 521 522 /** 523 * Test video snapshot for each available CamcorderProfile for a given camera. 524 * 525 * <p> 526 * Preview size is set to the video size. For the burst test, frame drop and jittering 527 * is not checked. 528 * </p> 529 * 530 * @param burstTest Perform burst capture or single capture. For burst capture 531 * {@value #BURST_VIDEO_SNAPSHOT_NUM} capture requests will be sent. 532 */ 533 private void videoSnapshotTestByCamera(boolean burstTest) 534 throws Exception { 535 final int NUM_SINGLE_SHOT_TEST = 5; 536 final int FRAMEDROP_TOLERANCE = 8; 537 final int FRAME_SIZE_15M = 15000000; 538 final float FRAME_DROP_TOLERENCE_FACTOR = 1.5f; 539 int kFrameDrop_Tolerence = FRAMEDROP_TOLERANCE; 540 541 for (int profileId : mCamcorderProfileList) { 542 int cameraId = Integer.parseInt(mCamera.getId()); 543 if (!CamcorderProfile.hasProfile(cameraId, profileId) || 544 allowedUnsupported(cameraId, profileId)) { 545 continue; 546 } 547 548 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 549 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 550 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 551 552 if (mStaticInfo.isHardwareLevelLegacy() && 553 (videoSz.getWidth() > maxPreviewSize.getWidth() || 554 videoSz.getHeight() > maxPreviewSize.getHeight())) { 555 // Skip. Legacy mode can only do recording up to max preview size 556 continue; 557 } 558 559 if (!mSupportedVideoSizes.contains(videoSz)) { 560 mCollector.addMessage("Video size " + videoSz.toString() + " for profile ID " + 561 profileId + " must be one of the camera device supported video size!"); 562 continue; 563 } 564 565 // For LEGACY, find closest supported smaller or equal JPEG size to the current video 566 // size; if no size is smaller than the video, pick the smallest JPEG size. The assert 567 // for video size above guarantees that for LIMITED or FULL, we select videoSz here. 568 // Also check for minFrameDuration here to make sure jpeg stream won't slow down 569 // video capture 570 Size videoSnapshotSz = mOrderedStillSizes.get(mOrderedStillSizes.size() - 1); 571 // Allow a bit tolerance so we don't fail for a few nano seconds of difference 572 final float FRAME_DURATION_TOLERANCE = 0.01f; 573 long videoFrameDuration = (long) (1e9 / profile.videoFrameRate * 574 (1.0 + FRAME_DURATION_TOLERANCE)); 575 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 576 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.JPEG); 577 for (int i = mOrderedStillSizes.size() - 2; i >= 0; i--) { 578 Size candidateSize = mOrderedStillSizes.get(i); 579 if (mStaticInfo.isHardwareLevelLegacy()) { 580 // Legacy level doesn't report min frame duration 581 if (candidateSize.getWidth() <= videoSz.getWidth() && 582 candidateSize.getHeight() <= videoSz.getHeight()) { 583 videoSnapshotSz = candidateSize; 584 } 585 } else { 586 Long jpegFrameDuration = minFrameDurationMap.get(candidateSize); 587 assertTrue("Cannot find minimum frame duration for jpeg size " + candidateSize, 588 jpegFrameDuration != null); 589 if (candidateSize.getWidth() <= videoSz.getWidth() && 590 candidateSize.getHeight() <= videoSz.getHeight() && 591 jpegFrameDuration <= videoFrameDuration) { 592 videoSnapshotSz = candidateSize; 593 } 594 } 595 } 596 597 /** 598 * Only test full res snapshot when below conditions are all true. 599 * 1. Camera is a FULL device 600 * 2. video size is up to max preview size, which will be bounded by 1080p. 601 * 3. Full resolution jpeg stream can keep up to video stream speed. 602 * When full res jpeg stream cannot keep up to video stream speed, search 603 * the largest jpeg size that can susptain video speed instead. 604 */ 605 if (mStaticInfo.isHardwareLevelFull() && 606 videoSz.getWidth() <= maxPreviewSize.getWidth() && 607 videoSz.getHeight() <= maxPreviewSize.getHeight()) { 608 for (Size jpegSize : mOrderedStillSizes) { 609 Long jpegFrameDuration = minFrameDurationMap.get(jpegSize); 610 assertTrue("Cannot find minimum frame duration for jpeg size " + jpegSize, 611 jpegFrameDuration != null); 612 if (jpegFrameDuration <= videoFrameDuration) { 613 videoSnapshotSz = jpegSize; 614 break; 615 } 616 if (jpegSize.equals(videoSz)) { 617 throw new AssertionFailedError( 618 "Cannot find adequate video snapshot size for video size" + 619 videoSz); 620 } 621 } 622 } 623 624 Log.i(TAG, "Testing video snapshot size " + videoSnapshotSz + 625 " for video size " + videoSz); 626 if (videoSnapshotSz.getWidth() * videoSnapshotSz.getHeight() > FRAME_SIZE_15M) 627 kFrameDrop_Tolerence = (int)(FRAMEDROP_TOLERANCE * FRAME_DROP_TOLERENCE_FACTOR); 628 629 createImageReader( 630 videoSnapshotSz, ImageFormat.JPEG, 631 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 632 633 if (VERBOSE) { 634 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 635 } 636 637 // Configure preview and recording surfaces. 638 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 639 if (DEBUG_DUMP) { 640 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 641 + videoSz.toString() + ".mp4"; 642 } 643 644 int numTestIterations = burstTest ? 1 : NUM_SINGLE_SHOT_TEST; 645 int totalDroppedFrames = 0; 646 647 for (int numTested = 0; numTested < numTestIterations; numTested++) { 648 prepareRecordingWithProfile(profile); 649 650 // prepare video snapshot 651 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 652 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(); 653 CaptureRequest.Builder videoSnapshotRequestBuilder = 654 mCamera.createCaptureRequest((mStaticInfo.isHardwareLevelLegacy()) ? 655 CameraDevice.TEMPLATE_RECORD : 656 CameraDevice.TEMPLATE_VIDEO_SNAPSHOT); 657 658 // prepare preview surface by using video size. 659 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate); 660 661 prepareVideoSnapshot(videoSnapshotRequestBuilder, imageListener); 662 CaptureRequest request = videoSnapshotRequestBuilder.build(); 663 664 // Start recording 665 startRecording(/* useMediaRecorder */true, resultListener, /*useVideoStab*/false); 666 long startTime = SystemClock.elapsedRealtime(); 667 668 // Record certain duration. 669 SystemClock.sleep(RECORDING_DURATION_MS / 2); 670 671 // take video snapshot 672 if (burstTest) { 673 List<CaptureRequest> requests = 674 new ArrayList<CaptureRequest>(BURST_VIDEO_SNAPSHOT_NUM); 675 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 676 requests.add(request); 677 } 678 mSession.captureBurst(requests, resultListener, mHandler); 679 } else { 680 mSession.capture(request, resultListener, mHandler); 681 } 682 683 // make sure recording is still going after video snapshot 684 SystemClock.sleep(RECORDING_DURATION_MS / 2); 685 686 // Stop recording and preview 687 int durationMs = stopRecording(/* useMediaRecorder */true); 688 // For non-burst test, use number of frames to also double check video frame rate. 689 // Burst video snapshot is allowed to cause frame rate drop, so do not use number 690 // of frames to estimate duration 691 if (!burstTest) { 692 durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 693 profile.videoFrameRate); 694 } 695 696 // Validation recorded video 697 validateRecording(videoSz, durationMs); 698 699 if (burstTest) { 700 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 701 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 702 validateVideoSnapshotCapture(image, videoSnapshotSz); 703 image.close(); 704 } 705 } else { 706 // validate video snapshot image 707 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 708 validateVideoSnapshotCapture(image, videoSnapshotSz); 709 710 // validate if there is framedrop around video snapshot 711 totalDroppedFrames += validateFrameDropAroundVideoSnapshot( 712 resultListener, image.getTimestamp()); 713 714 //TODO: validate jittering. Should move to PTS 715 //validateJittering(resultListener); 716 717 image.close(); 718 } 719 } 720 721 if (!burstTest) { 722 Log.w(TAG, String.format("Camera %d Video size %s: Number of dropped frames " + 723 "detected in %d trials is %d frames.", cameraId, videoSz.toString(), 724 numTestIterations, totalDroppedFrames)); 725 mCollector.expectLessOrEqual( 726 String.format( 727 "Camera %d Video size %s: Number of dropped frames %d must not" 728 + " be larger than %d", 729 cameraId, videoSz.toString(), totalDroppedFrames, 730 kFrameDrop_Tolerence), 731 kFrameDrop_Tolerence, totalDroppedFrames); 732 } 733 closeImageReader(); 734 } 735 } 736 737 /** 738 * Configure video snapshot request according to the still capture size 739 */ 740 private void prepareVideoSnapshot( 741 CaptureRequest.Builder requestBuilder, 742 ImageReader.OnImageAvailableListener imageListener) 743 throws Exception { 744 mReader.setOnImageAvailableListener(imageListener, mHandler); 745 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 746 requestBuilder.addTarget(mRecordingSurface); 747 assertNotNull("Preview surface must be non-null!", mPreviewSurface); 748 requestBuilder.addTarget(mPreviewSurface); 749 assertNotNull("Reader surface must be non-null!", mReaderSurface); 750 requestBuilder.addTarget(mReaderSurface); 751 } 752 753 /** 754 * Update preview size with video size. 755 * 756 * <p>Preview size will be capped with max preview size.</p> 757 * 758 * @param videoSize The video size used for preview. 759 * @param videoFrameRate The video frame rate 760 * 761 */ 762 private void updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate) { 763 if (mOrderedPreviewSizes == null) { 764 throw new IllegalStateException("supported preview size list is not initialized yet"); 765 } 766 final float FRAME_DURATION_TOLERANCE = 0.01f; 767 long videoFrameDuration = (long) (1e9 / videoFrameRate * 768 (1.0 + FRAME_DURATION_TOLERANCE)); 769 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 770 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.PRIVATE); 771 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 772 Size previewSize = null; 773 if (videoSize.getWidth() > maxPreviewSize.getWidth() || 774 videoSize.getHeight() > maxPreviewSize.getHeight()) { 775 for (Size s : mOrderedPreviewSizes) { 776 Long frameDuration = minFrameDurationMap.get(s); 777 if (mStaticInfo.isHardwareLevelLegacy()) { 778 // Legacy doesn't report min frame duration 779 frameDuration = new Long(0); 780 } 781 assertTrue("Cannot find minimum frame duration for private size" + s, 782 frameDuration != null); 783 if (frameDuration <= videoFrameDuration && 784 s.getWidth() <= videoSize.getWidth() && 785 s.getHeight() <= videoSize.getHeight()) { 786 Log.w(TAG, "Overwrite preview size from " + videoSize.toString() + 787 " to " + s.toString()); 788 previewSize = s; 789 break; 790 // If all preview size doesn't work then we fallback to video size 791 } 792 } 793 } 794 if (previewSize == null) { 795 previewSize = videoSize; 796 } 797 updatePreviewSurface(previewSize); 798 } 799 800 /** 801 * Configure MediaRecorder recording session with CamcorderProfile, prepare 802 * the recording surface. 803 */ 804 private void prepareRecordingWithProfile(CamcorderProfile profile) 805 throws Exception { 806 // Prepare MediaRecorder. 807 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 808 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 809 mMediaRecorder.setProfile(profile); 810 mMediaRecorder.setOutputFile(mOutMediaFileName); 811 if (mPersistentSurface != null) { 812 mMediaRecorder.setInputSurface(mPersistentSurface); 813 mRecordingSurface = mPersistentSurface; 814 } 815 mMediaRecorder.prepare(); 816 if (mPersistentSurface == null) { 817 mRecordingSurface = mMediaRecorder.getSurface(); 818 } 819 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 820 mVideoFrameRate = profile.videoFrameRate; 821 mVideoSize = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 822 } 823 824 /** 825 * Configure MediaRecorder recording session with CamcorderProfile, prepare 826 * the recording surface. Use AVC for video compression, AAC for audio compression. 827 * Both are required for android devices by android CDD. 828 */ 829 private void prepareRecording(Size sz, int videoFrameRate, int captureRate) 830 throws Exception { 831 // Prepare MediaRecorder. 832 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 833 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 834 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP); 835 mMediaRecorder.setOutputFile(mOutMediaFileName); 836 mMediaRecorder.setVideoEncodingBitRate(getVideoBitRate(sz)); 837 mMediaRecorder.setVideoFrameRate(videoFrameRate); 838 mMediaRecorder.setCaptureRate(captureRate); 839 mMediaRecorder.setVideoSize(sz.getWidth(), sz.getHeight()); 840 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 841 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC); 842 if (mPersistentSurface != null) { 843 mMediaRecorder.setInputSurface(mPersistentSurface); 844 mRecordingSurface = mPersistentSurface; 845 } 846 mMediaRecorder.prepare(); 847 if (mPersistentSurface == null) { 848 mRecordingSurface = mMediaRecorder.getSurface(); 849 } 850 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 851 mVideoFrameRate = videoFrameRate; 852 mVideoSize = sz; 853 } 854 855 private void startRecording(boolean useMediaRecorder, 856 CameraCaptureSession.CaptureCallback listener, boolean useVideoStab) throws Exception { 857 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 858 throw new IllegalArgumentException("Video stabilization is not supported"); 859 } 860 861 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 862 assertTrue("Both preview and recording surfaces should be valid", 863 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 864 outputSurfaces.add(mPreviewSurface); 865 outputSurfaces.add(mRecordingSurface); 866 // Video snapshot surface 867 if (mReaderSurface != null) { 868 outputSurfaces.add(mReaderSurface); 869 } 870 mSessionListener = new BlockingSessionCallback(); 871 mSession = configureCameraSession(mCamera, outputSurfaces, mSessionListener, mHandler); 872 873 CaptureRequest.Builder recordingRequestBuilder = 874 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 875 // Make sure camera output frame rate is set to correct value. 876 Range<Integer> fpsRange = Range.create(mVideoFrameRate, mVideoFrameRate); 877 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 878 if (useVideoStab) { 879 recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, 880 CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON); 881 } 882 recordingRequestBuilder.addTarget(mRecordingSurface); 883 recordingRequestBuilder.addTarget(mPreviewSurface); 884 mSession.setRepeatingRequest(recordingRequestBuilder.build(), listener, mHandler); 885 886 if (useMediaRecorder) { 887 mMediaRecorder.start(); 888 } else { 889 // TODO: need implement MediaCodec path. 890 } 891 mRecordingStartTime = SystemClock.elapsedRealtime(); 892 } 893 894 private void stopCameraStreaming() throws Exception { 895 if (VERBOSE) { 896 Log.v(TAG, "Stopping camera streaming and waiting for idle"); 897 } 898 // Stop repeating, wait for captures to complete, and disconnect from 899 // surfaces 900 mSession.close(); 901 mSessionListener.getStateWaiter().waitForState(SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS); 902 } 903 904 // Stop recording and return the estimated video duration in milliseconds. 905 private int stopRecording(boolean useMediaRecorder) throws Exception { 906 long stopRecordingTime = SystemClock.elapsedRealtime(); 907 if (useMediaRecorder) { 908 stopCameraStreaming(); 909 910 mMediaRecorder.stop(); 911 // Can reuse the MediaRecorder object after reset. 912 mMediaRecorder.reset(); 913 } else { 914 // TODO: need implement MediaCodec path. 915 } 916 if (mPersistentSurface == null && mRecordingSurface != null) { 917 mRecordingSurface.release(); 918 mRecordingSurface = null; 919 } 920 return (int) (stopRecordingTime - mRecordingStartTime); 921 } 922 923 private void releaseRecorder() { 924 if (mMediaRecorder != null) { 925 mMediaRecorder.release(); 926 mMediaRecorder = null; 927 } 928 } 929 930 private void validateRecording(Size sz, int expectedDurationMs) throws Exception { 931 File outFile = new File(mOutMediaFileName); 932 assertTrue("No video is recorded", outFile.exists()); 933 934 MediaExtractor extractor = new MediaExtractor(); 935 try { 936 extractor.setDataSource(mOutMediaFileName); 937 long durationUs = 0; 938 int width = -1, height = -1; 939 int numTracks = extractor.getTrackCount(); 940 final String VIDEO_MIME_TYPE = "video"; 941 for (int i = 0; i < numTracks; i++) { 942 MediaFormat format = extractor.getTrackFormat(i); 943 String mime = format.getString(MediaFormat.KEY_MIME); 944 if (mime.contains(VIDEO_MIME_TYPE)) { 945 Log.i(TAG, "video format is: " + format.toString()); 946 durationUs = format.getLong(MediaFormat.KEY_DURATION); 947 width = format.getInteger(MediaFormat.KEY_WIDTH); 948 height = format.getInteger(MediaFormat.KEY_HEIGHT); 949 break; 950 } 951 } 952 Size videoSz = new Size(width, height); 953 assertTrue("Video size doesn't match, expected " + sz.toString() + 954 " got " + videoSz.toString(), videoSz.equals(sz)); 955 int duration = (int) (durationUs / 1000); 956 if (VERBOSE) { 957 Log.v(TAG, String.format("Video duration: recorded %dms, expected %dms", 958 duration, expectedDurationMs)); 959 } 960 961 // TODO: Don't skip this for video snapshot 962 if (!mStaticInfo.isHardwareLevelLegacy()) { 963 assertTrue(String.format( 964 "Camera %s: Video duration doesn't match: recorded %dms, expected %dms.", 965 mCamera.getId(), duration, expectedDurationMs), 966 Math.abs(duration - expectedDurationMs) < 967 DURATION_MARGIN * expectedDurationMs); 968 } 969 } finally { 970 extractor.release(); 971 if (!DEBUG_DUMP) { 972 outFile.delete(); 973 } 974 } 975 } 976 977 /** 978 * Validate video snapshot capture image object sanity and test. 979 * 980 * <p> Check for size, format and jpeg decoding</p> 981 * 982 * @param image The JPEG image to be verified. 983 * @param size The JPEG capture size to be verified against. 984 */ 985 private void validateVideoSnapshotCapture(Image image, Size size) { 986 CameraTestUtils.validateImage(image, size.getWidth(), size.getHeight(), 987 ImageFormat.JPEG, /*filePath*/null); 988 } 989 990 /** 991 * Validate if video snapshot causes frame drop. 992 * Here frame drop is defined as frame duration >= 2 * expected frame duration. 993 * Return the estimated number of frames dropped during video snapshot 994 */ 995 private int validateFrameDropAroundVideoSnapshot( 996 SimpleCaptureCallback resultListener, long imageTimeStamp) { 997 double expectedDurationMs = 1000.0 / mVideoFrameRate; 998 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 999 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 1000 while (!resultListener.hasMoreResults()) { 1001 CaptureResult currentResult = 1002 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1003 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 1004 if (currentTS == imageTimeStamp) { 1005 // validate the timestamp before and after, then return 1006 CaptureResult nextResult = 1007 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1008 long nextTS = getValueNotNull(nextResult, CaptureResult.SENSOR_TIMESTAMP); 1009 double durationMs = (currentTS - prevTS) / 1000000.0; 1010 int totalFramesDropped = 0; 1011 1012 // Snapshots in legacy mode pause the preview briefly. Skip the duration 1013 // requirements for legacy mode unless this is fixed. 1014 if (!mStaticInfo.isHardwareLevelLegacy()) { 1015 mCollector.expectTrue( 1016 String.format( 1017 "Video %dx%d Frame drop detected before video snapshot: " + 1018 "duration %.2fms (expected %.2fms)", 1019 mVideoSize.getWidth(), mVideoSize.getHeight(), 1020 durationMs, expectedDurationMs 1021 ), 1022 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 1023 ); 1024 // Log a warning is there is any frame drop detected. 1025 if (durationMs >= expectedDurationMs * 2) { 1026 Log.w(TAG, String.format( 1027 "Video %dx%d Frame drop detected before video snapshot: " + 1028 "duration %.2fms (expected %.2fms)", 1029 mVideoSize.getWidth(), mVideoSize.getHeight(), 1030 durationMs, expectedDurationMs 1031 )); 1032 } 1033 1034 durationMs = (nextTS - currentTS) / 1000000.0; 1035 mCollector.expectTrue( 1036 String.format( 1037 "Video %dx%d Frame drop detected after video snapshot: " + 1038 "duration %.2fms (expected %.2fms)", 1039 mVideoSize.getWidth(), mVideoSize.getHeight(), 1040 durationMs, expectedDurationMs 1041 ), 1042 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 1043 ); 1044 // Log a warning is there is any frame drop detected. 1045 if (durationMs >= expectedDurationMs * 2) { 1046 Log.w(TAG, String.format( 1047 "Video %dx%d Frame drop detected after video snapshot: " + 1048 "duration %fms (expected %fms)", 1049 mVideoSize.getWidth(), mVideoSize.getHeight(), 1050 durationMs, expectedDurationMs 1051 )); 1052 } 1053 1054 double totalDurationMs = (nextTS - prevTS) / 1000000.0; 1055 // Minus 2 for the expected 2 frames interval 1056 totalFramesDropped = (int) (totalDurationMs / expectedDurationMs) - 2; 1057 if (totalFramesDropped < 0) { 1058 Log.w(TAG, "totalFrameDropped is " + totalFramesDropped + 1059 ". Video frame rate might be too fast."); 1060 } 1061 totalFramesDropped = Math.max(0, totalFramesDropped); 1062 } 1063 return totalFramesDropped; 1064 } 1065 prevTS = currentTS; 1066 } 1067 throw new AssertionFailedError( 1068 "Video snapshot timestamp does not match any of capture results!"); 1069 } 1070 1071 /** 1072 * Calculate a video bit rate based on the size. The bit rate is scaled 1073 * based on ratio of video size to 1080p size. 1074 */ 1075 private int getVideoBitRate(Size sz) { 1076 int rate = BIT_RATE_1080P; 1077 float scaleFactor = sz.getHeight() * sz.getWidth() / (float)(1920 * 1080); 1078 rate = (int)(rate * scaleFactor); 1079 1080 // Clamp to the MIN, MAX range. 1081 return Math.max(BIT_RATE_MIN, Math.min(BIT_RATE_MAX, rate)); 1082 } 1083} 1084