1e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch// Copyright 2014 The Chromium Authors. All rights reserved. 2e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch// Use of this source code is governed by a BSD-style license that can be 3e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch// found in the LICENSE file. 4e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 5e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochpackage org.chromium.media; 6e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 7e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochimport android.content.Context; 8e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochimport android.graphics.ImageFormat; 9e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochimport android.util.Log; 10e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 11e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochimport java.nio.ByteBuffer; 12e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochimport java.util.ArrayList; 13e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochimport java.util.Arrays; 14e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 15e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch/** 16e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch * This class extends the VideoCapture base class for manipulating a Tango 17e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch * device's cameras, namely the associated Depth (z-Buffer), Fisheye and back- 18e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch * facing 4MP video capture devices. These devices are differentiated via the 19e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch * |id| passed on constructor, according to the index correspondence in 20e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch * |s_CAM_PARAMS|; all devices |id| are index 0 towards the parent VideoCapture. 21e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch **/ 221320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci@SuppressWarnings("deprecation") 23e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdochpublic class VideoCaptureTango extends VideoCapture { 24e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private ByteBuffer mFrameBuffer = null; 25e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private final int mTangoCameraId; 26e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // The indexes must coincide with the s_CAM_PARAMS used below. 27e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int DEPTH_CAMERA_ID = 0; 28e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int FISHEYE_CAMERA_ID = 1; 29e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int FOURMP_CAMERA_ID = 2; 306e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) private static final VideoCaptureFactory.CamParams CAM_PARAMS[] = { 31e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch new VideoCaptureFactory.CamParams(DEPTH_CAMERA_ID, "depth", 320, 240), 32e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch new VideoCaptureFactory.CamParams(FISHEYE_CAMERA_ID, "fisheye", 640, 480), 33e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch new VideoCaptureFactory.CamParams(FOURMP_CAMERA_ID, "4MP", 1280, 720)}; 34e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 35e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // SuperFrame size definitions. Note that total size is the amount of lines 36e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // multiplied by 3/2 due to Chroma components following. 37e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_WIDTH = 1280; 38e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_HEIGHT = 1168; 39e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_FULL_HEIGHT = SF_HEIGHT * 3 / 2; 40e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_LINES_HEADER = 16; 41e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_LINES_FISHEYE = 240; 42e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_LINES_RESERVED = 80; // Spec says 96. 43e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_LINES_DEPTH = 60; 44e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_LINES_DEPTH_PADDED = 112; // Spec says 96. 45e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_LINES_BIGIMAGE = 720; 46e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final int SF_OFFSET_4MP_CHROMA = 112; 47e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 48e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final byte CHROMA_ZERO_LEVEL = 127; 49e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch private static final String TAG = "VideoCaptureTango"; 50e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 51e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch static int numberOfCameras() { 526e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) return CAM_PARAMS.length; 53e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 54e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 55e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch static VideoCaptureFactory.CamParams getCamParams(int index) { 566e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if (index >= CAM_PARAMS.length) return null; 576e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) return CAM_PARAMS[index]; 58e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 59e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 60e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch static CaptureFormat[] getDeviceSupportedFormats(int id) { 61e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch ArrayList<CaptureFormat> formatList = new ArrayList<CaptureFormat>(); 62e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch if (id == DEPTH_CAMERA_ID) { 63e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch formatList.add(new CaptureFormat(320, 180, 5, ImageFormat.YV12)); 64e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } else if (id == FISHEYE_CAMERA_ID) { 65e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch formatList.add(new CaptureFormat(640, 480, 30, ImageFormat.YV12)); 66e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } else if (id == FOURMP_CAMERA_ID) { 67e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch formatList.add(new CaptureFormat(1280, 720, 20, ImageFormat.YV12)); 68e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 69e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch return formatList.toArray(new CaptureFormat[formatList.size()]); 70e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 71e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 721320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci VideoCaptureTango(Context context, int id, long nativeVideoCaptureDeviceAndroid) { 731320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci // All Tango cameras are like the back facing one for the generic VideoCapture code. 74e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch super(context, 0, nativeVideoCaptureDeviceAndroid); 75e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mTangoCameraId = id; 76e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 77e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 78e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch @Override 791320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci protected void setCaptureParameters(int width, int height, int frameRate, 801320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci android.hardware.Camera.Parameters cameraParameters) { 811320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci mCaptureFormat = new CaptureFormat(CAM_PARAMS[mTangoCameraId].mWidth, 821320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci CAM_PARAMS[mTangoCameraId].mHeight, 831320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci frameRate, 841320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci ImageFormat.YV12); 851320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci // Connect Tango SuperFrame mode. Available sf modes are "all", 861320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci // "big-rgb", "small-rgb", "depth", "ir". 87e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch cameraParameters.set("sf-mode", "all"); 88e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 89e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 90e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch @Override 91e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch protected void allocateBuffers() { 92e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mFrameBuffer = ByteBuffer.allocateDirect( 93e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mCaptureFormat.mWidth * mCaptureFormat.mHeight * 3 / 2); 94e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // Prefill Chroma to their zero-equivalent for the cameras that only 95e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // provide Luma component. 96e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch Arrays.fill(mFrameBuffer.array(), CHROMA_ZERO_LEVEL); 97e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 98e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 99e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch @Override 1001320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci protected void setPreviewCallback(android.hardware.Camera.PreviewCallback cb) { 101e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mCamera.setPreviewCallback(cb); 102e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 103e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 104e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch @Override 1051320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci public void onPreviewFrame(byte[] data, android.hardware.Camera camera) { 106e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mPreviewBufferLock.lock(); 107e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch try { 1081320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci if (!mIsRunning) return; 1091320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 110e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch if (data.length == SF_WIDTH * SF_FULL_HEIGHT) { 111e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int rotation = getDeviceOrientation(); 112e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch if (rotation != mDeviceOrientation) { 113e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mDeviceOrientation = rotation; 114e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 1151320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci if (mCameraFacing == android.hardware.Camera.CameraInfo.CAMERA_FACING_BACK) { 116e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch rotation = 360 - rotation; 117e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 118e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch rotation = (mCameraOrientation + rotation) % 360; 119e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 120e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch if (mTangoCameraId == DEPTH_CAMERA_ID) { 121e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int sizeY = SF_WIDTH * SF_LINES_DEPTH; 122e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int startY = 1231320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci SF_WIDTH * (SF_LINES_HEADER + SF_LINES_FISHEYE + SF_LINES_RESERVED); 124e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // Depth is composed of 16b samples in which only 12b are 125e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // used. Throw away lowest 4 resolution bits. Android 126e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // platforms are big endian, LSB in lowest address. In this 127e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // case Chroma components are unused. No need to write them 128e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // explicitly since they're filled to 128 on creation. 129e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch byte depthsample; 130e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch for (int j = startY; j < startY + 2 * sizeY; j += 2) { 1311320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci depthsample = (byte) ((data[j + 1] << 4) | ((data[j] & 0xF0) >> 4)); 132e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mFrameBuffer.put(depthsample); 133e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 1341320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci for (int j = 0; j < mCaptureFormat.mWidth * mCaptureFormat.mHeight - sizeY; 1351320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci ++j) { 1361320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci mFrameBuffer.put((byte) 0); 1371320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci } 138e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } else if (mTangoCameraId == FISHEYE_CAMERA_ID) { 139e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int sizeY = SF_WIDTH * SF_LINES_FISHEYE; 140e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int startY = SF_WIDTH * SF_LINES_HEADER; 1411320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci // Fisheye is black and white so Chroma components are unused. No need to write 1421320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci // them explicitly since they're filled to 128 on creation. 1431320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci ByteBuffer.wrap(data, startY, sizeY).get(mFrameBuffer.array(), 0, sizeY); 144e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } else if (mTangoCameraId == FOURMP_CAMERA_ID) { 1451320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci int startY = SF_WIDTH * (SF_LINES_HEADER + SF_LINES_FISHEYE + 146e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch SF_LINES_RESERVED + SF_LINES_DEPTH_PADDED); 147e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int sizeY = SF_WIDTH * SF_LINES_BIGIMAGE; 148e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 149e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // The spec is completely inaccurate on the location, sizes 150e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // and format of these channels. 151e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int startU = SF_WIDTH * (SF_HEIGHT + SF_OFFSET_4MP_CHROMA); 152e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int sizeU = SF_WIDTH * SF_LINES_BIGIMAGE / 4; 1531320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci int startV = (SF_WIDTH * SF_HEIGHT * 5 / 4) + SF_WIDTH * SF_OFFSET_4MP_CHROMA; 154e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch int sizeV = SF_WIDTH * SF_LINES_BIGIMAGE / 4; 155e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch 156e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // Equivalent to the following |for| loop but much faster: 157e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // for (int i = START; i < START + SIZE; ++i) 158e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch // mFrameBuffer.put(data[i]); 159e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch ByteBuffer.wrap(data, startY, sizeY) 160e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch .get(mFrameBuffer.array(), 0, sizeY); 161e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch ByteBuffer.wrap(data, startU, sizeU) 162e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch .get(mFrameBuffer.array(), sizeY, sizeU); 163e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch ByteBuffer.wrap(data, startV, sizeV) 164e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch .get(mFrameBuffer.array(), sizeY + sizeU, sizeV); 165e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } else { 166e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch Log.e(TAG, "Unknown camera, #id: " + mTangoCameraId); 167e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch return; 168e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 169e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mFrameBuffer.rewind(); // Important! 1701320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci nativeOnFrameAvailable(mNativeVideoCaptureDeviceAndroid, mFrameBuffer.array(), 1711320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci mFrameBuffer.capacity(), rotation); 172e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 173e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } finally { 174e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch mPreviewBufferLock.unlock(); 175e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 176e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch } 177e5d81f57cb97b3b6b7fccc9c5610d21eb81db09dBen Murdoch} 178