Mosaic.java revision b28b9c0fa991bc97e8aa11da83d27f71fdfef6da
181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen/* 281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Copyright (C) 2011 The Android Open Source Project 381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Licensed under the Apache License, Version 2.0 (the "License"); 581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * you may not use this file except in compliance with the License. 681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * You may obtain a copy of the License at 781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * http://www.apache.org/licenses/LICENSE-2.0 981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 1081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Unless required by applicable law or agreed to in writing, software 1181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * distributed under the License is distributed on an "AS IS" BASIS, 1281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * See the License for the specific language governing permissions and 1481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * limitations under the License. 1581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 1681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 1781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chenpackage com.android.camera.panorama; 1881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 1981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen/** 2081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * The Java interface to JNI calls regarding mosaic stitching. 2181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 2281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * A high-level usage is: 2381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 2481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Mosaic mosaic = new Mosaic(); 2581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * mosaic.setSourceImageDimensions(width, height); 2681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * mosaic.reset(blendType); 2781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 2881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * while ((pixels = hasNextImage()) != null) { 2981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * mosaic.setSourceImage(pixels); 3081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * } 3181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 3281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * mosaic.createMosaic(highRes); 3381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * byte[] result = mosaic.getFinalMosaic(); 3481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 3581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 3681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chenpublic class Mosaic { 3781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 3881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * In this mode, the images are stitched together in the same spatial arrangement as acquired 3981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * i.e. if the user follows a curvy trajectory, the image boundary of the resulting mosaic will 4081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * be curved in the same manner. This mode is useful if the user wants to capture a mosaic as 4181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * if "painting" the scene using the smart-phone device and does not want any corrective warps 4281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * to distort the captured images. 4381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 4481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public static final int BLENDTYPE_FULL = 0; 4581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 4681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 4781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * This mode is the same as BLENDTYPE_FULL except that the resulting mosaic is rotated 4881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * to balance the first and last images to be approximately at the same vertical offset in the 4981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * output mosaic. This is useful when acquiring a mosaic by a typical panning-like motion to 5081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * remove a one-sided curve in the mosaic (typically due to the camera not staying horizontal 5181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * during the video capture) and convert it to a more symmetrical "smiley-face" like output. 5281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 5381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public static final int BLENDTYPE_PAN = 1; 5481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 5581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 5681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * This mode compensates for typical "smiley-face" like output in longer mosaics and creates 5781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * a rectangular mosaic with minimal black borders (by unwrapping the mosaic onto an imaginary 5881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * cylinder). If the user follows a curved trajectory (instead of a perfect panning trajectory), 5981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * the resulting mosaic here may suffer from some image distortions in trying to map the 6081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * trajectory to a cylinder. 6181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 6281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public static final int BLENDTYPE_CYLINDERPAN = 2; 6381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 6481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 6581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * This mode is basically BLENDTYPE_CYLINDERPAN plus doing a rectangle cropping before returning 6681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * the mosaic. The mode is useful for making the resulting mosaic have a rectangle shape. 6781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 6881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public static final int BLENDTYPE_HORIZONTAL =3; 6981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 70e1178a73fd5756771d25d0b8375452450f509e99mbansal /** 71b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * This strip type will use the default thin strips where the strips are 72b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * spaced according to the image capture rate. 73b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal */ 74b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal public static final int STRIPTYPE_THIN = 0; 75b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal 76b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal /** 77b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * This strip type will use wider strips for blending. The strip separation 78b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * is controlled by a threshold on the native side. Since the strips are 79b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * wider, there is an additional cross-fade blending step to make the seam 80b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * boundaries smoother. Since this mode uses lesser image frames, it is 81b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * computationally more efficient than the thin strip mode. 82b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal */ 83b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal public static final int STRIPTYPE_WIDE = 1; 84b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal 85b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal /** 86e1178a73fd5756771d25d0b8375452450f509e99mbansal * Return flags returned by createMosaic() are one of the following. 87e1178a73fd5756771d25d0b8375452450f509e99mbansal */ 88e1178a73fd5756771d25d0b8375452450f509e99mbansal public static final int MOSAIC_RET_OK = 1; 89e1178a73fd5756771d25d0b8375452450f509e99mbansal public static final int MOSAIC_RET_ERROR = -1; 90e1178a73fd5756771d25d0b8375452450f509e99mbansal public static final int MOSAIC_RET_CANCELLED = -2; 91e1178a73fd5756771d25d0b8375452450f509e99mbansal 9281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen static { 9381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen System.loadLibrary("jni_mosaic"); 9481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen } 9581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 9681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 9781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Allocate memory for the image frames at the given resolution. 9881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 9981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @param width width of the input frames in pixels 10081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @param height height of the input frames in pixels 10181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 102b062b806441939e162e19be0baeb6513592c66b3mbansal public native void allocateMosaicMemory(int width, int height); 103b062b806441939e162e19be0baeb6513592c66b3mbansal 104b062b806441939e162e19be0baeb6513592c66b3mbansal /** 105b062b806441939e162e19be0baeb6513592c66b3mbansal * Free memory allocated by allocateMosaicMemory. 106b062b806441939e162e19be0baeb6513592c66b3mbansal * 107b062b806441939e162e19be0baeb6513592c66b3mbansal */ 108b062b806441939e162e19be0baeb6513592c66b3mbansal public native void freeMosaicMemory(); 10981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 11081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 11181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Pass the input image frame to the native layer. Each time the a new 11281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * source image t is set, the transformation matrix from the first source 11381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * image to t is computed and returned. 11481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 11581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @param pixels source image of NV21 format. 11681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @return Float array of length 10; first 9 entries correspond to the 3x3 11781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * transformation matrix between the first frame and the passed frame, 11881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * and the last entry is the number of the passed frame, 11981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * where the counting starts from 1. 12081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 12181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public native float[] setSourceImage(byte[] pixels); 12281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 12381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 12441a2e9735136f372de95652d0828600282c8e967mbansal * This is an alternative to the setSourceImage function above. This should 12541a2e9735136f372de95652d0828600282c8e967mbansal * be called when the image data is already on the native side in a fixed 12641a2e9735136f372de95652d0828600282c8e967mbansal * byte array. In implementation, this array is filled by the GL thread 12741a2e9735136f372de95652d0828600282c8e967mbansal * using glReadPixels directly from GPU memory (where it is accessed by 12841a2e9735136f372de95652d0828600282c8e967mbansal * an associated SurfaceTexture). 12941a2e9735136f372de95652d0828600282c8e967mbansal * 13041a2e9735136f372de95652d0828600282c8e967mbansal * @return Float array of length 10; first 9 entries correspond to the 3x3 13141a2e9735136f372de95652d0828600282c8e967mbansal * transformation matrix between the first frame and the passed frame, 13241a2e9735136f372de95652d0828600282c8e967mbansal * and the last entry is the number of the passed frame, 13341a2e9735136f372de95652d0828600282c8e967mbansal * where the counting starts from 1. 13441a2e9735136f372de95652d0828600282c8e967mbansal */ 13541a2e9735136f372de95652d0828600282c8e967mbansal public native float[] setSourceImageFromGPU(); 13641a2e9735136f372de95652d0828600282c8e967mbansal 13741a2e9735136f372de95652d0828600282c8e967mbansal /** 13881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Set the type of blending. 13981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 14081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @param type the blending type defined in the class. {BLENDTYPE_FULL, 14181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * BLENDTYPE_PAN, BLENDTYPE_CYLINDERPAN, BLENDTYPE_HORIZONTAL} 14281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 14381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public native void setBlendingType(int type); 14481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 14581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 146b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * Set the type of strips to use for blending. 147b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * @param type the blending strip type to use {STRIPTYPE_THIN, 148b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal * STRIPTYPE_WIDE}. 149b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal */ 150b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal public native void setStripType(int type); 151b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal 152b28b9c0fa991bc97e8aa11da83d27f71fdfef6dambansal /** 15381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Tell the native layer to create the final mosaic after all the input frame 15481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * data have been collected. 15581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * The case of generating high-resolution mosaic may take dozens of seconds to finish. 15681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 15781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @param value True means generating a high-resolution mosaic - 15881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * which is based on the original images set in setSourceImage(). 15981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * False means generating a low-resolution version - 16081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * which is based on 1/4 downscaled images from the original images. 161e1178a73fd5756771d25d0b8375452450f509e99mbansal * @return Returns a status code suggesting if the mosaic building was 162e1178a73fd5756771d25d0b8375452450f509e99mbansal * successful, in error, or was cancelled by the user. 16381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 164e1178a73fd5756771d25d0b8375452450f509e99mbansal public native int createMosaic(boolean value); 16581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 16681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 16781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Get the data for the created mosaic. 16881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 16981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @return Returns an integer array which contains the final mosaic in the ARGB_8888 format. 17081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * The first MosaicWidth*MosaicHeight values contain the image data, followed by 2 17181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * integers corresponding to the values MosaicWidth and MosaicHeight respectively. 17281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 17381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public native int[] getFinalMosaic(); 17481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 17581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 17681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Get the data for the created mosaic. 17781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 17881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * @return Returns a byte array which contains the final mosaic in the NV21 format. 17981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * The first MosaicWidth*MosaicHeight*1.5 values contain the image data, followed by 18081f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * 8 bytes which pack the MosaicWidth and MosaicHeight integers into 4 bytes each 18181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * respectively. 18281f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 18381f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public native byte[] getFinalMosaicNV21(); 18481f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen 18581f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen /** 18681f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Reset the state of the frame arrays which maintain the captured frame data. 18781f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen * Also re-initializes the native mosaic object to make it ready for capturing a new mosaic. 18881f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen */ 18981f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen public native void reset(); 19050b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal 19150b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal /** 19250b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal * Get the progress status of the mosaic computation process. 19350b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal * @param hires Boolean flag to select whether to report progress of the 19450b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal * low-res or high-res mosaicer. 195e1178a73fd5756771d25d0b8375452450f509e99mbansal * @param cancelComputation Boolean flag to allow cancelling the 196e1178a73fd5756771d25d0b8375452450f509e99mbansal * mosaic computation when needed from the GUI end. 19750b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal * @return Returns a number from 0-100 where 50 denotes that the mosaic 19850b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal * computation is 50% done. 19950b3c890986aadb3780b4da8c0b8dbb0f1422ebambansal */ 200e1178a73fd5756771d25d0b8375452450f509e99mbansal public native int reportProgress(boolean hires, boolean cancelComputation); 20181f844f803ea2feeb61eb9270fbdd5592620d42eWei-Ta Chen} 202