PanoramaActivity.java revision 41a2e9735136f372de95652d0828600282c8e967 
1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.camera.panorama;
18
19import com.android.camera.CameraDisabledException;
20import com.android.camera.CameraHardwareException;
21import com.android.camera.CameraHolder;
22import com.android.camera.MenuHelper;
23import com.android.camera.ModePicker;
24import com.android.camera.R;
25import com.android.camera.ShutterButton;
26import com.android.camera.Storage;
27import com.android.camera.Util;
28
29import android.app.Activity;
30import android.content.Context;
31import android.graphics.Bitmap;
32import android.graphics.BitmapFactory;
33import android.graphics.ImageFormat;
34import android.graphics.PixelFormat;
35import android.graphics.Rect;
36import android.graphics.SurfaceTexture;
37import android.graphics.YuvImage;
38import android.hardware.Camera;
39import android.hardware.Sensor;
40import android.hardware.SensorEvent;
41import android.hardware.SensorEventListener;
42import android.hardware.SensorManager;
43import android.hardware.Camera.Parameters;
44import android.hardware.Camera.Size;
45import android.os.Bundle;
46import android.os.Handler;
47import android.os.Message;
48import android.util.Log;
49import android.view.SurfaceView;
50import android.view.View;
51import android.view.WindowManager;
52import android.widget.ImageView;
53
54import java.io.ByteArrayOutputStream;
55import java.util.List;
56
57/**
58 * Activity to handle panorama capturing.
59 */
60public class PanoramaActivity extends Activity implements
61        ModePicker.OnModeChangeListener,
62        SurfaceTexture.OnFrameAvailableListener {
63    public static final int DEFAULT_SWEEP_ANGLE = 60;
64    public static final int DEFAULT_BLEND_MODE = Mosaic.BLENDTYPE_HORIZONTAL;
65    public static final int DEFAULT_CAPTURE_PIXELS = 960 * 720;
66
67    private static final int MSG_FINAL_MOSAIC_READY = 1;
68    private static final int MSG_RESET_TO_PREVIEW = 2;
69
70    private static final String TAG = "PanoramaActivity";
71    private static final int PREVIEW_STOPPED = 0;
72    private static final int PREVIEW_ACTIVE = 1;
73    private static final int CAPTURE_VIEWFINDER = 0;
74    private static final int CAPTURE_MOSAIC = 1;
75
76    // Ratio of nanosecond to second
77    private static final float NS2S = 1.0f / 1000000000.0f;
78
79    private boolean mPausing;
80
81    private View mPanoControlLayout;
82    private View mCaptureLayout;
83    private View mReviewLayout;
84    private SurfaceView mPreview;
85    private ImageView mReview;
86    private CaptureView mCaptureView;
87    private MosaicRendererSurfaceView mRealTimeMosaicView;
88    private ShutterButton mShutterButton;
89
90    private byte[] mFinalJpegData;
91
92    private int mPreviewWidth;
93    private int mPreviewHeight;
94    private Camera mCameraDevice;
95    private int mCameraState;
96    private int mCaptureState;
97    private SensorManager mSensorManager;
98    private Sensor mSensor;
99    private ModePicker mModePicker;
100    private MosaicFrameProcessor mMosaicFrameProcessor;
101    private String mCurrentImagePath = null;
102    private long mTimeTaken;
103    private Handler mMainHandler;
104    private SurfaceTexture mSurface;
105    private boolean mUseSurfaceTexture = true;
106
107    private boolean mThreadRunning;
108
109    @Override
110    public void onCreate(Bundle icicle) {
111        super.onCreate(icicle);
112
113        getWindow().setFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON,
114                WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
115
116        createContentView();
117
118        mSensorManager = (SensorManager) getSystemService(Context.SENSOR_SERVICE);
119        mSensor = mSensorManager.getDefaultSensor(Sensor.TYPE_GYROSCOPE);
120        if (mSensor == null) {
121            mSensor = mSensorManager.getDefaultSensor(Sensor.TYPE_ORIENTATION);
122        }
123
124        mMainHandler = new Handler() {
125            @Override
126            public void handleMessage(Message msg) {
127                switch (msg.what) {
128                    case MSG_FINAL_MOSAIC_READY:
129                        mThreadRunning = false;
130                        showFinalMosaic((Bitmap) msg.obj);
131                        break;
132                    case MSG_RESET_TO_PREVIEW:
133                        mThreadRunning = false;
134                        resetToPreview();
135                        break;
136                }
137                clearMosaicFrameProcessorIfNeeded();
138            }
139        };
140    }
141
142    public void createSurfaceTextureAndStartPreview(int textureID)
143    {
144        /*
145         * Create the SurfaceTexture that will feed this textureID, and pass it to the camera
146         */
147        mSurface = new SurfaceTexture(textureID);
148        mSurface.setOnFrameAvailableListener(this);
149        startPreview();
150        Log.i(TAG, "Created Surface Texture");
151    }
152
153    public SurfaceTexture getSurfaceTexture()
154    {
155        return mSurface;
156    }
157
158    private void setupCamera() {
159        openCamera();
160        Parameters parameters = mCameraDevice.getParameters();
161        setupCaptureParams(parameters);
162        configureCamera(parameters);
163    }
164
165    private void releaseCamera() {
166        if (mCameraDevice != null) {
167            mCameraDevice.setPreviewCallbackWithBuffer(null);
168            CameraHolder.instance().release();
169            mCameraDevice = null;
170            mCameraState = PREVIEW_STOPPED;
171        }
172    }
173
174    private void openCamera() {
175        try {
176            mCameraDevice = Util.openCamera(this, CameraHolder.instance().getBackCameraId());
177        } catch (CameraHardwareException e) {
178            Util.showErrorAndFinish(this, R.string.cannot_connect_camera);
179            return;
180        } catch (CameraDisabledException e) {
181            Util.showErrorAndFinish(this, R.string.camera_disabled);
182            return;
183        }
184    }
185
186    private boolean findBestPreviewSize(List<Size> supportedSizes, boolean need4To3,
187            boolean needSmaller) {
188        int pixelsDiff = DEFAULT_CAPTURE_PIXELS;
189        boolean hasFound = false;
190        for (Size size : supportedSizes) {
191            int h = size.height;
192            int w = size.width;
193            // we only want 4:3 format.
194            int d = DEFAULT_CAPTURE_PIXELS - h * w;
195            if (needSmaller && d < 0) { // no bigger preview than 960x720.
196                continue;
197            }
198            if (need4To3 && (h * 4 != w * 3)) {
199                continue;
200            }
201            d = Math.abs(d);
202            if (d < pixelsDiff) {
203                mPreviewWidth = w;
204                mPreviewHeight = h;
205                pixelsDiff = d;
206                hasFound = true;
207            }
208        }
209        return hasFound;
210    }
211
212    private void setupCaptureParams(Parameters parameters) {
213        List<Size> supportedSizes = parameters.getSupportedPreviewSizes();
214        if (!findBestPreviewSize(supportedSizes, true, true)) {
215            Log.w(TAG, "No 4:3 ratio preview size supported.");
216            if (!findBestPreviewSize(supportedSizes, false, true)) {
217                Log.w(TAG, "Can't find a supported preview size smaller than 960x720.");
218                findBestPreviewSize(supportedSizes, false, false);
219            }
220        }
221        Log.v(TAG, "preview h = " + mPreviewHeight + " , w = " + mPreviewWidth);
222        parameters.setPreviewSize(mPreviewWidth, mPreviewHeight);
223
224        List<int[]> frameRates = parameters.getSupportedPreviewFpsRange();
225        int last = frameRates.size() - 1;
226        int minFps = (frameRates.get(last))[Parameters.PREVIEW_FPS_MIN_INDEX];
227        int maxFps = (frameRates.get(last))[Parameters.PREVIEW_FPS_MAX_INDEX];
228        parameters.setPreviewFpsRange(minFps, maxFps);
229        Log.v(TAG, "preview fps: " + minFps + ", " + maxFps);
230
231        parameters.setRecordingHint(false);
232    }
233
234    public int getPreviewBufSize() {
235        PixelFormat pixelInfo = new PixelFormat();
236        PixelFormat.getPixelFormatInfo(mCameraDevice.getParameters().getPreviewFormat(), pixelInfo);
237        // TODO: remove this extra 32 byte after the driver bug is fixed.
238        return (mPreviewWidth * mPreviewHeight * pixelInfo.bitsPerPixel / 8) + 32;
239    }
240
241    private void configureCamera(Parameters parameters) {
242        mCameraDevice.setParameters(parameters);
243
244        int orientation = Util.getDisplayOrientation(Util.getDisplayRotation(this),
245                CameraHolder.instance().getBackCameraId());
246        mCameraDevice.setDisplayOrientation(orientation);
247
248        if(!mUseSurfaceTexture) {
249            int bufSize = getPreviewBufSize();
250            Log.v(TAG, "BufSize = " + bufSize);
251            for (int i = 0; i < 10; i++) {
252                try {
253                    mCameraDevice.addCallbackBuffer(new byte[bufSize]);
254                } catch (OutOfMemoryError e) {
255                    Log.v(TAG, "Buffer allocation failed: buffer " + i);
256                    break;
257                }
258            }
259        }
260    }
261
262    private boolean switchToOtherMode(int mode) {
263        if (isFinishing()) {
264            return false;
265        }
266        MenuHelper.gotoMode(mode, this);
267        finish();
268        return true;
269    }
270
271    public boolean onModeChanged(int mode) {
272        if (mode != ModePicker.MODE_PANORAMA) {
273            return switchToOtherMode(mode);
274        } else {
275            return true;
276        }
277    }
278
279    public void runViewFinder() {
280        mRealTimeMosaicView.setWarping(false);
281
282        // First update the surface texture...
283        mRealTimeMosaicView.updateSurfaceTexture();
284        // ...then call preprocess to render it to low-res and high-res RGB textures
285        mRealTimeMosaicView.preprocess();
286
287        mRealTimeMosaicView.setReady();
288        mRealTimeMosaicView.requestRender();
289    }
290
291    public void runMosaicCapture() {
292        mRealTimeMosaicView.setWarping(true);
293
294        // Lock the condition variable
295        mRealTimeMosaicView.lockPreviewReadyFlag();
296        // First update the surface texture...
297        mRealTimeMosaicView.updateSurfaceTexture();
298        // ...then call preprocess to render it to low-res and high-res RGB textures
299        mRealTimeMosaicView.preprocess();
300        // Now, transfer the textures from GPU to CPU memory for processing
301        mRealTimeMosaicView.transferGPUtoCPU();
302        // Wait on the condition variable (will be opened when GPU->CPU transfer is done).
303        mRealTimeMosaicView.waitUntilPreviewReady();
304
305        mMosaicFrameProcessor.processFrame(null);
306    }
307
308    synchronized public void onFrameAvailable(SurfaceTexture surface) {
309        /* For simplicity, SurfaceTexture calls here when it has new
310         * data available.  Call may come in from some random thread,
311         * so let's be safe and use synchronize. No OpenGL calls can be done here.
312         */
313        if (mCaptureState == CAPTURE_VIEWFINDER) {
314            runViewFinder();
315        } else {
316            runMosaicCapture();
317        }
318    }
319
320    public void startCapture() {
321        // Reset values so we can do this again.
322        mTimeTaken = System.currentTimeMillis();
323        mCaptureState = CAPTURE_MOSAIC;
324
325        mMosaicFrameProcessor.setProgressListener(new MosaicFrameProcessor.ProgressListener() {
326            @Override
327            public void onProgress(boolean isFinished, float translationRate, int traversedAngleX,
328                    int traversedAngleY) {
329                if (isFinished) {
330                    stopCapture();
331                } else {
332                    updateProgress(translationRate, traversedAngleX, traversedAngleY);
333                }
334            }
335        });
336
337        if (!mUseSurfaceTexture) {
338            // Preview callback used whenever new viewfinder frame is available
339            mCameraDevice.setPreviewCallbackWithBuffer(new Camera.PreviewCallback() {
340                @Override
341                public void onPreviewFrame(final byte[] data, Camera camera) {
342                    mMosaicFrameProcessor.processFrame(data);
343                    // The returned buffer needs be added back to callback buffer
344                    // again.
345                    camera.addCallbackBuffer(data);
346                }
347            });
348        }
349
350        mCaptureLayout.setVisibility(View.VISIBLE);
351        mPreview.setVisibility(View.INVISIBLE);  // will be re-used, invisible is better than gone.
352        mRealTimeMosaicView.setVisibility(View.VISIBLE);
353        mPanoControlLayout.setVisibility(View.GONE);
354
355    }
356
357    private void stopCapture() {
358        mCaptureState = CAPTURE_VIEWFINDER;
359
360        mMosaicFrameProcessor.setProgressListener(null);
361        stopPreview();
362
363        if (!mUseSurfaceTexture) {
364            mCameraDevice.setPreviewCallbackWithBuffer(null);
365        }
366
367        mSurface.setOnFrameAvailableListener(null);
368
369        // TODO: show some dialog for long computation.
370        if (!mThreadRunning) {
371            mThreadRunning = true;
372            Thread t = new Thread() {
373                @Override
374                public void run() {
375                    generateAndStoreFinalMosaic(false);
376                }
377            };
378            t.start();
379        }
380    }
381
382    private void updateProgress(float translationRate, int traversedAngleX, int traversedAngleY) {
383
384        mRealTimeMosaicView.setReady();
385        mRealTimeMosaicView.requestRender();
386
387        if (translationRate > 150) {
388            // TODO: remove the text and draw implications according to the UI
389            // spec.
390            mCaptureView.setStatusText("S L O W   D O W N");
391            mCaptureView.setSweepAngle(Math.max(traversedAngleX, traversedAngleY) + 1);
392            mCaptureView.invalidate();
393        } else {
394            mCaptureView.setStatusText("");
395            mCaptureView.setSweepAngle(Math.max(traversedAngleX, traversedAngleY) + 1);
396            mCaptureView.invalidate();
397        }
398    }
399
400    private void createContentView() {
401        setContentView(R.layout.panorama);
402
403        mCaptureState = CAPTURE_VIEWFINDER;
404
405        mCaptureLayout = (View) findViewById(R.id.pano_capture_layout);
406        mReviewLayout = (View) findViewById(R.id.pano_review_layout);
407
408        mPreview = (SurfaceView) findViewById(R.id.pano_preview_view);
409
410        mCaptureView = (CaptureView) findViewById(R.id.pano_capture_view);
411        mCaptureView.setStartAngle(-DEFAULT_SWEEP_ANGLE / 2);
412        mReview = (ImageView) findViewById(R.id.pano_reviewarea);
413
414        mRealTimeMosaicView = (MosaicRendererSurfaceView) findViewById(R.id.pano_renderer);
415        mRealTimeMosaicView.setUIObject(this);
416
417        mShutterButton = (ShutterButton) findViewById(R.id.pano_shutter_button);
418        mShutterButton.setOnClickListener(new View.OnClickListener() {
419            @Override
420            public void onClick(View v) {
421                if (mPausing || mThreadRunning) return;
422                startCapture();
423            }
424        });
425
426        mPanoControlLayout = (View) findViewById(R.id.pano_control_layout);
427
428        mModePicker = (ModePicker) findViewById(R.id.mode_picker);
429        mModePicker.setVisibility(View.VISIBLE);
430        mModePicker.setOnModeChangeListener(this);
431        mModePicker.setCurrentMode(ModePicker.MODE_PANORAMA);
432
433        mRealTimeMosaicView.setVisibility(View.VISIBLE);
434    }
435
436    @OnClickAttr
437    public void onStopButtonClicked(View v) {
438        if (mPausing || mThreadRunning) return;
439        stopCapture();
440    }
441
442    @OnClickAttr
443    public void onOkButtonClicked(View v) {
444        if (mPausing || mThreadRunning) return;
445        mThreadRunning = true;
446        Thread t = new Thread() {
447            @Override
448            public void run() {
449                saveFinalMosaic();
450            }
451        };
452        t.start();
453    }
454
455    @OnClickAttr
456    public void onRetakeButtonClicked(View v) {
457        if (mPausing || mThreadRunning) return;
458        resetToPreview();
459    }
460
461    private void resetToPreview() {
462        mCaptureState = CAPTURE_VIEWFINDER;
463
464        mReviewLayout.setVisibility(View.GONE);
465        mPreview.setVisibility(View.VISIBLE);
466        mPanoControlLayout.setVisibility(View.VISIBLE);
467        mCaptureLayout.setVisibility(View.GONE);
468        mMosaicFrameProcessor.reset();
469
470        mSurface.setOnFrameAvailableListener(this);
471
472        if (!mPausing) startPreview();
473
474        mRealTimeMosaicView.setVisibility(View.VISIBLE);
475    }
476
477    private void showFinalMosaic(Bitmap bitmap) {
478        if (bitmap != null) {
479            mReview.setImageBitmap(bitmap);
480            mCaptureLayout.setVisibility(View.GONE);
481            mPreview.setVisibility(View.INVISIBLE);
482            mReviewLayout.setVisibility(View.VISIBLE);
483            mCaptureView.setStatusText("");
484            mCaptureView.setSweepAngle(0);
485            mCaptureView.invalidate();
486        }
487    }
488
489    private void saveFinalMosaic() {
490        if (mFinalJpegData != null) {
491            Storage.addImage(getContentResolver(), mCurrentImagePath, mTimeTaken, null, 0,
492                    mFinalJpegData);
493            mFinalJpegData = null;
494        }
495        mMainHandler.sendMessage(mMainHandler.obtainMessage(MSG_RESET_TO_PREVIEW));
496    }
497
498    private void clearMosaicFrameProcessorIfNeeded() {
499        if (!mPausing || mThreadRunning) return;
500        mMosaicFrameProcessor.clear();
501    }
502
503    private void initMosaicFrameProcessorIfNeeded() {
504        if (mPausing || mThreadRunning) return;
505        if (mMosaicFrameProcessor == null) {
506            // Start the activity for the first time.
507            mMosaicFrameProcessor = new MosaicFrameProcessor(DEFAULT_SWEEP_ANGLE - 5,
508                    mPreviewWidth, mPreviewHeight, getPreviewBufSize(), mUseSurfaceTexture);
509        }
510        mMosaicFrameProcessor.initialize();
511    }
512
513    @Override
514    protected void onPause() {
515        super.onPause();
516        mSurface.setOnFrameAvailableListener(null);
517        releaseCamera();
518        mPausing = true;
519
520        mRealTimeMosaicView.onPause();
521        mCaptureView.onPause();
522        mSensorManager.unregisterListener(mListener);
523        clearMosaicFrameProcessorIfNeeded();
524        System.gc();
525    }
526
527    @Override
528    protected void onResume() {
529        super.onResume();
530
531        mPausing = false;
532        /*
533         * It is not necessary to get accelerometer events at a very high rate,
534         * by using a slower rate (SENSOR_DELAY_UI), we get an automatic
535         * low-pass filter, which "extracts" the gravity component of the
536         * acceleration. As an added benefit, we use less power and CPU
537         * resources.
538         */
539        mSensorManager.registerListener(mListener, mSensor, SensorManager.SENSOR_DELAY_UI);
540        mCaptureState = CAPTURE_VIEWFINDER;
541        setupCamera();
542        // Camera must be initialized before MosaicFrameProcessor is initialized. The preview size
543        // has to be decided by camera device.
544        initMosaicFrameProcessorIfNeeded();
545        mCaptureView.onResume();
546        mRealTimeMosaicView.onResume();
547    }
548
549    private final SensorEventListener mListener = new SensorEventListener() {
550        private float mCompassCurrX; // degrees
551        private float mCompassCurrY; // degrees
552        private float mTimestamp;
553
554        public void onSensorChanged(SensorEvent event) {
555            if (event.sensor.getType() == Sensor.TYPE_GYROSCOPE) {
556                if (mTimestamp != 0) {
557                    final float dT = (event.timestamp - mTimestamp) * NS2S;
558                    mCompassCurrX += event.values[1] * dT * 180.0f / Math.PI;
559                    mCompassCurrY += event.values[0] * dT * 180.0f / Math.PI;
560                }
561                mTimestamp = event.timestamp;
562
563            } else if (event.sensor.getType() == Sensor.TYPE_ORIENTATION) {
564                mCompassCurrX = event.values[0];
565                mCompassCurrY = event.values[1];
566            }
567
568            if (mMosaicFrameProcessor != null) {
569                mMosaicFrameProcessor.updateCompassValue(mCompassCurrX, mCompassCurrY);
570            }
571        }
572
573        @Override
574        public void onAccuracyChanged(Sensor sensor, int accuracy) {
575        }
576    };
577
578    public void generateAndStoreFinalMosaic(boolean highRes) {
579        mMosaicFrameProcessor.createMosaic(highRes);
580
581        mCurrentImagePath = PanoUtil.createName(
582                getResources().getString(R.string.pano_file_name_format), mTimeTaken);
583
584        if (highRes) {
585            mCurrentImagePath += "_HR";
586        } else {
587            mCurrentImagePath += "_LR";
588        }
589
590            byte[] imageData = mMosaicFrameProcessor.getFinalMosaicNV21();
591            int len = imageData.length - 8;
592
593            int width = (imageData[len + 0] << 24) + ((imageData[len + 1] & 0xFF) << 16)
594                    + ((imageData[len + 2] & 0xFF) << 8) + (imageData[len + 3] & 0xFF);
595            int height = (imageData[len + 4] << 24) + ((imageData[len + 5] & 0xFF) << 16)
596                    + ((imageData[len + 6] & 0xFF) << 8) + (imageData[len + 7] & 0xFF);
597            Log.v(TAG, "ImLength = " + (len) + ", W = " + width + ", H = " + height);
598
599            YuvImage yuvimage = new YuvImage(imageData, ImageFormat.NV21, width, height, null);
600            ByteArrayOutputStream out = new ByteArrayOutputStream();
601            yuvimage.compressToJpeg(new Rect(0, 0, width, height), 100, out);
602            try {
603                out.close();
604            } catch (Exception e) {
605                Log.e(TAG, "Exception in storing final mosaic", e);
606                return;
607            }
608
609            mFinalJpegData = out.toByteArray();
610            Bitmap bitmap = BitmapFactory.decodeByteArray(mFinalJpegData, 0, mFinalJpegData.length);
611            mMainHandler.sendMessage(mMainHandler.obtainMessage(MSG_FINAL_MOSAIC_READY, bitmap));
612            // Now's a good time to run the GC. Since we won't do any explicit
613            // allocation during the test, the GC should stay dormant and not
614            // influence our results.
615            System.runFinalization();
616            System.gc();
617    }
618
619    private void setPreviewTexture(SurfaceTexture surface) {
620        try {
621            mCameraDevice.setPreviewTexture(surface);
622        } catch (Throwable ex) {
623            releaseCamera();
624            throw new RuntimeException("setPreviewTexture failed", ex);
625        }
626    }
627
628    private void startPreview() {
629        // If we're previewing already, stop the preview first (this will blank
630        // the screen).
631        if (mCameraState != PREVIEW_STOPPED) stopPreview();
632
633        setPreviewTexture(mSurface);
634
635        try {
636            Log.v(TAG, "startPreview");
637            mCameraDevice.startPreview();
638        } catch (Throwable ex) {
639            releaseCamera();
640            throw new RuntimeException("startPreview failed", ex);
641        }
642        mCameraState = PREVIEW_ACTIVE;
643    }
644
645    private void stopPreview() {
646        if (mCameraDevice != null && mCameraState != PREVIEW_STOPPED) {
647            Log.v(TAG, "stopPreview");
648            mCameraDevice.stopPreview();
649        }
650        mCameraState = PREVIEW_STOPPED;
651    }
652}
653