android/mediadump/VideoDumpView.java

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.mediadump;

import java.io.IOException;
import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.FileOutputStream;
import java.io.File;

import java.lang.Integer;
import java.lang.Math;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.channels.FileChannel;
import java.nio.IntBuffer;
import java.util.Properties;

import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;

import android.app.Activity;
import android.content.Context;
import android.content.pm.ActivityInfo;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.GLUtils;
import android.opengl.Matrix;
import android.os.Bundle;
import android.util.Log;
import android.view.MotionEvent;
import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.View;
import android.widget.MediaController;
import android.widget.MediaController.MediaPlayerControl;

/**
 * A view to play a video, specified by VideoDumpConfig.VIDEO_URI, and dump the screen
 * into raw RGB files.
 * It uses a renderer to display each video frame over a surface texture, read pixels,
 * and writes the pixels into a rgb file on sdcard.
 * Those raw rgb files will be used to compare the quality distortion against
 * the original video. They can be viewed with the RgbPlayer app for debugging.
 */
class VideoDumpView extends GLSurfaceView implements MediaPlayerControl {
    private static final String TAG = "VideoDumpView";
    VideoDumpRenderer mRenderer;
    private MediaController mMediaController;
    private boolean mMediaControllerAttached = false;
    private MediaPlayer mMediaPlayer = null;
    private BufferedWriter mImageListWriter = null;

    // A serials of configuration constants.
    class VideoDumpConfig {
        // Currently we are running with a local copy of the video.
        // It should work with a "http://" sort of streaming url as well.
        public static final String VIDEO_URI = "/sdcard/mediadump/sample.mp4";
        public static final String ROOT_DIR = "/sdcard/mediadump/";
        public static final String IMAGES_LIST = "images.lst";
        public static final String IMAGE_PREFIX = "img";
        public static final String IMAGE_SUFFIX = ".rgb";
        public static final String PROPERTY_FILE = "prop.xml";

        // So far, glReadPixels only supports two (format, type) combinations
        //     GL_RGB  GL_UNSIGNED_SHORT_5_6_5   16 bits per pixel (default)
        //     GL_RGBA GL_UNSIGNED_BYTE          32 bits per pixel
        public static final int PIXEL_FORMAT = GLES20.GL_RGB;
        public static final int PIXEL_TYPE = PIXEL_FORMAT == GLES20.GL_RGBA
                ? GLES20.GL_UNSIGNED_BYTE : GLES20.GL_UNSIGNED_SHORT_5_6_5;
        public static final int BYTES_PER_PIXEL =
                PIXEL_FORMAT == GLES20.GL_RGBA ? 4 : 2;
        public static final boolean SET_CHOOSER
                = PIXEL_FORMAT == GLES20.GL_RGBA ? true : false;

        // On Motorola Xoom, it takes 100ms to read pixels and 180ms to write to a file
        // to dump a complete 720p(1280*720) video frame. It's much slower than the frame
        // playback interval (40ms). So we only dump a center block and it should be able
        // to catch all the e2e distortion. A reasonable size of the block is 256x256,
        // which takes 4ms to read pixels and 25 ms to write to a file.
        public static final int MAX_DUMP_WIDTH = 256;
        public static final int MAX_DUMP_HEIGHT = 256;

        // TODO: MediaPlayer doesn't give back the video frame rate and we'll need to
        // figure it by dividing the total number of frames by the duration.
        public static final int FRAME_RATE = 25;
    }

    public VideoDumpView(Context context) {
        super(context);
        setEGLContextClientVersion(2);
        // GLSurfaceView uses RGB_5_6_5 by default.
        if (VideoDumpConfig.SET_CHOOSER) {
            setEGLConfigChooser(8, 8, 8, 8, 8, 8);
        }
        mRenderer = new VideoDumpRenderer(context);
        setRenderer(mRenderer);
    }

    @Override
    public void onPause() {
        stopPlayback();
        super.onPause();
    }

    @Override
    public void onResume() {
        Log.d(TAG, "onResume");

        mMediaPlayer = new MediaPlayer();
        try {
            mMediaPlayer.setDataSource(VideoDumpConfig.VIDEO_URI);

            class RGBFilter implements FilenameFilter {
                public boolean accept(File dir, String name) {
                    return (name.endsWith(VideoDumpConfig.IMAGE_SUFFIX));
                }
            }
            File dump_dir = new File(VideoDumpConfig.ROOT_DIR);
            File[] dump_files = dump_dir.listFiles(new RGBFilter());
            for (File dump_file :dump_files) {
                dump_file.delete();
            }

            File image_list = new File(VideoDumpConfig.ROOT_DIR
                                       + VideoDumpConfig.IMAGES_LIST);
            image_list.delete();
            mImageListWriter = new BufferedWriter(new FileWriter(image_list));
        } catch (java.io.IOException e) {
            Log.e(TAG, e.getMessage(), e);
        }

        queueEvent(new Runnable(){
                public void run() {
                    mRenderer.setMediaPlayer(mMediaPlayer);
                    mRenderer.setImageListWriter(mImageListWriter);
                }});

        super.onResume();
    }

    public void start() {
        mMediaPlayer.start();
    }

    public void pause() {
        mMediaPlayer.pause();
        try {
            mImageListWriter.flush();
        } catch (java.io.IOException e) {
            Log.e(TAG, e.getMessage(), e);
        }
    }

    public void stopPlayback() {
        Log.d(TAG, "stopPlayback");

        if (mMediaPlayer != null) {
            mMediaPlayer.stop();
            mMediaPlayer.release();
            mMediaPlayer = null;
        }
        if (mImageListWriter != null) {
            try {
                mImageListWriter.flush();
                mImageListWriter.close();
            } catch (java.io.IOException e) {
                Log.e(TAG, e.getMessage(), e);
            }
        } else {
            Log.d(TAG, "image list file was not written successfully.");
        }
    }

    public void setMediaController(MediaController controller) {
        if (mMediaController != null) {
            mMediaController.hide();
        }
        mMediaController = controller;
    }

    private void attachMediaController() {
        if (mMediaPlayer != null && mMediaController != null) {
            if (!mMediaControllerAttached) {
                mMediaController.setMediaPlayer(this);
                View anchorView = this.getParent() instanceof View ?
                        (View)this.getParent() : this;
                mMediaController.setAnchorView(anchorView);
                mMediaController.setEnabled(true);
                mMediaControllerAttached = true;
            }
            mMediaController.show();
        }
    }

    private boolean isInPlaybackState() {
        return (mMediaPlayer != null && mMediaPlayer.isPlaying());
    }

    public boolean canPause () {
        return true;
    }

    public boolean canSeekBackward () {
        return true;
    }

    public boolean canSeekForward () {
        return true;
    }

    public int getBufferPercentage () {
        return 1;
    }

    public int getCurrentPosition () {
        if (isInPlaybackState()) {
            return mMediaPlayer.getCurrentPosition();
        }
        return 0;
    }

    public int getDuration () {
        return mMediaPlayer.getDuration();
    }

    public boolean isPlaying () {
        return isInPlaybackState() && mMediaPlayer.isPlaying();
    }

    public void seekTo (int pos) {
        mMediaPlayer.seekTo(pos);
    }

    @Override
    public boolean onTouchEvent(MotionEvent ev) {
        attachMediaController();
        return true;
    }

    /**
     * A renderer to read each video frame from a media player, draw it over a surface
     * texture, dump the on-screen pixels into a buffer, and writes the pixels into
     * a rgb file on sdcard.
     */
    private static class VideoDumpRenderer
        implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
        private static String TAG = "VideoDumpRenderer";

        /* All GL related fields from
         * http://developer.android.com/resources/samples/ApiDemos/src/com/example
         * /android/apis/graphics/GLES20TriangleRenderer.html
         */
        private static final int FLOAT_SIZE_BYTES = 4;
        private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
        private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
        private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
        private final float[] mTriangleVerticesData = {
            // X, Y, Z, U, V
            -1.0f, -1.0f, 0, 0.f, 0.f,
            1.0f, -1.0f, 0, 1.f, 0.f,
            -1.0f,  1.0f, 0, 0.f, 1.f,
            1.0f,  1.0f, 0, 1.f, 1.f,
        };

        private FloatBuffer mTriangleVertices;

        private final String mVertexShader =
                "uniform mat4 uMVPMatrix;\n" +
                "uniform mat4 uSTMatrix;\n" +
                "attribute vec4 aPosition;\n" +
                "attribute vec4 aTextureCoord;\n" +
                "varying vec2 vTextureCoord;\n" +
                "void main() {\n" +
                "  gl_Position = uMVPMatrix * aPosition;\n" +
                "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
                "}\n";

        private final String mFragmentShader =
                "#extension GL_OES_EGL_image_external : require\n" +
                "precision mediump float;\n" +
                "varying vec2 vTextureCoord;\n" +
                "uniform samplerExternalOES sTexture;\n" +
                "void main() {\n" +
                "  gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
                "}\n";

        private float[] mMVPMatrix = new float[16];
        private float[] mSTMatrix = new float[16];

        private int mProgram;
        private int mTextureID;
        private int muMVPMatrixHandle;
        private int muSTMatrixHandle;
        private int maPositionHandle;
        private int maTextureHandle;

        private SurfaceTexture mSurface;
        private boolean updateSurface = false;

        // Magic key
        private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;


        /**
         * Fields that reads video source and dumps to file.
         */
        // The media player that loads and decodes the video.
        // Not owned by this class.
        private MediaPlayer mMediaPlayer;
        // The frame number from media player.
        private int mFrameNumber = 0;
        // The frame number that is drawing on screen.
        private int mDrawNumber = 0;
        // The width and height of dumping block.
        private int mWidth = 0;
        private int mHeight = 0;
        // The offset of the dumping block.
        private int mStartX = 0;
        private int mStartY = 0;
        // A buffer to hold the dumping pixels.
        private ByteBuffer mBuffer = null;
        // A file writer to write the filenames of images.
        private BufferedWriter mImageListWriter;

        public VideoDumpRenderer(Context context) {
            mTriangleVertices = ByteBuffer.allocateDirect(
                mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
                    .order(ByteOrder.nativeOrder()).asFloatBuffer();
            mTriangleVertices.put(mTriangleVerticesData).position(0);

            Matrix.setIdentityM(mSTMatrix, 0);
        }

        public void setMediaPlayer(MediaPlayer player) {
            mMediaPlayer = player;
        }

        public void setImageListWriter(BufferedWriter imageListWriter) {
            mImageListWriter = imageListWriter;
        }

        /**
         * Called to draw the current frame.
         * This method is responsible for drawing the current frame.
         */
        public void onDrawFrame(GL10 glUnused) {
            boolean isNewFrame = false;
            int frameNumber = 0;

            synchronized(this) {
                if (updateSurface) {
                    isNewFrame = true;
                    frameNumber = mFrameNumber;
                    mSurface.updateTexImage();
                    mSurface.getTransformMatrix(mSTMatrix);
                    updateSurface = false;
                }
            }

            // Initial clear.
            GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
            GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);

            // Load the program, which is the basics rules to draw the vertexes and textures.
            GLES20.glUseProgram(mProgram);
            checkGlError("glUseProgram");

            // Activate the texture.
            GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
            GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);

            // Load the vertexes coordinates. Simple here since it only draw a rectangle
            // that fits the whole screen.
            mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
            GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
            checkGlError("glVertexAttribPointer maPosition");
            GLES20.glEnableVertexAttribArray(maPositionHandle);
            checkGlError("glEnableVertexAttribArray maPositionHandle");

            // Load the texture coordinates, which is essentially a rectangle that fits
            // the whole video frame.
            mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
            GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
            checkGlError("glVertexAttribPointer maTextureHandle");
            GLES20.glEnableVertexAttribArray(maTextureHandle);
            checkGlError("glEnableVertexAttribArray maTextureHandle");

            // Set up the GL matrices.
            Matrix.setIdentityM(mMVPMatrix, 0);
            GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
            GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

            // Draw a rectangle and render the video frame as a texture on it.
            GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
            checkGlError("glDrawArrays");
            GLES20.glFinish();

            if (isNewFrame) {  // avoid duplicates.
                Log.d(TAG, mDrawNumber + "/" + frameNumber + " before dumping "
                      + System.currentTimeMillis());
                DumpToFile(frameNumber);
                Log.d(TAG, mDrawNumber + "/" + frameNumber + " after  dumping "
                      + System.currentTimeMillis());

                mDrawNumber++;
            }
        }

        // Call the GL function that dumps the screen into a buffer, then write to a file.
        private void DumpToFile(int frameNumber) {
            GLES20.glReadPixels(mStartX, mStartY, mWidth, mHeight,
                                VideoDumpConfig.PIXEL_FORMAT,
                                VideoDumpConfig.PIXEL_TYPE,
                                mBuffer);
            checkGlError("glReadPixels");

            Log.d(TAG, mDrawNumber + "/" + frameNumber + " after  glReadPixels "
                  + System.currentTimeMillis());

            String filename =  VideoDumpConfig.ROOT_DIR + VideoDumpConfig.IMAGE_PREFIX
                    + frameNumber + VideoDumpConfig.IMAGE_SUFFIX;
            try {
                mImageListWriter.write(filename);
                mImageListWriter.newLine();
                FileOutputStream fos = new FileOutputStream(filename);
                fos.write(mBuffer.array());
                fos.close();
            } catch (java.io.IOException e) {
                Log.e(TAG, e.getMessage(), e);
            }
        }

        /**
         * Called when the surface changed size.
         * Called after the surface is created and whenever the OpenGL surface size changes.
         */
        public void onSurfaceChanged(GL10 glUnused, int width, int height) {
            Log.d(TAG, "Surface size: " + width + "x" + height);

            int video_width = mMediaPlayer.getVideoWidth();
            int video_height = mMediaPlayer.getVideoHeight();
            Log.d(TAG, "Video size: " + video_width
                  + "x" + video_height);

            // TODO: adjust video_width and video_height with the surface size.
            GLES20.glViewport(0, 0, video_width, video_height);

            mWidth = Math.min(VideoDumpConfig.MAX_DUMP_WIDTH, video_width);
            mHeight = Math.min(VideoDumpConfig.MAX_DUMP_HEIGHT, video_height);
            mStartX = video_width / mWidth / 2 * mWidth;
            mStartY = video_height / mHeight / 2 * mHeight;

            Log.d(TAG, "dumping block start at (" + mStartX + "," + mStartY + ") "
                  + "size " + mWidth + "x" + mHeight);

            int image_size = mWidth * mHeight * VideoDumpConfig.BYTES_PER_PIXEL;
            mBuffer = ByteBuffer.allocate(image_size);

            int bpp[] = new int[3];
            GLES20.glGetIntegerv(GLES20.GL_RED_BITS, bpp, 0);
            GLES20.glGetIntegerv(GLES20.GL_GREEN_BITS, bpp, 1);
            GLES20.glGetIntegerv(GLES20.GL_BLUE_BITS, bpp, 2);
            Log.d(TAG, "rgb bits: " + bpp[0] + "-" + bpp[1] + "-" + bpp[2]);

            // Save the properties into a xml file
            // so the RgbPlayer can understand the output format.
            Properties prop = new Properties();
            prop.setProperty("width", Integer.toString(mWidth));
            prop.setProperty("height", Integer.toString(mHeight));
            prop.setProperty("startX", Integer.toString(mStartX));
            prop.setProperty("startY", Integer.toString(mStartY));
            prop.setProperty("bytesPerPixel",
                             Integer.toString(VideoDumpConfig.BYTES_PER_PIXEL));
            prop.setProperty("frameRate", Integer.toString(VideoDumpConfig.FRAME_RATE));
            try {
                prop.storeToXML(new FileOutputStream(VideoDumpConfig.ROOT_DIR
                                                     + VideoDumpConfig.PROPERTY_FILE), "");
            } catch (java.io.IOException e) {
                Log.e(TAG, e.getMessage(), e);
            }
        }

        /**
         * Called when the surface is created or recreated.
         * Called when the rendering thread starts and whenever the EGL context is lost.
         * A place to put code to create resources that need to be created when the rendering
         * starts, and that need to be recreated when the EGL context is lost e.g. texture.
         * Note that when the EGL context is lost, all OpenGL resources associated with
         * that context will be automatically deleted.
         */
        public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
            Log.d(TAG, "onSurfaceCreated");

            /* Set up shaders and handles to their variables */
            mProgram = createProgram(mVertexShader, mFragmentShader);
            if (mProgram == 0) {
                return;
            }
            maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
            checkGlError("glGetAttribLocation aPosition");
            if (maPositionHandle == -1) {
                throw new RuntimeException("Could not get attrib location for aPosition");
            }
            maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
            checkGlError("glGetAttribLocation aTextureCoord");
            if (maTextureHandle == -1) {
                throw new RuntimeException("Could not get attrib location for aTextureCoord");
            }

            muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
            checkGlError("glGetUniformLocation uMVPMatrix");
            if (muMVPMatrixHandle == -1) {
                throw new RuntimeException("Could not get attrib location for uMVPMatrix");
            }

            muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
            checkGlError("glGetUniformLocation uSTMatrix");
            if (muSTMatrixHandle == -1) {
                throw new RuntimeException("Could not get attrib location for uSTMatrix");
            }


            // Create our texture. This has to be done each time the surface is created.
            int[] textures = new int[1];
            GLES20.glGenTextures(1, textures, 0);

            mTextureID = textures[0];
            GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
            checkGlError("glBindTexture mTextureID");

            // Can't do mipmapping with mediaplayer source
            GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
                                   GLES20.GL_NEAREST);
            GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
                                   GLES20.GL_LINEAR);
            // Clamp to edge is the only option
            GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
                                   GLES20.GL_CLAMP_TO_EDGE);
            GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
                                   GLES20.GL_CLAMP_TO_EDGE);
            checkGlError("glTexParameteri mTextureID");

            /*
             * Create the SurfaceTexture that will feed this textureID,
             * and pass it to the MediaPlayer
             */
            mSurface = new SurfaceTexture(mTextureID);
            mSurface.setOnFrameAvailableListener(this);

            Surface surface = new Surface(mSurface);
            mMediaPlayer.setSurface(surface);
            surface.release();

            try {
                mMediaPlayer.prepare();
            } catch (IOException t) {
                Log.e(TAG, "media player prepare failed");
            }

            synchronized(this) {
                updateSurface = false;
            }
        }

        synchronized public void onFrameAvailable(SurfaceTexture surface) {
            /* For simplicity, SurfaceTexture calls here when it has new
             * data available.  Call may come in from some random thread,
             * so let's be safe and use synchronize. No OpenGL calls can be done here.
             */
            mFrameNumber++;
            updateSurface = true;
        }

        private int loadShader(int shaderType, String source) {
            int shader = GLES20.glCreateShader(shaderType);
            if (shader != 0) {
                GLES20.glShaderSource(shader, source);
                GLES20.glCompileShader(shader);
                int[] compiled = new int[1];
                GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
                if (compiled[0] == 0) {
                    Log.e(TAG, "Could not compile shader " + shaderType + ":");
                    Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
                    GLES20.glDeleteShader(shader);
                    shader = 0;
                }
            }
            return shader;
        }

        private int createProgram(String vertexSource, String fragmentSource) {
            int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
            if (vertexShader == 0) {
                return 0;
            }
            int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
            if (pixelShader == 0) {
                return 0;
            }

            int program = GLES20.glCreateProgram();
            if (program != 0) {
                GLES20.glAttachShader(program, vertexShader);
                checkGlError("glAttachShader");
                GLES20.glAttachShader(program, pixelShader);
                checkGlError("glAttachShader");
                GLES20.glLinkProgram(program);
                int[] linkStatus = new int[1];
                GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
                if (linkStatus[0] != GLES20.GL_TRUE) {
                    Log.e(TAG, "Could not link program: ");
                    Log.e(TAG, GLES20.glGetProgramInfoLog(program));
                    GLES20.glDeleteProgram(program);
                    program = 0;
                }
            }
            return program;
        }

        private void checkGlError(String op) {
            int error;
            while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
                Log.e(TAG, op + ": glError " + error);
                throw new RuntimeException(op + ": glError " + error);
            }
        }

    }  // End of class VideoDumpRender.

}  // End of class VideoDumpView.