1/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package androidx.heifwriter;
18
19import static android.support.test.InstrumentationRegistry.getContext;
20
21import android.graphics.Bitmap;
22import android.graphics.ImageFormat;
23import android.media.MediaExtractor;
24import android.media.MediaFormat;
25import android.media.MediaMetadataRetriever;
26import android.opengl.GLES20;
27import android.os.Environment;
28import android.os.Handler;
29import android.os.HandlerThread;
30import android.os.Process;
31import android.support.test.filters.LargeTest;
32import android.support.test.runner.AndroidJUnit4;
33import android.util.Log;
34
35import static androidx.heifwriter.HeifWriter.INPUT_MODE_BITMAP;
36import static androidx.heifwriter.HeifWriter.INPUT_MODE_BUFFER;
37import static androidx.heifwriter.HeifWriter.INPUT_MODE_SURFACE;
38
39import androidx.annotation.NonNull;
40import androidx.annotation.Nullable;
41import androidx.heifwriter.test.R;
42
43import static org.junit.Assert.assertEquals;
44import static org.junit.Assert.assertTrue;
45
46import org.junit.After;
47import org.junit.Before;
48import org.junit.Test;
49import org.junit.runner.RunWith;
50
51import java.io.Closeable;
52import java.io.File;
53import java.io.FileInputStream;
54import java.io.FileOutputStream;
55import java.io.IOException;
56import java.io.InputStream;
57import java.io.OutputStream;
58import java.util.Arrays;
59
60/**
61 * Test {@link HeifWriter}.
62 */
63@RunWith(AndroidJUnit4.class)
64public class HeifWriterTest {
65    private static final String TAG = HeifWriterTest.class.getSimpleName();
66    private static final boolean DEBUG = false;
67    private static final boolean DUMP_YUV_INPUT = false;
68
69    private static byte[][] TEST_COLORS = {
70            {(byte) 255, (byte) 0, (byte) 0},
71            {(byte) 255, (byte) 0, (byte) 255},
72            {(byte) 255, (byte) 255, (byte) 255},
73            {(byte) 255, (byte) 255, (byte) 0},
74    };
75
76    private static final String TEST_HEIC = "test.heic";
77    private static final int[] IMAGE_RESOURCES = new int[] {
78            R.raw.test
79    };
80    private static final String[] IMAGE_FILENAMES = new String[] {
81            TEST_HEIC
82    };
83    private static final String OUTPUT_FILENAME = "output.heic";
84
85    private EglWindowSurface mInputEglSurface;
86    private Handler mHandler;
87    private int mInputIndex;
88
89    @Before
90    public void setUp() throws Exception {
91        for (int i = 0; i < IMAGE_RESOURCES.length; ++i) {
92            String outputPath = new File(Environment.getExternalStorageDirectory(),
93                    IMAGE_FILENAMES[i]).getAbsolutePath();
94
95            InputStream inputStream = null;
96            FileOutputStream outputStream = null;
97            try {
98                inputStream = getContext().getResources().openRawResource(IMAGE_RESOURCES[i]);
99                outputStream = new FileOutputStream(outputPath);
100                copy(inputStream, outputStream);
101            } finally {
102                closeQuietly(inputStream);
103                closeQuietly(outputStream);
104            }
105        }
106
107        HandlerThread handlerThread = new HandlerThread(
108                "HeifEncoderThread", Process.THREAD_PRIORITY_FOREGROUND);
109        handlerThread.start();
110        mHandler = new Handler(handlerThread.getLooper());
111    }
112
113    @After
114    public void tearDown() throws Exception {
115        for (int i = 0; i < IMAGE_RESOURCES.length; ++i) {
116            String imageFilePath =
117                    new File(Environment.getExternalStorageDirectory(), IMAGE_FILENAMES[i])
118                            .getAbsolutePath();
119            File imageFile = new File(imageFilePath);
120            if (imageFile.exists()) {
121                imageFile.delete();
122            }
123        }
124    }
125
126    @Test
127    @LargeTest
128    public void testInputBuffer_NoGrid_NoHandler() throws Throwable {
129        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BUFFER, false, false);
130        doTestForVariousNumberImages(builder);
131    }
132
133    @Test
134    @LargeTest
135    public void testInputBuffer_Grid_NoHandler() throws Throwable {
136        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BUFFER, true, false);
137        doTestForVariousNumberImages(builder);
138    }
139
140    @Test
141    @LargeTest
142    public void testInputBuffer_NoGrid_Handler() throws Throwable {
143        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BUFFER, false, true);
144        doTestForVariousNumberImages(builder);
145    }
146
147    @Test
148    @LargeTest
149    public void testInputBuffer_Grid_Handler() throws Throwable {
150        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BUFFER, true, true);
151        doTestForVariousNumberImages(builder);
152    }
153
154    @Test
155    @LargeTest
156    public void testInputSurface_NoGrid_NoHandler() throws Throwable {
157        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_SURFACE, false, false);
158        doTestForVariousNumberImages(builder);
159    }
160
161    @Test
162    @LargeTest
163    public void testInputSurface_Grid_NoHandler() throws Throwable {
164        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_SURFACE, true, false);
165        doTestForVariousNumberImages(builder);
166    }
167
168    @Test
169    @LargeTest
170    public void testInputSurface_NoGrid_Handler() throws Throwable {
171        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_SURFACE, false, true);
172        doTestForVariousNumberImages(builder);
173    }
174
175    @Test
176    @LargeTest
177    public void testInputSurface_Grid_Handler() throws Throwable {
178        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_SURFACE, true, true);
179        doTestForVariousNumberImages(builder);
180    }
181
182    @Test
183    @LargeTest
184    public void testInputBitmap_NoGrid_NoHandler() throws Throwable {
185        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BITMAP, false, false);
186        for (int i = 0; i < IMAGE_RESOURCES.length; ++i) {
187            String inputPath = new File(Environment.getExternalStorageDirectory(),
188                    IMAGE_FILENAMES[i]).getAbsolutePath();
189            doTestForVariousNumberImages(builder.setInputPath(inputPath));
190        }
191    }
192
193    @Test
194    @LargeTest
195    public void testInputBitmap_Grid_NoHandler() throws Throwable {
196        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BITMAP, true, false);
197        for (int i = 0; i < IMAGE_RESOURCES.length; ++i) {
198            String inputPath = new File(Environment.getExternalStorageDirectory(),
199                    IMAGE_FILENAMES[i]).getAbsolutePath();
200            doTestForVariousNumberImages(builder.setInputPath(inputPath));
201        }
202    }
203
204    @Test
205    @LargeTest
206    public void testInputBitmap_NoGrid_Handler() throws Throwable {
207        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BITMAP, false, true);
208        for (int i = 0; i < IMAGE_RESOURCES.length; ++i) {
209            String inputPath = new File(Environment.getExternalStorageDirectory(),
210                    IMAGE_FILENAMES[i]).getAbsolutePath();
211            doTestForVariousNumberImages(builder.setInputPath(inputPath));
212        }
213    }
214
215    @Test
216    @LargeTest
217    public void testInputBitmap_Grid_Handler() throws Throwable {
218        TestConfig.Builder builder = new TestConfig.Builder(INPUT_MODE_BITMAP, true, true);
219        for (int i = 0; i < IMAGE_RESOURCES.length; ++i) {
220            String inputPath = new File(Environment.getExternalStorageDirectory(),
221                    IMAGE_FILENAMES[i]).getAbsolutePath();
222            doTestForVariousNumberImages(builder.setInputPath(inputPath));
223        }
224    }
225
226    private void doTestForVariousNumberImages(TestConfig.Builder builder) throws Exception {
227        builder.setNumImages(4);
228        doTest(builder.setRotation(270).build());
229        doTest(builder.setRotation(180).build());
230        doTest(builder.setRotation(90).build());
231        doTest(builder.setRotation(0).build());
232        doTest(builder.setNumImages(1).build());
233        doTest(builder.setNumImages(8).build());
234    }
235
236    private void closeQuietly(Closeable closeable) {
237        if (closeable != null) {
238            try {
239                closeable.close();
240            } catch (RuntimeException rethrown) {
241                throw rethrown;
242            } catch (Exception ignored) {
243            }
244        }
245    }
246
247    private int copy(InputStream in, OutputStream out) throws IOException {
248        int total = 0;
249        byte[] buffer = new byte[8192];
250        int c;
251        while ((c = in.read(buffer)) != -1) {
252            total += c;
253            out.write(buffer, 0, c);
254        }
255        return total;
256    }
257
258    private static class TestConfig {
259        final int mInputMode;
260        final boolean mUseGrid;
261        final boolean mUseHandler;
262        final int mMaxNumImages;
263        final int mNumImages;
264        final int mWidth;
265        final int mHeight;
266        final int mRotation;
267        final int mQuality;
268        final String mInputPath;
269        final String mOutputPath;
270        final Bitmap[] mBitmaps;
271
272        TestConfig(int inputMode, boolean useGrid, boolean useHandler,
273                   int maxNumImages, int numImages, int width, int height,
274                   int rotation, int quality,
275                   String inputPath, String outputPath, Bitmap[] bitmaps) {
276            mInputMode = inputMode;
277            mUseGrid = useGrid;
278            mUseHandler = useHandler;
279            mMaxNumImages = maxNumImages;
280            mNumImages = numImages;
281            mWidth = width;
282            mHeight = height;
283            mRotation = rotation;
284            mQuality = quality;
285            mInputPath = inputPath;
286            mOutputPath = outputPath;
287            mBitmaps = bitmaps;
288        }
289
290        static class Builder {
291            final int mInputMode;
292            final boolean mUseGrid;
293            final boolean mUseHandler;
294            int mMaxNumImages;
295            int mNumImages;
296            int mWidth;
297            int mHeight;
298            int mRotation;
299            final int mQuality;
300            String mInputPath;
301            final String mOutputPath;
302            Bitmap[] mBitmaps;
303            boolean mNumImagesSetExplicitly;
304
305
306            Builder(int inputMode, boolean useGrids, boolean useHandler) {
307                mInputMode = inputMode;
308                mUseGrid = useGrids;
309                mUseHandler = useHandler;
310                mMaxNumImages = mNumImages = 4;
311                mWidth = 1920;
312                mHeight = 1080;
313                mRotation = 0;
314                mQuality = 100;
315                mOutputPath = new File(Environment.getExternalStorageDirectory(),
316                        OUTPUT_FILENAME).getAbsolutePath();
317            }
318
319            Builder setInputPath(String inputPath) {
320                mInputPath = (mInputMode == INPUT_MODE_BITMAP) ? inputPath : null;
321                return this;
322            }
323
324            Builder setNumImages(int numImages) {
325                mNumImagesSetExplicitly = true;
326                mNumImages = numImages;
327                return this;
328            }
329
330            Builder setRotation(int rotation) {
331                mRotation = rotation;
332                return this;
333            }
334
335            private void loadBitmapInputs() {
336                if (mInputMode != INPUT_MODE_BITMAP) {
337                    return;
338                }
339                MediaMetadataRetriever retriever = new MediaMetadataRetriever();
340                retriever.setDataSource(mInputPath);
341                String hasImage = retriever.extractMetadata(
342                        MediaMetadataRetriever.METADATA_KEY_HAS_IMAGE);
343                if (!"yes".equals(hasImage)) {
344                    throw new IllegalArgumentException("no bitmap found!");
345                }
346                mMaxNumImages = Math.min(mMaxNumImages, Integer.parseInt(retriever.extractMetadata(
347                        MediaMetadataRetriever.METADATA_KEY_IMAGE_COUNT)));
348                if (!mNumImagesSetExplicitly) {
349                    mNumImages = mMaxNumImages;
350                }
351                mBitmaps = new Bitmap[mMaxNumImages];
352                for (int i = 0; i < mBitmaps.length; i++) {
353                    mBitmaps[i] = retriever.getImageAtIndex(i);
354                }
355                mWidth = mBitmaps[0].getWidth();
356                mHeight = mBitmaps[0].getHeight();
357                retriever.release();
358            }
359
360            private void cleanupStaleOutputs() {
361                File outputFile = new File(mOutputPath);
362                if (outputFile.exists()) {
363                    outputFile.delete();
364                }
365            }
366
367            TestConfig build() {
368                cleanupStaleOutputs();
369                loadBitmapInputs();
370
371                return new TestConfig(mInputMode, mUseGrid, mUseHandler, mMaxNumImages, mNumImages,
372                        mWidth, mHeight, mRotation, mQuality, mInputPath, mOutputPath, mBitmaps);
373            }
374        }
375
376        @Override
377        public String toString() {
378            return "TestConfig"
379                    + ": mInputMode " + mInputMode
380                    + ", mUseGrid " + mUseGrid
381                    + ", mUseHandler " + mUseHandler
382                    + ", mMaxNumImages " + mMaxNumImages
383                    + ", mNumImages " + mNumImages
384                    + ", mWidth " + mWidth
385                    + ", mHeight " + mHeight
386                    + ", mRotation " + mRotation
387                    + ", mQuality " + mQuality
388                    + ", mInputPath " + mInputPath
389                    + ", mOutputPath " + mOutputPath;
390        }
391    }
392
393    private void doTest(TestConfig config) throws Exception {
394        int width = config.mWidth;
395        int height = config.mHeight;
396        int numImages = config.mNumImages;
397
398        mInputIndex = 0;
399        HeifWriter heifWriter = null;
400        FileInputStream inputStream = null;
401        FileOutputStream outputStream = null;
402        try {
403            if (DEBUG) Log.d(TAG, "started: " + config);
404
405            heifWriter = new HeifWriter.Builder(
406                    config.mOutputPath, width, height, config.mInputMode)
407                    .setRotation(config.mRotation)
408                    .setGridEnabled(config.mUseGrid)
409                    .setMaxImages(config.mMaxNumImages)
410                    .setQuality(config.mQuality)
411                    .setPrimaryIndex(config.mMaxNumImages - 1)
412                    .setHandler(config.mUseHandler ? mHandler : null)
413                    .build();
414
415            if (config.mInputMode == INPUT_MODE_SURFACE) {
416                mInputEglSurface = new EglWindowSurface(heifWriter.getInputSurface());
417            }
418
419            heifWriter.start();
420
421            if (config.mInputMode == INPUT_MODE_BUFFER) {
422                byte[] data = new byte[width * height * 3 / 2];
423
424                if (config.mInputPath != null) {
425                    inputStream = new FileInputStream(config.mInputPath);
426                }
427
428                if (DUMP_YUV_INPUT) {
429                    File outputFile = new File("/sdcard/input.yuv");
430                    outputFile.createNewFile();
431                    outputStream = new FileOutputStream(outputFile);
432                }
433
434                for (int i = 0; i < numImages; i++) {
435                    if (DEBUG) Log.d(TAG, "fillYuvBuffer: " + i);
436                    fillYuvBuffer(i, data, width, height, inputStream);
437                    if (DUMP_YUV_INPUT) {
438                        Log.d(TAG, "@@@ dumping input YUV");
439                        outputStream.write(data);
440                    }
441                    heifWriter.addYuvBuffer(ImageFormat.YUV_420_888, data);
442                }
443            } else if (config.mInputMode == INPUT_MODE_SURFACE) {
444                // The input surface is a surface texture using single buffer mode, draws will be
445                // blocked until onFrameAvailable is done with the buffer, which is dependant on
446                // how fast MediaCodec processes them, which is further dependent on how fast the
447                // MediaCodec callbacks are handled. We can't put draws on the same looper that
448                // handles MediaCodec callback, it will cause deadlock.
449                for (int i = 0; i < numImages; i++) {
450                    if (DEBUG) Log.d(TAG, "drawFrame: " + i);
451                    drawFrame(width, height);
452                }
453                heifWriter.setInputEndOfStreamTimestamp(
454                        1000 * computePresentationTime(numImages - 1));
455            } else if (config.mInputMode == INPUT_MODE_BITMAP) {
456                Bitmap[] bitmaps = config.mBitmaps;
457                for (int i = 0; i < Math.min(bitmaps.length, numImages); i++) {
458                    if (DEBUG) Log.d(TAG, "addBitmap: " + i);
459                    heifWriter.addBitmap(bitmaps[i]);
460                    bitmaps[i].recycle();
461                }
462            }
463
464            heifWriter.stop(3000);
465            verifyResult(config.mOutputPath, width, height, config.mRotation, config.mUseGrid,
466                    Math.min(numImages, config.mMaxNumImages));
467            if (DEBUG) Log.d(TAG, "finished: PASS");
468        } finally {
469            try {
470                if (outputStream != null) {
471                    outputStream.close();
472                }
473                if (inputStream != null) {
474                    inputStream.close();
475                }
476            } catch (IOException e) {}
477
478            if (heifWriter != null) {
479                heifWriter.close();
480                heifWriter = null;
481            }
482            if (mInputEglSurface != null) {
483                // This also releases the surface from encoder.
484                mInputEglSurface.release();
485                mInputEglSurface = null;
486            }
487        }
488    }
489
490    private long computePresentationTime(int frameIndex) {
491        return 132 + (long)frameIndex * 1000000;
492    }
493
494    private void fillYuvBuffer(int frameIndex, @NonNull byte[] data, int width, int height,
495                               @Nullable FileInputStream inputStream) throws IOException {
496        if (inputStream != null) {
497            inputStream.read(data);
498        } else {
499            byte[] color = TEST_COLORS[frameIndex % TEST_COLORS.length];
500            int sizeY = width * height;
501            Arrays.fill(data, 0, sizeY, color[0]);
502            Arrays.fill(data, sizeY, sizeY * 5 / 4, color[1]);
503            Arrays.fill(data, sizeY * 5 / 4, sizeY * 3 / 2, color[2]);
504        }
505    }
506
507    private void drawFrame(int width, int height) {
508        mInputEglSurface.makeCurrent();
509        generateSurfaceFrame(mInputIndex, width, height);
510        mInputEglSurface.setPresentationTime(1000 * computePresentationTime(mInputIndex));
511        mInputEglSurface.swapBuffers();
512        mInputIndex++;
513    }
514
515    private void generateSurfaceFrame(int frameIndex, int width, int height) {
516        frameIndex %= 4;
517
518        GLES20.glViewport(0, 0, width, height);
519        GLES20.glDisable(GLES20.GL_SCISSOR_TEST);
520        GLES20.glClearColor(1.0f, 0.0f, 0.0f, 1.0f);
521        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
522        GLES20.glEnable(GLES20.GL_SCISSOR_TEST);
523
524        int startX, startY;
525        int borderWidth = 16;
526        for (int i = 0; i < 7; i++) {
527            startX = (width - borderWidth * 2) * i / 7 + borderWidth;
528            GLES20.glScissor(startX, borderWidth,
529                    (width - borderWidth * 2) / 7, height - borderWidth * 2);
530            GLES20.glClearColor(((7 - i) & 0x4) * 0.16f,
531                    ((7 - i) & 0x2) * 0.32f,
532                    ((7 - i) & 0x1) * 0.64f,
533                    1.0f);
534            GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
535        }
536
537        startX = (width / 6) + (width / 6) * frameIndex;
538        startY = height / 4;
539        GLES20.glScissor(startX, startY, width / 6, height / 3);
540        GLES20.glClearColor(0.5f, 0.5f, 0.5f, 1.0f);
541        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
542        GLES20.glScissor(startX + borderWidth, startY + borderWidth,
543                width / 6 - borderWidth * 2, height / 3 - borderWidth * 2);
544        GLES20.glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
545        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
546    }
547
548    private void verifyResult(
549            String filename, int width, int height, int rotation, boolean useGrid, int numImages)
550            throws Exception {
551        MediaMetadataRetriever retriever = new MediaMetadataRetriever();
552        retriever.setDataSource(filename);
553        String hasImage = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_IMAGE);
554        if (!"yes".equals(hasImage)) {
555            throw new Exception("No images found in file " + filename);
556        }
557        assertEquals("Wrong image count", numImages,
558                Integer.parseInt(retriever.extractMetadata(
559                    MediaMetadataRetriever.METADATA_KEY_IMAGE_COUNT)));
560        assertEquals("Wrong width", width,
561                Integer.parseInt(retriever.extractMetadata(
562                    MediaMetadataRetriever.METADATA_KEY_IMAGE_WIDTH)));
563        assertEquals("Wrong height", height,
564                Integer.parseInt(retriever.extractMetadata(
565                    MediaMetadataRetriever.METADATA_KEY_IMAGE_HEIGHT)));
566        assertEquals("Wrong rotation", rotation,
567                Integer.parseInt(retriever.extractMetadata(
568                    MediaMetadataRetriever.METADATA_KEY_IMAGE_ROTATION)));
569        retriever.release();
570
571        if (useGrid) {
572            MediaExtractor extractor = new MediaExtractor();
573            extractor.setDataSource(filename);
574            MediaFormat format = extractor.getTrackFormat(0);
575            int gridWidth = format.getInteger(MediaFormat.KEY_TILE_WIDTH);
576            int gridHeight = format.getInteger(MediaFormat.KEY_TILE_HEIGHT);
577            int gridRows = format.getInteger(MediaFormat.KEY_GRID_ROWS);
578            int gridCols = format.getInteger(MediaFormat.KEY_GRID_COLUMNS);
579            assertTrue("Wrong grid width or cols",
580                    ((width + gridWidth - 1) / gridWidth) == gridCols);
581            assertTrue("Wrong grid height or rows",
582                    ((height + gridHeight - 1) / gridHeight) == gridRows);
583            extractor.release();
584        }
585    }
586}
587