1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/*
18*
19 */
20#include <string.h>
21#include <jni.h>
22#include <stdio.h>
23#include <stdlib.h>
24#include <time.h>
25#include <db_utilities_camera.h>
26
27#include "mosaic/AlignFeatures.h"
28#include "mosaic/Blend.h"
29#include "mosaic/Mosaic.h"
30#include "mosaic/Log.h"
31#define LOG_TAG "FEATURE_MOS_JNI"
32
33#ifdef __cplusplus
34extern "C" {
35#endif
36
37#include "mosaic_renderer_jni.h"
38
39char buffer[1024];
40
41const int MAX_FRAMES = 100;
42
43static double mTx;
44
45int tWidth[NR];
46int tHeight[NR];
47
48ImageType tImage[NR][MAX_FRAMES];// = {{ImageUtils::IMAGE_TYPE_NOIMAGE}}; // YVU24 format image
49Mosaic *mosaic[NR] = {NULL,NULL};
50ImageType resultYVU = ImageUtils::IMAGE_TYPE_NOIMAGE;
51ImageType resultBGR = ImageUtils::IMAGE_TYPE_NOIMAGE;
52float gTRS[11]; // 9 elements of the transformation, 1 for frame-number, 1 for alignment error code.
53// Variables to keep track of the mosaic computation progress for both LR & HR.
54float gProgress[NR];
55// Variables to be able to cancel the mosaic computation when the GUI says so.
56bool gCancelComputation[NR];
57
58int c;
59int width=0, height=0;
60int mosaicWidth=0, mosaicHeight=0;
61
62//int blendingType = Blend::BLEND_TYPE_FULL;
63//int blendingType = Blend::BLEND_TYPE_CYLPAN;
64int blendingType = Blend::BLEND_TYPE_HORZ;
65int stripType = Blend::STRIP_TYPE_THIN;
66bool high_res = false;
67bool quarter_res[NR] = {false,false};
68float thresh_still[NR] = {5.0f,0.0f};
69
70/* return current time in milliseconds*/
71
72#ifndef now_ms
73static double
74now_ms(void)
75{
76    //struct timespec res;
77    struct timeval res;
78    //clock_gettime(CLOCK_REALTIME, &res);
79    gettimeofday(&res, NULL);
80    return 1000.0*res.tv_sec + (double)res.tv_usec/1e3;
81}
82#endif
83
84
85static int frame_number_HR = 0;
86static int frame_number_LR = 0;
87
88int Init(int mID, int nmax)
89{
90        double  t0, t1, time_c;
91
92        if(mosaic[mID]!=NULL)
93        {
94                delete mosaic[mID];
95                mosaic[mID] = NULL;
96        }
97
98        mosaic[mID] = new Mosaic();
99
100        t0 = now_ms();
101
102        // When processing higher than 720x480 video, process low-res at
103        // quarter resolution
104        if(tWidth[LR]>180)
105            quarter_res[LR] = true;
106
107
108        // Check for initialization and if not, initialize
109        if (!mosaic[mID]->isInitialized())
110        {
111                mosaic[mID]->initialize(blendingType, stripType, tWidth[mID], tHeight[mID],
112                        nmax, quarter_res[mID], thresh_still[mID]);
113        }
114
115        t1 = now_ms();
116        time_c = t1 - t0;
117        LOGV("Init[%d]: %g ms [%d frames]",mID,time_c,nmax);
118        return 1;
119}
120
121void GenerateQuarterResImagePlanar(ImageType im, int input_w, int input_h,
122        ImageType &out)
123{
124    ImageType imp;
125    ImageType outp;
126
127    int count = 0;
128
129    for (int j = 0; j < input_h; j += H2L_FACTOR)
130    {
131        imp = im + j * input_w;
132        outp = out + (j / H2L_FACTOR) * (input_w / H2L_FACTOR);
133
134        for (int i = 0; i < input_w; i += H2L_FACTOR)
135        {
136            *outp++ = *(imp + i);
137            count++;
138        }
139    }
140
141    for (int j = input_h; j < 2 * input_h; j += H2L_FACTOR)
142    {
143        imp = im + j * input_w;
144        outp = out + (j / H2L_FACTOR) * (input_w / H2L_FACTOR);
145
146        for (int i = 0; i < input_w; i += H2L_FACTOR)
147        {
148            *outp++ = *(imp + i);
149            count++;
150        }
151    }
152
153    for (int j = 2 * input_h; j < 3 * input_h; j += H2L_FACTOR)
154    {
155        imp = im + j * input_w;
156        outp = out + (j / H2L_FACTOR) * (input_w / H2L_FACTOR);
157
158        for (int i = 0; i < input_w; i += H2L_FACTOR)
159        {
160            *outp++ = *(imp + i);
161            count++;
162        }
163    }
164}
165
166int AddFrame(int mID, int k, float* trs1d)
167{
168    double  t0, t1, time_c;
169    double trs[3][3];
170
171    int ret_code = mosaic[mID]->addFrame(tImage[mID][k]);
172
173    mosaic[mID]->getAligner()->getLastTRS(trs);
174
175    if(trs1d!=NULL)
176    {
177
178        trs1d[0] = trs[0][0];
179        trs1d[1] = trs[0][1];
180        trs1d[2] = trs[0][2];
181        trs1d[3] = trs[1][0];
182        trs1d[4] = trs[1][1];
183        trs1d[5] = trs[1][2];
184        trs1d[6] = trs[2][0];
185        trs1d[7] = trs[2][1];
186        trs1d[8] = trs[2][2];
187    }
188
189    return ret_code;
190}
191
192int Finalize(int mID)
193{
194    double  t0, t1, time_c;
195
196    t0 = now_ms();
197    // Create the mosaic
198    int ret = mosaic[mID]->createMosaic(gProgress[mID], gCancelComputation[mID]);
199    t1 = now_ms();
200    time_c = t1 - t0;
201    LOGV("CreateMosaic: %g ms",time_c);
202
203    // Get back the result
204    resultYVU = mosaic[mID]->getMosaic(mosaicWidth, mosaicHeight);
205
206    return ret;
207}
208
209void YUV420toYVU24(ImageType yvu24, ImageType yuv420sp, int width, int height)
210{
211    int frameSize = width * height;
212
213    ImageType oyp = yvu24;
214    ImageType ovp = yvu24+frameSize;
215    ImageType oup = yvu24+frameSize+frameSize;
216
217    for (int j = 0, yp = 0; j < height; j++)
218    {
219        unsigned char u = 0, v = 0;
220        int uvp = frameSize + (j >> 1) * width;
221        for (int i = 0; i < width; i++, yp++)
222        {
223            *oyp++ = yuv420sp[yp];
224            //int y = (0xff & (int)yuv420sp[yp]) -16;
225            //yvu24p[yp] = (y<0)?0:y;
226
227            if ((i & 1) == 0)
228            {
229                v = yuv420sp[uvp++];
230                u = yuv420sp[uvp++];
231            }
232
233            *ovp++ = v;
234            *oup++ = u;
235        }
236    }
237}
238
239void YUV420toYVU24_NEW(ImageType yvu24, ImageType yuv420sp, int width,
240        int height)
241{
242    int frameSize = width * height;
243
244    ImageType oyp = yvu24;
245    ImageType ovp = yvu24 + frameSize;
246    ImageType oup = yvu24 + frameSize + frameSize;
247
248    memcpy(yvu24, yuv420sp, frameSize * sizeof(unsigned char));
249
250    for (int j = 0; j < height; j += 2)
251    {
252        unsigned char u = 0, v = 0;
253        int uvp = frameSize + (j >> 1) * width;
254        ovp = yvu24 + frameSize + j * width;
255        oup = ovp + frameSize;
256
257        ImageType iuvp = yuv420sp + uvp;
258
259        for (int i = 0; i < width; i += 2)
260        {
261            v = *iuvp++;
262            u = *iuvp++;
263
264            *ovp++ = v;
265            *oup++ = u;
266
267            *ovp++ = v;
268            *oup++ = u;
269
270        }
271        memcpy(ovp, ovp - width, width * sizeof(unsigned char));
272        memcpy(oup, oup - width, width * sizeof(unsigned char));
273    }
274}
275
276
277JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_allocateMosaicMemory(
278        JNIEnv* env, jobject thiz, jint width, jint height)
279{
280    tWidth[HR] = width;
281    tHeight[HR] = height;
282    tWidth[LR] = int(width / H2L_FACTOR);
283    tHeight[LR] = int(height / H2L_FACTOR);
284
285    for(int i=0; i<MAX_FRAMES; i++)
286    {
287            tImage[LR][i] = ImageUtils::allocateImage(tWidth[LR], tHeight[LR],
288                    ImageUtils::IMAGE_TYPE_NUM_CHANNELS);
289            tImage[HR][i] = ImageUtils::allocateImage(tWidth[HR], tHeight[HR],
290                    ImageUtils::IMAGE_TYPE_NUM_CHANNELS);
291    }
292
293    AllocateTextureMemory(tWidth[HR], tHeight[HR], tWidth[LR], tHeight[LR]);
294}
295
296JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_freeMosaicMemory(
297        JNIEnv* env, jobject thiz)
298{
299    for(int i = 0; i < MAX_FRAMES; i++)
300    {
301        ImageUtils::freeImage(tImage[LR][i]);
302        ImageUtils::freeImage(tImage[HR][i]);
303    }
304
305    FreeTextureMemory();
306}
307
308
309void decodeYUV444SP(unsigned char* rgb, unsigned char* yuv420sp, int width,
310        int height)
311{
312    int frameSize = width * height;
313
314    for (int j = 0, yp = 0; j < height; j++)
315    {
316        int vp = frameSize + j * width, u = 0, v = 0;
317        int up = vp + frameSize;
318
319        for (int i = 0; i < width; i++, yp++, vp++, up++)
320        {
321            int y = (0xff & ((int) yuv420sp[yp])) - 16;
322            if (y < 0) y = 0;
323
324            v = (0xff & yuv420sp[vp]) - 128;
325            u = (0xff & yuv420sp[up]) - 128;
326
327            int y1192 = 1192 * y;
328            int r = (y1192 + 1634 * v);
329            int g = (y1192 - 833 * v - 400 * u);
330            int b = (y1192 + 2066 * u);
331
332            if (r < 0) r = 0; else if (r > 262143) r = 262143;
333            if (g < 0) g = 0; else if (g > 262143) g = 262143;
334            if (b < 0) b = 0; else if (b > 262143) b = 262143;
335
336            //rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
337            int p = j*width*3+i*3;
338            rgb[p+0] = (r<<6 & 0xFF0000)>>16;
339            rgb[p+1] = (g>>2 & 0xFF00)>>8;
340            rgb[p+2] =  b>>10 & 0xFF;
341        }
342    }
343}
344
345static int count = 0;
346
347void ConvertYVUAiToPlanarYVU(unsigned char *planar, unsigned char *in, int width,
348        int height)
349{
350    int planeSize = width * height;
351    unsigned char* Yptr = planar;
352    unsigned char* Vptr = planar + planeSize;
353    unsigned char* Uptr = Vptr + planeSize;
354
355    for (int i = 0; i < planeSize; i++)
356    {
357        *Yptr++ = *in++;
358        *Vptr++ = *in++;
359        *Uptr++ = *in++;
360        in++;   // Alpha
361    }
362}
363
364JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceImageFromGPU(
365        JNIEnv* env, jobject thiz)
366{
367    double  t0, t1, time_c;
368    t0 = now_ms();
369    int ret_code;
370
371    if(frame_number_HR<MAX_FRAMES && frame_number_LR<MAX_FRAMES)
372    {
373        double last_tx = mTx;
374
375        sem_wait(&gPreviewImage_semaphore);
376        ConvertYVUAiToPlanarYVU(tImage[LR][frame_number_LR], gPreviewImage[LR],
377                tWidth[LR], tHeight[LR]);
378
379        sem_post(&gPreviewImage_semaphore);
380
381        ret_code = AddFrame(LR, frame_number_LR, gTRS);
382
383        if(ret_code == Mosaic::MOSAIC_RET_OK || ret_code == Mosaic::MOSAIC_RET_FEW_INLIERS)
384        {
385            // Copy into HR buffer only if this is a valid frame
386            sem_wait(&gPreviewImage_semaphore);
387            ConvertYVUAiToPlanarYVU(tImage[HR][frame_number_HR], gPreviewImage[HR],
388                    tWidth[HR], tHeight[HR]);
389            sem_post(&gPreviewImage_semaphore);
390
391            frame_number_LR++;
392            frame_number_HR++;
393        }
394    }
395    else
396    {
397        gTRS[1] = gTRS[2] = gTRS[3] = gTRS[5] = gTRS[6] = gTRS[7] = 0.0f;
398        gTRS[0] = gTRS[4] = gTRS[8] = 1.0f;
399    }
400
401    UpdateWarpTransformation(gTRS);
402
403    gTRS[9] = frame_number_HR;
404    gTRS[10] = ret_code;
405
406    jfloatArray bytes = env->NewFloatArray(11);
407    if(bytes != 0)
408    {
409        env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS);
410    }
411    return bytes;
412}
413
414
415
416JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceImage(
417        JNIEnv* env, jobject thiz, jbyteArray photo_data)
418{
419    double  t0, t1, time_c;
420    t0 = now_ms();
421
422    int ret_code;
423
424    if(frame_number_HR<MAX_FRAMES && frame_number_LR<MAX_FRAMES)
425    {
426        jbyte *pixels = env->GetByteArrayElements(photo_data, 0);
427
428        YUV420toYVU24_NEW(tImage[HR][frame_number_HR], (ImageType)pixels,
429                tWidth[HR], tHeight[HR]);
430
431        env->ReleaseByteArrayElements(photo_data, pixels, 0);
432
433        double last_tx = mTx;
434
435        t0 = now_ms();
436        GenerateQuarterResImagePlanar(tImage[HR][frame_number_HR], tWidth[HR],
437                tHeight[HR], tImage[LR][frame_number_LR]);
438
439
440        sem_wait(&gPreviewImage_semaphore);
441        decodeYUV444SP(gPreviewImage[LR], tImage[LR][frame_number_LR],
442                gPreviewImageWidth[LR], gPreviewImageHeight[LR]);
443        sem_post(&gPreviewImage_semaphore);
444
445        ret_code = AddFrame(LR, frame_number_LR, gTRS);
446
447        if(ret_code == Mosaic::MOSAIC_RET_OK || ret_code == Mosaic::MOSAIC_RET_FEW_INLIERS)
448        {
449            frame_number_LR++;
450            frame_number_HR++;
451        }
452
453    }
454    else
455    {
456        gTRS[1] = gTRS[2] = gTRS[3] = gTRS[5] = gTRS[6] = gTRS[7] = 0.0f;
457        gTRS[0] = gTRS[4] = gTRS[8] = 1.0f;
458    }
459
460    UpdateWarpTransformation(gTRS);
461
462    gTRS[9] = frame_number_HR;
463    gTRS[10] = ret_code;
464
465    jfloatArray bytes = env->NewFloatArray(11);
466    if(bytes != 0)
467    {
468        env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS);
469    }
470    return bytes;
471}
472
473JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setBlendingType(
474        JNIEnv* env, jobject thiz, jint type)
475{
476    blendingType = int(type);
477}
478
479JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setStripType(
480        JNIEnv* env, jobject thiz, jint type)
481{
482    stripType = int(type);
483}
484
485JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_reset(
486        JNIEnv* env, jobject thiz)
487{
488    frame_number_HR = 0;
489    frame_number_LR = 0;
490
491    gProgress[LR] = 0.0;
492    gProgress[HR] = 0.0;
493
494    gCancelComputation[LR] = false;
495    gCancelComputation[HR] = false;
496
497    Init(LR,MAX_FRAMES);
498}
499
500JNIEXPORT jint JNICALL Java_com_android_camera_panorama_Mosaic_reportProgress(
501        JNIEnv* env, jobject thiz, jboolean hires, jboolean cancel_computation)
502{
503    if(bool(hires))
504        gCancelComputation[HR] = cancel_computation;
505    else
506        gCancelComputation[LR] = cancel_computation;
507
508    if(bool(hires))
509        return (jint) gProgress[HR];
510    else
511        return (jint) gProgress[LR];
512}
513
514JNIEXPORT jint JNICALL Java_com_android_camera_panorama_Mosaic_createMosaic(
515        JNIEnv* env, jobject thiz, jboolean value)
516{
517    high_res = bool(value);
518
519    int ret;
520
521    if(high_res)
522    {
523        LOGV("createMosaic() - High-Res Mode");
524        double  t0, t1, time_c;
525
526        gProgress[HR] = 0.0;
527        t0 = now_ms();
528
529        Init(HR, frame_number_HR);
530
531        for(int k = 0; k < frame_number_HR; k++)
532        {
533            if (gCancelComputation[HR])
534                break;
535            AddFrame(HR, k, NULL);
536            gProgress[HR] += TIME_PERCENT_ALIGN/frame_number_HR;
537        }
538
539        if (gCancelComputation[HR])
540        {
541            ret = Mosaic::MOSAIC_RET_CANCELLED;
542        }
543        else
544        {
545            gProgress[HR] = TIME_PERCENT_ALIGN;
546
547            t1 = now_ms();
548            time_c = t1 - t0;
549            LOGV("AlignAll - %d frames [HR]: %g ms", frame_number_HR, time_c);
550
551            ret = Finalize(HR);
552
553            gProgress[HR] = 100.0;
554        }
555
556        high_res = false;
557    }
558    else
559    {
560        LOGV("createMosaic() - Low-Res Mode");
561        gProgress[LR] = TIME_PERCENT_ALIGN;
562
563        ret = Finalize(LR);
564
565        gProgress[LR] = 100.0;
566    }
567
568    return (jint) ret;
569}
570
571JNIEXPORT jintArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaic(
572        JNIEnv* env, jobject thiz)
573{
574    int y,x;
575    int width = mosaicWidth;
576    int height = mosaicHeight;
577    int imageSize = width * height;
578
579    // Convert back to RGB24
580    resultBGR = ImageUtils::allocateImage(mosaicWidth, mosaicHeight,
581            ImageUtils::IMAGE_TYPE_NUM_CHANNELS);
582    ImageUtils::yvu2bgr(resultBGR, resultYVU, mosaicWidth, mosaicHeight);
583
584    LOGV("MosBytes: %d, W = %d, H = %d", imageSize, width, height);
585
586    int* image = new int[imageSize];
587    int* dims = new int[2];
588
589    for(y=0; y<height; y++)
590    {
591        for(x=0; x<width; x++)
592        {
593            image[y*width+x] = (0xFF<<24) | (resultBGR[y*width*3+x*3+2]<<16)|
594                    (resultBGR[y*width*3+x*3+1]<<8)| (resultBGR[y*width*3+x*3]);
595        }
596    }
597
598    dims[0] = width;
599    dims[1] = height;
600
601    ImageUtils::freeImage(resultBGR);
602
603    jintArray bytes = env->NewIntArray(imageSize+2);
604    if (bytes == 0) {
605        LOGE("Error in creating the image.");
606        delete[] image;
607        return 0;
608    }
609    env->SetIntArrayRegion(bytes, 0, imageSize, (jint*) image);
610    env->SetIntArrayRegion(bytes, imageSize, 2, (jint*) dims);
611    delete[] image;
612    delete[] dims;
613    return bytes;
614}
615
616JNIEXPORT jbyteArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaicNV21(
617        JNIEnv* env, jobject thiz)
618{
619    int y,x;
620    int width;
621    int height;
622
623    width = mosaicWidth;
624    height = mosaicHeight;
625
626    int imageSize = 1.5*width * height;
627
628    // Convert YVU to NV21 format in-place
629    ImageType V = resultYVU+mosaicWidth*mosaicHeight;
630    ImageType U = V+mosaicWidth*mosaicHeight;
631    for(int j=0; j<mosaicHeight/2; j++)
632    {
633        for(int i=0; i<mosaicWidth; i+=2)
634        {
635            V[j*mosaicWidth+i] = V[(2*j)*mosaicWidth+i];        // V
636            V[j*mosaicWidth+i+1] = U[(2*j)*mosaicWidth+i];        // U
637        }
638    }
639
640    LOGV("MosBytes: %d, W = %d, H = %d", imageSize, width, height);
641
642    unsigned char* dims = new unsigned char[8];
643
644    dims[0] = (unsigned char)(width >> 24);
645    dims[1] = (unsigned char)(width >> 16);
646    dims[2] = (unsigned char)(width >> 8);
647    dims[3] = (unsigned char)width;
648
649    dims[4] = (unsigned char)(height >> 24);
650    dims[5] = (unsigned char)(height >> 16);
651    dims[6] = (unsigned char)(height >> 8);
652    dims[7] = (unsigned char)height;
653
654    jbyteArray bytes = env->NewByteArray(imageSize+8);
655    if (bytes == 0) {
656        LOGE("Error in creating the image.");
657        ImageUtils::freeImage(resultYVU);
658        return 0;
659    }
660    env->SetByteArrayRegion(bytes, 0, imageSize, (jbyte*) resultYVU);
661    env->SetByteArrayRegion(bytes, imageSize, 8, (jbyte*) dims);
662    delete[] dims;
663    ImageUtils::freeImage(resultYVU);
664    return bytes;
665}
666
667#ifdef __cplusplus
668}
669#endif
670