1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include <GLES2/gl2.h> 18#include <GLES2/gl2ext.h> 19#include <jni.h> 20#include <math.h> 21#include <stdio.h> 22#include <stdlib.h> 23#include "db_utilities_camera.h" 24#include "mosaic/ImageUtils.h" 25#include "mosaic_renderer/FrameBuffer.h" 26#include "mosaic_renderer/WarpRenderer.h" 27#include "mosaic_renderer/SurfaceTextureRenderer.h" 28#include "mosaic_renderer/YVURenderer.h" 29 30#include "mosaic/Log.h" 31#define LOG_TAG "MosaicRenderer" 32 33#include "mosaic_renderer_jni.h" 34 35// Texture handle 36GLuint gSurfaceTextureID[1]; 37 38bool gWarpImage = true; 39 40// Low-Res input image frame in YUVA format for preview rendering and processing 41// and high-res YUVA input image for processing. 42unsigned char* gPreviewImage[NR]; 43// Low-Res & high-res preview image width 44int gPreviewImageWidth[NR]; 45// Low-Res & high-res preview image height 46int gPreviewImageHeight[NR]; 47 48// Semaphore to protect simultaneous read/writes from gPreviewImage 49sem_t gPreviewImage_semaphore; 50 51// Off-screen preview FBO width (large enough to store the entire 52// preview mosaic). FBO is frame buffer object. 53int gPreviewFBOWidth; 54// Off-screen preview FBO height (large enough to store the entire 55// preview mosaic). 56int gPreviewFBOHeight; 57 58// gK is the transformation to map the canonical {-1,1} vertex coordinate system 59// to the {0,gPreviewImageWidth[LR]} input image frame coordinate system before 60// applying the given affine transformation trs. gKm is the corresponding 61// transformation for going to the {0,gPreviewFBOWidth}. 62double gK[9]; 63double gKinv[9]; 64double gKm[9]; 65double gKminv[9]; 66 67// Shader to copy input SurfaceTexture into and RGBA FBO. The two shaders 68// render to the textures with dimensions corresponding to the low-res and 69// high-res image frames. 70SurfaceTextureRenderer gSurfTexRenderer[NR]; 71// Off-screen FBOs to store the low-res and high-res RGBA copied out from 72// the SurfaceTexture by the gSurfTexRenderers. 73FrameBuffer gBufferInput[NR]; 74 75// Shader to convert RGBA textures into YVU textures for processing 76YVURenderer gYVURenderer[NR]; 77// Off-screen FBOs to store the low-res and high-res YVU textures for processing 78FrameBuffer gBufferInputYVU[NR]; 79 80// Shader to translate the flip-flop FBO - gBuffer[1-current] -> gBuffer[current] 81WarpRenderer gWarper1; 82// Shader to add warped current frame to the flip-flop FBO - gBuffer[current] 83WarpRenderer gWarper2; 84// Off-screen FBOs (flip-flop) to store the result of gWarper1 & gWarper2 85FrameBuffer gBuffer[2]; 86 87// Shader to warp and render the preview FBO to the screen 88WarpRenderer gPreview; 89 90// Index of the gBuffer FBO gWarper1 is going to write into 91int gCurrentFBOIndex = 0; 92 93// 3x3 Matrices holding the transformation of this frame (gThisH1t) and of 94// the last frame (gLastH1t) w.r.t the first frame. 95double gThisH1t[9]; 96double gLastH1t[9]; 97 98// Variables to represent the fixed position of the top-left corner of the 99// current frame in the previewFBO 100double gCenterOffsetX = 0.0f; 101double gCenterOffsetY = 0.0f; 102 103// X-Offset of the viewfinder (current frame) w.r.t 104// (gCenterOffsetX, gCenterOffsetY). This offset varies with time and is 105// used to pan the viewfinder across the UI layout. 106double gPanOffset = 0.0f; 107 108// Variables tracking the translation value for the current frame and the 109// last frame (both w.r.t the first frame). The difference between these 110// values is used to control the panning speed of the viewfinder display 111// on the UI screen. 112double gThisTx = 0.0f; 113double gLastTx = 0.0f; 114 115// These are the scale factors used by the gPreview shader to ensure that 116// the image frame is correctly scaled to the full UI layout height while 117// maintaining its aspect ratio 118double gUILayoutScalingX = 1.0f; 119double gUILayoutScalingY = 1.0f; 120 121// Whether the view that we will render preview FBO onto is in landscape or portrait 122// orientation. 123bool gIsLandscapeOrientation = true; 124 125// State of the viewfinder. Set to false when the viewfinder hits the UI edge. 126bool gPanViewfinder = true; 127 128// Affine transformation in GL 4x4 format (column-major) to warp the 129// last frame mosaic into the current frame coordinate system. 130GLfloat g_dAffinetransGL[16]; 131double g_dAffinetrans[16]; 132 133// Affine transformation in GL 4x4 format (column-major) to translate the 134// preview FBO across the screen (viewfinder panning). 135GLfloat g_dAffinetransPanGL[16]; 136double g_dAffinetransPan[16]; 137 138// XY translation in GL 4x4 format (column-major) to center the current 139// preview mosaic in the preview FBO 140GLfloat g_dTranslationToFBOCenterGL[16]; 141double g_dTranslationToFBOCenter[16]; 142 143// GL 4x4 Identity transformation 144GLfloat g_dAffinetransIdentGL[] = { 145 1., 0., 0., 0., 146 0., 1., 0., 0., 147 0., 0., 1., 0., 148 0., 0., 0., 1.}; 149 150// GL 4x4 Rotation transformation (column-majored): 90 degree 151GLfloat g_dAffinetransRotation90GL[] = { 152 0., 1., 0., 0., 153 -1., 0., 0., 0., 154 0., 0., 1., 0., 155 0., 0., 0., 1.}; 156 157// 3x3 Rotation transformation (row-majored): 90 degree 158double gRotation90[] = { 159 0., -1., 0., 160 1., 0., 0., 161 0., 0., 1.,}; 162 163 164float g_dIdent3x3[] = { 165 1.0, 0.0, 0.0, 166 0.0, 1.0, 0.0, 167 0.0, 0.0, 1.0}; 168 169const int GL_TEXTURE_EXTERNAL_OES_ENUM = 0x8D65; 170 171static void printGLString(const char *name, GLenum s) { 172 const char *v = (const char *) glGetString(s); 173 LOGI("GL %s = %s", name, v); 174} 175 176void checkFramebufferStatus(const char* name) { 177 GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER); 178 if (status == 0) { 179 LOGE("Checking completeness of Framebuffer:%s", name); 180 checkGlError("checkFramebufferStatus (is the target \"GL_FRAMEBUFFER\"?)"); 181 } else if (status != GL_FRAMEBUFFER_COMPLETE) { 182 const char* msg = "not listed"; 183 switch (status) { 184 case GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT: msg = "attachment"; break; 185 case GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS: msg = "dimensions"; break; 186 case GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT: msg = "missing attachment"; break; 187 case GL_FRAMEBUFFER_UNSUPPORTED: msg = "unsupported"; break; 188 } 189 LOGE("Framebuffer: %s is INCOMPLETE: %s, %x", name, msg, status); 190 } 191} 192 193// @return false if there was an error 194bool checkGLErrorDetail(const char* file, int line, const char* op) { 195 GLint error = glGetError(); 196 const char* err_msg = "NOT_LISTED"; 197 if (error != 0) { 198 switch (error) { 199 case GL_INVALID_VALUE: err_msg = "NOT_LISTED_YET"; break; 200 case GL_INVALID_OPERATION: err_msg = "INVALID_OPERATION"; break; 201 case GL_INVALID_ENUM: err_msg = "INVALID_ENUM"; break; 202 } 203 LOGE("Error after %s(). glError: %s (0x%x) in line %d of %s", op, err_msg, error, line, file); 204 return false; 205 } 206 return true; 207} 208 209void bindSurfaceTexture(GLuint texId) 210{ 211 glBindTexture(GL_TEXTURE_EXTERNAL_OES_ENUM, texId); 212 213 // Can't do mipmapping with camera source 214 glTexParameterf(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_MIN_FILTER, 215 GL_LINEAR); 216 glTexParameterf(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_MAG_FILTER, 217 GL_LINEAR); 218 // Clamp to edge is the only option 219 glTexParameteri(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_WRAP_S, 220 GL_CLAMP_TO_EDGE); 221 glTexParameteri(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_WRAP_T, 222 GL_CLAMP_TO_EDGE); 223} 224 225void ClearPreviewImage(int mID) 226{ 227 unsigned char* ptr = gPreviewImage[mID]; 228 for(int j = 0, i = 0; 229 j < gPreviewImageWidth[mID] * gPreviewImageHeight[mID] * 4; 230 j += 4) 231 { 232 ptr[i++] = 0; 233 ptr[i++] = 0; 234 ptr[i++] = 0; 235 ptr[i++] = 255; 236 } 237 238} 239 240void ConvertAffine3x3toGL4x4(double *matGL44, double *mat33) 241{ 242 matGL44[0] = mat33[0]; 243 matGL44[1] = mat33[3]; 244 matGL44[2] = 0.0; 245 matGL44[3] = mat33[6]; 246 247 matGL44[4] = mat33[1]; 248 matGL44[5] = mat33[4]; 249 matGL44[6] = 0.0; 250 matGL44[7] = mat33[7]; 251 252 matGL44[8] = 0; 253 matGL44[9] = 0; 254 matGL44[10] = 1.0; 255 matGL44[11] = 0.0; 256 257 matGL44[12] = mat33[2]; 258 matGL44[13] = mat33[5]; 259 matGL44[14] = 0.0; 260 matGL44[15] = mat33[8]; 261} 262 263bool continuePanningFBO(double panOffset) { 264 double normalizedScreenLimitLeft = -1.0 + VIEWPORT_BORDER_FACTOR_HORZ * 2.0; 265 double normalizedScreenLimitRight = 1.0 - VIEWPORT_BORDER_FACTOR_HORZ * 2.0; 266 double normalizedXPositionOnScreenLeft; 267 double normalizedXPositionOnScreenRight; 268 269 // Compute the position of the current frame in the screen coordinate system 270 if (gIsLandscapeOrientation) { 271 normalizedXPositionOnScreenLeft = (2.0 * 272 (gCenterOffsetX + panOffset) / gPreviewFBOWidth - 1.0) * 273 gUILayoutScalingX; 274 normalizedXPositionOnScreenRight = (2.0 * 275 ((gCenterOffsetX + panOffset) + gPreviewImageWidth[HR]) / 276 gPreviewFBOWidth - 1.0) * gUILayoutScalingX; 277 } else { 278 normalizedXPositionOnScreenLeft = (2.0 * 279 (gCenterOffsetX + panOffset) / gPreviewFBOWidth - 1.0) * 280 gUILayoutScalingY; 281 normalizedXPositionOnScreenRight = (2.0 * 282 ((gCenterOffsetX + panOffset) + gPreviewImageWidth[HR]) / 283 gPreviewFBOWidth - 1.0) * gUILayoutScalingY; 284 } 285 286 // Stop the viewfinder panning if we hit the maximum border allowed for 287 // this UI layout 288 if (normalizedXPositionOnScreenRight > normalizedScreenLimitRight || 289 normalizedXPositionOnScreenLeft < normalizedScreenLimitLeft) { 290 return false; 291 } else { 292 return true; 293 } 294} 295 296// This function computes fills the 4x4 matrices g_dAffinetrans, 297// and g_dAffinetransPan using the specified 3x3 affine 298// transformation between the first captured frame and the current frame. 299// The computed g_dAffinetrans is such that it warps the preview mosaic in 300// the last frame's coordinate system into the coordinate system of the 301// current frame. Thus, applying this transformation will create the current 302// frame mosaic but with the current frame missing. This frame will then be 303// pasted in by gWarper2 after translating it by g_dTranslationToFBOCenter. 304// The computed g_dAffinetransPan is such that it offsets the computed preview 305// mosaic horizontally to make the viewfinder pan within the UI layout. 306void UpdateWarpTransformation(float *trs) 307{ 308 double H[9], Hp[9], Htemp1[9], Htemp2[9], T[9]; 309 310 for(int i = 0; i < 9; i++) 311 { 312 gThisH1t[i] = trs[i]; 313 } 314 315 // Alignment is done based on low-res data. 316 // To render the preview mosaic, the translation of the high-res mosaic is estimated to 317 // H2L_FACTOR x low-res-based tranlation. 318 gThisH1t[2] *= H2L_FACTOR; 319 gThisH1t[5] *= H2L_FACTOR; 320 321 db_Identity3x3(T); 322 T[2] = -gCenterOffsetX; 323 T[5] = -gCenterOffsetY; 324 325 // H = ( inv(gThisH1t) * gLastH1t ) * T 326 db_Identity3x3(Htemp1); 327 db_Identity3x3(Htemp2); 328 db_Identity3x3(H); 329 db_InvertAffineTransform(Htemp1, gThisH1t); 330 db_Multiply3x3_3x3(Htemp2, Htemp1, gLastH1t); 331 db_Multiply3x3_3x3(H, Htemp2, T); 332 333 for(int i = 0; i < 9; i++) 334 { 335 gLastH1t[i] = gThisH1t[i]; 336 } 337 338 // Move the origin such that the frame is centered in the previewFBO 339 // i.e. H = inv(T) * H 340 H[2] += gCenterOffsetX; 341 H[5] += gCenterOffsetY; 342 343 // Hp = inv(Km) * H * Km 344 // Km moves the coordinate system from openGL to image pixels so 345 // that the alignment transform H can be applied to them. 346 // inv(Km) moves the coordinate system back to openGL normalized 347 // coordinates so that the shader can correctly render it. 348 db_Identity3x3(Htemp1); 349 db_Multiply3x3_3x3(Htemp1, H, gKm); 350 db_Multiply3x3_3x3(Hp, gKminv, Htemp1); 351 352 ConvertAffine3x3toGL4x4(g_dAffinetrans, Hp); 353 354 //////////////////////////////////////////////// 355 ////// Compute g_dAffinetransPan now... ////// 356 //////////////////////////////////////////////// 357 358 gThisTx = trs[2]; 359 360 if(gPanViewfinder) 361 { 362 gPanOffset += (gThisTx - gLastTx) * VIEWFINDER_PAN_FACTOR_HORZ; 363 } 364 365 gLastTx = gThisTx; 366 gPanViewfinder = continuePanningFBO(gPanOffset); 367 368 db_Identity3x3(H); 369 H[2] = gPanOffset; 370 371 // Hp = inv(Km) * H * Km 372 db_Identity3x3(Htemp1); 373 db_Multiply3x3_3x3(Htemp1, H, gKm); 374 db_Multiply3x3_3x3(Hp, gKminv, Htemp1); 375 376 if (gIsLandscapeOrientation) { 377 ConvertAffine3x3toGL4x4(g_dAffinetransPan, Hp); 378 } else { 379 // rotate Hp by 90 degress. 380 db_Multiply3x3_3x3(Htemp1, gRotation90, Hp); 381 ConvertAffine3x3toGL4x4(g_dAffinetransPan, Htemp1); 382 } 383} 384 385void AllocateTextureMemory(int widthHR, int heightHR, int widthLR, int heightLR) 386{ 387 gPreviewImageWidth[HR] = widthHR; 388 gPreviewImageHeight[HR] = heightHR; 389 390 gPreviewImageWidth[LR] = widthLR; 391 gPreviewImageHeight[LR] = heightLR; 392 393 sem_wait(&gPreviewImage_semaphore); 394 gPreviewImage[LR] = ImageUtils::allocateImage(gPreviewImageWidth[LR], 395 gPreviewImageHeight[LR], 4); 396 gPreviewImage[HR] = ImageUtils::allocateImage(gPreviewImageWidth[HR], 397 gPreviewImageHeight[HR], 4); 398 sem_post(&gPreviewImage_semaphore); 399 400 gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageWidth[HR]; 401 gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageHeight[HR]; 402 403 // The origin is such that the current frame will sit with its center 404 // at the center of the previewFBO 405 gCenterOffsetX = (gPreviewFBOWidth / 2 - gPreviewImageWidth[HR] / 2); 406 gCenterOffsetY = (gPreviewFBOHeight / 2 - gPreviewImageHeight[HR] / 2); 407 408 gPanOffset = 0.0f; 409 410 db_Identity3x3(gThisH1t); 411 db_Identity3x3(gLastH1t); 412 413 gPanViewfinder = true; 414 415 int w = gPreviewImageWidth[HR]; 416 int h = gPreviewImageHeight[HR]; 417 418 int wm = gPreviewFBOWidth; 419 int hm = gPreviewFBOHeight; 420 421 // K is the transformation to map the canonical [-1,1] vertex coordinate 422 // system to the [0,w] image coordinate system before applying the given 423 // affine transformation trs. 424 gKm[0] = wm / 2.0 - 0.5; 425 gKm[1] = 0.0; 426 gKm[2] = wm / 2.0 - 0.5; 427 gKm[3] = 0.0; 428 gKm[4] = hm / 2.0 - 0.5; 429 gKm[5] = hm / 2.0 - 0.5; 430 gKm[6] = 0.0; 431 gKm[7] = 0.0; 432 gKm[8] = 1.0; 433 434 gK[0] = w / 2.0 - 0.5; 435 gK[1] = 0.0; 436 gK[2] = w / 2.0 - 0.5; 437 gK[3] = 0.0; 438 gK[4] = h / 2.0 - 0.5; 439 gK[5] = h / 2.0 - 0.5; 440 gK[6] = 0.0; 441 gK[7] = 0.0; 442 gK[8] = 1.0; 443 444 db_Identity3x3(gKinv); 445 db_InvertCalibrationMatrix(gKinv, gK); 446 447 db_Identity3x3(gKminv); 448 db_InvertCalibrationMatrix(gKminv, gKm); 449 450 ////////////////////////////////////////// 451 ////// Compute g_Translation now... ////// 452 ////////////////////////////////////////// 453 double T[9], Tp[9], Ttemp[9]; 454 455 db_Identity3x3(T); 456 T[2] = gCenterOffsetX; 457 T[5] = gCenterOffsetY; 458 459 // Tp = inv(K) * T * K 460 db_Identity3x3(Ttemp); 461 db_Multiply3x3_3x3(Ttemp, T, gK); 462 db_Multiply3x3_3x3(Tp, gKinv, Ttemp); 463 464 ConvertAffine3x3toGL4x4(g_dTranslationToFBOCenter, Tp); 465 466 UpdateWarpTransformation(g_dIdent3x3); 467} 468 469void FreeTextureMemory() 470{ 471 sem_wait(&gPreviewImage_semaphore); 472 ImageUtils::freeImage(gPreviewImage[LR]); 473 ImageUtils::freeImage(gPreviewImage[HR]); 474 sem_post(&gPreviewImage_semaphore); 475} 476 477extern "C" 478{ 479 JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved); 480 JNIEXPORT void JNICALL JNI_OnUnload(JavaVM* vm, void* reserved); 481 JNIEXPORT jint JNICALL Java_com_android_camera_MosaicRenderer_init( 482 JNIEnv * env, jobject obj); 483 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_reset( 484 JNIEnv * env, jobject obj, jint width, jint height, 485 jboolean isLandscapeOrientation); 486 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_preprocess( 487 JNIEnv * env, jobject obj, jfloatArray stMatrix); 488 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_transferGPUtoCPU( 489 JNIEnv * env, jobject obj); 490 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_step( 491 JNIEnv * env, jobject obj); 492 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_updateMatrix( 493 JNIEnv * env, jobject obj); 494 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_setWarping( 495 JNIEnv * env, jobject obj, jboolean flag); 496}; 497 498 499JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) 500{ 501 sem_init(&gPreviewImage_semaphore, 0, 1); 502 503 return JNI_VERSION_1_4; 504} 505 506 507JNIEXPORT void JNICALL JNI_OnUnload(JavaVM* vm, void* reserved) 508{ 509 sem_destroy(&gPreviewImage_semaphore); 510} 511JNIEXPORT jint JNICALL Java_com_android_camera_MosaicRenderer_init( 512 JNIEnv * env, jobject obj) 513{ 514 gSurfTexRenderer[LR].InitializeGLProgram(); 515 gSurfTexRenderer[HR].InitializeGLProgram(); 516 gYVURenderer[LR].InitializeGLProgram(); 517 gYVURenderer[HR].InitializeGLProgram(); 518 gWarper1.InitializeGLProgram(); 519 gWarper2.InitializeGLProgram(); 520 gPreview.InitializeGLProgram(); 521 gBuffer[0].InitializeGLContext(); 522 gBuffer[1].InitializeGLContext(); 523 gBufferInput[LR].InitializeGLContext(); 524 gBufferInput[HR].InitializeGLContext(); 525 gBufferInputYVU[LR].InitializeGLContext(); 526 gBufferInputYVU[HR].InitializeGLContext(); 527 528 glBindFramebuffer(GL_FRAMEBUFFER, 0); 529 530 glGenTextures(1, gSurfaceTextureID); 531 // bind the surface texture 532 bindSurfaceTexture(gSurfaceTextureID[0]); 533 534 return (jint) gSurfaceTextureID[0]; 535} 536 537// width: the width of the view 538// height: the height of the view 539// isLandscape: whether the device is in landscape or portrait. Android 540// Compatibility Definition Document specifies that the long side of the 541// camera aligns with the long side of the screen. 542void calculateUILayoutScaling(int width, int height, bool isLandscape) { 543 if (isLandscape) { 544 // __________ ________ 545 // | | => |________| 546 // |__________| => (View) 547 // (Preview FBO) 548 // 549 // Scale the preview FBO's height to the height of view and 550 // maintain the aspect ratio of the current frame on the screen. 551 gUILayoutScalingY = PREVIEW_FBO_HEIGHT_SCALE; 552 553 // Note that OpenGL scales a texture to view's width and height automatically. 554 // The "width / height" inverts the scaling, so as to maintain the aspect ratio 555 // of the current frame. 556 gUILayoutScalingX = ((float) gPreviewFBOWidth / gPreviewFBOHeight) 557 / ((float) width / height) * PREVIEW_FBO_HEIGHT_SCALE; 558 } else { 559 // ___ 560 // __________ | | ______ 561 // | | => | | => |______| 562 // |__________| => | | => (View) 563 // (Preview FBO) | | 564 // |___| 565 // 566 // Scale the preview FBO's height to the width of view and 567 // maintain the aspect ratio of the current frame on the screen. 568 // In preview, Java_com_android_camera_MosaicRenderer_step rotates the 569 // preview FBO by 90 degrees. In capture, UpdateWarpTransformation 570 // rotates the preview FBO. 571 gUILayoutScalingY = PREVIEW_FBO_WIDTH_SCALE; 572 573 // Note that OpenGL scales a texture to view's width and height automatically. 574 // The "height / width" inverts the scaling, so as to maintain the aspect ratio 575 // of the current frame. 576 gUILayoutScalingX = ((float) gPreviewFBOHeight / gPreviewFBOWidth) 577 / ((float) width / height) * PREVIEW_FBO_WIDTH_SCALE; 578 } 579} 580 581JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_reset( 582 JNIEnv * env, jobject obj, jint width, jint height, jboolean isLandscapeOrientation) 583{ 584 gIsLandscapeOrientation = isLandscapeOrientation; 585 calculateUILayoutScaling(width, height, gIsLandscapeOrientation); 586 587 gBuffer[0].Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA); 588 gBuffer[1].Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA); 589 590 gBufferInput[LR].Init(gPreviewImageWidth[LR], 591 gPreviewImageHeight[LR], GL_RGBA); 592 593 gBufferInput[HR].Init(gPreviewImageWidth[HR], 594 gPreviewImageHeight[HR], GL_RGBA); 595 596 gBufferInputYVU[LR].Init(gPreviewImageWidth[LR], 597 gPreviewImageHeight[LR], GL_RGBA); 598 599 gBufferInputYVU[HR].Init(gPreviewImageWidth[HR], 600 gPreviewImageHeight[HR], GL_RGBA); 601 602 // bind the surface texture 603 bindSurfaceTexture(gSurfaceTextureID[0]); 604 605 // To speed up, there is no need to clear the destination buffers 606 // (offscreen/screen buffers) of gSurfTexRenderer, gYVURenderer 607 // and gPreview because we always fill the whole destination buffers 608 // when we draw something to those offscreen/screen buffers. 609 gSurfTexRenderer[LR].SetupGraphics(&gBufferInput[LR]); 610 gSurfTexRenderer[LR].SetViewportMatrix(1, 1, 1, 1); 611 gSurfTexRenderer[LR].SetScalingMatrix(1.0f, -1.0f); 612 gSurfTexRenderer[LR].SetInputTextureName(gSurfaceTextureID[0]); 613 gSurfTexRenderer[LR].SetInputTextureType(GL_TEXTURE_EXTERNAL_OES_ENUM); 614 615 gSurfTexRenderer[HR].SetupGraphics(&gBufferInput[HR]); 616 gSurfTexRenderer[HR].SetViewportMatrix(1, 1, 1, 1); 617 gSurfTexRenderer[HR].SetScalingMatrix(1.0f, -1.0f); 618 gSurfTexRenderer[HR].SetInputTextureName(gSurfaceTextureID[0]); 619 gSurfTexRenderer[HR].SetInputTextureType(GL_TEXTURE_EXTERNAL_OES_ENUM); 620 621 gYVURenderer[LR].SetupGraphics(&gBufferInputYVU[LR]); 622 gYVURenderer[LR].SetInputTextureName(gBufferInput[LR].GetTextureName()); 623 gYVURenderer[LR].SetInputTextureType(GL_TEXTURE_2D); 624 625 gYVURenderer[HR].SetupGraphics(&gBufferInputYVU[HR]); 626 gYVURenderer[HR].SetInputTextureName(gBufferInput[HR].GetTextureName()); 627 gYVURenderer[HR].SetInputTextureType(GL_TEXTURE_2D); 628 629 // gBuffer[1-gCurrentFBOIndex] --> gWarper1 --> gBuffer[gCurrentFBOIndex] 630 gWarper1.SetupGraphics(&gBuffer[gCurrentFBOIndex]); 631 632 // Clear the destination buffer of gWarper1. 633 gWarper1.Clear(0.0, 0.0, 0.0, 1.0); 634 gWarper1.SetViewportMatrix(1, 1, 1, 1); 635 gWarper1.SetScalingMatrix(1.0f, 1.0f); 636 gWarper1.SetInputTextureName(gBuffer[1 - gCurrentFBOIndex].GetTextureName()); 637 gWarper1.SetInputTextureType(GL_TEXTURE_2D); 638 639 // gBufferInput[HR] --> gWarper2 --> gBuffer[gCurrentFBOIndex] 640 gWarper2.SetupGraphics(&gBuffer[gCurrentFBOIndex]); 641 642 // gWarp2's destination buffer is the same to gWarp1's. No need to clear it 643 // again. 644 gWarper2.SetViewportMatrix(gPreviewImageWidth[HR], 645 gPreviewImageHeight[HR], gBuffer[gCurrentFBOIndex].GetWidth(), 646 gBuffer[gCurrentFBOIndex].GetHeight()); 647 gWarper2.SetScalingMatrix(1.0f, 1.0f); 648 gWarper2.SetInputTextureName(gBufferInput[HR].GetTextureName()); 649 gWarper2.SetInputTextureType(GL_TEXTURE_2D); 650 651 // gBuffer[gCurrentFBOIndex] --> gPreview --> Screen 652 gPreview.SetupGraphics(width, height); 653 gPreview.SetViewportMatrix(1, 1, 1, 1); 654 655 // Scale the previewFBO so that the viewfinder window fills the layout height 656 // while maintaining the image aspect ratio 657 gPreview.SetScalingMatrix(gUILayoutScalingX, -1.0f * gUILayoutScalingY); 658 gPreview.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName()); 659 gPreview.SetInputTextureType(GL_TEXTURE_2D); 660} 661 662JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_preprocess( 663 JNIEnv * env, jobject obj, jfloatArray stMatrix) 664{ 665 jfloat *stmat = env->GetFloatArrayElements(stMatrix, 0); 666 667 gSurfTexRenderer[LR].SetSTMatrix((float*) stmat); 668 gSurfTexRenderer[HR].SetSTMatrix((float*) stmat); 669 670 env->ReleaseFloatArrayElements(stMatrix, stmat, 0); 671 672 gSurfTexRenderer[LR].DrawTexture(g_dAffinetransIdentGL); 673 gSurfTexRenderer[HR].DrawTexture(g_dAffinetransIdentGL); 674} 675 676#ifndef now_ms 677#include <time.h> 678static double 679now_ms(void) 680{ 681 //struct timespec res; 682 struct timeval res; 683 //clock_gettime(CLOCK_REALTIME, &res); 684 gettimeofday(&res, NULL); 685 return 1000.0*res.tv_sec + (double)res.tv_usec/1e3; 686} 687#endif 688 689 690 691JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_transferGPUtoCPU( 692 JNIEnv * env, jobject obj) 693{ 694 double t0, t1, time_c; 695 696 gYVURenderer[LR].DrawTexture(); 697 gYVURenderer[HR].DrawTexture(); 698 699 sem_wait(&gPreviewImage_semaphore); 700 // Bind to the input LR FBO and read the Low-Res data from there... 701 glBindFramebuffer(GL_FRAMEBUFFER, gBufferInputYVU[LR].GetFrameBufferName()); 702 t0 = now_ms(); 703 glReadPixels(0, 704 0, 705 gBufferInput[LR].GetWidth(), 706 gBufferInput[LR].GetHeight(), 707 GL_RGBA, 708 GL_UNSIGNED_BYTE, 709 gPreviewImage[LR]); 710 711 checkGlError("glReadPixels LR (MosaicRenderer.transferGPUtoCPU())"); 712 713 // Bind to the input HR FBO and read the high-res data from there... 714 glBindFramebuffer(GL_FRAMEBUFFER, gBufferInputYVU[HR].GetFrameBufferName()); 715 t0 = now_ms(); 716 glReadPixels(0, 717 0, 718 gBufferInput[HR].GetWidth(), 719 gBufferInput[HR].GetHeight(), 720 GL_RGBA, 721 GL_UNSIGNED_BYTE, 722 gPreviewImage[HR]); 723 724 checkGlError("glReadPixels HR (MosaicRenderer.transferGPUtoCPU())"); 725 726 sem_post(&gPreviewImage_semaphore); 727} 728 729JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_step( 730 JNIEnv * env, jobject obj) 731{ 732 if(!gWarpImage) // ViewFinder 733 { 734 gWarper2.SetupGraphics(&gBuffer[gCurrentFBOIndex]); 735 gPreview.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName()); 736 737 gWarper2.DrawTexture(g_dTranslationToFBOCenterGL); 738 739 if (gIsLandscapeOrientation) { 740 gPreview.DrawTexture(g_dAffinetransIdentGL); 741 } else { 742 gPreview.DrawTexture(g_dAffinetransRotation90GL); 743 } 744 } 745 else 746 { 747 gWarper1.SetupGraphics(&gBuffer[gCurrentFBOIndex]); 748 // Clear the destination so that we can paint on it afresh 749 gWarper1.Clear(0.0, 0.0, 0.0, 1.0); 750 gWarper1.SetInputTextureName( 751 gBuffer[1 - gCurrentFBOIndex].GetTextureName()); 752 gWarper2.SetupGraphics(&gBuffer[gCurrentFBOIndex]); 753 gPreview.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName()); 754 755 gWarper1.DrawTexture(g_dAffinetransGL); 756 gWarper2.DrawTexture(g_dTranslationToFBOCenterGL); 757 gPreview.DrawTexture(g_dAffinetransPanGL); 758 759 gCurrentFBOIndex = 1 - gCurrentFBOIndex; 760 } 761} 762 763JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_setWarping( 764 JNIEnv * env, jobject obj, jboolean flag) 765{ 766 // TODO: Review this logic 767 if(gWarpImage != (bool) flag) //switching from viewfinder to capture or vice-versa 768 { 769 // Clear gBuffer[0] 770 gWarper1.SetupGraphics(&gBuffer[0]); 771 gWarper1.Clear(0.0, 0.0, 0.0, 1.0); 772 // Clear gBuffer[1] 773 gWarper1.SetupGraphics(&gBuffer[1]); 774 gWarper1.Clear(0.0, 0.0, 0.0, 1.0); 775 // Clear the screen to black. 776 gPreview.Clear(0.0, 0.0, 0.0, 1.0); 777 778 gLastTx = 0.0f; 779 gPanOffset = 0.0f; 780 gPanViewfinder = true; 781 782 db_Identity3x3(gThisH1t); 783 db_Identity3x3(gLastH1t); 784 // Make sure g_dAffinetransGL and g_dAffinetransPanGL are updated. 785 // Otherwise, the first frame after setting the flag to true will be 786 // incorrectly drawn. 787 if ((bool) flag) { 788 UpdateWarpTransformation(g_dIdent3x3); 789 } 790 } 791 792 gWarpImage = (bool)flag; 793} 794 795JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_updateMatrix( 796 JNIEnv * env, jobject obj) 797{ 798 for(int i=0; i<16; i++) 799 { 800 g_dAffinetransGL[i] = g_dAffinetrans[i]; 801 g_dAffinetransPanGL[i] = g_dAffinetransPan[i]; 802 g_dTranslationToFBOCenterGL[i] = g_dTranslationToFBOCenter[i]; 803 } 804} 805