Blend.cpp revision a369cd3153170975a5e5bd6fa5544dcd6f22ee85
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17/////////////////////////////////////////////////// 18// Blend.cpp 19// $Id: Blend.cpp,v 1.22 2011/06/24 04:22:14 mbansal Exp $ 20 21#include <string.h> 22 23#include "Interp.h" 24#include "Blend.h" 25 26#include "Geometry.h" 27#include "trsMatrix.h" 28 29#include "Log.h" 30#define LOG_TAG "BLEND" 31 32Blend::Blend() 33{ 34 m_wb.blendingType = BLEND_TYPE_NONE; 35} 36 37Blend::~Blend() 38{ 39 if (m_pFrameVPyr) free(m_pFrameVPyr); 40 if (m_pFrameUPyr) free(m_pFrameUPyr); 41 if (m_pFrameYPyr) free(m_pFrameYPyr); 42} 43 44int Blend::initialize(int blendingType, int stripType, int frame_width, int frame_height) 45{ 46 this->width = frame_width; 47 this->height = frame_height; 48 this->m_wb.blendingType = blendingType; 49 this->m_wb.stripType = stripType; 50 51 m_wb.blendRange = m_wb.blendRangeUV = BLEND_RANGE_DEFAULT; 52 m_wb.nlevs = m_wb.blendRange; 53 m_wb.nlevsC = m_wb.blendRangeUV; 54 55 if (m_wb.nlevs <= 0) m_wb.nlevs = 1; // Need levels for YUV processing 56 if (m_wb.nlevsC > m_wb.nlevs) m_wb.nlevsC = m_wb.nlevs; 57 58 m_wb.roundoffOverlap = 1.5; 59 60 m_pFrameYPyr = NULL; 61 m_pFrameUPyr = NULL; 62 m_pFrameVPyr = NULL; 63 64 m_pFrameYPyr = PyramidShort::allocatePyramidPacked(m_wb.nlevs, (unsigned short) width, (unsigned short) height, BORDER); 65 m_pFrameUPyr = PyramidShort::allocatePyramidPacked(m_wb.nlevsC, (unsigned short) (width), (unsigned short) (height), BORDER); 66 m_pFrameVPyr = PyramidShort::allocatePyramidPacked(m_wb.nlevsC, (unsigned short) (width), (unsigned short) (height), BORDER); 67 68 if (!m_pFrameYPyr || !m_pFrameUPyr || !m_pFrameVPyr) 69 { 70 LOGE("Error: Could not allocate pyramids for blending"); 71 return BLEND_RET_ERROR_MEMORY; 72 } 73 74 return BLEND_RET_OK; 75} 76 77inline double max(double a, double b) { return a > b ? a : b; } 78inline double min(double a, double b) { return a < b ? a : b; } 79 80void Blend::AlignToMiddleFrame(MosaicFrame **frames, int frames_size) 81{ 82 // Unwarp this frame and Warp the others to match 83 MosaicFrame *mb = NULL; 84 MosaicFrame *ref = frames[int(frames_size/2)]; // Middle frame 85 86 double invtrs[3][3]; 87 inv33d(ref->trs, invtrs); 88 89 for(int mfit = 0; mfit < frames_size; mfit++) 90 { 91 mb = frames[mfit]; 92 double temp[3][3]; 93 mult33d(temp, invtrs, mb->trs); 94 memcpy(mb->trs, temp, sizeof(temp)); 95 normProjMat33d(mb->trs); 96 } 97} 98 99int Blend::runBlend(MosaicFrame **oframes, MosaicFrame **rframes, 100 int frames_size, 101 ImageType &imageMosaicYVU, int &mosaicWidth, int &mosaicHeight, 102 float &progress, bool &cancelComputation) 103{ 104 int ret; 105 int numCenters; 106 107 MosaicFrame **frames; 108 109 // For THIN strip mode, accept all frames for blending 110 if (m_wb.stripType == STRIP_TYPE_THIN) 111 { 112 frames = oframes; 113 } 114 else // For WIDE strip mode, first select the relevant frames to blend. 115 { 116 SelectRelevantFrames(oframes, frames_size, rframes, frames_size); 117 frames = rframes; 118 } 119 120 ComputeBlendParameters(frames, frames_size, true); 121 numCenters = frames_size; 122 123 if (numCenters == 0) 124 { 125 LOGE("Error: No frames to blend"); 126 return BLEND_RET_ERROR; 127 } 128 129 if (!(m_AllSites = m_Triangulator.allocMemory(numCenters))) 130 { 131 return BLEND_RET_ERROR_MEMORY; 132 } 133 134 // Bounding rectangle (real numbers) of the final mosaic computed by projecting 135 // each input frame into the mosaic coordinate system. 136 BlendRect global_rect; 137 138 global_rect.lft = global_rect.bot = 2e30; // min values 139 global_rect.rgt = global_rect.top = -2e30; // max values 140 MosaicFrame *mb = NULL; 141 double halfwidth = width / 2.0; 142 double halfheight = height / 2.0; 143 144 double z, x0, y0, x1, y1, x2, y2, x3, y3; 145 146 // Corners of the left-most and right-most frames respectively in the 147 // mosaic coordinate system. 148 double xLeftCorners[2] = {2e30, 2e30}; 149 double xRightCorners[2] = {-2e30, -2e30}; 150 151 // Determine the extents of the final mosaic 152 CSite *csite = m_AllSites ; 153 for(int mfit = 0; mfit < frames_size; mfit++) 154 { 155 mb = frames[mfit]; 156 157 // Compute clipping for this frame's rect 158 FrameToMosaicRect(mb->width, mb->height, mb->trs, mb->brect); 159 // Clip global rect using this frame's rect 160 ClipRect(mb->brect, global_rect); 161 162 // Calculate the corner points 163 FrameToMosaic(mb->trs, 0.0, 0.0, x0, y0); 164 FrameToMosaic(mb->trs, 0.0, mb->height-1.0, x1, y1); 165 FrameToMosaic(mb->trs, mb->width-1.0, mb->height-1.0, x2, y2); 166 FrameToMosaic(mb->trs, mb->width-1.0, 0.0, x3, y3); 167 168 if(x0 < xLeftCorners[0] || x1 < xLeftCorners[1]) // If either of the left corners is lower 169 { 170 xLeftCorners[0] = x0; 171 xLeftCorners[1] = x1; 172 } 173 174 if(x3 > xRightCorners[0] || x2 > xRightCorners[1]) // If either of the right corners is higher 175 { 176 xRightCorners[0] = x3; 177 xRightCorners[1] = x2; 178 } 179 180 // Compute the centroid of the warped region 181 FindQuadCentroid(x0, y0, x1, y1, x2, y2, x3, y3, csite->getVCenter().x, csite->getVCenter().y); 182 183 csite->setMb(mb); 184 csite++; 185 } 186 187 // Get origin and sizes 188 189 // Bounding rectangle (int numbers) of the final mosaic computed by projecting 190 // each input frame into the mosaic coordinate system. 191 MosaicRect fullRect; 192 193 fullRect.left = (int) floor(global_rect.lft); // min-x 194 fullRect.top = (int) floor(global_rect.bot); // min-y 195 fullRect.right = (int) ceil(global_rect.rgt); // max-x 196 fullRect.bottom = (int) ceil(global_rect.top);// max-y 197 Mwidth = (unsigned short) (fullRect.right - fullRect.left + 1); 198 Mheight = (unsigned short) (fullRect.bottom - fullRect.top + 1); 199 200 int xLeftMost, xRightMost; 201 202 // Rounding up, so that we don't include the gray border. 203 xLeftMost = max(0, max(xLeftCorners[0], xLeftCorners[1]) - fullRect.left + 1); 204 xRightMost = min(Mwidth - 1, min(xRightCorners[0], xRightCorners[1]) - fullRect.left - 1); 205 206 // Make sure image width is multiple of 4 207 Mwidth = (unsigned short) ((Mwidth + 3) & ~3); 208 Mheight = (unsigned short) ((Mheight + 3) & ~3); // Round up. 209 210 if (Mwidth < width || Mheight < height || xRightMost <= xLeftMost) 211 { 212 LOGE("RunBlend: aborting - consistency check failed, w=%d, h=%d, xLeftMost=%d, xRightMost=%d", Mwidth, Mheight, xLeftMost, xRightMost); 213 return BLEND_RET_ERROR; 214 } 215 216 YUVinfo *imgMos = YUVinfo::allocateImage(Mwidth, Mheight); 217 if (imgMos == NULL) 218 { 219 LOGE("RunBlend: aborting - couldn't alloc %d x %d mosaic image", Mwidth, Mheight); 220 return BLEND_RET_ERROR_MEMORY; 221 } 222 223 // Set the Y image to 255 so we can distinguish when frame idx are written to it 224 memset(imgMos->Y.ptr[0], 255, (imgMos->Y.width * imgMos->Y.height)); 225 // Set the v and u images to black 226 memset(imgMos->V.ptr[0], 128, (imgMos->V.width * imgMos->V.height) << 1); 227 228 // Do the triangulation. It returns a sorted list of edges 229 SEdgeVector *edge; 230 int n = m_Triangulator.triangulate(&edge, numCenters, width, height); 231 m_Triangulator.linkNeighbors(edge, n, numCenters); 232 233 // Bounding rectangle that determines the positioning of the rectangle that is 234 // cropped out of the computed mosaic to get rid of the gray borders. 235 MosaicRect cropping_rect; 236 237 cropping_rect.left = xLeftMost; 238 cropping_rect.right = xRightMost; 239 240 // Do merging and blending : 241 ret = DoMergeAndBlend(frames, numCenters, width, height, *imgMos, fullRect, 242 cropping_rect, progress, cancelComputation); 243 244 if (m_wb.blendingType == BLEND_TYPE_HORZ) 245 CropFinalMosaic(*imgMos, cropping_rect); 246 247 248 m_Triangulator.freeMemory(); // note: can be called even if delaunay_alloc() wasn't successful 249 250 imageMosaicYVU = imgMos->Y.ptr[0]; 251 252 253 if (m_wb.blendingType == BLEND_TYPE_HORZ) 254 { 255 mosaicWidth = cropping_rect.right - cropping_rect.left + 1; 256 mosaicHeight = cropping_rect.bottom - cropping_rect.top + 1; 257 } 258 else 259 { 260 mosaicWidth = Mwidth; 261 mosaicHeight = Mheight; 262 } 263 264 return ret; 265} 266 267 268int Blend::FillFramePyramid(MosaicFrame *mb) 269{ 270 ImageType mbY, mbU, mbV; 271 // Lay this image, centered into the temporary buffer 272 mbY = mb->image; 273 mbU = mb->getU(); 274 mbV = mb->getV(); 275 276 int h, w; 277 278 for(h=0; h<height; h++) 279 { 280 ImageTypeShort yptr = m_pFrameYPyr->ptr[h]; 281 ImageTypeShort uptr = m_pFrameUPyr->ptr[h]; 282 ImageTypeShort vptr = m_pFrameVPyr->ptr[h]; 283 284 for(w=0; w<width; w++) 285 { 286 yptr[w] = (short) ((*(mbY++)) << 3); 287 uptr[w] = (short) ((*(mbU++)) << 3); 288 vptr[w] = (short) ((*(mbV++)) << 3); 289 } 290 } 291 292 // Spread the image through the border 293 PyramidShort::BorderSpread(m_pFrameYPyr, BORDER, BORDER, BORDER, BORDER); 294 PyramidShort::BorderSpread(m_pFrameUPyr, BORDER, BORDER, BORDER, BORDER); 295 PyramidShort::BorderSpread(m_pFrameVPyr, BORDER, BORDER, BORDER, BORDER); 296 297 // Generate Laplacian pyramids 298 if (!PyramidShort::BorderReduce(m_pFrameYPyr, m_wb.nlevs) || !PyramidShort::BorderExpand(m_pFrameYPyr, m_wb.nlevs, -1) || 299 !PyramidShort::BorderReduce(m_pFrameUPyr, m_wb.nlevsC) || !PyramidShort::BorderExpand(m_pFrameUPyr, m_wb.nlevsC, -1) || 300 !PyramidShort::BorderReduce(m_pFrameVPyr, m_wb.nlevsC) || !PyramidShort::BorderExpand(m_pFrameVPyr, m_wb.nlevsC, -1)) 301 { 302 LOGE("Error: Could not generate Laplacian pyramids"); 303 return BLEND_RET_ERROR; 304 } 305 else 306 { 307 return BLEND_RET_OK; 308 } 309} 310 311int Blend::DoMergeAndBlend(MosaicFrame **frames, int nsite, 312 int width, int height, YUVinfo &imgMos, MosaicRect &rect, 313 MosaicRect &cropping_rect, float &progress, bool &cancelComputation) 314{ 315 m_pMosaicYPyr = NULL; 316 m_pMosaicUPyr = NULL; 317 m_pMosaicVPyr = NULL; 318 319 m_pMosaicYPyr = PyramidShort::allocatePyramidPacked(m_wb.nlevs,(unsigned short)rect.Width(),(unsigned short)rect.Height(),BORDER); 320 m_pMosaicUPyr = PyramidShort::allocatePyramidPacked(m_wb.nlevsC,(unsigned short)rect.Width(),(unsigned short)rect.Height(),BORDER); 321 m_pMosaicVPyr = PyramidShort::allocatePyramidPacked(m_wb.nlevsC,(unsigned short)rect.Width(),(unsigned short)rect.Height(),BORDER); 322 if (!m_pMosaicYPyr || !m_pMosaicUPyr || !m_pMosaicVPyr) 323 { 324 LOGE("Error: Could not allocate pyramids for blending"); 325 return BLEND_RET_ERROR_MEMORY; 326 } 327 328 MosaicFrame *mb; 329 330 CSite *esite = m_AllSites + nsite; 331 int site_idx; 332 333 // First go through each frame and for each mosaic pixel determine which frame it should come from 334 site_idx = 0; 335 for(CSite *csite = m_AllSites; csite < esite; csite++) 336 { 337 if(cancelComputation) 338 { 339 if (m_pMosaicVPyr) free(m_pMosaicVPyr); 340 if (m_pMosaicUPyr) free(m_pMosaicUPyr); 341 if (m_pMosaicYPyr) free(m_pMosaicYPyr); 342 return BLEND_RET_CANCELLED; 343 } 344 345 mb = csite->getMb(); 346 347 mb->vcrect = mb->brect; 348 ClipBlendRect(csite, mb->vcrect); 349 350 ComputeMask(csite, mb->vcrect, mb->brect, rect, imgMos, site_idx); 351 352 site_idx++; 353 } 354 355 ////////// imgMos.Y, imgMos.V, imgMos.U are used as follows ////////////// 356 ////////////////////// THIN STRIP MODE /////////////////////////////////// 357 358 // imgMos.Y is used to store the index of the image from which each pixel 359 // in the output mosaic can be read out for the thin-strip mode. Thus, 360 // there is no special handling for pixels around the seam. Also, imgMos.Y 361 // is set to 255 wherever we can't get its value from any input image e.g. 362 // in the gray border areas. imgMos.V and imgMos.U are set to 128 for the 363 // thin-strip mode. 364 365 ////////////////////// WIDE STRIP MODE /////////////////////////////////// 366 367 // imgMos.Y is used the same way as the thin-strip mode. 368 // imgMos.V is used to store the index of the neighboring image which 369 // should contribute to the color of an output pixel in a band around 370 // the seam. Thus, in this band, we will crossfade between the color values 371 // from the image index imgMos.Y and image index imgMos.V. imgMos.U is 372 // used to store the weight (multiplied by 100) that each image will 373 // contribute to the blending process. Thus, we start at 99% contribution 374 // from the first image, then go to 50% contribution from each image at 375 // the seam. Then, the contribution from the second image goes up to 99%. 376 377 // For WIDE mode, set the pixel masks to guide the blender to cross-fade 378 // between the images on either side of each seam: 379 if (m_wb.stripType == STRIP_TYPE_WIDE) 380 { 381 // Set the number of pixels around the seam to cross-fade between 382 // the two component images, 383 int tw = STRIP_CROSS_FADE_WIDTH * width; 384 385 for(int y = 0; y < imgMos.Y.height; y++) 386 { 387 for(int x = tw; x < imgMos.Y.width - tw + 1; ) 388 { 389 // Determine where the seam is... 390 if (imgMos.Y.ptr[y][x] != imgMos.Y.ptr[y][x+1] && 391 imgMos.Y.ptr[y][x] != 255 && 392 imgMos.Y.ptr[y][x+1] != 255) 393 { 394 // Find the image indices on both sides of the seam 395 unsigned char idx1 = imgMos.Y.ptr[y][x]; 396 unsigned char idx2 = imgMos.Y.ptr[y][x+1]; 397 398 for (int o = tw; o >= 0; o--) 399 { 400 // Set the image index to use for cross-fading 401 imgMos.V.ptr[y][x - o] = idx2; 402 // Set the intensity weights to use for cross-fading 403 imgMos.U.ptr[y][x - o] = 50 + (99 - 50) * o / tw; 404 } 405 406 for (int o = 1; o <= tw; o++) 407 { 408 // Set the image index to use for cross-fading 409 imgMos.V.ptr[y][x + o] = idx1; 410 // Set the intensity weights to use for cross-fading 411 imgMos.U.ptr[y][x + o] = imgMos.U.ptr[y][x - o]; 412 } 413 414 x += (tw + 1); 415 } 416 else 417 { 418 x++; 419 } 420 } 421 } 422 } 423 424 // Now perform the actual blending using the frame assignment determined above 425 site_idx = 0; 426 for(CSite *csite = m_AllSites; csite < esite; csite++) 427 { 428 if(cancelComputation) 429 { 430 if (m_pMosaicVPyr) free(m_pMosaicVPyr); 431 if (m_pMosaicUPyr) free(m_pMosaicUPyr); 432 if (m_pMosaicYPyr) free(m_pMosaicYPyr); 433 return BLEND_RET_CANCELLED; 434 } 435 436 mb = csite->getMb(); 437 438 439 if(FillFramePyramid(mb)!=BLEND_RET_OK) 440 return BLEND_RET_ERROR; 441 442 ProcessPyramidForThisFrame(csite, mb->vcrect, mb->brect, rect, imgMos, mb->trs, site_idx); 443 444 progress += TIME_PERCENT_BLEND/nsite; 445 446 site_idx++; 447 } 448 449 450 // Blend 451 PerformFinalBlending(imgMos, cropping_rect); 452 453 if (m_pMosaicVPyr) free(m_pMosaicVPyr); 454 if (m_pMosaicUPyr) free(m_pMosaicUPyr); 455 if (m_pMosaicYPyr) free(m_pMosaicYPyr); 456 457 progress += TIME_PERCENT_FINAL; 458 459 return BLEND_RET_OK; 460} 461 462void Blend::CropFinalMosaic(YUVinfo &imgMos, MosaicRect &cropping_rect) 463{ 464 int i, j, k; 465 ImageType yimg; 466 ImageType uimg; 467 ImageType vimg; 468 469 470 yimg = imgMos.Y.ptr[0]; 471 uimg = imgMos.U.ptr[0]; 472 vimg = imgMos.V.ptr[0]; 473 474 k = 0; 475 for (j = cropping_rect.top; j <= cropping_rect.bottom; j++) 476 { 477 for (i = cropping_rect.left; i <= cropping_rect.right; i++) 478 { 479 yimg[k] = yimg[j*imgMos.Y.width+i]; 480 k++; 481 } 482 } 483 for (j = cropping_rect.top; j <= cropping_rect.bottom; j++) 484 { 485 for (i = cropping_rect.left; i <= cropping_rect.right; i++) 486 { 487 yimg[k] = vimg[j*imgMos.Y.width+i]; 488 k++; 489 } 490 } 491 for (j = cropping_rect.top; j <= cropping_rect.bottom; j++) 492 { 493 for (i = cropping_rect.left; i <= cropping_rect.right; i++) 494 { 495 yimg[k] = uimg[j*imgMos.Y.width+i]; 496 k++; 497 } 498 } 499} 500 501int Blend::PerformFinalBlending(YUVinfo &imgMos, MosaicRect &cropping_rect) 502{ 503 if (!PyramidShort::BorderExpand(m_pMosaicYPyr, m_wb.nlevs, 1) || !PyramidShort::BorderExpand(m_pMosaicUPyr, m_wb.nlevsC, 1) || 504 !PyramidShort::BorderExpand(m_pMosaicVPyr, m_wb.nlevsC, 1)) 505 { 506 LOGE("Error: Could not BorderExpand!"); 507 return BLEND_RET_ERROR; 508 } 509 510 ImageTypeShort myimg; 511 ImageTypeShort muimg; 512 ImageTypeShort mvimg; 513 ImageType yimg; 514 ImageType uimg; 515 ImageType vimg; 516 517 int cx = (int)imgMos.Y.width/2; 518 int cy = (int)imgMos.Y.height/2; 519 520 // 2D boolean array that contains true wherever the mosaic image data is 521 // invalid (i.e. in the gray border). 522 bool **b = new bool*[imgMos.Y.height]; 523 524 for(int j=0; j<imgMos.Y.height; j++) 525 { 526 b[j] = new bool[imgMos.Y.width]; 527 } 528 529 // Copy the resulting image into the full image using the mask 530 int i, j; 531 532 yimg = imgMos.Y.ptr[0]; 533 uimg = imgMos.U.ptr[0]; 534 vimg = imgMos.V.ptr[0]; 535 536 for (j = 0; j < imgMos.Y.height; j++) 537 { 538 myimg = m_pMosaicYPyr->ptr[j]; 539 muimg = m_pMosaicUPyr->ptr[j]; 540 mvimg = m_pMosaicVPyr->ptr[j]; 541 542 for (i = 0; i<imgMos.Y.width; i++) 543 { 544 // A final mask was set up previously, 545 // if the value is zero skip it, otherwise replace it. 546 if (*yimg <255) 547 { 548 short value = (short) ((*myimg) >> 3); 549 if (value < 0) value = 0; 550 else if (value > 255) value = 255; 551 *yimg = (unsigned char) value; 552 553 value = (short) ((*muimg) >> 3); 554 if (value < 0) value = 0; 555 else if (value > 255) value = 255; 556 *uimg = (unsigned char) value; 557 558 value = (short) ((*mvimg) >> 3); 559 if (value < 0) value = 0; 560 else if (value > 255) value = 255; 561 *vimg = (unsigned char) value; 562 563 b[j][i] = false; 564 565 } 566 else 567 { // set border color in here 568 *yimg = (unsigned char) 96; 569 *uimg = (unsigned char) 128; 570 *vimg = (unsigned char) 128; 571 572 b[j][i] = true; 573 } 574 575 yimg++; 576 uimg++; 577 vimg++; 578 myimg++; 579 muimg++; 580 mvimg++; 581 } 582 } 583 584 //Scan through each row and increment top if the row contains any gray 585 for (j = 0; j < imgMos.Y.height; j++) 586 { 587 for (i = cropping_rect.left; i < cropping_rect.right; i++) 588 { 589 if (b[j][i]) 590 { 591 break; // to next row 592 } 593 } 594 595 if (i == cropping_rect.right) //no gray pixel in this row! 596 { 597 cropping_rect.top = j; 598 break; 599 } 600 } 601 602 //Scan through each row and decrement bottom if the row contains any gray 603 for (j = imgMos.Y.height-1; j >= 0; j--) 604 { 605 for (i = cropping_rect.left; i < cropping_rect.right; i++) 606 { 607 if (b[j][i]) 608 { 609 break; // to next row 610 } 611 } 612 613 if (i == cropping_rect.right) //no gray pixel in this row! 614 { 615 cropping_rect.bottom = j; 616 break; 617 } 618 } 619 620 for(int j=0; j<imgMos.Y.height; j++) 621 { 622 delete b[j]; 623 } 624 625 delete b; 626 627 return BLEND_RET_OK; 628} 629 630void Blend::ComputeMask(CSite *csite, BlendRect &vcrect, BlendRect &brect, MosaicRect &rect, YUVinfo &imgMos, int site_idx) 631{ 632 PyramidShort *dptr = m_pMosaicYPyr; 633 634 int nC = m_wb.nlevsC; 635 int l = (int) ((vcrect.lft - rect.left)); 636 int b = (int) ((vcrect.bot - rect.top)); 637 int r = (int) ((vcrect.rgt - rect.left)); 638 int t = (int) ((vcrect.top - rect.top)); 639 640 if (vcrect.lft == brect.lft) 641 l = (l <= 0) ? -BORDER : l - BORDER; 642 else if (l < -BORDER) 643 l = -BORDER; 644 645 if (vcrect.bot == brect.bot) 646 b = (b <= 0) ? -BORDER : b - BORDER; 647 else if (b < -BORDER) 648 b = -BORDER; 649 650 if (vcrect.rgt == brect.rgt) 651 r = (r >= dptr->width) ? dptr->width + BORDER - 1 : r + BORDER; 652 else if (r >= dptr->width + BORDER) 653 r = dptr->width + BORDER - 1; 654 655 if (vcrect.top == brect.top) 656 t = (t >= dptr->height) ? dptr->height + BORDER - 1 : t + BORDER; 657 else if (t >= dptr->height + BORDER) 658 t = dptr->height + BORDER - 1; 659 660 // Walk the Region of interest and populate the pyramid 661 for (int j = b; j <= t; j++) 662 { 663 int jj = j; 664 double sj = jj + rect.top; 665 666 for (int i = l; i <= r; i++) 667 { 668 int ii = i; 669 // project point and then triangulate to neighbors 670 double si = ii + rect.left; 671 672 double dself = hypotSq(csite->getVCenter().x - si, csite->getVCenter().y - sj); 673 int inMask = ((unsigned) ii < imgMos.Y.width && 674 (unsigned) jj < imgMos.Y.height) ? 1 : 0; 675 676 if(!inMask) 677 continue; 678 679 // scan the neighbors to see if this is a valid position 680 unsigned char mask = (unsigned char) 255; 681 SEdgeVector *ce; 682 int ecnt; 683 for (ce = csite->getNeighbor(), ecnt = csite->getNumNeighbors(); ecnt--; ce++) 684 { 685 double d1 = hypotSq(m_AllSites[ce->second].getVCenter().x - si, 686 m_AllSites[ce->second].getVCenter().y - sj); 687 if (d1 < dself) 688 { 689 break; 690 } 691 } 692 693 if (ecnt >= 0) continue; 694 695 imgMos.Y.ptr[jj][ii] = (unsigned char)site_idx; 696 } 697 } 698} 699 700void Blend::ProcessPyramidForThisFrame(CSite *csite, BlendRect &vcrect, BlendRect &brect, MosaicRect &rect, YUVinfo &imgMos, double trs[3][3], int site_idx) 701{ 702 // Put the Region of interest (for all levels) into m_pMosaicYPyr 703 double inv_trs[3][3]; 704 inv33d(trs, inv_trs); 705 706 // Process each pyramid level 707 PyramidShort *sptr = m_pFrameYPyr; 708 PyramidShort *suptr = m_pFrameUPyr; 709 PyramidShort *svptr = m_pFrameVPyr; 710 711 PyramidShort *dptr = m_pMosaicYPyr; 712 PyramidShort *duptr = m_pMosaicUPyr; 713 PyramidShort *dvptr = m_pMosaicVPyr; 714 715 int dscale = 0; // distance scale for the current level 716 int nC = m_wb.nlevsC; 717 for (int n = m_wb.nlevs; n--; dscale++, dptr++, sptr++, dvptr++, duptr++, svptr++, suptr++, nC--) 718 { 719 int l = (int) ((vcrect.lft - rect.left) / (1 << dscale)); 720 int b = (int) ((vcrect.bot - rect.top) / (1 << dscale)); 721 int r = (int) ((vcrect.rgt - rect.left) / (1 << dscale) + .5); 722 int t = (int) ((vcrect.top - rect.top) / (1 << dscale) + .5); 723 724 if (vcrect.lft == brect.lft) 725 l = (l <= 0) ? -BORDER : l - BORDER; 726 else if (l < -BORDER) 727 l = -BORDER; 728 729 if (vcrect.bot == brect.bot) 730 b = (b <= 0) ? -BORDER : b - BORDER; 731 else if (b < -BORDER) 732 b = -BORDER; 733 734 if (vcrect.rgt == brect.rgt) 735 r = (r >= dptr->width) ? dptr->width + BORDER - 1 : r + BORDER; 736 else if (r >= dptr->width + BORDER) 737 r = dptr->width + BORDER - 1; 738 739 if (vcrect.top == brect.top) 740 t = (t >= dptr->height) ? dptr->height + BORDER - 1 : t + BORDER; 741 else if (t >= dptr->height + BORDER) 742 t = dptr->height + BORDER - 1; 743 744 // Walk the Region of interest and populate the pyramid 745 for (int j = b; j <= t; j++) 746 { 747 int jj = (j << dscale); 748 double sj = jj + rect.top; 749 750 for (int i = l; i <= r; i++) 751 { 752 int ii = (i << dscale); 753 // project point and then triangulate to neighbors 754 double si = ii + rect.left; 755 756 int inMask = ((unsigned) ii < imgMos.Y.width && 757 (unsigned) jj < imgMos.Y.height) ? 1 : 0; 758 759 if(inMask && imgMos.Y.ptr[jj][ii] != site_idx && 760 imgMos.V.ptr[jj][ii] != site_idx && 761 imgMos.Y.ptr[jj][ii] != 255) 762 continue; 763 764 // Setup weights for cross-fading 765 // Weight of the intensity already in the output pixel 766 double wt0 = 0.0; 767 // Weight of the intensity from the input pixel (current frame) 768 double wt1 = 1.0; 769 770 if (m_wb.stripType == STRIP_TYPE_WIDE) 771 { 772 if(inMask && imgMos.Y.ptr[jj][ii] != 255) 773 { 774 if(imgMos.V.ptr[jj][ii] == 128) // Not on a seam 775 { 776 wt0 = 0.0; 777 wt1 = 1.0; 778 } 779 else 780 { 781 wt0 = 1.0; 782 wt1 = ((imgMos.Y.ptr[jj][ii] == site_idx) ? 783 (double)imgMos.U.ptr[jj][ii] / 100.0 : 784 1.0 - (double)imgMos.U.ptr[jj][ii] / 100.0); 785 } 786 } 787 } 788 789 // Project this mosaic point into the original frame coordinate space 790 double xx, yy; 791 792 MosaicToFrame(inv_trs, si, sj, xx, yy); 793 794 if (xx < 0.0 || yy < 0.0 || xx > width - 1.0 || yy > height - 1.0) 795 { 796 if(inMask) 797 { 798 imgMos.Y.ptr[jj][ii] = 255; 799 wt0 = 0.0f; 800 wt1 = 1.0f; 801 } 802 } 803 804 xx /= (1 << dscale); 805 yy /= (1 << dscale); 806 807 808 int x1 = (xx >= 0.0) ? (int) xx : (int) floor(xx); 809 int y1 = (yy >= 0.0) ? (int) yy : (int) floor(yy); 810 811 // Final destination in extended pyramid 812#ifndef LINEAR_INTERP 813 if(inSegment(x1, sptr->width, BORDER-1) && 814 inSegment(y1, sptr->height, BORDER-1)) 815 { 816 double xfrac = xx - x1; 817 double yfrac = yy - y1; 818 dptr->ptr[j][i] = (short) (wt0 * dptr->ptr[j][i] + .5 + 819 wt1 * ciCalc(sptr, x1, y1, xfrac, yfrac)); 820 if (dvptr >= m_pMosaicVPyr && nC > 0) 821 { 822 duptr->ptr[j][i] = (short) (wt0 * duptr->ptr[j][i] + .5 + 823 wt1 * ciCalc(suptr, x1, y1, xfrac, yfrac)); 824 dvptr->ptr[j][i] = (short) (wt0 * dvptr->ptr[j][i] + .5 + 825 wt1 * ciCalc(svptr, x1, y1, xfrac, yfrac)); 826 } 827 } 828#else 829 if(inSegment(x1, sptr->width, BORDER) && inSegment(y1, sptr->height, BORDER)) 830 { 831 int x2 = x1 + 1; 832 int y2 = y1 + 1; 833 double xfrac = xx - x1; 834 double yfrac = yy - y1; 835 double y1val = sptr->ptr[y1][x1] + 836 (sptr->ptr[y1][x2] - sptr->ptr[y1][x1]) * xfrac; 837 double y2val = sptr->ptr[y2][x1] + 838 (sptr->ptr[y2][x2] - sptr->ptr[y2][x1]) * xfrac; 839 dptr->ptr[j][i] = (short) (y1val + yfrac * (y2val - y1val)); 840 841 if (dvptr >= m_pMosaicVPyr && nC > 0) 842 { 843 y1val = suptr->ptr[y1][x1] + 844 (suptr->ptr[y1][x2] - suptr->ptr[y1][x1]) * xfrac; 845 y2val = suptr->ptr[y2][x1] + 846 (suptr->ptr[y2][x2] - suptr->ptr[y2][x1]) * xfrac; 847 848 duptr->ptr[j][i] = (short) (y1val + yfrac * (y2val - y1val)); 849 850 y1val = svptr->ptr[y1][x1] + 851 (svptr->ptr[y1][x2] - svptr->ptr[y1][x1]) * xfrac; 852 y2val = svptr->ptr[y2][x1] + 853 (svptr->ptr[y2][x2] - svptr->ptr[y2][x1]) * xfrac; 854 855 dvptr->ptr[j][i] = (short) (y1val + yfrac * (y2val - y1val)); 856 } 857 } 858#endif 859 else 860 { 861 clipToSegment(x1, sptr->width, BORDER); 862 clipToSegment(y1, sptr->height, BORDER); 863 864 dptr->ptr[j][i] = (short) (wt0 * dptr->ptr[j][i] + 0.5 + 865 wt1 * sptr->ptr[y1][x1] ); 866 if (dvptr >= m_pMosaicVPyr && nC > 0) 867 { 868 dvptr->ptr[j][i] = (short) (wt0 * dvptr->ptr[j][i] + 869 0.5 + wt1 * svptr->ptr[y1][x1] ); 870 duptr->ptr[j][i] = (short) (wt0 * duptr->ptr[j][i] + 871 0.5 + wt1 * suptr->ptr[y1][x1] ); 872 } 873 } 874 } 875 } 876 } 877} 878 879void Blend::MosaicToFrame(double trs[3][3], double x, double y, double &wx, double &wy) 880{ 881 double X, Y, z; 882 if (m_wb.theta == 0.0) 883 { 884 X = x; 885 Y = y; 886 } 887 else if (m_wb.horizontal) 888 { 889 double alpha = x * m_wb.direction / m_wb.width; 890 double length = (y - alpha * m_wb.correction) * m_wb.direction + m_wb.radius; 891 double deltaTheta = m_wb.theta * alpha; 892 double sinTheta = sin(deltaTheta); 893 double cosTheta = sqrt(1.0 - sinTheta * sinTheta) * m_wb.direction; 894 X = length * sinTheta + m_wb.x; 895 Y = length * cosTheta + m_wb.y; 896 } 897 else 898 { 899 double alpha = y * m_wb.direction / m_wb.width; 900 double length = (x - alpha * m_wb.correction) * m_wb.direction + m_wb.radius; 901 double deltaTheta = m_wb.theta * alpha; 902 double sinTheta = sin(deltaTheta); 903 double cosTheta = sqrt(1.0 - sinTheta * sinTheta) * m_wb.direction; 904 Y = length * sinTheta + m_wb.y; 905 X = length * cosTheta + m_wb.x; 906 } 907 z = ProjZ(trs, X, Y, 1.0); 908 wx = ProjX(trs, X, Y, z, 1.0); 909 wy = ProjY(trs, X, Y, z, 1.0); 910} 911 912void Blend::FrameToMosaic(double trs[3][3], double x, double y, double &wx, double &wy) 913{ 914 // Project into the intermediate Mosaic coordinate system 915 double z = ProjZ(trs, x, y, 1.0); 916 double X = ProjX(trs, x, y, z, 1.0); 917 double Y = ProjY(trs, x, y, z, 1.0); 918 919 if (m_wb.theta == 0.0) 920 { 921 // No rotation, then this is all we need to do. 922 wx = X; 923 wy = Y; 924 } 925 else if (m_wb.horizontal) 926 { 927 double deltaX = X - m_wb.x; 928 double deltaY = Y - m_wb.y; 929 double length = sqrt(deltaX * deltaX + deltaY * deltaY); 930 double deltaTheta = asin(deltaX / length); 931 double alpha = deltaTheta / m_wb.theta; 932 wx = alpha * m_wb.width * m_wb.direction; 933 wy = (length - m_wb.radius) * m_wb.direction + alpha * m_wb.correction; 934 } 935 else 936 { 937 double deltaX = X - m_wb.x; 938 double deltaY = Y - m_wb.y; 939 double length = sqrt(deltaX * deltaX + deltaY * deltaY); 940 double deltaTheta = asin(deltaY / length); 941 double alpha = deltaTheta / m_wb.theta; 942 wy = alpha * m_wb.width * m_wb.direction; 943 wx = (length - m_wb.radius) * m_wb.direction + alpha * m_wb.correction; 944 } 945} 946 947 948 949// Clip the region of interest as small as possible by using the Voronoi edges of 950// the neighbors 951void Blend::ClipBlendRect(CSite *csite, BlendRect &brect) 952{ 953 SEdgeVector *ce; 954 int ecnt; 955 for (ce = csite->getNeighbor(), ecnt = csite->getNumNeighbors(); ecnt--; ce++) 956 { 957 // calculate the Voronoi bisector intersection 958 const double epsilon = 1e-5; 959 double dx = (m_AllSites[ce->second].getVCenter().x - m_AllSites[ce->first].getVCenter().x); 960 double dy = (m_AllSites[ce->second].getVCenter().y - m_AllSites[ce->first].getVCenter().y); 961 double xmid = m_AllSites[ce->first].getVCenter().x + dx/2.0; 962 double ymid = m_AllSites[ce->first].getVCenter().y + dy/2.0; 963 double inter; 964 965 if (dx > epsilon) 966 { 967 // neighbor is on right 968 if ((inter = m_wb.roundoffOverlap + xmid - dy * (((dy >= 0.0) ? brect.bot : brect.top) - ymid) / dx) < brect.rgt) 969 brect.rgt = inter; 970 } 971 else if (dx < -epsilon) 972 { 973 // neighbor is on left 974 if ((inter = -m_wb.roundoffOverlap + xmid - dy * (((dy >= 0.0) ? brect.bot : brect.top) - ymid) / dx) > brect.lft) 975 brect.lft = inter; 976 } 977 if (dy > epsilon) 978 { 979 // neighbor is above 980 if ((inter = m_wb.roundoffOverlap + ymid - dx * (((dx >= 0.0) ? brect.lft : brect.rgt) - xmid) / dy) < brect.top) 981 brect.top = inter; 982 } 983 else if (dy < -epsilon) 984 { 985 // neighbor is below 986 if ((inter = -m_wb.roundoffOverlap + ymid - dx * (((dx >= 0.0) ? brect.lft : brect.rgt) - xmid) / dy) > brect.bot) 987 brect.bot = inter; 988 } 989 } 990} 991 992void Blend::FrameToMosaicRect(int width, int height, double trs[3][3], BlendRect &brect) 993{ 994 // We need to walk the perimeter since the borders can be bent. 995 brect.lft = brect.bot = 2e30; 996 brect.rgt = brect.top = -2e30; 997 double xpos, ypos; 998 double lasty = height - 1.0; 999 double lastx = width - 1.0; 1000 int i; 1001 1002 for (i = width; i--;) 1003 { 1004 1005 FrameToMosaic(trs, (double) i, 0.0, xpos, ypos); 1006 ClipRect(xpos, ypos, brect); 1007 FrameToMosaic(trs, (double) i, lasty, xpos, ypos); 1008 ClipRect(xpos, ypos, brect); 1009 } 1010 for (i = height; i--;) 1011 { 1012 FrameToMosaic(trs, 0.0, (double) i, xpos, ypos); 1013 ClipRect(xpos, ypos, brect); 1014 FrameToMosaic(trs, lastx, (double) i, xpos, ypos); 1015 ClipRect(xpos, ypos, brect); 1016 } 1017} 1018 1019void Blend::SelectRelevantFrames(MosaicFrame **frames, int frames_size, 1020 MosaicFrame **relevant_frames, int &relevant_frames_size) 1021{ 1022 MosaicFrame *first = frames[0]; 1023 MosaicFrame *last = frames[frames_size-1]; 1024 MosaicFrame *mb; 1025 1026 double fxpos = first->trs[0][2], fypos = first->trs[1][2]; 1027 1028 double midX = last->width / 2.0; 1029 double midY = last->height / 2.0; 1030 double z = ProjZ(first->trs, midX, midY, 1.0); 1031 double firstX, firstY; 1032 double prevX = firstX = ProjX(first->trs, midX, midY, z, 1.0); 1033 double prevY = firstY = ProjY(first->trs, midX, midY, z, 1.0); 1034 1035 relevant_frames[0] = first; // Add first frame by default 1036 relevant_frames_size = 1; 1037 1038 for (int i = 0; i < frames_size - 1; i++) 1039 { 1040 mb = frames[i]; 1041 double currX, currY; 1042 z = ProjZ(mb->trs, midX, midY, 1.0); 1043 currX = ProjX(mb->trs, midX, midY, z, 1.0); 1044 currY = ProjY(mb->trs, midX, midY, z, 1.0); 1045 double deltaX = currX - prevX; 1046 double deltaY = currY - prevY; 1047 double center2centerDist = sqrt(deltaY * deltaY + deltaX * deltaX); 1048 1049 if (fabs(deltaX) > STRIP_SEPARATION_THRESHOLD * last->width) 1050 { 1051 relevant_frames[relevant_frames_size] = mb; 1052 relevant_frames_size++; 1053 1054 prevX = currX; 1055 prevY = currY; 1056 } 1057 } 1058 1059 // Add last frame by default 1060 relevant_frames[relevant_frames_size] = last; 1061 relevant_frames_size++; 1062} 1063 1064void Blend::ComputeBlendParameters(MosaicFrame **frames, int frames_size, int is360) 1065{ 1066 // For FULL and PAN modes, we do not unwarp the mosaic into a rectangular coordinate system 1067 // and so we set the theta to 0 and return. 1068 if (m_wb.blendingType != BLEND_TYPE_CYLPAN && m_wb.blendingType != BLEND_TYPE_HORZ) 1069 { 1070 m_wb.theta = 0.0; 1071 return; 1072 } 1073 1074 MosaicFrame *first = frames[0]; 1075 MosaicFrame *last = frames[frames_size-1]; 1076 MosaicFrame *mb; 1077 1078 double lxpos = last->trs[0][2], lypos = last->trs[1][2]; 1079 double fxpos = first->trs[0][2], fypos = first->trs[1][2]; 1080 1081 // Calculate warp to produce proper stitching. 1082 // get x, y displacement 1083 double midX = last->width / 2.0; 1084 double midY = last->height / 2.0; 1085 double z = ProjZ(first->trs, midX, midY, 1.0); 1086 double firstX, firstY; 1087 double prevX = firstX = ProjX(first->trs, midX, midY, z, 1.0); 1088 double prevY = firstY = ProjY(first->trs, midX, midY, z, 1.0); 1089 1090 double arcLength, lastTheta; 1091 m_wb.theta = lastTheta = arcLength = 0.0; 1092 1093 // Step through all the frames to compute the total arc-length of the cone 1094 // swept while capturing the mosaic (in the original conical coordinate system). 1095 for (int i = 0; i < frames_size; i++) 1096 { 1097 mb = frames[i]; 1098 double currX, currY; 1099 z = ProjZ(mb->trs, midX, midY, 1.0); 1100 currX = ProjX(mb->trs, midX, midY, z, 1.0); 1101 currY = ProjY(mb->trs, midX, midY, z, 1.0); 1102 double deltaX = currX - prevX; 1103 double deltaY = currY - prevY; 1104 1105 // The arcLength is computed by summing the lengths of the chords 1106 // connecting the pairwise projected image centers of the input image frames. 1107 arcLength += sqrt(deltaY * deltaY + deltaX * deltaX); 1108 1109 if (!is360) 1110 { 1111 double thisTheta = asin(mb->trs[1][0]); 1112 m_wb.theta += thisTheta - lastTheta; 1113 lastTheta = thisTheta; 1114 } 1115 1116 prevX = currX; 1117 prevY = currY; 1118 } 1119 1120 // Stretch this to end at the proper alignment i.e. the width of the 1121 // rectangle is determined by the arcLength computed above and the cone 1122 // sector angle is determined using the rotation of the last frame. 1123 m_wb.width = arcLength; 1124 if (is360) m_wb.theta = asin(last->trs[1][0]); 1125 1126 // If there is no rotation, we're done. 1127 if (m_wb.theta != 0.0) 1128 { 1129 double dx = prevX - firstX; 1130 double dy = prevY - firstY; 1131 1132 // If the mosaic was captured by sweeping horizontally 1133 if (abs(lxpos - fxpos) > abs(lypos - fypos)) 1134 { 1135 m_wb.horizontal = 1; 1136 // Calculate radius position to make ends exactly the same Y offset 1137 double radiusTheta = dx / cos(3.14159 / 2.0 - m_wb.theta); 1138 m_wb.radius = dy + radiusTheta * cos(m_wb.theta); 1139 if (m_wb.radius < 0.0) m_wb.radius = -m_wb.radius; 1140 } 1141 else 1142 { 1143 m_wb.horizontal = 0; 1144 // Calculate radius position to make ends exactly the same Y offset 1145 double radiusTheta = dy / cos(3.14159 / 2.0 - m_wb.theta); 1146 m_wb.radius = dx + radiusTheta * cos(m_wb.theta); 1147 if (m_wb.radius < 0.0) m_wb.radius = -m_wb.radius; 1148 } 1149 1150 // Determine major direction 1151 if (m_wb.horizontal) 1152 { 1153 // Horizontal strip 1154 // m_wb.x,y record the origin of the rectangle coordinate system. 1155 if (is360) m_wb.x = firstX; 1156 else 1157 { 1158 if (lxpos - fxpos < 0) 1159 { 1160 m_wb.x = firstX + midX; 1161 z = ProjZ(last->trs, 0.0, midY, 1.0); 1162 prevX = ProjX(last->trs, 0.0, midY, z, 1.0); 1163 prevY = ProjY(last->trs, 0.0, midY, z, 1.0); 1164 } 1165 else 1166 { 1167 m_wb.x = firstX - midX; 1168 z = ProjZ(last->trs, last->width - 1.0, midY, 1.0); 1169 prevX = ProjX(last->trs, last->width - 1.0, midY, z, 1.0); 1170 prevY = ProjY(last->trs, last->width - 1.0, midY, z, 1.0); 1171 } 1172 } 1173 dy = prevY - firstY; 1174 if (dy < 0.0) m_wb.direction = 1.0; 1175 else m_wb.direction = -1.0; 1176 m_wb.y = firstY - m_wb.radius * m_wb.direction; 1177 if (dy * m_wb.theta > 0.0) m_wb.width = -m_wb.width; 1178 } 1179 else 1180 { 1181 // Vertical strip 1182 if (is360) m_wb.y = firstY; 1183 else 1184 { 1185 if (lypos - fypos < 0) 1186 { 1187 m_wb.x = firstY + midY; 1188 z = ProjZ(last->trs, midX, 0.0, 1.0); 1189 prevX = ProjX(last->trs, midX, 0.0, z, 1.0); 1190 prevY = ProjY(last->trs, midX, 0.0, z, 1.0); 1191 } 1192 else 1193 { 1194 m_wb.x = firstX - midX; 1195 z = ProjZ(last->trs, midX, last->height - 1.0, 1.0); 1196 prevX = ProjX(last->trs, midX, last->height - 1.0, z, 1.0); 1197 prevY = ProjY(last->trs, midX, last->height - 1.0, z, 1.0); 1198 } 1199 } 1200 dx = prevX - firstX; 1201 if (dx < 0.0) m_wb.direction = 1.0; 1202 else m_wb.direction = -1.0; 1203 m_wb.x = firstX - m_wb.radius * m_wb.direction; 1204 if (dx * m_wb.theta > 0.0) m_wb.width = -m_wb.width; 1205 } 1206 1207 // Calculate the correct correction factor 1208 double deltaX = prevX - m_wb.x; 1209 double deltaY = prevY - m_wb.y; 1210 double length = sqrt(deltaX * deltaX + deltaY * deltaY); 1211 double deltaTheta = (m_wb.horizontal) ? deltaX : deltaY; 1212 deltaTheta = asin(deltaTheta / length); 1213 m_wb.correction = ((m_wb.radius - length) * m_wb.direction) / 1214 (deltaTheta / m_wb.theta); 1215 } 1216} 1217