JpegProcessor.cpp revision 8aa0f0619ea867e8fb240cf27913d4f8ae767385
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define LOG_TAG "Camera2-JpegProcessor" 18#define ATRACE_TAG ATRACE_TAG_CAMERA 19//#define LOG_NDEBUG 0 20 21#include <netinet/in.h> 22 23#include <binder/MemoryBase.h> 24#include <binder/MemoryHeapBase.h> 25#include <utils/Log.h> 26#include <utils/Trace.h> 27#include <gui/Surface.h> 28 29#include "common/CameraDeviceBase.h" 30#include "api1/Camera2Client.h" 31#include "api1/client2/Camera2Heap.h" 32#include "api1/client2/CaptureSequencer.h" 33#include "api1/client2/JpegProcessor.h" 34 35namespace android { 36namespace camera2 { 37 38JpegProcessor::JpegProcessor( 39 sp<Camera2Client> client, 40 wp<CaptureSequencer> sequencer): 41 Thread(false), 42 mDevice(client->getCameraDevice()), 43 mSequencer(sequencer), 44 mId(client->getCameraId()), 45 mCaptureAvailable(false), 46 mCaptureStreamId(NO_STREAM) { 47} 48 49JpegProcessor::~JpegProcessor() { 50 ALOGV("%s: Exit", __FUNCTION__); 51 deleteStream(); 52} 53 54void JpegProcessor::onFrameAvailable() { 55 Mutex::Autolock l(mInputMutex); 56 if (!mCaptureAvailable) { 57 mCaptureAvailable = true; 58 mCaptureAvailableSignal.signal(); 59 } 60} 61 62status_t JpegProcessor::updateStream(const Parameters ¶ms) { 63 ATRACE_CALL(); 64 ALOGV("%s", __FUNCTION__); 65 status_t res; 66 67 Mutex::Autolock l(mInputMutex); 68 69 sp<CameraDeviceBase> device = mDevice.promote(); 70 if (device == 0) { 71 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 72 return INVALID_OPERATION; 73 } 74 75 // Find out buffer size for JPEG 76 camera_metadata_ro_entry_t maxJpegSize = 77 params.staticInfo(ANDROID_JPEG_MAX_SIZE); 78 if (maxJpegSize.count == 0) { 79 ALOGE("%s: Camera %d: Can't find ANDROID_JPEG_MAX_SIZE!", 80 __FUNCTION__, mId); 81 return INVALID_OPERATION; 82 } 83 84 if (mCaptureConsumer == 0) { 85 // Create CPU buffer queue endpoint 86 sp<IGraphicBufferProducer> producer; 87 sp<IGraphicBufferConsumer> consumer; 88 BufferQueue::createBufferQueue(&producer, &consumer); 89 mCaptureConsumer = new CpuConsumer(consumer, 1); 90 mCaptureConsumer->setFrameAvailableListener(this); 91 mCaptureConsumer->setName(String8("Camera2Client::CaptureConsumer")); 92 mCaptureWindow = new Surface(producer); 93 // Create memory for API consumption 94 mCaptureHeap = new MemoryHeapBase(maxJpegSize.data.i32[0], 0, 95 "Camera2Client::CaptureHeap"); 96 if (mCaptureHeap->getSize() == 0) { 97 ALOGE("%s: Camera %d: Unable to allocate memory for capture", 98 __FUNCTION__, mId); 99 return NO_MEMORY; 100 } 101 } 102 103 if (mCaptureStreamId != NO_STREAM) { 104 // Check if stream parameters have to change 105 uint32_t currentWidth, currentHeight; 106 res = device->getStreamInfo(mCaptureStreamId, 107 ¤tWidth, ¤tHeight, 0); 108 if (res != OK) { 109 ALOGE("%s: Camera %d: Error querying capture output stream info: " 110 "%s (%d)", __FUNCTION__, 111 mId, strerror(-res), res); 112 return res; 113 } 114 if (currentWidth != (uint32_t)params.pictureWidth || 115 currentHeight != (uint32_t)params.pictureHeight) { 116 ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed", 117 __FUNCTION__, mId, mCaptureStreamId); 118 res = device->deleteStream(mCaptureStreamId); 119 if (res == -EBUSY) { 120 ALOGV("%s: Camera %d: Device is busy, call updateStream again " 121 " after it becomes idle", __FUNCTION__, mId); 122 return res; 123 } else if (res != OK) { 124 ALOGE("%s: Camera %d: Unable to delete old output stream " 125 "for capture: %s (%d)", __FUNCTION__, 126 mId, strerror(-res), res); 127 return res; 128 } 129 mCaptureStreamId = NO_STREAM; 130 } 131 } 132 133 if (mCaptureStreamId == NO_STREAM) { 134 // Create stream for HAL production 135 res = device->createStream(mCaptureWindow, 136 params.pictureWidth, params.pictureHeight, 137 HAL_PIXEL_FORMAT_BLOB, maxJpegSize.data.i32[0], 138 &mCaptureStreamId); 139 if (res != OK) { 140 ALOGE("%s: Camera %d: Can't create output stream for capture: " 141 "%s (%d)", __FUNCTION__, mId, 142 strerror(-res), res); 143 return res; 144 } 145 146 } 147 return OK; 148} 149 150status_t JpegProcessor::deleteStream() { 151 ATRACE_CALL(); 152 153 Mutex::Autolock l(mInputMutex); 154 155 if (mCaptureStreamId != NO_STREAM) { 156 sp<CameraDeviceBase> device = mDevice.promote(); 157 if (device == 0) { 158 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 159 return INVALID_OPERATION; 160 } 161 162 device->deleteStream(mCaptureStreamId); 163 164 mCaptureHeap.clear(); 165 mCaptureWindow.clear(); 166 mCaptureConsumer.clear(); 167 168 mCaptureStreamId = NO_STREAM; 169 } 170 return OK; 171} 172 173int JpegProcessor::getStreamId() const { 174 Mutex::Autolock l(mInputMutex); 175 return mCaptureStreamId; 176} 177 178void JpegProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const { 179} 180 181bool JpegProcessor::threadLoop() { 182 status_t res; 183 184 { 185 Mutex::Autolock l(mInputMutex); 186 while (!mCaptureAvailable) { 187 res = mCaptureAvailableSignal.waitRelative(mInputMutex, 188 kWaitDuration); 189 if (res == TIMED_OUT) return true; 190 } 191 mCaptureAvailable = false; 192 } 193 194 do { 195 res = processNewCapture(); 196 } while (res == OK); 197 198 return true; 199} 200 201status_t JpegProcessor::processNewCapture() { 202 ATRACE_CALL(); 203 status_t res; 204 sp<Camera2Heap> captureHeap; 205 sp<MemoryBase> captureBuffer; 206 207 CpuConsumer::LockedBuffer imgBuffer; 208 209 { 210 Mutex::Autolock l(mInputMutex); 211 if (mCaptureStreamId == NO_STREAM) { 212 ALOGW("%s: Camera %d: No stream is available", __FUNCTION__, mId); 213 return INVALID_OPERATION; 214 } 215 216 res = mCaptureConsumer->lockNextBuffer(&imgBuffer); 217 if (res != OK) { 218 if (res != BAD_VALUE) { 219 ALOGE("%s: Camera %d: Error receiving still image buffer: " 220 "%s (%d)", __FUNCTION__, 221 mId, strerror(-res), res); 222 } 223 return res; 224 } 225 226 ALOGV("%s: Camera %d: Still capture available", __FUNCTION__, 227 mId); 228 229 if (imgBuffer.format != HAL_PIXEL_FORMAT_BLOB) { 230 ALOGE("%s: Camera %d: Unexpected format for still image: " 231 "%x, expected %x", __FUNCTION__, mId, 232 imgBuffer.format, 233 HAL_PIXEL_FORMAT_BLOB); 234 mCaptureConsumer->unlockBuffer(imgBuffer); 235 return OK; 236 } 237 238 // Find size of JPEG image 239 size_t jpegSize = findJpegSize(imgBuffer.data, imgBuffer.width); 240 if (jpegSize == 0) { // failed to find size, default to whole buffer 241 jpegSize = imgBuffer.width; 242 } 243 size_t heapSize = mCaptureHeap->getSize(); 244 if (jpegSize > heapSize) { 245 ALOGW("%s: JPEG image is larger than expected, truncating " 246 "(got %d, expected at most %d bytes)", 247 __FUNCTION__, jpegSize, heapSize); 248 jpegSize = heapSize; 249 } 250 251 // TODO: Optimize this to avoid memcopy 252 captureBuffer = new MemoryBase(mCaptureHeap, 0, jpegSize); 253 void* captureMemory = mCaptureHeap->getBase(); 254 memcpy(captureMemory, imgBuffer.data, jpegSize); 255 256 mCaptureConsumer->unlockBuffer(imgBuffer); 257 } 258 259 sp<CaptureSequencer> sequencer = mSequencer.promote(); 260 if (sequencer != 0) { 261 sequencer->onCaptureAvailable(imgBuffer.timestamp, captureBuffer); 262 } 263 264 return OK; 265} 266 267/* 268 * JPEG FILE FORMAT OVERVIEW. 269 * http://www.jpeg.org/public/jfif.pdf 270 * (JPEG is the image compression algorithm, actual file format is called JFIF) 271 * 272 * "Markers" are 2-byte patterns used to distinguish parts of JFIF files. The 273 * first byte is always 0xFF, and the second byte is between 0x01 and 0xFE 274 * (inclusive). Because every marker begins with the same byte, they are 275 * referred to by the second byte's value. 276 * 277 * JFIF files all begin with the Start of Image (SOI) marker, which is 0xD8. 278 * Following it, "segment" sections begin with other markers, followed by a 279 * 2-byte length (in network byte order), then the segment data. 280 * 281 * For our purposes we will ignore the data, and just use the length to skip to 282 * the next segment. This is necessary because the data inside segments are 283 * allowed to contain the End of Image marker (0xFF 0xD9), preventing us from 284 * naievely scanning until the end. 285 * 286 * After all the segments are processed, the jpeg compressed image stream begins. 287 * This can be considered an opaque format with one requirement: all 0xFF bytes 288 * in this stream must be followed with a 0x00 byte. This prevents any of the 289 * image data to be interpreted as a segment. The only exception to this is at 290 * the end of the image stream there is an End of Image (EOI) marker, which is 291 * 0xFF followed by a non-zero (0xD9) byte. 292 */ 293 294const uint8_t MARK = 0xFF; // First byte of marker 295const uint8_t SOI = 0xD8; // Start of Image 296const uint8_t EOI = 0xD9; // End of Image 297const size_t MARKER_LENGTH = 2; // length of a marker 298 299#pragma pack(push) 300#pragma pack(1) 301typedef struct segment { 302 uint8_t marker[MARKER_LENGTH]; 303 uint16_t length; 304} segment_t; 305#pragma pack(pop) 306 307/* HELPER FUNCTIONS */ 308 309// check for Start of Image marker 310bool checkJpegStart(uint8_t* buf) { 311 return buf[0] == MARK && buf[1] == SOI; 312} 313// check for End of Image marker 314bool checkJpegEnd(uint8_t *buf) { 315 return buf[0] == MARK && buf[1] == EOI; 316} 317// check for arbitrary marker, returns marker type (second byte) 318// returns 0 if no marker found. Note: 0x00 is not a valid marker type 319uint8_t checkJpegMarker(uint8_t *buf) { 320 if (buf[0] == MARK && buf[1] > 0 && buf[1] < 0xFF) { 321 return buf[1]; 322 } 323 return 0; 324} 325 326// Return the size of the JPEG, 0 indicates failure 327size_t JpegProcessor::findJpegSize(uint8_t* jpegBuffer, size_t maxSize) { 328 size_t size; 329 330 // First check for JPEG transport header at the end of the buffer 331 uint8_t *header = jpegBuffer + (maxSize - sizeof(struct camera2_jpeg_blob)); 332 struct camera2_jpeg_blob *blob = (struct camera2_jpeg_blob*)(header); 333 if (blob->jpeg_blob_id == CAMERA2_JPEG_BLOB_ID) { 334 size = blob->jpeg_size; 335 if (size > 0 && size <= maxSize - sizeof(struct camera2_jpeg_blob)) { 336 // Verify SOI and EOI markers 337 size_t offset = size - MARKER_LENGTH; 338 uint8_t *end = jpegBuffer + offset; 339 if (checkJpegStart(jpegBuffer) && checkJpegEnd(end)) { 340 ALOGV("Found JPEG transport header, img size %d", size); 341 return size; 342 } else { 343 ALOGW("Found JPEG transport header with bad Image Start/End"); 344 } 345 } else { 346 ALOGW("Found JPEG transport header with bad size %d", size); 347 } 348 } 349 350 // Check Start of Image 351 if ( !checkJpegStart(jpegBuffer) ) { 352 ALOGE("Could not find start of JPEG marker"); 353 return 0; 354 } 355 356 // Read JFIF segment markers, skip over segment data 357 size = 0; 358 while (size <= maxSize - MARKER_LENGTH) { 359 segment_t *segment = (segment_t*)(jpegBuffer + size); 360 uint8_t type = checkJpegMarker(segment->marker); 361 if (type == 0) { // invalid marker, no more segments, begin JPEG data 362 ALOGV("JPEG stream found beginning at offset %d", size); 363 break; 364 } 365 if (type == EOI || size > maxSize - sizeof(segment_t)) { 366 ALOGE("Got premature End before JPEG data, offset %d", size); 367 return 0; 368 } 369 size_t length = ntohs(segment->length); 370 ALOGV("JFIF Segment, type %x length %x", type, length); 371 size += length + MARKER_LENGTH; 372 } 373 374 // Find End of Image 375 // Scan JPEG buffer until End of Image (EOI) 376 bool foundEnd = false; 377 for ( ; size <= maxSize - MARKER_LENGTH; size++) { 378 if ( checkJpegEnd(jpegBuffer + size) ) { 379 foundEnd = true; 380 size += MARKER_LENGTH; 381 break; 382 } 383 } 384 if (!foundEnd) { 385 ALOGE("Could not find end of JPEG marker"); 386 return 0; 387 } 388 389 if (size > maxSize) { 390 ALOGW("JPEG size %d too large, reducing to maxSize %d", size, maxSize); 391 size = maxSize; 392 } 393 ALOGV("Final JPEG size %d", size); 394 return size; 395} 396 397}; // namespace camera2 398}; // namespace android 399