FrameProcessor.cpp revision cb0652e5a850b2fcd919e977247e87239efaf70e 
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "Camera2-FrameProcessor"
18#define ATRACE_TAG ATRACE_TAG_CAMERA
19//#define LOG_NDEBUG 0
20
21#include <utils/Log.h>
22#include <utils/Trace.h>
23
24#include "common/CameraDeviceBase.h"
25#include "api1/Camera2Client.h"
26#include "api1/client2/FrameProcessor.h"
27
28namespace android {
29namespace camera2 {
30
31FrameProcessor::FrameProcessor(wp<CameraDeviceBase> device,
32                               sp<Camera2Client> client) :
33    FrameProcessorBase(device),
34    mClient(client),
35    mLastFrameNumberOfFaces(0),
36    mLast3AFrameNumber(-1) {
37
38    sp<CameraDeviceBase> d = device.promote();
39    mSynthesize3ANotify = !(d->willNotify3A());
40
41    {
42        SharedParameters::Lock l(client->getParameters());
43        mUsePartialQuirk = l.mParameters.quirks.partialResults;
44
45        // Initialize starting 3A state
46        m3aState.afTriggerId = l.mParameters.afTriggerCounter;
47        m3aState.aeTriggerId = l.mParameters.precaptureTriggerCounter;
48        // Check if lens is fixed-focus
49        if (l.mParameters.focusMode == Parameters::FOCUS_MODE_FIXED) {
50            m3aState.afMode = ANDROID_CONTROL_AF_MODE_OFF;
51        }
52    }
53}
54
55FrameProcessor::~FrameProcessor() {
56}
57
58bool FrameProcessor::processSingleFrame(CaptureResult &frame,
59                                        const sp<CameraDeviceBase> &device) {
60
61    sp<Camera2Client> client = mClient.promote();
62    if (!client.get()) {
63        return false;
64    }
65
66    bool partialResult = false;
67    if (mUsePartialQuirk) {
68        camera_metadata_entry_t entry;
69        entry = frame.mMetadata.find(ANDROID_QUIRKS_PARTIAL_RESULT);
70        if (entry.count > 0 &&
71                entry.data.u8[0] == ANDROID_QUIRKS_PARTIAL_RESULT_PARTIAL) {
72            partialResult = true;
73        }
74    }
75
76    if (!partialResult && processFaceDetect(frame.mMetadata, client) != OK) {
77        return false;
78    }
79
80    if (mSynthesize3ANotify) {
81        process3aState(frame.mMetadata, client);
82    }
83
84    return FrameProcessorBase::processSingleFrame(frame, device);
85}
86
87status_t FrameProcessor::processFaceDetect(const CameraMetadata &frame,
88        const sp<Camera2Client> &client) {
89    status_t res = BAD_VALUE;
90    ATRACE_CALL();
91    camera_metadata_ro_entry_t entry;
92    bool enableFaceDetect;
93
94    {
95        SharedParameters::Lock l(client->getParameters());
96        enableFaceDetect = l.mParameters.enableFaceDetect;
97    }
98    entry = frame.find(ANDROID_STATISTICS_FACE_DETECT_MODE);
99
100    // TODO: This should be an error once implementations are compliant
101    if (entry.count == 0) {
102        return OK;
103    }
104
105    uint8_t faceDetectMode = entry.data.u8[0];
106
107    camera_frame_metadata metadata;
108    Vector<camera_face_t> faces;
109    metadata.number_of_faces = 0;
110
111    if (enableFaceDetect &&
112        faceDetectMode != ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) {
113
114        SharedParameters::Lock l(client->getParameters());
115        entry = frame.find(ANDROID_STATISTICS_FACE_RECTANGLES);
116        if (entry.count == 0) {
117            // No faces this frame
118            /* warning: locks SharedCameraCallbacks */
119            callbackFaceDetection(client, metadata);
120            return OK;
121        }
122        metadata.number_of_faces = entry.count / 4;
123        if (metadata.number_of_faces >
124                l.mParameters.fastInfo.maxFaces) {
125            ALOGE("%s: Camera %d: More faces than expected! (Got %d, max %d)",
126                    __FUNCTION__, client->getCameraId(),
127                    metadata.number_of_faces, l.mParameters.fastInfo.maxFaces);
128            return res;
129        }
130        const int32_t *faceRects = entry.data.i32;
131
132        entry = frame.find(ANDROID_STATISTICS_FACE_SCORES);
133        if (entry.count == 0) {
134            ALOGE("%s: Camera %d: Unable to read face scores",
135                    __FUNCTION__, client->getCameraId());
136            return res;
137        }
138        const uint8_t *faceScores = entry.data.u8;
139
140        const int32_t *faceLandmarks = NULL;
141        const int32_t *faceIds = NULL;
142
143        if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) {
144            entry = frame.find(ANDROID_STATISTICS_FACE_LANDMARKS);
145            if (entry.count == 0) {
146                ALOGE("%s: Camera %d: Unable to read face landmarks",
147                        __FUNCTION__, client->getCameraId());
148                return res;
149            }
150            faceLandmarks = entry.data.i32;
151
152            entry = frame.find(ANDROID_STATISTICS_FACE_IDS);
153
154            if (entry.count == 0) {
155                ALOGE("%s: Camera %d: Unable to read face IDs",
156                        __FUNCTION__, client->getCameraId());
157                return res;
158            }
159            faceIds = entry.data.i32;
160        }
161
162        faces.setCapacity(metadata.number_of_faces);
163
164        size_t maxFaces = metadata.number_of_faces;
165        for (size_t i = 0; i < maxFaces; i++) {
166            if (faceScores[i] == 0) {
167                metadata.number_of_faces--;
168                continue;
169            }
170            if (faceScores[i] > 100) {
171                ALOGW("%s: Face index %zu with out of range score %d",
172                        __FUNCTION__, i, faceScores[i]);
173            }
174
175            camera_face_t face;
176
177            face.rect[0] = l.mParameters.arrayXToNormalized(faceRects[i*4 + 0]);
178            face.rect[1] = l.mParameters.arrayYToNormalized(faceRects[i*4 + 1]);
179            face.rect[2] = l.mParameters.arrayXToNormalized(faceRects[i*4 + 2]);
180            face.rect[3] = l.mParameters.arrayYToNormalized(faceRects[i*4 + 3]);
181
182            face.score = faceScores[i];
183            if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) {
184                face.id = faceIds[i];
185                face.left_eye[0] =
186                    l.mParameters.arrayXToNormalized(faceLandmarks[i*6 + 0]);
187                face.left_eye[1] =
188                    l.mParameters.arrayYToNormalized(faceLandmarks[i*6 + 1]);
189                face.right_eye[0] =
190                    l.mParameters.arrayXToNormalized(faceLandmarks[i*6 + 2]);
191                face.right_eye[1] =
192                    l.mParameters.arrayYToNormalized(faceLandmarks[i*6 + 3]);
193                face.mouth[0] =
194                    l.mParameters.arrayXToNormalized(faceLandmarks[i*6 + 4]);
195                face.mouth[1] =
196                    l.mParameters.arrayYToNormalized(faceLandmarks[i*6 + 5]);
197            } else {
198                face.id = 0;
199                face.left_eye[0] = face.left_eye[1] = -2000;
200                face.right_eye[0] = face.right_eye[1] = -2000;
201                face.mouth[0] = face.mouth[1] = -2000;
202            }
203            faces.push_back(face);
204        }
205
206        metadata.faces = faces.editArray();
207    }
208
209    /* warning: locks SharedCameraCallbacks */
210    callbackFaceDetection(client, metadata);
211
212    return OK;
213}
214
215status_t FrameProcessor::process3aState(const CameraMetadata &frame,
216        const sp<Camera2Client> &client) {
217
218    ATRACE_CALL();
219    camera_metadata_ro_entry_t entry;
220    int cameraId = client->getCameraId();
221
222    entry = frame.find(ANDROID_REQUEST_FRAME_COUNT);
223    int32_t frameNumber = entry.data.i32[0];
224
225    // Don't send 3A notifications for the same frame number twice
226    if (frameNumber <= mLast3AFrameNumber) {
227        ALOGV("%s: Already sent 3A for frame number %d, skipping",
228                __FUNCTION__, frameNumber);
229        return OK;
230    }
231
232    mLast3AFrameNumber = frameNumber;
233
234    // Get 3A states from result metadata
235    bool gotAllStates = true;
236
237    AlgState new3aState;
238
239    // TODO: Also use AE mode, AE trigger ID
240
241    gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AF_MODE,
242            &new3aState.afMode, frameNumber, cameraId);
243
244    gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AWB_MODE,
245            &new3aState.awbMode, frameNumber, cameraId);
246
247    gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AE_STATE,
248            &new3aState.aeState, frameNumber, cameraId);
249
250    gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AF_STATE,
251            &new3aState.afState, frameNumber, cameraId);
252
253    gotAllStates &= get3aResult<uint8_t>(frame, ANDROID_CONTROL_AWB_STATE,
254            &new3aState.awbState, frameNumber, cameraId);
255
256    gotAllStates &= get3aResult<int32_t>(frame, ANDROID_CONTROL_AF_TRIGGER_ID,
257            &new3aState.afTriggerId, frameNumber, cameraId);
258
259    gotAllStates &= get3aResult<int32_t>(frame, ANDROID_CONTROL_AE_PRECAPTURE_ID,
260            &new3aState.aeTriggerId, frameNumber, cameraId);
261
262    if (!gotAllStates) return BAD_VALUE;
263
264    if (new3aState.aeState != m3aState.aeState) {
265        ALOGV("%s: Camera %d: AE state %d->%d",
266                __FUNCTION__, cameraId,
267                m3aState.aeState, new3aState.aeState);
268        client->notifyAutoExposure(new3aState.aeState, new3aState.aeTriggerId);
269    }
270
271    if (new3aState.afState != m3aState.afState ||
272        new3aState.afMode != m3aState.afMode ||
273        new3aState.afTriggerId != m3aState.afTriggerId) {
274        ALOGV("%s: Camera %d: AF state %d->%d. AF mode %d->%d. Trigger %d->%d",
275                __FUNCTION__, cameraId,
276                m3aState.afState, new3aState.afState,
277                m3aState.afMode, new3aState.afMode,
278                m3aState.afTriggerId, new3aState.afTriggerId);
279        client->notifyAutoFocus(new3aState.afState, new3aState.afTriggerId);
280    }
281    if (new3aState.awbState != m3aState.awbState ||
282        new3aState.awbMode != m3aState.awbMode) {
283        ALOGV("%s: Camera %d: AWB state %d->%d. AWB mode %d->%d",
284                __FUNCTION__, cameraId,
285                m3aState.awbState, new3aState.awbState,
286                m3aState.awbMode, new3aState.awbMode);
287        client->notifyAutoWhitebalance(new3aState.awbState,
288                new3aState.aeTriggerId);
289    }
290
291    m3aState = new3aState;
292
293    return OK;
294}
295
296template<typename Src, typename T>
297bool FrameProcessor::get3aResult(const CameraMetadata& result, int32_t tag,
298        T* value, int32_t frameNumber, int cameraId) {
299    camera_metadata_ro_entry_t entry;
300    if (value == NULL) {
301        ALOGE("%s: Camera %d: Value to write to is NULL",
302                __FUNCTION__, cameraId);
303        return false;
304    }
305
306    entry = result.find(tag);
307    if (entry.count == 0) {
308        ALOGE("%s: Camera %d: No %s provided by HAL for frame %d!",
309                __FUNCTION__, cameraId,
310                get_camera_metadata_tag_name(tag), frameNumber);
311        return false;
312    } else {
313        switch(sizeof(Src)){
314            case sizeof(uint8_t):
315                *value = static_cast<T>(entry.data.u8[0]);
316                break;
317            case sizeof(int32_t):
318                *value = static_cast<T>(entry.data.i32[0]);
319                break;
320            default:
321                ALOGE("%s: Camera %d: Unsupported source",
322                        __FUNCTION__, cameraId);
323                return false;
324        }
325    }
326    return true;
327}
328
329
330void FrameProcessor::callbackFaceDetection(sp<Camera2Client> client,
331                                     const camera_frame_metadata &metadata) {
332
333    camera_frame_metadata *metadata_ptr =
334        const_cast<camera_frame_metadata*>(&metadata);
335
336    /**
337     * Filter out repeated 0-face callbacks,
338     * but not when the last frame was >0
339     */
340    if (metadata.number_of_faces != 0 ||
341        mLastFrameNumberOfFaces != metadata.number_of_faces) {
342
343        Camera2Client::SharedCameraCallbacks::Lock
344            l(client->mSharedCameraCallbacks);
345        if (l.mRemoteCallback != NULL) {
346            l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_METADATA,
347                                            NULL,
348                                            metadata_ptr);
349        }
350    }
351
352    mLastFrameNumberOfFaces = metadata.number_of_faces;
353}
354
355}; // namespace camera2
356}; // namespace android
357