1/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H
18#define ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H
19
20#include <android/hardware/camera/device/3.2/ICameraDevice.h>
21#include <android/hardware/camera/device/3.4/ICameraDeviceSession.h>
22#include <fmq/MessageQueue.h>
23#include <hidl/MQDescriptor.h>
24#include <hidl/Status.h>
25#include <include/convert.h>
26#include <chrono>
27#include <condition_variable>
28#include <list>
29#include <unordered_map>
30#include <unordered_set>
31#include "CameraMetadata.h"
32#include "HandleImporter.h"
33#include "Exif.h"
34#include "utils/KeyedVector.h"
35#include "utils/Mutex.h"
36#include "utils/Thread.h"
37#include "android-base/unique_fd.h"
38#include "ExternalCameraUtils.h"
39
40namespace android {
41namespace hardware {
42namespace camera {
43namespace device {
44namespace V3_4 {
45namespace implementation {
46
47using ::android::hardware::camera::device::V3_2::BufferCache;
48using ::android::hardware::camera::device::V3_2::BufferStatus;
49using ::android::hardware::camera::device::V3_2::CameraMetadata;
50using ::android::hardware::camera::device::V3_2::CaptureRequest;
51using ::android::hardware::camera::device::V3_2::CaptureResult;
52using ::android::hardware::camera::device::V3_2::ErrorCode;
53using ::android::hardware::camera::device::V3_2::ICameraDeviceCallback;
54using ::android::hardware::camera::device::V3_2::MsgType;
55using ::android::hardware::camera::device::V3_2::NotifyMsg;
56using ::android::hardware::camera::device::V3_2::RequestTemplate;
57using ::android::hardware::camera::device::V3_2::Stream;
58using ::android::hardware::camera::device::V3_4::StreamConfiguration;
59using ::android::hardware::camera::device::V3_2::StreamConfigurationMode;
60using ::android::hardware::camera::device::V3_2::StreamRotation;
61using ::android::hardware::camera::device::V3_2::StreamType;
62using ::android::hardware::camera::device::V3_2::DataspaceFlags;
63using ::android::hardware::camera::device::V3_2::CameraBlob;
64using ::android::hardware::camera::device::V3_2::CameraBlobId;
65using ::android::hardware::camera::device::V3_4::HalStreamConfiguration;
66using ::android::hardware::camera::device::V3_4::ICameraDeviceSession;
67using ::android::hardware::camera::common::V1_0::Status;
68using ::android::hardware::camera::common::V1_0::helper::HandleImporter;
69using ::android::hardware::camera::common::V1_0::helper::ExifUtils;
70using ::android::hardware::camera::external::common::ExternalCameraConfig;
71using ::android::hardware::camera::external::common::Size;
72using ::android::hardware::camera::external::common::SizeHasher;
73using ::android::hardware::graphics::common::V1_0::BufferUsage;
74using ::android::hardware::graphics::common::V1_0::Dataspace;
75using ::android::hardware::graphics::common::V1_0::PixelFormat;
76using ::android::hardware::kSynchronizedReadWrite;
77using ::android::hardware::MessageQueue;
78using ::android::hardware::MQDescriptorSync;
79using ::android::hardware::Return;
80using ::android::hardware::Void;
81using ::android::hardware::hidl_vec;
82using ::android::hardware::hidl_string;
83using ::android::sp;
84using ::android::Mutex;
85using ::android::base::unique_fd;
86
87struct ExternalCameraDeviceSession : public virtual RefBase {
88
89    ExternalCameraDeviceSession(const sp<ICameraDeviceCallback>&,
90            const ExternalCameraConfig& cfg,
91            const std::vector<SupportedV4L2Format>& sortedFormats,
92            const CroppingType& croppingType,
93            const common::V1_0::helper::CameraMetadata& chars,
94            const std::string& cameraId,
95            unique_fd v4l2Fd);
96    virtual ~ExternalCameraDeviceSession();
97    // Call by CameraDevice to dump active device states
98    void dumpState(const native_handle_t*);
99    // Caller must use this method to check if CameraDeviceSession ctor failed
100    bool isInitFailed() { return mInitFail; }
101    bool isClosed();
102
103    // Retrieve the HIDL interface, split into its own class to avoid inheritance issues when
104    // dealing with minor version revs and simultaneous implementation and interface inheritance
105    virtual sp<ICameraDeviceSession> getInterface() {
106        return new TrampolineSessionInterface_3_4(this);
107    }
108
109    static const int kMaxProcessedStream = 2;
110    static const int kMaxStallStream = 1;
111    static const uint32_t kMaxBytesPerPixel = 2;
112
113protected:
114
115    // Methods from ::android::hardware::camera::device::V3_2::ICameraDeviceSession follow
116
117    Return<void> constructDefaultRequestSettings(
118            RequestTemplate,
119            ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb);
120
121    Return<void> configureStreams(
122            const V3_2::StreamConfiguration&,
123            ICameraDeviceSession::configureStreams_cb);
124
125    Return<void> getCaptureRequestMetadataQueue(
126        ICameraDeviceSession::getCaptureRequestMetadataQueue_cb);
127
128    Return<void> getCaptureResultMetadataQueue(
129        ICameraDeviceSession::getCaptureResultMetadataQueue_cb);
130
131    Return<void> processCaptureRequest(
132            const hidl_vec<CaptureRequest>&,
133            const hidl_vec<BufferCache>&,
134            ICameraDeviceSession::processCaptureRequest_cb);
135
136    Return<Status> flush();
137    Return<void> close();
138
139    Return<void> configureStreams_3_3(
140            const V3_2::StreamConfiguration&,
141            ICameraDeviceSession::configureStreams_3_3_cb);
142
143    Return<void> configureStreams_3_4(
144            const V3_4::StreamConfiguration& requestedConfiguration,
145            ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb);
146
147    Return<void> processCaptureRequest_3_4(
148            const hidl_vec<V3_4::CaptureRequest>& requests,
149            const hidl_vec<V3_2::BufferCache>& cachesToRemove,
150            ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb);
151
152protected:
153    struct HalStreamBuffer {
154        int32_t streamId;
155        uint64_t bufferId;
156        uint32_t width;
157        uint32_t height;
158        PixelFormat format;
159        V3_2::BufferUsageFlags usage;
160        buffer_handle_t* bufPtr;
161        int acquireFence;
162        bool fenceTimeout;
163    };
164
165    struct HalRequest {
166        uint32_t frameNumber;
167        common::V1_0::helper::CameraMetadata setting;
168        sp<V4L2Frame> frameIn;
169        nsecs_t shutterTs;
170        std::vector<HalStreamBuffer> buffers;
171    };
172
173    Status constructDefaultRequestSettingsRaw(RequestTemplate type,
174            V3_2::CameraMetadata *outMetadata);
175
176    bool initialize();
177    Status initStatus() const;
178    status_t initDefaultRequests();
179    status_t fillCaptureResult(common::V1_0::helper::CameraMetadata& md, nsecs_t timestamp);
180    Status configureStreams(const V3_2::StreamConfiguration&, V3_3::HalStreamConfiguration* out);
181    // fps = 0.0 means default, which is
182    // slowest fps that is at least 30, or fastest fps if 30 is not supported
183    int configureV4l2StreamLocked(const SupportedV4L2Format& fmt, double fps = 0.0);
184    int v4l2StreamOffLocked();
185    int setV4l2FpsLocked(double fps);
186
187    // TODO: change to unique_ptr for better tracking
188    sp<V4L2Frame> dequeueV4l2FrameLocked(/*out*/nsecs_t* shutterTs); // Called with mLock hold
189    void enqueueV4l2Frame(const sp<V4L2Frame>&);
190
191    // Check if input Stream is one of supported stream setting on this device
192    bool isSupported(const Stream&);
193
194    // Validate and import request's output buffers and acquire fence
195    Status importRequest(
196            const CaptureRequest& request,
197            hidl_vec<buffer_handle_t*>& allBufPtrs,
198            hidl_vec<int>& allFences);
199    static void cleanupInflightFences(
200            hidl_vec<int>& allFences, size_t numFences);
201    void cleanupBuffersLocked(int id);
202    void updateBufferCaches(const hidl_vec<BufferCache>& cachesToRemove);
203
204    Status processOneCaptureRequest(const CaptureRequest& request);
205
206    Status processCaptureResult(std::shared_ptr<HalRequest>&);
207    Status processCaptureRequestError(const std::shared_ptr<HalRequest>&);
208    void notifyShutter(uint32_t frameNumber, nsecs_t shutterTs);
209    void notifyError(uint32_t frameNumber, int32_t streamId, ErrorCode ec);
210    void invokeProcessCaptureResultCallback(
211            hidl_vec<CaptureResult> &results, bool tryWriteFmq);
212    static void freeReleaseFences(hidl_vec<CaptureResult>&);
213
214    Size getMaxJpegResolution() const;
215    Size getMaxThumbResolution() const;
216
217    ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const;
218
219    int waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk);
220
221    class OutputThread : public android::Thread {
222    public:
223        OutputThread(wp<ExternalCameraDeviceSession> parent, CroppingType);
224        ~OutputThread();
225
226        Status allocateIntermediateBuffers(
227                const Size& v4lSize, const Size& thumbSize,
228                const hidl_vec<Stream>& streams);
229        Status submitRequest(const std::shared_ptr<HalRequest>&);
230        void flush();
231        void dump(int fd);
232        virtual bool threadLoop() override;
233
234        void setExifMakeModel(const std::string& make, const std::string& model);
235    private:
236        static const uint32_t FLEX_YUV_GENERIC = static_cast<uint32_t>('F') |
237                static_cast<uint32_t>('L') << 8 | static_cast<uint32_t>('E') << 16 |
238                static_cast<uint32_t>('X') << 24;
239        // returns FLEX_YUV_GENERIC for formats other than YV12/YU12/NV12/NV21
240        static uint32_t getFourCcFromLayout(const YCbCrLayout&);
241        static int getCropRect(
242                CroppingType ct, const Size& inSize, const Size& outSize, IMapper::Rect* out);
243
244        static const int kFlushWaitTimeoutSec = 3; // 3 sec
245        static const int kReqWaitTimeoutMs = 33;   // 33ms
246        static const int kReqWaitTimesMax = 90;    // 33ms * 90 ~= 3 sec
247
248        void waitForNextRequest(std::shared_ptr<HalRequest>* out);
249        void signalRequestDone();
250
251        int cropAndScaleLocked(
252                sp<AllocatedFrame>& in, const Size& outSize,
253                YCbCrLayout* out);
254
255        int cropAndScaleThumbLocked(
256                sp<AllocatedFrame>& in, const Size& outSize,
257                YCbCrLayout* out);
258
259        int formatConvertLocked(const YCbCrLayout& in, const YCbCrLayout& out,
260                Size sz, uint32_t format);
261
262        static int encodeJpegYU12(const Size &inSz,
263                const YCbCrLayout& inLayout, int jpegQuality,
264                const void *app1Buffer, size_t app1Size,
265                void *out, size_t maxOutSize,
266                size_t &actualCodeSize);
267
268        int createJpegLocked(HalStreamBuffer &halBuf, const std::shared_ptr<HalRequest>& req);
269
270        const wp<ExternalCameraDeviceSession> mParent;
271        const CroppingType mCroppingType;
272
273        mutable std::mutex mRequestListLock;      // Protect acccess to mRequestList,
274                                                  // mProcessingRequest and mProcessingFrameNumer
275        std::condition_variable mRequestCond;     // signaled when a new request is submitted
276        std::condition_variable mRequestDoneCond; // signaled when a request is done processing
277        std::list<std::shared_ptr<HalRequest>> mRequestList;
278        bool mProcessingRequest = false;
279        uint32_t mProcessingFrameNumer = 0;
280
281        // V4L2 frameIn
282        // (MJPG decode)-> mYu12Frame
283        // (Scale)-> mScaledYu12Frames
284        // (Format convert) -> output gralloc frames
285        mutable std::mutex mBufferLock; // Protect access to intermediate buffers
286        sp<AllocatedFrame> mYu12Frame;
287        sp<AllocatedFrame> mYu12ThumbFrame;
288        std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mIntermediateBuffers;
289        std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mScaledYu12Frames;
290        YCbCrLayout mYu12FrameLayout;
291        YCbCrLayout mYu12ThumbFrameLayout;
292
293        std::string mExifMake;
294        std::string mExifModel;
295    };
296
297    // Protect (most of) HIDL interface methods from synchronized-entering
298    mutable Mutex mInterfaceLock;
299
300    mutable Mutex mLock; // Protect all private members except otherwise noted
301    const sp<ICameraDeviceCallback> mCallback;
302    const ExternalCameraConfig& mCfg;
303    const common::V1_0::helper::CameraMetadata mCameraCharacteristics;
304    const std::vector<SupportedV4L2Format> mSupportedFormats;
305    const CroppingType mCroppingType;
306    const std::string& mCameraId;
307
308    // Not protected by mLock, this is almost a const.
309    // Setup in constructor, reset in close() after OutputThread is joined
310    unique_fd mV4l2Fd;
311
312    // device is closed either
313    //    - closed by user
314    //    - init failed
315    //    - camera disconnected
316    bool mClosed = false;
317    bool mInitFail = false;
318    bool mFirstRequest = false;
319    common::V1_0::helper::CameraMetadata mLatestReqSetting;
320
321    bool mV4l2Streaming = false;
322    SupportedV4L2Format mV4l2StreamingFmt;
323    double mV4l2StreamingFps = 0.0;
324    size_t mV4L2BufferCount = 0;
325
326    static const int kBufferWaitTimeoutSec = 3; // TODO: handle long exposure (or not allowing)
327    std::mutex mV4l2BufferLock; // protect the buffer count and condition below
328    std::condition_variable mV4L2BufferReturned;
329    size_t mNumDequeuedV4l2Buffers = 0;
330    uint32_t mMaxV4L2BufferSize = 0;
331
332    // Not protected by mLock (but might be used when mLock is locked)
333    sp<OutputThread> mOutputThread;
334
335    // Stream ID -> Camera3Stream cache
336    std::unordered_map<int, Stream> mStreamMap;
337
338    std::mutex mInflightFramesLock; // protect mInflightFrames
339    std::unordered_set<uint32_t>  mInflightFrames;
340
341    // buffers currently circulating between HAL and camera service
342    // key: bufferId sent via HIDL interface
343    // value: imported buffer_handle_t
344    // Buffer will be imported during processCaptureRequest and will be freed
345    // when the its stream is deleted or camera device session is closed
346    typedef std::unordered_map<uint64_t, buffer_handle_t> CirculatingBuffers;
347    // Stream ID -> circulating buffers map
348    std::map<int, CirculatingBuffers> mCirculatingBuffers;
349
350    std::mutex mAfTriggerLock; // protect mAfTrigger
351    bool mAfTrigger = false;
352
353    static HandleImporter sHandleImporter;
354
355    /* Beginning of members not changed after initialize() */
356    using RequestMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>;
357    std::unique_ptr<RequestMetadataQueue> mRequestMetadataQueue;
358    using ResultMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>;
359    std::shared_ptr<ResultMetadataQueue> mResultMetadataQueue;
360
361    // Protect against invokeProcessCaptureResultCallback()
362    Mutex mProcessCaptureResultLock;
363
364    std::unordered_map<RequestTemplate, CameraMetadata> mDefaultRequests;
365
366    const Size mMaxThumbResolution;
367    const Size mMaxJpegResolution;
368    /* End of members not changed after initialize() */
369
370private:
371
372    struct TrampolineSessionInterface_3_4 : public ICameraDeviceSession {
373        TrampolineSessionInterface_3_4(sp<ExternalCameraDeviceSession> parent) :
374                mParent(parent) {}
375
376        virtual Return<void> constructDefaultRequestSettings(
377                RequestTemplate type,
378                V3_3::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb) override {
379            return mParent->constructDefaultRequestSettings(type, _hidl_cb);
380        }
381
382        virtual Return<void> configureStreams(
383                const V3_2::StreamConfiguration& requestedConfiguration,
384                V3_3::ICameraDeviceSession::configureStreams_cb _hidl_cb) override {
385            return mParent->configureStreams(requestedConfiguration, _hidl_cb);
386        }
387
388        virtual Return<void> processCaptureRequest(const hidl_vec<V3_2::CaptureRequest>& requests,
389                const hidl_vec<V3_2::BufferCache>& cachesToRemove,
390                V3_3::ICameraDeviceSession::processCaptureRequest_cb _hidl_cb) override {
391            return mParent->processCaptureRequest(requests, cachesToRemove, _hidl_cb);
392        }
393
394        virtual Return<void> getCaptureRequestMetadataQueue(
395                V3_3::ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb) override  {
396            return mParent->getCaptureRequestMetadataQueue(_hidl_cb);
397        }
398
399        virtual Return<void> getCaptureResultMetadataQueue(
400                V3_3::ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb) override  {
401            return mParent->getCaptureResultMetadataQueue(_hidl_cb);
402        }
403
404        virtual Return<Status> flush() override {
405            return mParent->flush();
406        }
407
408        virtual Return<void> close() override {
409            return mParent->close();
410        }
411
412        virtual Return<void> configureStreams_3_3(
413                const V3_2::StreamConfiguration& requestedConfiguration,
414                configureStreams_3_3_cb _hidl_cb) override {
415            return mParent->configureStreams_3_3(requestedConfiguration, _hidl_cb);
416        }
417
418        virtual Return<void> configureStreams_3_4(
419                const V3_4::StreamConfiguration& requestedConfiguration,
420                configureStreams_3_4_cb _hidl_cb) override {
421            return mParent->configureStreams_3_4(requestedConfiguration, _hidl_cb);
422        }
423
424        virtual Return<void> processCaptureRequest_3_4(const hidl_vec<V3_4::CaptureRequest>& requests,
425                const hidl_vec<V3_2::BufferCache>& cachesToRemove,
426                ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb) override {
427            return mParent->processCaptureRequest_3_4(requests, cachesToRemove, _hidl_cb);
428        }
429
430    private:
431        sp<ExternalCameraDeviceSession> mParent;
432    };
433};
434
435}  // namespace implementation
436}  // namespace V3_4
437}  // namespace device
438}  // namespace camera
439}  // namespace hardware
440}  // namespace android
441
442#endif  // ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H
443