1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <queue> 21 22 #include <gui/IProducerListener.h> 23 #include <gui/CpuConsumer.h> 24 25 #include <media/hardware/VideoAPI.h> 26 #include <media/MediaCodecBuffer.h> 27 #include <media/stagefright/foundation/ALooper.h> 28 #include <media/stagefright/foundation/AMessage.h> 29 #include <media/stagefright/MediaCodec.h> 30 #include <media/stagefright/MediaMuxer.h> 31 32 #include "CompositeStream.h" 33 34 namespace android { 35 namespace camera3 { 36 37 class HeicCompositeStream : public CompositeStream, public Thread, 38 public CpuConsumer::FrameAvailableListener { 39 public: 40 HeicCompositeStream(sp<CameraDeviceBase> device, 41 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 42 ~HeicCompositeStream() override; 43 44 static bool isHeicCompositeStream(const sp<Surface> &surface); 45 46 status_t createInternalStreams(const std::vector<sp<Surface>>& consumers, 47 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 48 camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId, 49 std::vector<int> *surfaceIds, int streamSetId, bool isShared) override; 50 51 status_t deleteInternalStreams() override; 52 53 status_t configureStream() override; 54 55 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 56 int32_t* /*out*/currentStreamId) override; 57 58 status_t insertCompositeStreamIds(std::vector<int32_t>* compositeStreamIds /*out*/) override; 59 60 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 61 getStreamId()62 int getStreamId() override { return mMainImageStreamId; } 63 64 // Use onShutter to keep track of frame number <-> timestamp mapping. 65 void onBufferReleased(const BufferInfo& bufferInfo) override; 66 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 67 const CameraMetadata& settings) override; 68 69 // CpuConsumer listener implementation 70 void onFrameAvailable(const BufferItem& item) override; 71 72 // Return stream information about the internal camera streams 73 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 74 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 75 76 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 77 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr); 78 static bool isInMemoryTempFileSupported(); 79 protected: 80 81 bool threadLoop() override; 82 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 83 void onResultError(const CaptureResultExtras& resultExtras) override; 84 void onRequestError(const CaptureResultExtras& resultExtras) override; 85 86 private: 87 // 88 // HEIC/HEVC Codec related structures, utility functions, and callbacks 89 // 90 struct CodecOutputBufferInfo { 91 int32_t index; 92 int32_t offset; 93 int32_t size; 94 int64_t timeUs; 95 uint32_t flags; 96 }; 97 98 struct CodecInputBufferInfo { 99 int32_t index; 100 int64_t timeUs; 101 size_t tileIndex; 102 }; 103 104 class CodecCallbackHandler : public AHandler { 105 public: CodecCallbackHandler(wp<HeicCompositeStream> parent)106 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent) { 107 mParent = parent; 108 } 109 virtual void onMessageReceived(const sp<AMessage> &msg); 110 private: 111 wp<HeicCompositeStream> mParent; 112 }; 113 114 enum { 115 kWhatCallbackNotify, 116 }; 117 118 bool mUseHeic; 119 sp<MediaCodec> mCodec; 120 sp<ALooper> mCodecLooper, mCallbackLooper; 121 sp<CodecCallbackHandler> mCodecCallbackHandler; 122 sp<AMessage> mAsyncNotify; 123 sp<AMessage> mFormat; 124 size_t mNumOutputTiles; 125 126 int32_t mOutputWidth, mOutputHeight; 127 size_t mMaxHeicBufferSize; 128 int32_t mGridWidth, mGridHeight; 129 size_t mGridRows, mGridCols; 130 bool mUseGrid; // Whether to use framework YUV frame tiling. 131 132 static const int64_t kNoFrameDropMaxPtsGap = -1000000; 133 static const int32_t kNoGridOpRate = 30; 134 static const int32_t kGridOpRate = 120; 135 136 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo); 137 void onHeicInputFrameAvailable(int32_t index); // Only called for YUV input mode. 138 void onHeicFormatChanged(sp<AMessage>& newFormat); 139 void onHeicCodecError(); 140 141 status_t initializeCodec(uint32_t width, uint32_t height, 142 const sp<CameraDeviceBase>& cameraDevice); 143 void deinitCodec(); 144 145 // 146 // Composite stream related structures, utility functions and callbacks. 147 // 148 struct InputFrame { 149 int32_t orientation; 150 int32_t quality; 151 152 CpuConsumer::LockedBuffer appSegmentBuffer; 153 std::vector<CodecOutputBufferInfo> codecOutputBuffers; 154 std::unique_ptr<CameraMetadata> result; 155 156 // Fields that are only applicable to HEVC tiling. 157 CpuConsumer::LockedBuffer yuvBuffer; 158 std::vector<CodecInputBufferInfo> codecInputBuffers; 159 160 bool error; // Main input image buffer error 161 bool exifError; // Exif/APP_SEGMENT buffer error 162 int64_t timestamp; 163 int32_t requestId; 164 165 sp<AMessage> format; 166 sp<MediaMuxer> muxer; 167 int fenceFd; 168 int fileFd; 169 ssize_t trackIndex; 170 ANativeWindowBuffer *anb; 171 172 bool appSegmentWritten; 173 size_t pendingOutputTiles; 174 size_t codecInputCounter; 175 InputFrameInputFrame176 InputFrame() : orientation(0), quality(kDefaultJpegQuality), error(false), 177 exifError(false), timestamp(-1), requestId(-1), fenceFd(-1), 178 fileFd(-1), trackIndex(-1), anb(nullptr), appSegmentWritten(false), 179 pendingOutputTiles(0), codecInputCounter(0) { } 180 }; 181 182 void compilePendingInputLocked(); 183 // Find first complete and valid frame with smallest frame number 184 bool getNextReadyInputLocked(int64_t *frameNumber /*out*/); 185 // Find next failing frame number with smallest frame number and return respective frame number 186 int64_t getNextFailingInputLocked(); 187 188 status_t processInputFrame(int64_t frameNumber, InputFrame &inputFrame); 189 status_t processCodecInputFrame(InputFrame &inputFrame); 190 status_t startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame); 191 status_t processAppSegment(int64_t frameNumber, InputFrame &inputFrame); 192 status_t processOneCodecOutputFrame(int64_t frameNumber, InputFrame &inputFrame); 193 status_t processCompletedInputFrame(int64_t frameNumber, InputFrame &inputFrame); 194 195 void releaseInputFrameLocked(int64_t frameNumber, InputFrame *inputFrame /*out*/); 196 void releaseInputFramesLocked(); 197 198 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 199 size_t* app1SegmentSize); 200 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 201 const CpuConsumer::LockedBuffer& yuvBuffer, 202 size_t top, size_t left, size_t width, size_t height); 203 void initCopyRowFunction(int32_t width); 204 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 205 void updateCodecQualityLocked(int32_t quality); 206 207 static const nsecs_t kWaitDuration = 10000000; // 10 ms 208 static const int32_t kDefaultJpegQuality = 99; 209 static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 210 static const android_dataspace kAppSegmentDataSpace = 211 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 212 static const android_dataspace kHeifDataSpace = 213 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 214 // Use the limit of pipeline depth in the API sepc as maximum number of acquired 215 // app segment buffers. 216 static const uint32_t kMaxAcquiredAppSegment = 8; 217 218 int mAppSegmentStreamId, mAppSegmentSurfaceId; 219 sp<CpuConsumer> mAppSegmentConsumer; 220 sp<Surface> mAppSegmentSurface; 221 size_t mAppSegmentMaxSize; 222 std::queue<int64_t> mAppSegmentFrameNumbers; 223 CameraMetadata mStaticInfo; 224 225 int mMainImageStreamId, mMainImageSurfaceId; 226 sp<Surface> mMainImageSurface; 227 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 228 bool mYuvBufferAcquired; // Only applicable to HEVC codec 229 std::queue<int64_t> mMainImageFrameNumbers; 230 231 static const int32_t kMaxOutputSurfaceProducerCount = 1; 232 sp<Surface> mOutputSurface; 233 sp<ProducerListener> mProducerListener; 234 int32_t mDequeuedOutputBufferCnt; 235 236 // Map from frame number to JPEG setting of orientation+quality 237 struct HeicSettings { 238 int32_t orientation; 239 int32_t quality; 240 int64_t timestamp; 241 int32_t requestId; 242 bool shutterNotified; 243 HeicSettingsHeicSettings244 HeicSettings() : orientation(0), quality(95), timestamp(0), 245 requestId(-1), shutterNotified(false) {} HeicSettingsHeicSettings246 HeicSettings(int32_t _orientation, int32_t _quality) : 247 orientation(_orientation), 248 quality(_quality), timestamp(0), 249 requestId(-1), shutterNotified(false) {} 250 251 }; 252 std::map<int64_t, HeicSettings> mSettingsByFrameNumber; 253 254 // Keep all incoming APP segment Blob buffer pending further processing. 255 std::vector<int64_t> mInputAppSegmentBuffers; 256 int32_t mLockedAppSegmentBufferCnt; 257 258 // Keep all incoming HEIC blob buffer pending further processing. 259 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; 260 std::queue<int64_t> mCodecOutputBufferFrameNumbers; 261 size_t mCodecOutputCounter; 262 int32_t mQuality; 263 264 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 265 std::vector<int64_t> mInputYuvBuffers; 266 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 267 std::vector<int32_t> mCodecInputBuffers; 268 269 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 270 int64_t mGridTimestampUs; 271 272 // Indexed by frame number. In most common use case, entries are accessed in order. 273 std::map<int64_t, InputFrame> mPendingInputFrames; 274 275 // Function pointer of libyuv row copy. 276 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 277 278 // A set of APP_SEGMENT error frame numbers 279 std::set<int64_t> mExifErrorFrameNumbers; 280 void flagAnExifErrorFrameNumber(int64_t frameNumber); 281 282 // The status id for tracking the active/idle status of this composite stream 283 int mStatusId; 284 void markTrackerIdle(); 285 }; 286 287 }; // namespace camera3 288 }; // namespace android 289 290 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 291