1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Camera3-HeicCompositeStream"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
20 //#define LOG_NDEBUG 0
21 
22 #include <linux/memfd.h>
23 #include <pthread.h>
24 #include <sys/syscall.h>
25 
26 #include <aidl/android/hardware/camera/device/CameraBlob.h>
27 #include <aidl/android/hardware/camera/device/CameraBlobId.h>
28 #include <libyuv.h>
29 #include <gui/Surface.h>
30 #include <utils/Log.h>
31 #include <utils/Trace.h>
32 #include <camera/StringUtils.h>
33 
34 #include <mediadrm/ICrypto.h>
35 #include <media/MediaCodecBuffer.h>
36 #include <media/stagefright/foundation/ABuffer.h>
37 #include <media/stagefright/foundation/MediaDefs.h>
38 #include <media/stagefright/MediaCodecConstants.h>
39 
40 #include "common/CameraDeviceBase.h"
41 #include "utils/ExifUtils.h"
42 #include "utils/SessionConfigurationUtils.h"
43 #include "HeicEncoderInfoManager.h"
44 #include "HeicCompositeStream.h"
45 
46 using aidl::android::hardware::camera::device::CameraBlob;
47 using aidl::android::hardware::camera::device::CameraBlobId;
48 
49 namespace android {
50 namespace camera3 {
51 
HeicCompositeStream(sp<CameraDeviceBase> device,wp<hardware::camera2::ICameraDeviceCallbacks> cb)52 HeicCompositeStream::HeicCompositeStream(sp<CameraDeviceBase> device,
53         wp<hardware::camera2::ICameraDeviceCallbacks> cb) :
54         CompositeStream(device, cb),
55         mUseHeic(false),
56         mNumOutputTiles(1),
57         mOutputWidth(0),
58         mOutputHeight(0),
59         mMaxHeicBufferSize(0),
60         mGridWidth(HeicEncoderInfoManager::kGridWidth),
61         mGridHeight(HeicEncoderInfoManager::kGridHeight),
62         mGridRows(1),
63         mGridCols(1),
64         mUseGrid(false),
65         mAppSegmentStreamId(-1),
66         mAppSegmentSurfaceId(-1),
67         mMainImageStreamId(-1),
68         mMainImageSurfaceId(-1),
69         mYuvBufferAcquired(false),
70         mProducerListener(new ProducerListener()),
71         mDequeuedOutputBufferCnt(0),
72         mCodecOutputCounter(0),
73         mQuality(-1),
74         mGridTimestampUs(0),
75         mStatusId(StatusTracker::NO_STATUS_ID) {
76 }
77 
~HeicCompositeStream()78 HeicCompositeStream::~HeicCompositeStream() {
79     // Call deinitCodec in case stream hasn't been deleted yet to avoid any
80     // memory/resource leak.
81     deinitCodec();
82 
83     mInputAppSegmentBuffers.clear();
84     mCodecOutputBuffers.clear();
85 
86     mAppSegmentStreamId = -1;
87     mAppSegmentSurfaceId = -1;
88     mAppSegmentConsumer.clear();
89     mAppSegmentSurface.clear();
90 
91     mMainImageStreamId = -1;
92     mMainImageSurfaceId = -1;
93     mMainImageConsumer.clear();
94     mMainImageSurface.clear();
95 }
96 
isHeicCompositeStreamInfo(const OutputStreamInfo & streamInfo)97 bool HeicCompositeStream::isHeicCompositeStreamInfo(const OutputStreamInfo& streamInfo) {
98     return ((streamInfo.dataSpace == static_cast<android_dataspace_t>(HAL_DATASPACE_HEIF)) &&
99             (streamInfo.format == HAL_PIXEL_FORMAT_BLOB));
100 }
101 
isHeicCompositeStream(const sp<Surface> & surface)102 bool HeicCompositeStream::isHeicCompositeStream(const sp<Surface> &surface) {
103     ANativeWindow *anw = surface.get();
104     status_t err;
105     int format;
106     if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) {
107         std::string msg = fmt::sprintf("Failed to query Surface format: %s (%d)", strerror(-err),
108                 err);
109         ALOGE("%s: %s", __FUNCTION__, msg.c_str());
110         return false;
111     }
112 
113     int dataspace;
114     if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE, &dataspace)) != OK) {
115         std::string msg = fmt::sprintf("Failed to query Surface dataspace: %s (%d)", strerror(-err),
116                 err);
117         ALOGE("%s: %s", __FUNCTION__, msg.c_str());
118         return false;
119     }
120 
121     return ((format == HAL_PIXEL_FORMAT_BLOB) && (dataspace == HAL_DATASPACE_HEIF));
122 }
123 
createInternalStreams(const std::vector<sp<Surface>> & consumers,bool,uint32_t width,uint32_t height,int format,camera_stream_rotation_t rotation,int * id,const std::string & physicalCameraId,const std::unordered_set<int32_t> & sensorPixelModesUsed,std::vector<int> * surfaceIds,int,bool,int32_t colorSpace,int64_t,int64_t,bool useReadoutTimestamp)124 status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface>>& consumers,
125         bool /*hasDeferredConsumer*/, uint32_t width, uint32_t height, int format,
126         camera_stream_rotation_t rotation, int *id, const std::string& physicalCameraId,
127         const std::unordered_set<int32_t> &sensorPixelModesUsed,
128         std::vector<int> *surfaceIds,
129         int /*streamSetId*/, bool /*isShared*/, int32_t colorSpace,
130         int64_t /*dynamicProfile*/, int64_t /*streamUseCase*/, bool useReadoutTimestamp) {
131     sp<CameraDeviceBase> device = mDevice.promote();
132     if (!device.get()) {
133         ALOGE("%s: Invalid camera device!", __FUNCTION__);
134         return NO_INIT;
135     }
136 
137     status_t res = initializeCodec(width, height, device);
138     if (res != OK) {
139         ALOGE("%s: Failed to initialize HEIC/HEVC codec: %s (%d)",
140                 __FUNCTION__, strerror(-res), res);
141         return NO_INIT;
142     }
143 
144     sp<IGraphicBufferProducer> producer;
145     sp<IGraphicBufferConsumer> consumer;
146     BufferQueue::createBufferQueue(&producer, &consumer);
147     mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment);
148     mAppSegmentConsumer->setFrameAvailableListener(this);
149     mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
150     mAppSegmentSurface = new Surface(producer);
151 
152     mStaticInfo = device->info();
153 
154     res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,
155             kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId,
156             sensorPixelModesUsed, surfaceIds, camera3::CAMERA3_STREAM_SET_ID_INVALID,
157             /*isShared*/false, /*isMultiResolution*/false,
158             /*consumerUsage*/0, ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
159             ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
160             OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
161             OutputConfiguration::MIRROR_MODE_AUTO,
162             colorSpace,
163             useReadoutTimestamp);
164     if (res == OK) {
165         mAppSegmentSurfaceId = (*surfaceIds)[0];
166     } else {
167         ALOGE("%s: Failed to create JPEG App segment stream: %s (%d)", __FUNCTION__,
168                 strerror(-res), res);
169         return res;
170     }
171 
172     if (!mUseGrid) {
173         res = mCodec->createInputSurface(&producer);
174         if (res != OK) {
175             ALOGE("%s: Failed to create input surface for Heic codec: %s (%d)",
176                     __FUNCTION__, strerror(-res), res);
177             return res;
178         }
179     } else {
180         BufferQueue::createBufferQueue(&producer, &consumer);
181         mMainImageConsumer = new CpuConsumer(consumer, 1);
182         mMainImageConsumer->setFrameAvailableListener(this);
183         mMainImageConsumer->setName(String8("Camera3-HeicComposite-HevcInputYUVStream"));
184     }
185     mMainImageSurface = new Surface(producer);
186 
187     res = mCodec->start();
188     if (res != OK) {
189         ALOGE("%s: Failed to start codec: %s (%d)", __FUNCTION__,
190                 strerror(-res), res);
191         return res;
192     }
193 
194     std::vector<int> sourceSurfaceId;
195     //Use YUV_888 format if framework tiling is needed.
196     int srcStreamFmt = mUseGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
197             HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
198     res = device->createStream(mMainImageSurface, width, height, srcStreamFmt, kHeifDataSpace,
199             rotation, id, physicalCameraId, sensorPixelModesUsed, &sourceSurfaceId,
200             camera3::CAMERA3_STREAM_SET_ID_INVALID, /*isShared*/false, /*isMultiResolution*/false,
201             /*consumerUsage*/0, ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD,
202             ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT,
203             OutputConfiguration::TIMESTAMP_BASE_DEFAULT,
204             OutputConfiguration::MIRROR_MODE_AUTO,
205             colorSpace,
206             useReadoutTimestamp);
207     if (res == OK) {
208         mMainImageSurfaceId = sourceSurfaceId[0];
209         mMainImageStreamId = *id;
210     } else {
211         ALOGE("%s: Failed to create main image stream: %s (%d)", __FUNCTION__,
212                 strerror(-res), res);
213         return res;
214     }
215 
216     mOutputSurface = consumers[0];
217     res = registerCompositeStreamListener(mMainImageStreamId);
218     if (res != OK) {
219         ALOGE("%s: Failed to register HAL main image stream: %s (%d)", __FUNCTION__,
220                 strerror(-res), res);
221         return res;
222     }
223 
224     res = registerCompositeStreamListener(mAppSegmentStreamId);
225     if (res != OK) {
226         ALOGE("%s: Failed to register HAL app segment stream: %s (%d)", __FUNCTION__,
227                 strerror(-res), res);
228         return res;
229     }
230 
231     initCopyRowFunction(width);
232     return res;
233 }
234 
deleteInternalStreams()235 status_t HeicCompositeStream::deleteInternalStreams() {
236     requestExit();
237     auto res = join();
238     if (res != OK) {
239         ALOGE("%s: Failed to join with the main processing thread: %s (%d)", __FUNCTION__,
240                 strerror(-res), res);
241     }
242 
243     deinitCodec();
244 
245     if (mAppSegmentStreamId >= 0) {
246         // Camera devices may not be valid after switching to offline mode.
247         // In this case, all offline streams including internal composite streams
248         // are managed and released by the offline session.
249         sp<CameraDeviceBase> device = mDevice.promote();
250         if (device.get() != nullptr) {
251             res = device->deleteStream(mAppSegmentStreamId);
252         }
253 
254         mAppSegmentStreamId = -1;
255     }
256 
257     if (mOutputSurface != nullptr) {
258         mOutputSurface->disconnect(NATIVE_WINDOW_API_CAMERA);
259         mOutputSurface.clear();
260     }
261 
262     sp<StatusTracker> statusTracker = mStatusTracker.promote();
263     if (statusTracker != nullptr && mStatusId != StatusTracker::NO_STATUS_ID) {
264         statusTracker->removeComponent(mStatusId);
265         mStatusId = StatusTracker::NO_STATUS_ID;
266     }
267 
268     if (mPendingInputFrames.size() > 0) {
269         ALOGW("%s: mPendingInputFrames has %zu stale entries",
270                 __FUNCTION__, mPendingInputFrames.size());
271         mPendingInputFrames.clear();
272     }
273 
274     return res;
275 }
276 
onBufferReleased(const BufferInfo & bufferInfo)277 void HeicCompositeStream::onBufferReleased(const BufferInfo& bufferInfo) {
278     Mutex::Autolock l(mMutex);
279 
280     if (bufferInfo.mError) return;
281 
282     if (bufferInfo.mStreamId == mMainImageStreamId) {
283         mMainImageFrameNumbers.push(bufferInfo.mFrameNumber);
284         mCodecOutputBufferFrameNumbers.push(bufferInfo.mFrameNumber);
285         ALOGV("%s: [%" PRId64 "]: Adding main image frame number (%zu frame numbers in total)",
286                 __FUNCTION__, bufferInfo.mFrameNumber, mMainImageFrameNumbers.size());
287     } else if (bufferInfo.mStreamId == mAppSegmentStreamId) {
288         mAppSegmentFrameNumbers.push(bufferInfo.mFrameNumber);
289         ALOGV("%s: [%" PRId64 "]: Adding app segment frame number (%zu frame numbers in total)",
290                 __FUNCTION__, bufferInfo.mFrameNumber, mAppSegmentFrameNumbers.size());
291     }
292 }
293 
294 // We need to get the settings early to handle the case where the codec output
295 // arrives earlier than result metadata.
onBufferRequestForFrameNumber(uint64_t frameNumber,int streamId,const CameraMetadata & settings)296 void HeicCompositeStream::onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId,
297         const CameraMetadata& settings) {
298     ATRACE_ASYNC_BEGIN("HEIC capture", frameNumber);
299 
300     Mutex::Autolock l(mMutex);
301     if (mErrorState || (streamId != getStreamId())) {
302         return;
303     }
304 
305     mPendingCaptureResults.emplace(frameNumber, CameraMetadata());
306 
307     camera_metadata_ro_entry entry;
308 
309     int32_t orientation = 0;
310     entry = settings.find(ANDROID_JPEG_ORIENTATION);
311     if (entry.count == 1) {
312         orientation = entry.data.i32[0];
313     }
314 
315     int32_t quality = kDefaultJpegQuality;
316     entry = settings.find(ANDROID_JPEG_QUALITY);
317     if (entry.count == 1) {
318         quality = entry.data.i32[0];
319     }
320 
321     mSettingsByFrameNumber[frameNumber] = {orientation, quality};
322 }
323 
onFrameAvailable(const BufferItem & item)324 void HeicCompositeStream::onFrameAvailable(const BufferItem& item) {
325     if (item.mDataSpace == static_cast<android_dataspace>(kAppSegmentDataSpace)) {
326         ALOGV("%s: JPEG APP segments buffer with ts: %" PRIu64 " ms. arrived!",
327                 __func__, ns2ms(item.mTimestamp));
328 
329         Mutex::Autolock l(mMutex);
330         if (!mErrorState) {
331             mInputAppSegmentBuffers.push_back(item.mTimestamp);
332             mInputReadyCondition.signal();
333         }
334     } else if (item.mDataSpace == kHeifDataSpace) {
335         ALOGV("%s: YUV_888 buffer with ts: %" PRIu64 " ms. arrived!",
336                 __func__, ns2ms(item.mTimestamp));
337 
338         Mutex::Autolock l(mMutex);
339         if (!mUseGrid) {
340             ALOGE("%s: YUV_888 internal stream is only supported for HEVC tiling",
341                     __FUNCTION__);
342             return;
343         }
344         if (!mErrorState) {
345             mInputYuvBuffers.push_back(item.mTimestamp);
346             mInputReadyCondition.signal();
347         }
348     } else {
349         ALOGE("%s: Unexpected data space: 0x%x", __FUNCTION__, item.mDataSpace);
350     }
351 }
352 
getCompositeStreamInfo(const OutputStreamInfo & streamInfo,const CameraMetadata & ch,std::vector<OutputStreamInfo> * compositeOutput)353 status_t HeicCompositeStream::getCompositeStreamInfo(const OutputStreamInfo &streamInfo,
354             const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/) {
355     if (compositeOutput == nullptr) {
356         return BAD_VALUE;
357     }
358 
359     compositeOutput->clear();
360 
361     bool useGrid, useHeic;
362     bool isSizeSupported = isSizeSupportedByHeifEncoder(
363             streamInfo.width, streamInfo.height, &useHeic, &useGrid, nullptr);
364     if (!isSizeSupported) {
365         // Size is not supported by either encoder.
366         return OK;
367     }
368 
369     compositeOutput->insert(compositeOutput->end(), 2, streamInfo);
370 
371     // JPEG APPS segments Blob stream info
372     (*compositeOutput)[0].width = calcAppSegmentMaxSize(ch);
373     (*compositeOutput)[0].height = 1;
374     (*compositeOutput)[0].format = HAL_PIXEL_FORMAT_BLOB;
375     (*compositeOutput)[0].dataSpace = kAppSegmentDataSpace;
376     (*compositeOutput)[0].consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
377 
378     // YUV/IMPLEMENTATION_DEFINED stream info
379     (*compositeOutput)[1].width = streamInfo.width;
380     (*compositeOutput)[1].height = streamInfo.height;
381     (*compositeOutput)[1].format = useGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
382             HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
383     (*compositeOutput)[1].dataSpace = kHeifDataSpace;
384     (*compositeOutput)[1].consumerUsage = useHeic ? GRALLOC_USAGE_HW_IMAGE_ENCODER :
385             useGrid ? GRALLOC_USAGE_SW_READ_OFTEN : GRALLOC_USAGE_HW_VIDEO_ENCODER;
386 
387     return NO_ERROR;
388 }
389 
isSizeSupportedByHeifEncoder(int32_t width,int32_t height,bool * useHeic,bool * useGrid,int64_t * stall,AString * hevcName)390 bool HeicCompositeStream::isSizeSupportedByHeifEncoder(int32_t width, int32_t height,
391         bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName) {
392     static HeicEncoderInfoManager& heicManager = HeicEncoderInfoManager::getInstance();
393     return heicManager.isSizeSupported(width, height, useHeic, useGrid, stall, hevcName);
394 }
395 
isInMemoryTempFileSupported()396 bool HeicCompositeStream::isInMemoryTempFileSupported() {
397     int memfd = syscall(__NR_memfd_create, "HEIF-try-memfd", MFD_CLOEXEC);
398     if (memfd == -1) {
399         if (errno != ENOSYS) {
400             ALOGE("%s: Failed to create tmpfs file. errno %d", __FUNCTION__, errno);
401         }
402         return false;
403     }
404     close(memfd);
405     return true;
406 }
407 
onHeicOutputFrameAvailable(const CodecOutputBufferInfo & outputBufferInfo)408 void HeicCompositeStream::onHeicOutputFrameAvailable(
409         const CodecOutputBufferInfo& outputBufferInfo) {
410     Mutex::Autolock l(mMutex);
411 
412     ALOGV("%s: index %d, offset %d, size %d, time %" PRId64 ", flags 0x%x",
413             __FUNCTION__, outputBufferInfo.index, outputBufferInfo.offset,
414             outputBufferInfo.size, outputBufferInfo.timeUs, outputBufferInfo.flags);
415 
416     if (!mErrorState) {
417         if ((outputBufferInfo.size > 0) &&
418                 ((outputBufferInfo.flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) == 0)) {
419             mCodecOutputBuffers.push_back(outputBufferInfo);
420             mInputReadyCondition.signal();
421         } else {
422             ALOGV("%s: Releasing output buffer: size %d flags: 0x%x ", __FUNCTION__,
423                 outputBufferInfo.size, outputBufferInfo.flags);
424             mCodec->releaseOutputBuffer(outputBufferInfo.index);
425         }
426     } else {
427         mCodec->releaseOutputBuffer(outputBufferInfo.index);
428     }
429 }
430 
onHeicInputFrameAvailable(int32_t index)431 void HeicCompositeStream::onHeicInputFrameAvailable(int32_t index) {
432     Mutex::Autolock l(mMutex);
433 
434     if (!mUseGrid) {
435         ALOGE("%s: Codec YUV input mode must only be used for Hevc tiling mode", __FUNCTION__);
436         return;
437     }
438 
439     mCodecInputBuffers.push_back(index);
440     mInputReadyCondition.signal();
441 }
442 
onHeicFormatChanged(sp<AMessage> & newFormat)443 void HeicCompositeStream::onHeicFormatChanged(sp<AMessage>& newFormat) {
444     if (newFormat == nullptr) {
445         ALOGE("%s: newFormat must not be null!", __FUNCTION__);
446         return;
447     }
448 
449     Mutex::Autolock l(mMutex);
450 
451     AString mime;
452     AString mimeHeic(MIMETYPE_IMAGE_ANDROID_HEIC);
453     newFormat->findString(KEY_MIME, &mime);
454     if (mime != mimeHeic) {
455         // For HEVC codec, below keys need to be filled out or overwritten so that the
456         // muxer can handle them as HEIC output image.
457         newFormat->setString(KEY_MIME, mimeHeic);
458         newFormat->setInt32(KEY_WIDTH, mOutputWidth);
459         newFormat->setInt32(KEY_HEIGHT, mOutputHeight);
460     }
461 
462     if (mUseGrid || mUseHeic) {
463         int32_t gridRows, gridCols, tileWidth, tileHeight;
464         if (newFormat->findInt32(KEY_GRID_ROWS, &gridRows) &&
465                 newFormat->findInt32(KEY_GRID_COLUMNS, &gridCols) &&
466                 newFormat->findInt32(KEY_TILE_WIDTH, &tileWidth) &&
467                 newFormat->findInt32(KEY_TILE_HEIGHT, &tileHeight)) {
468             mGridWidth = tileWidth;
469             mGridHeight = tileHeight;
470             mGridRows = gridRows;
471             mGridCols = gridCols;
472         } else {
473             newFormat->setInt32(KEY_TILE_WIDTH, mGridWidth);
474             newFormat->setInt32(KEY_TILE_HEIGHT, mGridHeight);
475             newFormat->setInt32(KEY_GRID_ROWS, mGridRows);
476             newFormat->setInt32(KEY_GRID_COLUMNS, mGridCols);
477         }
478         int32_t left, top, right, bottom;
479         if (newFormat->findRect("crop", &left, &top, &right, &bottom)) {
480             newFormat->setRect("crop", 0, 0, mOutputWidth - 1, mOutputHeight - 1);
481         }
482     }
483     newFormat->setInt32(KEY_IS_DEFAULT, 1 /*isPrimary*/);
484 
485     int32_t gridRows, gridCols;
486     if (newFormat->findInt32(KEY_GRID_ROWS, &gridRows) &&
487             newFormat->findInt32(KEY_GRID_COLUMNS, &gridCols)) {
488         mNumOutputTiles = gridRows * gridCols;
489     } else {
490         mNumOutputTiles = 1;
491     }
492 
493     mFormat = newFormat;
494 
495     ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles);
496     mInputReadyCondition.signal();
497 }
498 
onHeicCodecError()499 void HeicCompositeStream::onHeicCodecError() {
500     Mutex::Autolock l(mMutex);
501     mErrorState = true;
502 }
503 
configureStream()504 status_t HeicCompositeStream::configureStream() {
505     if (isRunning()) {
506         // Processing thread is already running, nothing more to do.
507         return NO_ERROR;
508     }
509 
510     if (mOutputSurface.get() == nullptr) {
511         ALOGE("%s: No valid output surface set!", __FUNCTION__);
512         return NO_INIT;
513     }
514 
515     auto res = mOutputSurface->connect(NATIVE_WINDOW_API_CAMERA, mProducerListener);
516     if (res != OK) {
517         ALOGE("%s: Unable to connect to native window for stream %d",
518                 __FUNCTION__, mMainImageStreamId);
519         return res;
520     }
521 
522     if ((res = native_window_set_buffers_format(mOutputSurface.get(), HAL_PIXEL_FORMAT_BLOB))
523             != OK) {
524         ALOGE("%s: Unable to configure stream buffer format for stream %d", __FUNCTION__,
525                 mMainImageStreamId);
526         return res;
527     }
528 
529     ANativeWindow *anwConsumer = mOutputSurface.get();
530     int maxConsumerBuffers;
531     if ((res = anwConsumer->query(anwConsumer, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS,
532                     &maxConsumerBuffers)) != OK) {
533         ALOGE("%s: Unable to query consumer undequeued"
534                 " buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
535         return res;
536     }
537 
538     // Cannot use SourceSurface buffer count since it could be codec's 512*512 tile
539     // buffer count.
540     if ((res = native_window_set_buffer_count(
541                     anwConsumer, kMaxOutputSurfaceProducerCount + maxConsumerBuffers)) != OK) {
542         ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
543         return res;
544     }
545 
546     if ((res = native_window_set_buffers_dimensions(anwConsumer, mMaxHeicBufferSize, 1)) != OK) {
547         ALOGE("%s: Unable to set buffer dimension %zu x 1 for stream %d: %s (%d)",
548                 __FUNCTION__, mMaxHeicBufferSize, mMainImageStreamId, strerror(-res), res);
549         return res;
550     }
551 
552     sp<camera3::StatusTracker> statusTracker = mStatusTracker.promote();
553     if (statusTracker != nullptr) {
554         std::string name = std::string("HeicStream ") + std::to_string(getStreamId());
555         mStatusId = statusTracker->addComponent(name);
556     }
557 
558     run("HeicCompositeStreamProc");
559 
560     return NO_ERROR;
561 }
562 
insertGbp(SurfaceMap * outSurfaceMap,Vector<int32_t> * outputStreamIds,int32_t * currentStreamId)563 status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap,
564         Vector<int32_t>* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) {
565     if (outSurfaceMap->find(mAppSegmentStreamId) == outSurfaceMap->end()) {
566         outputStreamIds->push_back(mAppSegmentStreamId);
567     }
568     (*outSurfaceMap)[mAppSegmentStreamId].push_back(mAppSegmentSurfaceId);
569 
570     if (outSurfaceMap->find(mMainImageStreamId) == outSurfaceMap->end()) {
571         outputStreamIds->push_back(mMainImageStreamId);
572     }
573     (*outSurfaceMap)[mMainImageStreamId].push_back(mMainImageSurfaceId);
574 
575     if (currentStreamId != nullptr) {
576         *currentStreamId = mMainImageStreamId;
577     }
578 
579     return NO_ERROR;
580 }
581 
insertCompositeStreamIds(std::vector<int32_t> * compositeStreamIds)582 status_t HeicCompositeStream::insertCompositeStreamIds(
583         std::vector<int32_t>* compositeStreamIds /*out*/) {
584     if (compositeStreamIds == nullptr) {
585         return BAD_VALUE;
586     }
587 
588     compositeStreamIds->push_back(mAppSegmentStreamId);
589     compositeStreamIds->push_back(mMainImageStreamId);
590 
591     return OK;
592 }
593 
onShutter(const CaptureResultExtras & resultExtras,nsecs_t timestamp)594 void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) {
595     Mutex::Autolock l(mMutex);
596     if (mErrorState) {
597         return;
598     }
599 
600     if (mSettingsByFrameNumber.find(resultExtras.frameNumber) != mSettingsByFrameNumber.end()) {
601         ALOGV("%s: [%" PRId64 "]: timestamp %" PRId64 ", requestId %d", __FUNCTION__,
602                 resultExtras.frameNumber, timestamp, resultExtras.requestId);
603         mSettingsByFrameNumber[resultExtras.frameNumber].shutterNotified = true;
604         mSettingsByFrameNumber[resultExtras.frameNumber].timestamp = timestamp;
605         mSettingsByFrameNumber[resultExtras.frameNumber].requestId = resultExtras.requestId;
606         mInputReadyCondition.signal();
607     }
608 }
609 
compilePendingInputLocked()610 void HeicCompositeStream::compilePendingInputLocked() {
611     auto i = mSettingsByFrameNumber.begin();
612     while (i != mSettingsByFrameNumber.end()) {
613         if (i->second.shutterNotified) {
614             mPendingInputFrames[i->first].orientation = i->second.orientation;
615             mPendingInputFrames[i->first].quality = i->second.quality;
616             mPendingInputFrames[i->first].timestamp = i->second.timestamp;
617             mPendingInputFrames[i->first].requestId = i->second.requestId;
618             ALOGV("%s: [%" PRId64 "]: timestamp is %" PRId64, __FUNCTION__,
619                     i->first, i->second.timestamp);
620             i = mSettingsByFrameNumber.erase(i);
621 
622             // Set encoder quality if no inflight encoding
623             if (mPendingInputFrames.size() == 1) {
624                 sp<StatusTracker> statusTracker = mStatusTracker.promote();
625                 if (statusTracker != nullptr) {
626                     statusTracker->markComponentActive(mStatusId);
627                     ALOGV("%s: Mark component as active", __FUNCTION__);
628                 }
629 
630                 int32_t newQuality = mPendingInputFrames.begin()->second.quality;
631                 updateCodecQualityLocked(newQuality);
632             }
633         } else {
634             i++;
635         }
636     }
637 
638     while (!mInputAppSegmentBuffers.empty() && mAppSegmentFrameNumbers.size() > 0) {
639         CpuConsumer::LockedBuffer imgBuffer;
640         auto it = mInputAppSegmentBuffers.begin();
641         auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer);
642         if (res == NOT_ENOUGH_DATA) {
643             // Can not lock any more buffers.
644             break;
645         } else if ((res != OK) || (*it != imgBuffer.timestamp)) {
646             if (res != OK) {
647                 ALOGE("%s: Error locking JPEG_APP_SEGMENTS image buffer: %s (%d)", __FUNCTION__,
648                         strerror(-res), res);
649             } else {
650                 ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64
651                         " received buffer with time stamp: %" PRId64, __FUNCTION__,
652                         *it, imgBuffer.timestamp);
653                 mAppSegmentConsumer->unlockBuffer(imgBuffer);
654             }
655             mPendingInputFrames[*it].error = true;
656             mInputAppSegmentBuffers.erase(it);
657             continue;
658         }
659 
660         if (mPendingInputFrames.find(mAppSegmentFrameNumbers.front()) == mPendingInputFrames.end()) {
661             ALOGE("%s: mPendingInputFrames doesn't contain frameNumber %" PRId64, __FUNCTION__,
662                     mAppSegmentFrameNumbers.front());
663             mInputAppSegmentBuffers.erase(it);
664             mAppSegmentFrameNumbers.pop();
665             continue;
666         }
667 
668         int64_t frameNumber = mAppSegmentFrameNumbers.front();
669         // If mPendingInputFrames doesn't contain the expected frame number, the captured
670         // input app segment frame must have been dropped via a buffer error.  Simply
671         // return the buffer to the buffer queue.
672         if ((mPendingInputFrames.find(frameNumber) == mPendingInputFrames.end()) ||
673                 (mPendingInputFrames[frameNumber].error)) {
674             mAppSegmentConsumer->unlockBuffer(imgBuffer);
675         } else {
676             mPendingInputFrames[frameNumber].appSegmentBuffer = imgBuffer;
677         }
678         mInputAppSegmentBuffers.erase(it);
679         mAppSegmentFrameNumbers.pop();
680     }
681 
682     while (!mInputYuvBuffers.empty() && !mYuvBufferAcquired && mMainImageFrameNumbers.size() > 0) {
683         CpuConsumer::LockedBuffer imgBuffer;
684         auto it = mInputYuvBuffers.begin();
685         auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer);
686         if (res == NOT_ENOUGH_DATA) {
687             // Can not lock any more buffers.
688             break;
689         } else if (res != OK) {
690             ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__,
691                     strerror(-res), res);
692             mPendingInputFrames[*it].error = true;
693             mInputYuvBuffers.erase(it);
694             continue;
695         } else if (*it != imgBuffer.timestamp) {
696             ALOGW("%s: Expecting YUV_888 buffer with time stamp: %" PRId64 " received buffer with "
697                     "time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp);
698             mPendingInputFrames[*it].error = true;
699             mInputYuvBuffers.erase(it);
700             continue;
701         }
702 
703         if (mPendingInputFrames.find(mMainImageFrameNumbers.front()) == mPendingInputFrames.end()) {
704             ALOGE("%s: mPendingInputFrames doesn't contain frameNumber %" PRId64, __FUNCTION__,
705                     mMainImageFrameNumbers.front());
706             mInputYuvBuffers.erase(it);
707             mMainImageFrameNumbers.pop();
708             continue;
709         }
710 
711         int64_t frameNumber = mMainImageFrameNumbers.front();
712         // If mPendingInputFrames doesn't contain the expected frame number, the captured
713         // input main image must have been dropped via a buffer error. Simply
714         // return the buffer to the buffer queue.
715         if ((mPendingInputFrames.find(frameNumber) == mPendingInputFrames.end()) ||
716                 (mPendingInputFrames[frameNumber].error)) {
717             mMainImageConsumer->unlockBuffer(imgBuffer);
718         } else {
719             mPendingInputFrames[frameNumber].yuvBuffer = imgBuffer;
720             mYuvBufferAcquired = true;
721         }
722         mInputYuvBuffers.erase(it);
723         mMainImageFrameNumbers.pop();
724     }
725 
726     while (!mCodecOutputBuffers.empty()) {
727         auto it = mCodecOutputBuffers.begin();
728         // Assume encoder input to output is FIFO, use a queue to look up
729         // frameNumber when handling codec outputs.
730         int64_t bufferFrameNumber = -1;
731         if (mCodecOutputBufferFrameNumbers.empty()) {
732             ALOGV("%s: Failed to find buffer frameNumber for codec output buffer!", __FUNCTION__);
733             break;
734         } else {
735             // Direct mapping between camera frame number and codec timestamp (in us).
736             bufferFrameNumber = mCodecOutputBufferFrameNumbers.front();
737             mCodecOutputCounter++;
738             if (mCodecOutputCounter == mNumOutputTiles) {
739                 mCodecOutputBufferFrameNumbers.pop();
740                 mCodecOutputCounter = 0;
741             }
742 
743             mPendingInputFrames[bufferFrameNumber].codecOutputBuffers.push_back(*it);
744             ALOGV("%s: [%" PRId64 "]: Pushing codecOutputBuffers (frameNumber %" PRId64 ")",
745                     __FUNCTION__, bufferFrameNumber, it->timeUs);
746         }
747         mCodecOutputBuffers.erase(it);
748     }
749 
750     while (!mCaptureResults.empty()) {
751         auto it = mCaptureResults.begin();
752         // Negative frame number indicates that something went wrong during the capture result
753         // collection process.
754         int64_t frameNumber = std::get<0>(it->second);
755         if (it->first >= 0 &&
756                 mPendingInputFrames.find(frameNumber) != mPendingInputFrames.end()) {
757             if (mPendingInputFrames[frameNumber].timestamp == it->first) {
758                 mPendingInputFrames[frameNumber].result =
759                         std::make_unique<CameraMetadata>(std::get<1>(it->second));
760             } else {
761                 ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "
762                         "shutter and capture result! before: %" PRId64 ", after: %" PRId64,
763                         __FUNCTION__, mPendingInputFrames[frameNumber].timestamp,
764                         it->first);
765             }
766         }
767         mCaptureResults.erase(it);
768     }
769 
770     // mErrorFrameNumbers stores frame number of dropped buffers.
771     auto it = mErrorFrameNumbers.begin();
772     while (it != mErrorFrameNumbers.end()) {
773         if (mPendingInputFrames.find(*it) != mPendingInputFrames.end()) {
774             mPendingInputFrames[*it].error = true;
775         } else {
776             //Error callback is guaranteed to arrive after shutter notify, which
777             //results in mPendingInputFrames being populated.
778             ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,
779                     *it);
780         }
781         it = mErrorFrameNumbers.erase(it);
782     }
783 
784     // mExifErrorFrameNumbers stores the frame number of dropped APP_SEGMENT buffers
785     it = mExifErrorFrameNumbers.begin();
786     while (it != mExifErrorFrameNumbers.end()) {
787         if (mPendingInputFrames.find(*it) != mPendingInputFrames.end()) {
788             mPendingInputFrames[*it].exifError = true;
789         }
790         it = mExifErrorFrameNumbers.erase(it);
791     }
792 
793     // Distribute codec input buffers to be filled out from YUV output
794     for (auto it = mPendingInputFrames.begin();
795             it != mPendingInputFrames.end() && mCodecInputBuffers.size() > 0; it++) {
796         InputFrame& inputFrame(it->second);
797         if (inputFrame.codecInputCounter < mGridRows * mGridCols) {
798             // Available input tiles that are required for the current input
799             // image.
800             size_t newInputTiles = std::min(mCodecInputBuffers.size(),
801                     mGridRows * mGridCols - inputFrame.codecInputCounter);
802             for (size_t i = 0; i < newInputTiles; i++) {
803                 CodecInputBufferInfo inputInfo =
804                         { mCodecInputBuffers[0], mGridTimestampUs++, inputFrame.codecInputCounter };
805                 inputFrame.codecInputBuffers.push_back(inputInfo);
806 
807                 mCodecInputBuffers.erase(mCodecInputBuffers.begin());
808                 inputFrame.codecInputCounter++;
809             }
810             break;
811         }
812     }
813 }
814 
getNextReadyInputLocked(int64_t * frameNumber)815 bool HeicCompositeStream::getNextReadyInputLocked(int64_t *frameNumber /*out*/) {
816     if (frameNumber == nullptr) {
817         return false;
818     }
819 
820     bool newInputAvailable = false;
821     for (auto& it : mPendingInputFrames) {
822         // New input is considered to be available only if:
823         // 1. input buffers are ready, or
824         // 2. App segment and muxer is created, or
825         // 3. A codec output tile is ready, and an output buffer is available.
826         // This makes sure that muxer gets created only when an output tile is
827         // generated, because right now we only handle 1 HEIC output buffer at a
828         // time (max dequeued buffer count is 1).
829         bool appSegmentReady =
830                 (it.second.appSegmentBuffer.data != nullptr || it.second.exifError) &&
831                 !it.second.appSegmentWritten && it.second.result != nullptr &&
832                 it.second.muxer != nullptr;
833         bool codecOutputReady = !it.second.codecOutputBuffers.empty();
834         bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&
835                 (!it.second.codecInputBuffers.empty());
836         bool hasOutputBuffer = it.second.muxer != nullptr ||
837                 (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
838         if ((!it.second.error) &&
839                 (appSegmentReady || (codecOutputReady && hasOutputBuffer) || codecInputReady)) {
840             *frameNumber = it.first;
841             if (it.second.format == nullptr && mFormat != nullptr) {
842                 it.second.format = mFormat->dup();
843             }
844             newInputAvailable = true;
845             break;
846         }
847     }
848 
849     return newInputAvailable;
850 }
851 
getNextFailingInputLocked()852 int64_t HeicCompositeStream::getNextFailingInputLocked() {
853     int64_t res = -1;
854 
855     for (const auto& it : mPendingInputFrames) {
856         if (it.second.error) {
857             res = it.first;
858             break;
859         }
860     }
861 
862     return res;
863 }
864 
processInputFrame(int64_t frameNumber,InputFrame & inputFrame)865 status_t HeicCompositeStream::processInputFrame(int64_t frameNumber,
866         InputFrame &inputFrame) {
867     ATRACE_CALL();
868     status_t res = OK;
869 
870     bool appSegmentReady =
871             (inputFrame.appSegmentBuffer.data != nullptr || inputFrame.exifError) &&
872             !inputFrame.appSegmentWritten && inputFrame.result != nullptr &&
873             inputFrame.muxer != nullptr;
874     bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;
875     bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&
876             !inputFrame.codecInputBuffers.empty();
877     bool hasOutputBuffer = inputFrame.muxer != nullptr ||
878             (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
879 
880     ALOGV("%s: [%" PRId64 "]: appSegmentReady %d, codecOutputReady %d, codecInputReady %d,"
881             " dequeuedOutputBuffer %d, timestamp %" PRId64, __FUNCTION__, frameNumber,
882             appSegmentReady, codecOutputReady, codecInputReady, mDequeuedOutputBufferCnt,
883             inputFrame.timestamp);
884 
885     // Handle inputs for Hevc tiling
886     if (codecInputReady) {
887         res = processCodecInputFrame(inputFrame);
888         if (res != OK) {
889             ALOGE("%s: Failed to process codec input frame: %s (%d)", __FUNCTION__,
890                     strerror(-res), res);
891             return res;
892         }
893     }
894 
895     if (!(codecOutputReady && hasOutputBuffer) && !appSegmentReady) {
896         return OK;
897     }
898 
899     // Initialize and start muxer if not yet done so. In this case,
900     // codecOutputReady must be true. Otherwise, appSegmentReady is guaranteed
901     // to be false, and the function must have returned early.
902     if (inputFrame.muxer == nullptr) {
903         res = startMuxerForInputFrame(frameNumber, inputFrame);
904         if (res != OK) {
905             ALOGE("%s: Failed to create and start muxer: %s (%d)", __FUNCTION__,
906                     strerror(-res), res);
907             return res;
908         }
909     }
910 
911     // Write JPEG APP segments data to the muxer.
912     if (appSegmentReady) {
913         res = processAppSegment(frameNumber, inputFrame);
914         if (res != OK) {
915             ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
916                     strerror(-res), res);
917             return res;
918         }
919     }
920 
921     // Write media codec bitstream buffers to muxer.
922     while (!inputFrame.codecOutputBuffers.empty()) {
923         res = processOneCodecOutputFrame(frameNumber, inputFrame);
924         if (res != OK) {
925             ALOGE("%s: Failed to process codec output frame: %s (%d)", __FUNCTION__,
926                     strerror(-res), res);
927             return res;
928         }
929     }
930 
931     if (inputFrame.pendingOutputTiles == 0) {
932         if (inputFrame.appSegmentWritten) {
933             res = processCompletedInputFrame(frameNumber, inputFrame);
934             if (res != OK) {
935                 ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,
936                         strerror(-res), res);
937                 return res;
938             }
939         }
940     }
941 
942     return res;
943 }
944 
startMuxerForInputFrame(int64_t frameNumber,InputFrame & inputFrame)945 status_t HeicCompositeStream::startMuxerForInputFrame(int64_t frameNumber, InputFrame &inputFrame) {
946     sp<ANativeWindow> outputANW = mOutputSurface;
947 
948     auto res = outputANW->dequeueBuffer(mOutputSurface.get(), &inputFrame.anb, &inputFrame.fenceFd);
949     if (res != OK) {
950         ALOGE("%s: Error retrieving output buffer: %s (%d)", __FUNCTION__, strerror(-res),
951                 res);
952         return res;
953     }
954     mDequeuedOutputBufferCnt++;
955 
956     // Combine current thread id, stream id and timestamp to uniquely identify image.
957     std::ostringstream tempOutputFile;
958     tempOutputFile << "HEIF-" << pthread_self() << "-"
959             << getStreamId() << "-" << frameNumber;
960     inputFrame.fileFd = syscall(__NR_memfd_create, tempOutputFile.str().c_str(), MFD_CLOEXEC);
961     if (inputFrame.fileFd < 0) {
962         ALOGE("%s: Failed to create file %s. Error no is %d", __FUNCTION__,
963                 tempOutputFile.str().c_str(), errno);
964         return NO_INIT;
965     }
966     inputFrame.muxer = MediaMuxer::create(inputFrame.fileFd, MediaMuxer::OUTPUT_FORMAT_HEIF);
967     if (inputFrame.muxer == nullptr) {
968         ALOGE("%s: Failed to create MediaMuxer for file fd %d",
969                 __FUNCTION__, inputFrame.fileFd);
970         return NO_INIT;
971     }
972 
973     res = inputFrame.muxer->setOrientationHint(inputFrame.orientation);
974     if (res != OK) {
975         ALOGE("%s: Failed to setOrientationHint: %s (%d)", __FUNCTION__,
976                 strerror(-res), res);
977         return res;
978     }
979 
980     ssize_t trackId = inputFrame.muxer->addTrack(inputFrame.format);
981     if (trackId < 0) {
982         ALOGE("%s: Failed to addTrack to the muxer: %zd", __FUNCTION__, trackId);
983         return NO_INIT;
984     }
985 
986     inputFrame.trackIndex = trackId;
987     inputFrame.pendingOutputTiles = mNumOutputTiles;
988 
989     res = inputFrame.muxer->start();
990     if (res != OK) {
991         ALOGE("%s: Failed to start MediaMuxer: %s (%d)",
992                 __FUNCTION__, strerror(-res), res);
993         return res;
994     }
995 
996     ALOGV("%s: [%" PRId64 "]: Muxer started for inputFrame", __FUNCTION__,
997             frameNumber);
998     return OK;
999 }
1000 
processAppSegment(int64_t frameNumber,InputFrame & inputFrame)1001 status_t HeicCompositeStream::processAppSegment(int64_t frameNumber, InputFrame &inputFrame) {
1002     size_t app1Size = 0;
1003     size_t appSegmentSize = 0;
1004     if (!inputFrame.exifError) {
1005         appSegmentSize = findAppSegmentsSize(inputFrame.appSegmentBuffer.data,
1006                 inputFrame.appSegmentBuffer.width * inputFrame.appSegmentBuffer.height,
1007                 &app1Size);
1008         if (appSegmentSize == 0) {
1009             ALOGE("%s: Failed to find JPEG APP segment size", __FUNCTION__);
1010             return NO_INIT;
1011         }
1012     }
1013 
1014     std::unique_ptr<ExifUtils> exifUtils(ExifUtils::create());
1015     auto exifRes = inputFrame.exifError ?
1016             exifUtils->initializeEmpty() :
1017             exifUtils->initialize(inputFrame.appSegmentBuffer.data, app1Size);
1018     if (!exifRes) {
1019         ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);
1020         return BAD_VALUE;
1021     }
1022     exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,
1023             mOutputWidth, mOutputHeight);
1024     if (!exifRes) {
1025         ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);
1026         return BAD_VALUE;
1027     }
1028     exifRes = exifUtils->setOrientation(inputFrame.orientation);
1029     if (!exifRes) {
1030         ALOGE("%s: ExifUtils failed to set orientation", __FUNCTION__);
1031         return BAD_VALUE;
1032     }
1033     exifRes = exifUtils->generateApp1();
1034     if (!exifRes) {
1035         ALOGE("%s: ExifUtils failed to generate APP1 segment", __FUNCTION__);
1036         return BAD_VALUE;
1037     }
1038 
1039     unsigned int newApp1Length = exifUtils->getApp1Length();
1040     const uint8_t *newApp1Segment = exifUtils->getApp1Buffer();
1041 
1042     //Assemble the APP1 marker buffer required by MediaCodec
1043     uint8_t kExifApp1Marker[] = {'E', 'x', 'i', 'f', 0xFF, 0xE1, 0x00, 0x00};
1044     kExifApp1Marker[6] = static_cast<uint8_t>(newApp1Length >> 8);
1045     kExifApp1Marker[7] = static_cast<uint8_t>(newApp1Length & 0xFF);
1046     size_t appSegmentBufferSize = sizeof(kExifApp1Marker) +
1047             appSegmentSize - app1Size + newApp1Length;
1048     uint8_t* appSegmentBuffer = new uint8_t[appSegmentBufferSize];
1049     memcpy(appSegmentBuffer, kExifApp1Marker, sizeof(kExifApp1Marker));
1050     memcpy(appSegmentBuffer + sizeof(kExifApp1Marker), newApp1Segment, newApp1Length);
1051     if (appSegmentSize - app1Size > 0) {
1052         memcpy(appSegmentBuffer + sizeof(kExifApp1Marker) + newApp1Length,
1053                 inputFrame.appSegmentBuffer.data + app1Size, appSegmentSize - app1Size);
1054     }
1055 
1056     sp<ABuffer> aBuffer = new ABuffer(appSegmentBuffer, appSegmentBufferSize);
1057     auto res = inputFrame.muxer->writeSampleData(aBuffer, inputFrame.trackIndex,
1058             inputFrame.timestamp, MediaCodec::BUFFER_FLAG_MUXER_DATA);
1059     delete[] appSegmentBuffer;
1060 
1061     if (res != OK) {
1062         ALOGE("%s: Failed to write JPEG APP segments to muxer: %s (%d)",
1063                 __FUNCTION__, strerror(-res), res);
1064         return res;
1065     }
1066 
1067     ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu",
1068           __FUNCTION__, frameNumber, appSegmentSize, inputFrame.appSegmentBuffer.width,
1069           inputFrame.appSegmentBuffer.height, app1Size);
1070 
1071     inputFrame.appSegmentWritten = true;
1072     // Release the buffer now so any pending input app segments can be processed
1073     mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer);
1074     inputFrame.appSegmentBuffer.data = nullptr;
1075     inputFrame.exifError = false;
1076 
1077     return OK;
1078 }
1079 
processCodecInputFrame(InputFrame & inputFrame)1080 status_t HeicCompositeStream::processCodecInputFrame(InputFrame &inputFrame) {
1081     for (auto& inputBuffer : inputFrame.codecInputBuffers) {
1082         sp<MediaCodecBuffer> buffer;
1083         auto res = mCodec->getInputBuffer(inputBuffer.index, &buffer);
1084         if (res != OK) {
1085             ALOGE("%s: Error getting codec input buffer: %s (%d)", __FUNCTION__,
1086                     strerror(-res), res);
1087             return res;
1088         }
1089 
1090         // Copy one tile from source to destination.
1091         size_t tileX = inputBuffer.tileIndex % mGridCols;
1092         size_t tileY = inputBuffer.tileIndex / mGridCols;
1093         size_t top = mGridHeight * tileY;
1094         size_t left = mGridWidth * tileX;
1095         size_t width = (tileX == static_cast<size_t>(mGridCols) - 1) ?
1096                 mOutputWidth - tileX * mGridWidth : mGridWidth;
1097         size_t height = (tileY == static_cast<size_t>(mGridRows) - 1) ?
1098                 mOutputHeight - tileY * mGridHeight : mGridHeight;
1099         ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu,"
1100                 " timeUs %" PRId64, __FUNCTION__, tileX, tileY, top, left, width, height,
1101                 inputBuffer.timeUs);
1102 
1103         res = copyOneYuvTile(buffer, inputFrame.yuvBuffer, top, left, width, height);
1104         if (res != OK) {
1105             ALOGE("%s: Failed to copy YUV tile %s (%d)", __FUNCTION__,
1106                     strerror(-res), res);
1107             return res;
1108         }
1109 
1110         res = mCodec->queueInputBuffer(inputBuffer.index, 0, buffer->capacity(),
1111                 inputBuffer.timeUs, 0, nullptr /*errorDetailMsg*/);
1112         if (res != OK) {
1113             ALOGE("%s: Failed to queueInputBuffer to Codec: %s (%d)",
1114                     __FUNCTION__, strerror(-res), res);
1115             return res;
1116         }
1117     }
1118 
1119     inputFrame.codecInputBuffers.clear();
1120     return OK;
1121 }
1122 
processOneCodecOutputFrame(int64_t frameNumber,InputFrame & inputFrame)1123 status_t HeicCompositeStream::processOneCodecOutputFrame(int64_t frameNumber,
1124         InputFrame &inputFrame) {
1125     auto it = inputFrame.codecOutputBuffers.begin();
1126     sp<MediaCodecBuffer> buffer;
1127     status_t res = mCodec->getOutputBuffer(it->index, &buffer);
1128     if (res != OK) {
1129         ALOGE("%s: Error getting Heic codec output buffer at index %d: %s (%d)",
1130                 __FUNCTION__, it->index, strerror(-res), res);
1131         return res;
1132     }
1133     if (buffer == nullptr) {
1134         ALOGE("%s: Invalid Heic codec output buffer at index %d",
1135                 __FUNCTION__, it->index);
1136         return BAD_VALUE;
1137     }
1138 
1139     sp<ABuffer> aBuffer = new ABuffer(buffer->data(), buffer->size());
1140     res = inputFrame.muxer->writeSampleData(
1141             aBuffer, inputFrame.trackIndex, inputFrame.timestamp, 0 /*flags*/);
1142     if (res != OK) {
1143         ALOGE("%s: Failed to write buffer index %d to muxer: %s (%d)",
1144                 __FUNCTION__, it->index, strerror(-res), res);
1145         return res;
1146     }
1147 
1148     mCodec->releaseOutputBuffer(it->index);
1149     if (inputFrame.pendingOutputTiles == 0) {
1150         ALOGW("%s: Codec generated more tiles than expected!", __FUNCTION__);
1151     } else {
1152         inputFrame.pendingOutputTiles--;
1153     }
1154 
1155     inputFrame.codecOutputBuffers.erase(inputFrame.codecOutputBuffers.begin());
1156 
1157     ALOGV("%s: [%" PRId64 "]: Output buffer index %d",
1158         __FUNCTION__, frameNumber, it->index);
1159     return OK;
1160 }
1161 
processCompletedInputFrame(int64_t frameNumber,InputFrame & inputFrame)1162 status_t HeicCompositeStream::processCompletedInputFrame(int64_t frameNumber,
1163         InputFrame &inputFrame) {
1164     sp<ANativeWindow> outputANW = mOutputSurface;
1165     inputFrame.muxer->stop();
1166 
1167     // Copy the content of the file to memory.
1168     sp<GraphicBuffer> gb = GraphicBuffer::from(inputFrame.anb);
1169     void* dstBuffer;
1170     GraphicBufferLocker gbLocker(gb);
1171     auto res = gbLocker.lockAsync(&dstBuffer, inputFrame.fenceFd);
1172     if (res != OK) {
1173         ALOGE("%s: Error trying to lock output buffer fence: %s (%d)", __FUNCTION__,
1174                 strerror(-res), res);
1175         return res;
1176     }
1177 
1178     off_t fSize = lseek(inputFrame.fileFd, 0, SEEK_END);
1179     if (static_cast<size_t>(fSize) > mMaxHeicBufferSize - sizeof(CameraBlob)) {
1180         ALOGE("%s: Error: MediaMuxer output size %ld is larger than buffer sizer %zu",
1181                 __FUNCTION__, fSize, mMaxHeicBufferSize - sizeof(CameraBlob));
1182         return BAD_VALUE;
1183     }
1184 
1185     lseek(inputFrame.fileFd, 0, SEEK_SET);
1186     ssize_t bytesRead = read(inputFrame.fileFd, dstBuffer, fSize);
1187     if (bytesRead < fSize) {
1188         ALOGE("%s: Only %zd of %ld bytes read", __FUNCTION__, bytesRead, fSize);
1189         return BAD_VALUE;
1190     }
1191 
1192     close(inputFrame.fileFd);
1193     inputFrame.fileFd = -1;
1194 
1195     // Fill in HEIC header
1196     // Must be in sync with CAMERA3_HEIC_BLOB_ID in android_media_Utils.cpp
1197     uint8_t *header = static_cast<uint8_t*>(dstBuffer) + mMaxHeicBufferSize - sizeof(CameraBlob);
1198     CameraBlob blobHeader = {
1199         .blobId = static_cast<CameraBlobId>(0x00FE),
1200         .blobSizeBytes = static_cast<int32_t>(fSize)
1201     };
1202     memcpy(header, &blobHeader, sizeof(CameraBlob));
1203 
1204     res = native_window_set_buffers_timestamp(mOutputSurface.get(), inputFrame.timestamp);
1205     if (res != OK) {
1206         ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
1207                __FUNCTION__, getStreamId(), strerror(-res), res);
1208         return res;
1209     }
1210 
1211     res = outputANW->queueBuffer(mOutputSurface.get(), inputFrame.anb, /*fence*/ -1);
1212     if (res != OK) {
1213         ALOGE("%s: Failed to queueBuffer to Heic stream: %s (%d)", __FUNCTION__,
1214                 strerror(-res), res);
1215         return res;
1216     }
1217     inputFrame.anb = nullptr;
1218     mDequeuedOutputBufferCnt--;
1219 
1220     ALOGV("%s: [%" PRId64 "]", __FUNCTION__, frameNumber);
1221     ATRACE_ASYNC_END("HEIC capture", frameNumber);
1222     return OK;
1223 }
1224 
1225 
releaseInputFrameLocked(int64_t frameNumber,InputFrame * inputFrame)1226 void HeicCompositeStream::releaseInputFrameLocked(int64_t frameNumber,
1227         InputFrame *inputFrame /*out*/) {
1228     if (inputFrame == nullptr) {
1229         return;
1230     }
1231 
1232     if (inputFrame->appSegmentBuffer.data != nullptr) {
1233         mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer);
1234         inputFrame->appSegmentBuffer.data = nullptr;
1235     }
1236 
1237     while (!inputFrame->codecOutputBuffers.empty()) {
1238         auto it = inputFrame->codecOutputBuffers.begin();
1239         ALOGV("%s: releaseOutputBuffer index %d", __FUNCTION__, it->index);
1240         mCodec->releaseOutputBuffer(it->index);
1241         inputFrame->codecOutputBuffers.erase(it);
1242     }
1243 
1244     if (inputFrame->yuvBuffer.data != nullptr) {
1245         mMainImageConsumer->unlockBuffer(inputFrame->yuvBuffer);
1246         inputFrame->yuvBuffer.data = nullptr;
1247         mYuvBufferAcquired = false;
1248     }
1249 
1250     while (!inputFrame->codecInputBuffers.empty()) {
1251         auto it = inputFrame->codecInputBuffers.begin();
1252         inputFrame->codecInputBuffers.erase(it);
1253     }
1254 
1255     if (inputFrame->error || mErrorState) {
1256         ALOGV("%s: notifyError called for frameNumber %" PRId64, __FUNCTION__, frameNumber);
1257         notifyError(frameNumber, inputFrame->requestId);
1258     }
1259 
1260     if (inputFrame->fileFd >= 0) {
1261         close(inputFrame->fileFd);
1262         inputFrame->fileFd = -1;
1263     }
1264 
1265     if (inputFrame->anb != nullptr) {
1266         sp<ANativeWindow> outputANW = mOutputSurface;
1267         outputANW->cancelBuffer(mOutputSurface.get(), inputFrame->anb, /*fence*/ -1);
1268         inputFrame->anb = nullptr;
1269 
1270         mDequeuedOutputBufferCnt--;
1271     }
1272 }
1273 
releaseInputFramesLocked()1274 void HeicCompositeStream::releaseInputFramesLocked() {
1275     auto it = mPendingInputFrames.begin();
1276     bool inputFrameDone = false;
1277     while (it != mPendingInputFrames.end()) {
1278         auto& inputFrame = it->second;
1279         if (inputFrame.error ||
1280                 (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) {
1281             releaseInputFrameLocked(it->first, &inputFrame);
1282             it = mPendingInputFrames.erase(it);
1283             inputFrameDone = true;
1284         } else {
1285             it++;
1286         }
1287     }
1288 
1289     // Update codec quality based on first upcoming input frame.
1290     // Note that when encoding is in surface mode, currently there is  no
1291     // way for camera service to synchronize quality setting on a per-frame
1292     // basis: we don't get notification when codec is ready to consume a new
1293     // input frame. So we update codec quality on a best-effort basis.
1294     if (inputFrameDone) {
1295         auto firstPendingFrame = mPendingInputFrames.begin();
1296         if (firstPendingFrame != mPendingInputFrames.end()) {
1297             updateCodecQualityLocked(firstPendingFrame->second.quality);
1298         } else {
1299             markTrackerIdle();
1300         }
1301     }
1302 }
1303 
initializeCodec(uint32_t width,uint32_t height,const sp<CameraDeviceBase> & cameraDevice)1304 status_t HeicCompositeStream::initializeCodec(uint32_t width, uint32_t height,
1305         const sp<CameraDeviceBase>& cameraDevice) {
1306     ALOGV("%s", __FUNCTION__);
1307 
1308     bool useGrid = false;
1309     AString hevcName;
1310     bool isSizeSupported = isSizeSupportedByHeifEncoder(width, height,
1311             &mUseHeic, &useGrid, nullptr, &hevcName);
1312     if (!isSizeSupported) {
1313         ALOGE("%s: Encoder doesnt' support size %u x %u!",
1314                 __FUNCTION__, width, height);
1315         return BAD_VALUE;
1316     }
1317 
1318     // Create Looper for MediaCodec.
1319     auto desiredMime = mUseHeic ? MIMETYPE_IMAGE_ANDROID_HEIC : MIMETYPE_VIDEO_HEVC;
1320     mCodecLooper = new ALooper;
1321     mCodecLooper->setName("Camera3-HeicComposite-MediaCodecLooper");
1322     status_t res = mCodecLooper->start(
1323             false,   // runOnCallingThread
1324             false,    // canCallJava
1325             PRIORITY_AUDIO);
1326     if (res != OK) {
1327         ALOGE("%s: Failed to start codec looper: %s (%d)",
1328                 __FUNCTION__, strerror(-res), res);
1329         return NO_INIT;
1330     }
1331 
1332     // Create HEIC/HEVC codec.
1333     if (mUseHeic) {
1334         mCodec = MediaCodec::CreateByType(mCodecLooper, desiredMime, true /*encoder*/);
1335     } else {
1336         mCodec = MediaCodec::CreateByComponentName(mCodecLooper, hevcName);
1337     }
1338     if (mCodec == nullptr) {
1339         ALOGE("%s: Failed to create codec for %s", __FUNCTION__, desiredMime);
1340         return NO_INIT;
1341     }
1342 
1343     // Create Looper and handler for Codec callback.
1344     mCodecCallbackHandler = new CodecCallbackHandler(this);
1345     if (mCodecCallbackHandler == nullptr) {
1346         ALOGE("%s: Failed to create codec callback handler", __FUNCTION__);
1347         return NO_MEMORY;
1348     }
1349     mCallbackLooper = new ALooper;
1350     mCallbackLooper->setName("Camera3-HeicComposite-MediaCodecCallbackLooper");
1351     res = mCallbackLooper->start(
1352             false,   // runOnCallingThread
1353             false,    // canCallJava
1354             PRIORITY_AUDIO);
1355     if (res != OK) {
1356         ALOGE("%s: Failed to start media callback looper: %s (%d)",
1357                 __FUNCTION__, strerror(-res), res);
1358         return NO_INIT;
1359     }
1360     mCallbackLooper->registerHandler(mCodecCallbackHandler);
1361 
1362     mAsyncNotify = new AMessage(kWhatCallbackNotify, mCodecCallbackHandler);
1363     res = mCodec->setCallback(mAsyncNotify);
1364     if (res != OK) {
1365         ALOGE("%s: Failed to set MediaCodec callback: %s (%d)", __FUNCTION__,
1366                 strerror(-res), res);
1367         return res;
1368     }
1369 
1370     // Create output format and configure the Codec.
1371     sp<AMessage> outputFormat = new AMessage();
1372     outputFormat->setString(KEY_MIME, desiredMime);
1373     outputFormat->setInt32(KEY_BITRATE_MODE, BITRATE_MODE_CQ);
1374     outputFormat->setInt32(KEY_QUALITY, kDefaultJpegQuality);
1375     // Ask codec to skip timestamp check and encode all frames.
1376     outputFormat->setInt64(KEY_MAX_PTS_GAP_TO_ENCODER, kNoFrameDropMaxPtsGap);
1377 
1378     int32_t gridWidth, gridHeight, gridRows, gridCols;
1379     if (useGrid || mUseHeic) {
1380         gridWidth = HeicEncoderInfoManager::kGridWidth;
1381         gridHeight = HeicEncoderInfoManager::kGridHeight;
1382         gridRows = (height + gridHeight - 1)/gridHeight;
1383         gridCols = (width + gridWidth - 1)/gridWidth;
1384 
1385         if (mUseHeic) {
1386             outputFormat->setInt32(KEY_TILE_WIDTH, gridWidth);
1387             outputFormat->setInt32(KEY_TILE_HEIGHT, gridHeight);
1388             outputFormat->setInt32(KEY_GRID_COLUMNS, gridCols);
1389             outputFormat->setInt32(KEY_GRID_ROWS, gridRows);
1390         }
1391 
1392     } else {
1393         gridWidth = width;
1394         gridHeight = height;
1395         gridRows = 1;
1396         gridCols = 1;
1397     }
1398 
1399     outputFormat->setInt32(KEY_WIDTH, !useGrid ? width : gridWidth);
1400     outputFormat->setInt32(KEY_HEIGHT, !useGrid ? height : gridHeight);
1401     outputFormat->setInt32(KEY_I_FRAME_INTERVAL, 0);
1402     outputFormat->setInt32(KEY_COLOR_FORMAT,
1403             useGrid ? COLOR_FormatYUV420Flexible : COLOR_FormatSurface);
1404     outputFormat->setInt32(KEY_FRAME_RATE, useGrid ? gridRows * gridCols : kNoGridOpRate);
1405     // This only serves as a hint to encoder when encoding is not real-time.
1406     outputFormat->setInt32(KEY_OPERATING_RATE, useGrid ? kGridOpRate : kNoGridOpRate);
1407 
1408     res = mCodec->configure(outputFormat, nullptr /*nativeWindow*/,
1409             nullptr /*crypto*/, CONFIGURE_FLAG_ENCODE);
1410     if (res != OK) {
1411         ALOGE("%s: Failed to configure codec: %s (%d)", __FUNCTION__,
1412                 strerror(-res), res);
1413         return res;
1414     }
1415 
1416     mGridWidth = gridWidth;
1417     mGridHeight = gridHeight;
1418     mGridRows = gridRows;
1419     mGridCols = gridCols;
1420     mUseGrid = useGrid;
1421     mOutputWidth = width;
1422     mOutputHeight = height;
1423     mAppSegmentMaxSize = calcAppSegmentMaxSize(cameraDevice->info());
1424     mMaxHeicBufferSize =
1425         ALIGN(mOutputWidth, HeicEncoderInfoManager::kGridWidth) *
1426         ALIGN(mOutputHeight, HeicEncoderInfoManager::kGridHeight) * 3 / 2 + mAppSegmentMaxSize;
1427 
1428     return OK;
1429 }
1430 
deinitCodec()1431 void HeicCompositeStream::deinitCodec() {
1432     ALOGV("%s", __FUNCTION__);
1433     if (mCodec != nullptr) {
1434         mCodec->stop();
1435         mCodec->release();
1436         mCodec.clear();
1437     }
1438 
1439     if (mCodecLooper != nullptr) {
1440         mCodecLooper->stop();
1441         mCodecLooper.clear();
1442     }
1443 
1444     if (mCallbackLooper != nullptr) {
1445         mCallbackLooper->stop();
1446         mCallbackLooper.clear();
1447     }
1448 
1449     mAsyncNotify.clear();
1450     mFormat.clear();
1451 }
1452 
1453 // Return the size of the complete list of app segment, 0 indicates failure
findAppSegmentsSize(const uint8_t * appSegmentBuffer,size_t maxSize,size_t * app1SegmentSize)1454 size_t HeicCompositeStream::findAppSegmentsSize(const uint8_t* appSegmentBuffer,
1455         size_t maxSize, size_t *app1SegmentSize) {
1456     if (appSegmentBuffer == nullptr || app1SegmentSize == nullptr) {
1457         ALOGE("%s: Invalid input appSegmentBuffer %p, app1SegmentSize %p",
1458                 __FUNCTION__, appSegmentBuffer, app1SegmentSize);
1459         return 0;
1460     }
1461 
1462     size_t expectedSize = 0;
1463     // First check for EXIF transport header at the end of the buffer
1464     const uint8_t *header = appSegmentBuffer + (maxSize - sizeof(CameraBlob));
1465     const CameraBlob *blob = (const CameraBlob*)(header);
1466     if (blob->blobId != CameraBlobId::JPEG_APP_SEGMENTS) {
1467         ALOGE("%s: Invalid EXIF blobId %d", __FUNCTION__, blob->blobId);
1468         return 0;
1469     }
1470 
1471     expectedSize = blob->blobSizeBytes;
1472     if (expectedSize == 0 || expectedSize > maxSize - sizeof(CameraBlob)) {
1473         ALOGE("%s: Invalid blobSize %zu.", __FUNCTION__, expectedSize);
1474         return 0;
1475     }
1476 
1477     uint32_t totalSize = 0;
1478 
1479     // Verify APP1 marker (mandatory)
1480     uint8_t app1Marker[] = {0xFF, 0xE1};
1481     if (memcmp(appSegmentBuffer, app1Marker, sizeof(app1Marker))) {
1482         ALOGE("%s: Invalid APP1 marker: %x, %x", __FUNCTION__,
1483                 appSegmentBuffer[0], appSegmentBuffer[1]);
1484         return 0;
1485     }
1486     totalSize += sizeof(app1Marker);
1487 
1488     uint16_t app1Size = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1489             appSegmentBuffer[totalSize+1];
1490     totalSize += app1Size;
1491 
1492     ALOGV("%s: Expected APP segments size %zu, APP1 segment size %u",
1493             __FUNCTION__, expectedSize, app1Size);
1494     while (totalSize < expectedSize) {
1495         if (appSegmentBuffer[totalSize] != 0xFF ||
1496                 appSegmentBuffer[totalSize+1] <= 0xE1 ||
1497                 appSegmentBuffer[totalSize+1] > 0xEF) {
1498             // Invalid APPn marker
1499             ALOGE("%s: Invalid APPn marker: %x, %x", __FUNCTION__,
1500                     appSegmentBuffer[totalSize], appSegmentBuffer[totalSize+1]);
1501             return 0;
1502         }
1503         totalSize += 2;
1504 
1505         uint16_t appnSize = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1506                 appSegmentBuffer[totalSize+1];
1507         totalSize += appnSize;
1508     }
1509 
1510     if (totalSize != expectedSize) {
1511         ALOGE("%s: Invalid JPEG APP segments: totalSize %u vs expected size %zu",
1512                 __FUNCTION__, totalSize, expectedSize);
1513         return 0;
1514     }
1515 
1516     *app1SegmentSize = app1Size + sizeof(app1Marker);
1517     return expectedSize;
1518 }
1519 
copyOneYuvTile(sp<MediaCodecBuffer> & codecBuffer,const CpuConsumer::LockedBuffer & yuvBuffer,size_t top,size_t left,size_t width,size_t height)1520 status_t HeicCompositeStream::copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer,
1521         const CpuConsumer::LockedBuffer& yuvBuffer,
1522         size_t top, size_t left, size_t width, size_t height) {
1523     ATRACE_CALL();
1524 
1525     // Get stride information for codecBuffer
1526     sp<ABuffer> imageData;
1527     if (!codecBuffer->meta()->findBuffer("image-data", &imageData)) {
1528         ALOGE("%s: Codec input buffer is not for image data!", __FUNCTION__);
1529         return BAD_VALUE;
1530     }
1531     if (imageData->size() != sizeof(MediaImage2)) {
1532         ALOGE("%s: Invalid codec input image size %zu, expected %zu",
1533                 __FUNCTION__, imageData->size(), sizeof(MediaImage2));
1534         return BAD_VALUE;
1535     }
1536     MediaImage2* imageInfo = reinterpret_cast<MediaImage2*>(imageData->data());
1537     if (imageInfo->mType != MediaImage2::MEDIA_IMAGE_TYPE_YUV ||
1538             imageInfo->mBitDepth != 8 ||
1539             imageInfo->mBitDepthAllocated != 8 ||
1540             imageInfo->mNumPlanes != 3) {
1541         ALOGE("%s: Invalid codec input image info: mType %d, mBitDepth %d, "
1542                 "mBitDepthAllocated %d, mNumPlanes %d!", __FUNCTION__,
1543                 imageInfo->mType, imageInfo->mBitDepth,
1544                 imageInfo->mBitDepthAllocated, imageInfo->mNumPlanes);
1545         return BAD_VALUE;
1546     }
1547 
1548     ALOGV("%s: yuvBuffer chromaStep %d, chromaStride %d",
1549             __FUNCTION__, yuvBuffer.chromaStep, yuvBuffer.chromaStride);
1550     ALOGV("%s: U offset %u, V offset %u, U rowInc %d, V rowInc %d, U colInc %d, V colInc %d",
1551             __FUNCTION__, imageInfo->mPlane[MediaImage2::U].mOffset,
1552             imageInfo->mPlane[MediaImage2::V].mOffset,
1553             imageInfo->mPlane[MediaImage2::U].mRowInc,
1554             imageInfo->mPlane[MediaImage2::V].mRowInc,
1555             imageInfo->mPlane[MediaImage2::U].mColInc,
1556             imageInfo->mPlane[MediaImage2::V].mColInc);
1557 
1558     // Y
1559     for (auto row = top; row < top+height; row++) {
1560         uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::Y].mOffset +
1561                 imageInfo->mPlane[MediaImage2::Y].mRowInc * (row - top);
1562         mFnCopyRow(yuvBuffer.data+row*yuvBuffer.stride+left, dst, width);
1563     }
1564 
1565     // U is Cb, V is Cr
1566     bool codecUPlaneFirst = imageInfo->mPlane[MediaImage2::V].mOffset >
1567             imageInfo->mPlane[MediaImage2::U].mOffset;
1568     uint32_t codecUvOffsetDiff = codecUPlaneFirst ?
1569             imageInfo->mPlane[MediaImage2::V].mOffset - imageInfo->mPlane[MediaImage2::U].mOffset :
1570             imageInfo->mPlane[MediaImage2::U].mOffset - imageInfo->mPlane[MediaImage2::V].mOffset;
1571     bool isCodecUvSemiplannar = (codecUvOffsetDiff == 1) &&
1572             (imageInfo->mPlane[MediaImage2::U].mRowInc ==
1573             imageInfo->mPlane[MediaImage2::V].mRowInc) &&
1574             (imageInfo->mPlane[MediaImage2::U].mColInc == 2) &&
1575             (imageInfo->mPlane[MediaImage2::V].mColInc == 2);
1576     bool isCodecUvPlannar =
1577             ((codecUPlaneFirst && codecUvOffsetDiff >=
1578                     imageInfo->mPlane[MediaImage2::U].mRowInc * imageInfo->mHeight/2) ||
1579             ((!codecUPlaneFirst && codecUvOffsetDiff >=
1580                     imageInfo->mPlane[MediaImage2::V].mRowInc * imageInfo->mHeight/2))) &&
1581             imageInfo->mPlane[MediaImage2::U].mColInc == 1 &&
1582             imageInfo->mPlane[MediaImage2::V].mColInc == 1;
1583     bool cameraUPlaneFirst = yuvBuffer.dataCr > yuvBuffer.dataCb;
1584 
1585     if (isCodecUvSemiplannar && yuvBuffer.chromaStep == 2 &&
1586             (codecUPlaneFirst == cameraUPlaneFirst)) {
1587         // UV semiplannar
1588         // The chrome plane could be either Cb first, or Cr first. Take the
1589         // smaller address.
1590         uint8_t *src = std::min(yuvBuffer.dataCb, yuvBuffer.dataCr);
1591         MediaImage2::PlaneIndex dstPlane = codecUvOffsetDiff > 0 ? MediaImage2::U : MediaImage2::V;
1592         for (auto row = top/2; row < (top+height)/2; row++) {
1593             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[dstPlane].mOffset +
1594                     imageInfo->mPlane[dstPlane].mRowInc * (row - top/2);
1595             mFnCopyRow(src+row*yuvBuffer.chromaStride+left, dst, width);
1596         }
1597     } else if (isCodecUvPlannar && yuvBuffer.chromaStep == 1) {
1598         // U plane
1599         for (auto row = top/2; row < (top+height)/2; row++) {
1600             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::U].mOffset +
1601                     imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2);
1602             mFnCopyRow(yuvBuffer.dataCb+row*yuvBuffer.chromaStride+left/2, dst, width/2);
1603         }
1604 
1605         // V plane
1606         for (auto row = top/2; row < (top+height)/2; row++) {
1607             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::V].mOffset +
1608                     imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2);
1609             mFnCopyRow(yuvBuffer.dataCr+row*yuvBuffer.chromaStride+left/2, dst, width/2);
1610         }
1611     } else {
1612         // Convert between semiplannar and plannar, or when UV orders are
1613         // different.
1614         uint8_t *dst = codecBuffer->data();
1615         for (auto row = top/2; row < (top+height)/2; row++) {
1616             for (auto col = left/2; col < (left+width)/2; col++) {
1617                 // U/Cb
1618                 int32_t dstIndex = imageInfo->mPlane[MediaImage2::U].mOffset +
1619                         imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2) +
1620                         imageInfo->mPlane[MediaImage2::U].mColInc * (col - left/2);
1621                 int32_t srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1622                 dst[dstIndex] = yuvBuffer.dataCb[srcIndex];
1623 
1624                 // V/Cr
1625                 dstIndex = imageInfo->mPlane[MediaImage2::V].mOffset +
1626                         imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2) +
1627                         imageInfo->mPlane[MediaImage2::V].mColInc * (col - left/2);
1628                 srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1629                 dst[dstIndex] = yuvBuffer.dataCr[srcIndex];
1630             }
1631         }
1632     }
1633     return OK;
1634 }
1635 
initCopyRowFunction(int32_t width)1636 void HeicCompositeStream::initCopyRowFunction([[maybe_unused]] int32_t width)
1637 {
1638     using namespace libyuv;
1639 
1640     mFnCopyRow = CopyRow_C;
1641 #if defined(HAS_COPYROW_SSE2)
1642     if (TestCpuFlag(kCpuHasSSE2)) {
1643         mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
1644     }
1645 #endif
1646 #if defined(HAS_COPYROW_AVX)
1647     if (TestCpuFlag(kCpuHasAVX)) {
1648         mFnCopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
1649     }
1650 #endif
1651 #if defined(HAS_COPYROW_ERMS)
1652     if (TestCpuFlag(kCpuHasERMS)) {
1653         mFnCopyRow = CopyRow_ERMS;
1654     }
1655 #endif
1656 #if defined(HAS_COPYROW_NEON)
1657     if (TestCpuFlag(kCpuHasNEON)) {
1658         mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
1659     }
1660 #endif
1661 #if defined(HAS_COPYROW_MIPS)
1662     if (TestCpuFlag(kCpuHasMIPS)) {
1663         mFnCopyRow = CopyRow_MIPS;
1664     }
1665 #endif
1666 }
1667 
calcAppSegmentMaxSize(const CameraMetadata & info)1668 size_t HeicCompositeStream::calcAppSegmentMaxSize(const CameraMetadata& info) {
1669     camera_metadata_ro_entry_t entry = info.find(ANDROID_HEIC_INFO_MAX_JPEG_APP_SEGMENTS_COUNT);
1670     size_t maxAppsSegment = 1;
1671     if (entry.count > 0) {
1672         maxAppsSegment = entry.data.u8[0] < 1 ? 1 :
1673                 entry.data.u8[0] > 16 ? 16 : entry.data.u8[0];
1674     }
1675     return maxAppsSegment * (2 + 0xFFFF) + sizeof(CameraBlob);
1676 }
1677 
updateCodecQualityLocked(int32_t quality)1678 void HeicCompositeStream::updateCodecQualityLocked(int32_t quality) {
1679     if (quality != mQuality) {
1680         sp<AMessage> qualityParams = new AMessage;
1681         qualityParams->setInt32(PARAMETER_KEY_VIDEO_BITRATE, quality);
1682         status_t res = mCodec->setParameters(qualityParams);
1683         if (res != OK) {
1684             ALOGE("%s: Failed to set codec quality: %s (%d)",
1685                     __FUNCTION__, strerror(-res), res);
1686         } else {
1687             mQuality = quality;
1688         }
1689     }
1690 }
1691 
threadLoop()1692 bool HeicCompositeStream::threadLoop() {
1693     int64_t frameNumber = -1;
1694     bool newInputAvailable = false;
1695 
1696     {
1697         Mutex::Autolock l(mMutex);
1698         if (mErrorState) {
1699             // In case we landed in error state, return any pending buffers and
1700             // halt all further processing.
1701             compilePendingInputLocked();
1702             releaseInputFramesLocked();
1703             return false;
1704         }
1705 
1706 
1707         while (!newInputAvailable) {
1708             compilePendingInputLocked();
1709             newInputAvailable = getNextReadyInputLocked(&frameNumber);
1710 
1711             if (!newInputAvailable) {
1712                 auto failingFrameNumber = getNextFailingInputLocked();
1713                 if (failingFrameNumber >= 0) {
1714                     releaseInputFrameLocked(failingFrameNumber,
1715                             &mPendingInputFrames[failingFrameNumber]);
1716 
1717                     // It's okay to remove the entry from mPendingInputFrames
1718                     // because:
1719                     // 1. Only one internal stream (main input) is critical in
1720                     // backing the output stream.
1721                     // 2. If captureResult/appSegment arrives after the entry is
1722                     // removed, they are simply skipped.
1723                     mPendingInputFrames.erase(failingFrameNumber);
1724                     if (mPendingInputFrames.size() == 0) {
1725                         markTrackerIdle();
1726                     }
1727                     return true;
1728                 }
1729 
1730                 auto ret = mInputReadyCondition.waitRelative(mMutex, kWaitDuration);
1731                 if (ret == TIMED_OUT) {
1732                     return true;
1733                 } else if (ret != OK) {
1734                     ALOGE("%s: Timed wait on condition failed: %s (%d)", __FUNCTION__,
1735                             strerror(-ret), ret);
1736                     return false;
1737                 }
1738             }
1739         }
1740     }
1741 
1742     auto res = processInputFrame(frameNumber, mPendingInputFrames[frameNumber]);
1743     Mutex::Autolock l(mMutex);
1744     if (res != OK) {
1745         ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ", frameNumber: %"
1746                 PRId64 ": %s (%d)", __FUNCTION__, mPendingInputFrames[frameNumber].timestamp,
1747                 frameNumber, strerror(-res), res);
1748         mPendingInputFrames[frameNumber].error = true;
1749     }
1750 
1751     releaseInputFramesLocked();
1752 
1753     return true;
1754 }
1755 
flagAnExifErrorFrameNumber(int64_t frameNumber)1756 void HeicCompositeStream::flagAnExifErrorFrameNumber(int64_t frameNumber) {
1757     Mutex::Autolock l(mMutex);
1758     mExifErrorFrameNumbers.emplace(frameNumber);
1759     mInputReadyCondition.signal();
1760 }
1761 
onStreamBufferError(const CaptureResultExtras & resultExtras)1762 bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultExtras) {
1763     bool res = false;
1764     int64_t frameNumber = resultExtras.frameNumber;
1765 
1766     // Buffer errors concerning internal composite streams should not be directly visible to
1767     // camera clients. They must only receive a single buffer error with the public composite
1768     // stream id.
1769     if (resultExtras.errorStreamId == mAppSegmentStreamId) {
1770         ALOGV("%s: APP_SEGMENT frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1771         flagAnExifErrorFrameNumber(frameNumber);
1772         res = true;
1773     } else if (resultExtras.errorStreamId == mMainImageStreamId) {
1774         ALOGV("%s: YUV frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1775         flagAnErrorFrameNumber(frameNumber);
1776         res = true;
1777     }
1778 
1779     return res;
1780 }
1781 
onResultError(const CaptureResultExtras & resultExtras)1782 void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
1783     // For result error, since the APPS_SEGMENT buffer already contains EXIF,
1784     // simply skip using the capture result metadata to override EXIF.
1785     Mutex::Autolock l(mMutex);
1786 
1787     int64_t timestamp = -1;
1788     for (const auto& fn : mSettingsByFrameNumber) {
1789         if (fn.first == resultExtras.frameNumber) {
1790             timestamp = fn.second.timestamp;
1791             break;
1792         }
1793     }
1794     if (timestamp == -1) {
1795         for (const auto& inputFrame : mPendingInputFrames) {
1796             if (inputFrame.first == resultExtras.frameNumber) {
1797                 timestamp = inputFrame.second.timestamp;
1798                 break;
1799             }
1800         }
1801     }
1802 
1803     if (timestamp == -1) {
1804         ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);
1805         return;
1806     }
1807 
1808     mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));
1809     ALOGV("%s: timestamp %" PRId64 ", frameNumber %" PRId64, __FUNCTION__,
1810             timestamp, resultExtras.frameNumber);
1811     mInputReadyCondition.signal();
1812 }
1813 
onRequestError(const CaptureResultExtras & resultExtras)1814 void HeicCompositeStream::onRequestError(const CaptureResultExtras& resultExtras) {
1815     auto frameNumber = resultExtras.frameNumber;
1816     ALOGV("%s: frameNumber: %" PRId64, __FUNCTION__, frameNumber);
1817     Mutex::Autolock l(mMutex);
1818     auto numRequests = mSettingsByFrameNumber.erase(frameNumber);
1819     if (numRequests == 0) {
1820         // Pending request has been populated into mPendingInputFrames
1821         mErrorFrameNumbers.emplace(frameNumber);
1822         mInputReadyCondition.signal();
1823     } else {
1824         // REQUEST_ERROR was received without onShutter.
1825     }
1826 }
1827 
markTrackerIdle()1828 void HeicCompositeStream::markTrackerIdle() {
1829     sp<StatusTracker> statusTracker = mStatusTracker.promote();
1830     if (statusTracker != nullptr) {
1831         statusTracker->markComponentIdle(mStatusId, Fence::NO_FENCE);
1832         ALOGV("%s: Mark component as idle", __FUNCTION__);
1833     }
1834 }
1835 
onMessageReceived(const sp<AMessage> & msg)1836 void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {
1837     sp<HeicCompositeStream> parent = mParent.promote();
1838     if (parent == nullptr) return;
1839 
1840     switch (msg->what()) {
1841         case kWhatCallbackNotify: {
1842              int32_t cbID;
1843              if (!msg->findInt32("callbackID", &cbID)) {
1844                  ALOGE("kWhatCallbackNotify: callbackID is expected.");
1845                  break;
1846              }
1847 
1848              ALOGV("kWhatCallbackNotify: cbID = %d", cbID);
1849 
1850              switch (cbID) {
1851                  case MediaCodec::CB_INPUT_AVAILABLE: {
1852                      int32_t index;
1853                      if (!msg->findInt32("index", &index)) {
1854                          ALOGE("CB_INPUT_AVAILABLE: index is expected.");
1855                          break;
1856                      }
1857                      parent->onHeicInputFrameAvailable(index);
1858                      break;
1859                  }
1860 
1861                  case MediaCodec::CB_OUTPUT_AVAILABLE: {
1862                      int32_t index;
1863                      size_t offset;
1864                      size_t size;
1865                      int64_t timeUs;
1866                      int32_t flags;
1867 
1868                      if (!msg->findInt32("index", &index)) {
1869                          ALOGE("CB_OUTPUT_AVAILABLE: index is expected.");
1870                          break;
1871                      }
1872                      if (!msg->findSize("offset", &offset)) {
1873                          ALOGE("CB_OUTPUT_AVAILABLE: offset is expected.");
1874                          break;
1875                      }
1876                      if (!msg->findSize("size", &size)) {
1877                          ALOGE("CB_OUTPUT_AVAILABLE: size is expected.");
1878                          break;
1879                      }
1880                      if (!msg->findInt64("timeUs", &timeUs)) {
1881                          ALOGE("CB_OUTPUT_AVAILABLE: timeUs is expected.");
1882                          break;
1883                      }
1884                      if (!msg->findInt32("flags", &flags)) {
1885                          ALOGE("CB_OUTPUT_AVAILABLE: flags is expected.");
1886                          break;
1887                      }
1888 
1889                      CodecOutputBufferInfo bufferInfo = {
1890                          index,
1891                          (int32_t)offset,
1892                          (int32_t)size,
1893                          timeUs,
1894                          (uint32_t)flags};
1895 
1896                      parent->onHeicOutputFrameAvailable(bufferInfo);
1897                      break;
1898                  }
1899 
1900                  case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: {
1901                      sp<AMessage> format;
1902                      if (!msg->findMessage("format", &format)) {
1903                          ALOGE("CB_OUTPUT_FORMAT_CHANGED: format is expected.");
1904                          break;
1905                      }
1906                      // Here format is MediaCodec's internal copy of output format.
1907                      // Make a copy since onHeicFormatChanged() might modify it.
1908                      sp<AMessage> formatCopy;
1909                      if (format != nullptr) {
1910                          formatCopy = format->dup();
1911                      }
1912                      parent->onHeicFormatChanged(formatCopy);
1913                      break;
1914                  }
1915 
1916                  case MediaCodec::CB_ERROR: {
1917                      status_t err;
1918                      int32_t actionCode;
1919                      AString detail;
1920                      if (!msg->findInt32("err", &err)) {
1921                          ALOGE("CB_ERROR: err is expected.");
1922                          break;
1923                      }
1924                      if (!msg->findInt32("action", &actionCode)) {
1925                          ALOGE("CB_ERROR: action is expected.");
1926                          break;
1927                      }
1928                      msg->findString("detail", &detail);
1929                      ALOGE("Codec reported error(0x%x), actionCode(%d), detail(%s)",
1930                              err, actionCode, detail.c_str());
1931 
1932                      parent->onHeicCodecError();
1933                      break;
1934                  }
1935 
1936                  default: {
1937                      ALOGE("kWhatCallbackNotify: callbackID(%d) is unexpected.", cbID);
1938                      break;
1939                  }
1940              }
1941              break;
1942         }
1943 
1944         default:
1945             ALOGE("shouldn't be here");
1946             break;
1947     }
1948 }
1949 
1950 }; // namespace camera3
1951 }; // namespace android
1952