1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 // #define LOG_NDEBUG 0
18 #define LOG_TAG "GCH_RgbirdResultRequestProcessor"
19 #define ATRACE_TAG ATRACE_TAG_CAMERA
20 #include "rgbird_result_request_processor.h"
21 
22 #include <cutils/native_handle.h>
23 #include <cutils/properties.h>
24 #include <inttypes.h>
25 #include <log/log.h>
26 #include <sync/sync.h>
27 #include <utils/Trace.h>
28 
29 #include "hal_utils.h"
30 
31 namespace android {
32 namespace google_camera_hal {
33 
Create(const RgbirdResultRequestProcessorCreateData & create_data)34 std::unique_ptr<RgbirdResultRequestProcessor> RgbirdResultRequestProcessor::Create(
35     const RgbirdResultRequestProcessorCreateData& create_data) {
36   ATRACE_CALL();
37   auto result_processor = std::unique_ptr<RgbirdResultRequestProcessor>(
38       new RgbirdResultRequestProcessor(create_data));
39   if (result_processor == nullptr) {
40     ALOGE("%s: Creating RgbirdResultRequestProcessor failed.", __FUNCTION__);
41     return nullptr;
42   }
43 
44   // TODO(b/128633958): remove this after FLL syncing is verified
45   result_processor->force_internal_stream_ =
46       property_get_bool("persist.vendor.camera.rgbird.forceinternal", false);
47   if (result_processor->force_internal_stream_) {
48     ALOGI("%s: Force creating internal streams for IR pipelines", __FUNCTION__);
49   }
50 
51   // TODO(b/129910835): Change the controlling prop into some deterministic
52   // logic that controls when the front depth autocal will be triggered.
53   result_processor->rgb_ir_auto_cal_enabled_ =
54       property_get_bool("vendor.camera.frontdepth.enableautocal", true);
55   if (result_processor->rgb_ir_auto_cal_enabled_) {
56     ALOGI("%s: autocal is enabled.", __FUNCTION__);
57   }
58 
59   return result_processor;
60 }
61 
RgbirdResultRequestProcessor(const RgbirdResultRequestProcessorCreateData & create_data)62 RgbirdResultRequestProcessor::RgbirdResultRequestProcessor(
63     const RgbirdResultRequestProcessorCreateData& create_data)
64     : kRgbCameraId(create_data.rgb_camera_id),
65       kIr1CameraId(create_data.ir1_camera_id),
66       kIr2CameraId(create_data.ir2_camera_id),
67       rgb_raw_stream_id_(create_data.rgb_raw_stream_id),
68       is_hdrplus_supported_(create_data.is_hdrplus_supported),
69       rgb_internal_yuv_stream_id_(create_data.rgb_internal_yuv_stream_id) {
70 }
71 
SetResultCallback(ProcessCaptureResultFunc process_capture_result,NotifyFunc notify,ProcessBatchCaptureResultFunc)72 void RgbirdResultRequestProcessor::SetResultCallback(
73     ProcessCaptureResultFunc process_capture_result, NotifyFunc notify,
74     ProcessBatchCaptureResultFunc /*process_batch_capture_result*/) {
75   std::lock_guard<std::mutex> lock(callback_lock_);
76   process_capture_result_ = process_capture_result;
77   notify_ = notify;
78 }
79 
SaveFdForHdrplus(const CaptureRequest & request)80 void RgbirdResultRequestProcessor::SaveFdForHdrplus(
81     const CaptureRequest& request) {
82   // Enable face detect mode for internal use
83   if (request.settings != nullptr) {
84     uint8_t fd_mode;
85     status_t res = hal_utils::GetFdMode(request, &fd_mode);
86     if (res == OK) {
87       current_face_detect_mode_ = fd_mode;
88     }
89   }
90 
91   {
92     std::lock_guard<std::mutex> lock(face_detect_lock_);
93     requested_face_detect_modes_.emplace(request.frame_number,
94                                          current_face_detect_mode_);
95   }
96 }
97 
SaveLsForHdrplus(const CaptureRequest & request)98 void RgbirdResultRequestProcessor::SaveLsForHdrplus(
99     const CaptureRequest& request) {
100   if (request.settings != nullptr) {
101     uint8_t lens_shading_map_mode;
102     status_t res =
103         hal_utils::GetLensShadingMapMode(request, &lens_shading_map_mode);
104     if (res == OK) {
105       current_lens_shading_map_mode_ = lens_shading_map_mode;
106     }
107   }
108 
109   {
110     std::lock_guard<std::mutex> lock(lens_shading_lock_);
111     requested_lens_shading_map_modes_.emplace(request.frame_number,
112                                               current_lens_shading_map_mode_);
113   }
114 }
115 
HandleLsResultForHdrplus(uint32_t frameNumber,HalCameraMetadata * metadata)116 status_t RgbirdResultRequestProcessor::HandleLsResultForHdrplus(
117     uint32_t frameNumber, HalCameraMetadata* metadata) {
118   if (metadata == nullptr) {
119     ALOGE("%s: metadata is nullptr", __FUNCTION__);
120     return BAD_VALUE;
121   }
122   std::lock_guard<std::mutex> lock(lens_shading_lock_);
123   auto iter = requested_lens_shading_map_modes_.find(frameNumber);
124   if (iter == requested_lens_shading_map_modes_.end()) {
125     ALOGW("%s: can't find frame (%d)", __FUNCTION__, frameNumber);
126     return OK;
127   }
128 
129   if (iter->second == ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF) {
130     status_t res = hal_utils::RemoveLsInfoFromResult(metadata);
131     if (res != OK) {
132       ALOGW("%s: RemoveLsInfoFromResult fail", __FUNCTION__);
133     }
134   }
135   requested_lens_shading_map_modes_.erase(iter);
136 
137   return OK;
138 }
139 
IsAutocalRequest(uint32_t frame_number) const140 bool RgbirdResultRequestProcessor::IsAutocalRequest(uint32_t frame_number) const {
141   // TODO(b/129910835): Use the proper logic to control when internal yuv buffer
142   // needs to be passed to the depth process block. Even if the auto cal is
143   // enabled, there is no need to pass the internal yuv buffer for every
144   // request, not even every device session. This is also related to how the
145   // buffer is added into the request. Similar logic exists in realtime request
146   // processor. However, this logic can further filter and determine which
147   // requests contain the internal yuv stream buffers and send them to the depth
148   // process block. Current implementation only treat the kAutocalFrameNumber
149   // request as autocal request. This must be consistent with that of the
150   // rt_request_processor.
151   if (!rgb_ir_auto_cal_enabled_) {
152     return false;
153   }
154 
155   return frame_number == kAutocalFrameNumber;
156 }
157 
TryReturnInternalBufferForDepth(CaptureResult * result,bool * has_internal)158 void RgbirdResultRequestProcessor::TryReturnInternalBufferForDepth(
159     CaptureResult* result, bool* has_internal) {
160   ATRACE_CALL();
161   if (result == nullptr || has_internal == nullptr) {
162     ALOGE("%s: result or has_rgb_raw_output is nullptr", __FUNCTION__);
163     return;
164   }
165 
166   if (internal_stream_manager_ == nullptr) {
167     ALOGE("%s: internal_stream_manager_ nullptr", __FUNCTION__);
168     return;
169   }
170 
171   std::vector<StreamBuffer> modified_output_buffers;
172   for (uint32_t i = 0; i < result->output_buffers.size(); i++) {
173     if (rgb_internal_yuv_stream_id_ == result->output_buffers[i].stream_id &&
174         !IsAutocalRequest(result->frame_number)) {
175       *has_internal = true;
176       status_t res = internal_stream_manager_->ReturnStreamBuffer(
177           result->output_buffers[i]);
178       if (res != OK) {
179         ALOGW("%s: Failed to return RGB internal raw buffer for frame %d",
180               __FUNCTION__, result->frame_number);
181       }
182     } else {
183       modified_output_buffers.push_back(result->output_buffers[i]);
184     }
185   }
186 
187   if (!result->output_buffers.empty()) {
188     result->output_buffers = modified_output_buffers;
189   }
190 }
191 
HandleFdResultForHdrplus(uint32_t frameNumber,HalCameraMetadata * metadata)192 status_t RgbirdResultRequestProcessor::HandleFdResultForHdrplus(
193     uint32_t frameNumber, HalCameraMetadata* metadata) {
194   if (metadata == nullptr) {
195     ALOGE("%s: metadata is nullptr", __FUNCTION__);
196     return BAD_VALUE;
197   }
198   std::lock_guard<std::mutex> lock(face_detect_lock_);
199   auto iter = requested_face_detect_modes_.find(frameNumber);
200   if (iter == requested_face_detect_modes_.end()) {
201     ALOGW("%s: can't find frame (%d)", __FUNCTION__, frameNumber);
202     return OK;
203   }
204 
205   if (iter->second == ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) {
206     status_t res = hal_utils::RemoveFdInfoFromResult(metadata);
207     if (res != OK) {
208       ALOGW("%s: RestoreFdMetadataForHdrplus fail", __FUNCTION__);
209     }
210   }
211   requested_face_detect_modes_.erase(iter);
212 
213   return OK;
214 }
215 
AddPendingRequests(const std::vector<ProcessBlockRequest> &,const CaptureRequest & remaining_session_request)216 status_t RgbirdResultRequestProcessor::AddPendingRequests(
217     const std::vector<ProcessBlockRequest>& /*process_block_requests*/,
218     const CaptureRequest& remaining_session_request) {
219   ATRACE_CALL();
220   std::lock_guard<std::mutex> lock(depth_requests_mutex_);
221   for (auto stream_buffer : remaining_session_request.output_buffers) {
222     if (stream_buffer.acquire_fence != nullptr) {
223       stream_buffer.acquire_fence =
224           native_handle_clone(stream_buffer.acquire_fence);
225       if (stream_buffer.acquire_fence == nullptr) {
226         ALOGE("%s: Cloning acquire_fence of buffer failed", __FUNCTION__);
227         return UNKNOWN_ERROR;
228       }
229     }
230     if (depth_stream_id_ == stream_buffer.stream_id) {
231       ALOGV("%s: request %d has a depth buffer", __FUNCTION__,
232             remaining_session_request.frame_number);
233       auto capture_request = std::make_unique<CaptureRequest>();
234       capture_request->frame_number = remaining_session_request.frame_number;
235       if (remaining_session_request.settings != nullptr) {
236         capture_request->settings =
237             HalCameraMetadata::Clone(remaining_session_request.settings.get());
238       }
239       capture_request->input_buffers.clear();
240       capture_request->output_buffers.push_back(stream_buffer);
241       depth_requests_.emplace(remaining_session_request.frame_number,
242                               std::move(capture_request));
243       break;
244     }
245   }
246 
247   if (is_hdrplus_supported_) {
248     SaveFdForHdrplus(remaining_session_request);
249     SaveLsForHdrplus(remaining_session_request);
250   }
251   return OK;
252 }
253 
ProcessResultForHdrplus(CaptureResult * result,bool * rgb_raw_output)254 void RgbirdResultRequestProcessor::ProcessResultForHdrplus(CaptureResult* result,
255                                                            bool* rgb_raw_output) {
256   ATRACE_CALL();
257   if (result == nullptr || rgb_raw_output == nullptr) {
258     ALOGE("%s: result or rgb_raw_output is nullptr", __FUNCTION__);
259     return;
260   }
261 
262   if (internal_stream_manager_ == nullptr) {
263     ALOGE("%s: internal_stream_manager_ nullptr", __FUNCTION__);
264     return;
265   }
266 
267   // Return filled raw buffer to internal stream manager
268   // And remove raw buffer from result
269   status_t res;
270   std::vector<StreamBuffer> modified_output_buffers;
271   for (uint32_t i = 0; i < result->output_buffers.size(); i++) {
272     if (rgb_raw_stream_id_ == result->output_buffers[i].stream_id) {
273       *rgb_raw_output = true;
274       res = internal_stream_manager_->ReturnFilledBuffer(
275           result->frame_number, result->output_buffers[i]);
276       if (res != OK) {
277         ALOGW("%s: (%d)ReturnStreamBuffer fail", __FUNCTION__,
278               result->frame_number);
279       }
280     } else {
281       modified_output_buffers.push_back(result->output_buffers[i]);
282     }
283   }
284 
285   if (result->output_buffers.size() > 0) {
286     result->output_buffers = modified_output_buffers;
287   }
288 
289   if (result->result_metadata) {
290     res = internal_stream_manager_->ReturnMetadata(
291         rgb_raw_stream_id_, result->frame_number, result->result_metadata.get());
292     if (res != OK) {
293       ALOGW("%s: (%d)ReturnMetadata fail", __FUNCTION__, result->frame_number);
294     }
295 
296     res = HandleFdResultForHdrplus(result->frame_number,
297                                    result->result_metadata.get());
298     if (res != OK) {
299       ALOGE("%s: HandleFdResultForHdrplus(%d) fail", __FUNCTION__,
300             result->frame_number);
301       return;
302     }
303 
304     res = HandleLsResultForHdrplus(result->frame_number,
305                                    result->result_metadata.get());
306     if (res != OK) {
307       ALOGE("%s: HandleLsResultForHdrplus(%d) fail", __FUNCTION__,
308             result->frame_number);
309       return;
310     }
311   }
312 }
313 
ReturnInternalStreams(CaptureResult * result)314 status_t RgbirdResultRequestProcessor::ReturnInternalStreams(
315     CaptureResult* result) {
316   ATRACE_CALL();
317   if (result == nullptr) {
318     ALOGE("%s: block_result is null.", __FUNCTION__);
319     return UNKNOWN_ERROR;
320   }
321 
322   std::vector<StreamBuffer> modified_output_buffers;
323   for (auto& stream_buffer : result->output_buffers) {
324     if (framework_stream_id_set_.find(stream_buffer.stream_id) ==
325         framework_stream_id_set_.end()) {
326       status_t res = internal_stream_manager_->ReturnStreamBuffer(stream_buffer);
327       if (res != OK) {
328         ALOGE("%s: Failed to return stream buffer.", __FUNCTION__);
329         return UNKNOWN_ERROR;
330       }
331     } else {
332       modified_output_buffers.push_back(stream_buffer);
333     }
334   }
335   result->output_buffers = modified_output_buffers;
336   return OK;
337 }
338 
CheckFenceStatus(CaptureRequest * request)339 status_t RgbirdResultRequestProcessor::CheckFenceStatus(CaptureRequest* request) {
340   int fence_status = 0;
341 
342   if (request == nullptr) {
343     ALOGE("%s: request is null.", __FUNCTION__);
344     return UNKNOWN_ERROR;
345   }
346 
347   for (uint32_t i = 0; i < request->output_buffers.size(); i++) {
348     if (request->output_buffers[i].acquire_fence != nullptr) {
349       auto fence =
350           const_cast<native_handle_t*>(request->output_buffers[i].acquire_fence);
351       if (fence->numFds == 1) {
352         fence_status = sync_wait(fence->data[0], kSyncWaitTime);
353       }
354       if (0 != fence_status) {
355         ALOGE("%s: Fence check failed.", __FUNCTION__);
356         return UNKNOWN_ERROR;
357       }
358       native_handle_close(fence);
359       native_handle_delete(fence);
360       request->output_buffers[i].acquire_fence = nullptr;
361     }
362   }
363 
364   return OK;
365 }
366 
IsAutocalMetadataReadyLocked(const HalCameraMetadata & metadata)367 bool RgbirdResultRequestProcessor::IsAutocalMetadataReadyLocked(
368     const HalCameraMetadata& metadata) {
369   camera_metadata_ro_entry entry = {};
370   if (metadata.Get(VendorTagIds::kNonWarpedCropRegion, &entry) != OK) {
371     ALOGV("%s Get kNonWarpedCropRegion, tag fail.", __FUNCTION__);
372     return false;
373   }
374 
375   uint8_t fd_mode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
376   if (metadata.Get(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry) != OK) {
377     ALOGV("%s Get ANDROID_STATISTICS_FACE_DETECT_MODE tag fail.", __FUNCTION__);
378     return false;
379   } else {
380     fd_mode = *entry.data.u8;
381   }
382 
383   // If FD mode is off, don't need to check FD related metadata.
384   if (fd_mode != ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) {
385     if (metadata.Get(ANDROID_STATISTICS_FACE_RECTANGLES, &entry) != OK) {
386       ALOGV("%s Get ANDROID_STATISTICS_FACE_RECTANGLES tag fail.", __FUNCTION__);
387       return false;
388     }
389     if (metadata.Get(ANDROID_STATISTICS_FACE_SCORES, &entry) != OK) {
390       ALOGV("%s Get ANDROID_STATISTICS_FACE_SCORES tag fail.", __FUNCTION__);
391       return false;
392     }
393   }
394 
395   return true;
396 }
397 
VerifyAndSubmitDepthRequest(uint32_t frame_number)398 status_t RgbirdResultRequestProcessor::VerifyAndSubmitDepthRequest(
399     uint32_t frame_number) {
400   std::lock_guard<std::mutex> lock(depth_requests_mutex_);
401   if (depth_requests_.find(frame_number) == depth_requests_.end()) {
402     ALOGW("%s: Can not find depth request with frame number %u", __FUNCTION__,
403           frame_number);
404     return NAME_NOT_FOUND;
405   }
406 
407   uint32_t valid_input_buffer_num = 0;
408   auto& depth_request = depth_requests_[frame_number];
409   for (auto& input_buffer : depth_request->input_buffers) {
410     if (input_buffer.stream_id != kInvalidStreamId) {
411       valid_input_buffer_num++;
412     }
413   }
414 
415   if (IsAutocalRequest(frame_number)) {
416     if (valid_input_buffer_num != /*rgb+ir1+ir2*/ 3) {
417       // not all input buffers are ready, early return properly
418       ALOGV("%s: Not all input buffers are ready for frame %u", __FUNCTION__,
419             frame_number);
420       return OK;
421     }
422   } else {
423     // The input buffer for RGB pipeline could be a place holder to be
424     // consistent with the input buffer metadata.
425     if (valid_input_buffer_num != /*ir1+ir2*/ 2) {
426       // not all input buffers are ready, early return properly
427       ALOGV("%s: Not all input buffers are ready for frame %u", __FUNCTION__,
428             frame_number);
429       return OK;
430     }
431   }
432 
433   if (depth_request->input_buffer_metadata.empty()) {
434     // input buffer metadata is not ready(cloned) yet, early return properly
435     ALOGV("%s: Input buffer metadata is not ready for frame %u", __FUNCTION__,
436           frame_number);
437     return OK;
438   }
439 
440   // Check against all metadata needed before move on e.g. check against
441   // cropping info, FD result for internal YUV stream
442   status_t res = OK;
443   if (IsAutocalRequest(frame_number)) {
444     bool is_ready = false;
445     for (auto& metadata : depth_request->input_buffer_metadata) {
446       if (metadata != nullptr) {
447         is_ready = IsAutocalMetadataReadyLocked(*(metadata.get()));
448       }
449     }
450     if (!is_ready) {
451       ALOGV("%s: Not all AutoCal Metadata is ready for frame %u.", __FUNCTION__,
452             frame_number);
453       return OK;
454     }
455   }
456 
457   res = CheckFenceStatus(depth_request.get());
458   if (res != OK) {
459     ALOGE("%s:Fence status wait failed.", __FUNCTION__);
460     return UNKNOWN_ERROR;
461   }
462 
463   res = ProcessRequest(*depth_request.get());
464   if (res != OK) {
465     ALOGE("%s: Failed to submit process request to depth process block.",
466           __FUNCTION__);
467     return UNKNOWN_ERROR;
468   }
469 
470   depth_requests_.erase(frame_number);
471   return OK;
472 }
473 
TrySubmitDepthProcessBlockRequest(const ProcessBlockResult & block_result)474 status_t RgbirdResultRequestProcessor::TrySubmitDepthProcessBlockRequest(
475     const ProcessBlockResult& block_result) {
476   ATRACE_CALL();
477   uint32_t request_id = block_result.request_id;
478   CaptureResult* result = block_result.result.get();
479   uint32_t frame_number = result->frame_number;
480 
481   bool pending_request_updated = false;
482   for (auto& output_buffer : result->output_buffers) {
483     if (request_id == kIr1CameraId || request_id == kIr2CameraId ||
484         (request_id == kRgbCameraId &&
485          rgb_internal_yuv_stream_id_ == output_buffer.stream_id &&
486          IsAutocalRequest(frame_number))) {
487       std::lock_guard<std::mutex> lock(depth_requests_mutex_);
488 
489       // In case depth request is flushed
490       if (depth_requests_.find(frame_number) == depth_requests_.end()) {
491         ALOGV("%s: Can not find depth request with frame number %u",
492               __FUNCTION__, frame_number);
493         status_t res =
494             internal_stream_manager_->ReturnStreamBuffer(output_buffer);
495         if (res != OK) {
496           ALOGW(
497               "%s: Failed to return internal buffer for flushed depth request"
498               " %u",
499               __FUNCTION__, frame_number);
500         }
501         continue;
502       }
503 
504       // If input_buffer_metadata is not empty, the RGB pipeline result metadata
505       // must have been cloned(other entries for IRs set to nullptr). The
506       // yuv_internal_stream buffer has to be inserted into the corresponding
507       // entry in input_buffers. Or if this is not a AutoCal request, the stream
508       // id for the place holder of the RGB input buffer must be invalid. Refer
509       // the logic below for result metadata handling.
510       const auto& metadata_list =
511           depth_requests_[frame_number]->input_buffer_metadata;
512       auto& input_buffers = depth_requests_[frame_number]->input_buffers;
513       if (!metadata_list.empty()) {
514         uint32_t rgb_metadata_index = 0;
515         for (; rgb_metadata_index < metadata_list.size(); rgb_metadata_index++) {
516           // Only the RGB pipeline result metadata is needed and cloned
517           if (metadata_list[rgb_metadata_index] != nullptr) {
518             break;
519           }
520         }
521 
522         if (rgb_metadata_index == metadata_list.size()) {
523           ALOGE("%s: RGB result metadata not found. rgb_metadata_index %u",
524                 __FUNCTION__, rgb_metadata_index);
525           return UNKNOWN_ERROR;
526         }
527 
528         if (input_buffers.size() < kNumOfAutoCalInputBuffers) {
529           input_buffers.resize(kNumOfAutoCalInputBuffers);
530         }
531 
532         if (request_id == kRgbCameraId) {
533           if (input_buffers[rgb_metadata_index].stream_id != kInvalidStreamId) {
534             ALOGE("%s: YUV buffer already exists.", __FUNCTION__);
535             return UNKNOWN_ERROR;
536           }
537           input_buffers[rgb_metadata_index] = output_buffer;
538         } else {
539           for (uint32_t i_buffer = 0; i_buffer < input_buffers.size();
540                i_buffer++) {
541             if (input_buffers[i_buffer].stream_id == kInvalidStreamId &&
542                 rgb_metadata_index != i_buffer) {
543               input_buffers[i_buffer] = output_buffer;
544               break;
545             }
546           }
547         }
548       } else {
549         input_buffers.push_back(output_buffer);
550       }
551       pending_request_updated = true;
552     }
553   }
554 
555   if (result->result_metadata != nullptr && request_id == kRgbCameraId) {
556     std::lock_guard<std::mutex> lock(depth_requests_mutex_);
557 
558     // In case a depth request is flushed
559     if (depth_requests_.find(frame_number) == depth_requests_.end()) {
560       ALOGV("%s No depth request for Autocal", __FUNCTION__);
561       return OK;
562     }
563 
564     // If YUV buffer exists in the input_buffers, the RGB pipeline metadata
565     // needs to be inserted into the corresponding entry in
566     // input_buffer_metadata. Otherwise, insert the RGB pipeline metadata into
567     // the entry that is not reserved for any existing IR input buffer. Refer
568     // above logic for input buffer preparation.
569     auto& input_buffers = depth_requests_[frame_number]->input_buffers;
570     auto& metadata_list = depth_requests_[frame_number]->input_buffer_metadata;
571     metadata_list.resize(kNumOfAutoCalInputBuffers);
572     uint32_t yuv_buffer_index = 0;
573     for (; yuv_buffer_index < input_buffers.size(); yuv_buffer_index++) {
574       if (input_buffers[yuv_buffer_index].stream_id ==
575           rgb_internal_yuv_stream_id_) {
576         break;
577       }
578     }
579 
580     if (yuv_buffer_index >= kNumOfAutoCalInputBuffers) {
581       ALOGE("%s: input_buffers is full and YUV buffer not found.", __FUNCTION__);
582       return UNKNOWN_ERROR;
583     }
584 
585     metadata_list[yuv_buffer_index] =
586         HalCameraMetadata::Clone(result->result_metadata.get());
587     if (metadata_list[yuv_buffer_index] == nullptr) {
588       ALOGE("%s: clone RGB pipeline result metadata failed.", __FUNCTION__);
589       return UNKNOWN_ERROR;
590     }
591     pending_request_updated = true;
592 
593     // If metadata arrives after all IR buffers and there is not RGB buffer
594     if (input_buffers.size() < kNumOfAutoCalInputBuffers) {
595       input_buffers.resize(kNumOfAutoCalInputBuffers);
596     }
597   }
598 
599   if (pending_request_updated) {
600     status_t res = VerifyAndSubmitDepthRequest(frame_number);
601     if (res != OK) {
602       ALOGE("%s: Failed to verify and submit depth request.", __FUNCTION__);
603       return res;
604     }
605   }
606 
607   return OK;
608 }
609 
ProcessResult(ProcessBlockResult block_result)610 void RgbirdResultRequestProcessor::ProcessResult(ProcessBlockResult block_result) {
611   ATRACE_CALL();
612   std::lock_guard<std::mutex> lock(callback_lock_);
613   if (block_result.result == nullptr) {
614     ALOGW("%s: Received a nullptr result.", __FUNCTION__);
615     return;
616   }
617 
618   if (process_capture_result_ == nullptr) {
619     ALOGE("%s: process_capture_result_ is nullptr. Dropping a result.",
620           __FUNCTION__);
621     return;
622   }
623 
624   CaptureResult* result = block_result.result.get();
625 
626   bool has_internal_stream_buffer = false;
627   if (is_hdrplus_supported_) {
628     ProcessResultForHdrplus(result, &has_internal_stream_buffer);
629   } else if (depth_stream_id_ != -1) {
630     TryReturnInternalBufferForDepth(result, &has_internal_stream_buffer);
631   }
632 
633   status_t res = OK;
634   if (result->result_metadata) {
635     res = hal_utils::SetEnableZslMetadata(result->result_metadata.get(), false);
636     if (res != OK) {
637       ALOGW("%s: SetEnableZslMetadata (%d) fail", __FUNCTION__,
638             result->frame_number);
639     }
640   }
641 
642   // Don't send result to framework if only internal raw callback
643   if (has_internal_stream_buffer && result->result_metadata == nullptr &&
644       result->output_buffers.size() == 0 && result->input_buffers.size() == 0) {
645     return;
646   }
647 
648   // TODO(b/128633958): remove the following once FLL syncing is verified
649   {
650     std::lock_guard<std::mutex> lock(depth_requests_mutex_);
651     if (((force_internal_stream_) ||
652          (depth_requests_.find(result->frame_number) == depth_requests_.end())) &&
653         (depth_stream_id_ != -1)) {
654       res = ReturnInternalStreams(result);
655       if (res != OK) {
656         ALOGE("%s: Failed to return internal buffers.", __FUNCTION__);
657         return;
658       }
659     }
660   }
661 
662   // Save necessary data for depth process block request
663   res = TrySubmitDepthProcessBlockRequest(block_result);
664   if (res != OK) {
665     ALOGE("%s: Failed to submit depth process block request.", __FUNCTION__);
666     return;
667   }
668 
669   if (block_result.request_id != kRgbCameraId) {
670     return;
671   }
672 
673   // If internal yuv stream remains in the result output buffer list, it must
674   // be used by some other purposes and will be returned separately. It should
675   // not be returned through the process_capture_result_. So we remove them here.
676   if (!result->output_buffers.empty()) {
677     auto iter = result->output_buffers.begin();
678     while (iter != result->output_buffers.end()) {
679       if (iter->stream_id == rgb_internal_yuv_stream_id_) {
680         result->output_buffers.erase(iter);
681         break;
682       }
683       iter++;
684     }
685   }
686 
687   process_capture_result_(std::move(block_result.result));
688 }
689 
Notify(const ProcessBlockNotifyMessage & block_message)690 void RgbirdResultRequestProcessor::Notify(
691     const ProcessBlockNotifyMessage& block_message) {
692   ATRACE_CALL();
693   std::lock_guard<std::mutex> lock(callback_lock_);
694   if (notify_ == nullptr) {
695     ALOGE("%s: notify_ is nullptr. Dropping a message.", __FUNCTION__);
696     return;
697   }
698 
699   const NotifyMessage& message = block_message.message;
700   // Request ID is set to camera ID by RgbirdRtRequestProcessor.
701   uint32_t camera_id = block_message.request_id;
702   if (message.type == MessageType::kShutter && camera_id != kRgbCameraId) {
703     // Only send out shutters from the lead camera.
704     return;
705   }
706 
707   notify_(block_message.message);
708 }
709 
ConfigureStreams(InternalStreamManager * internal_stream_manager,const StreamConfiguration & stream_config,StreamConfiguration * process_block_stream_config)710 status_t RgbirdResultRequestProcessor::ConfigureStreams(
711     InternalStreamManager* internal_stream_manager,
712     const StreamConfiguration& stream_config,
713     StreamConfiguration* process_block_stream_config) {
714   ATRACE_CALL();
715   if (process_block_stream_config == nullptr) {
716     ALOGE("%s: process_block_stream_config is null.", __FUNCTION__);
717     return BAD_VALUE;
718   }
719 
720   if (internal_stream_manager == nullptr) {
721     ALOGE("%s: internal_stream_manager is null.", __FUNCTION__);
722     return BAD_VALUE;
723   }
724   internal_stream_manager_ = internal_stream_manager;
725 
726   if (is_hdrplus_supported_) {
727     return OK;
728   }
729 
730   process_block_stream_config->streams.clear();
731   Stream depth_stream = {};
732   for (auto& stream : stream_config.streams) {
733     // stream_config passed to this ConfigureStreams must contain only framework
734     // output and internal input streams
735     if (stream.stream_type == StreamType::kOutput) {
736       if (utils::IsDepthStream(stream)) {
737         ALOGI("%s: Depth stream id: %u observed by RgbirdResReqProcessor.",
738               __FUNCTION__, stream.id);
739         depth_stream_id_ = stream.id;
740         depth_stream = stream;
741       }
742       // record all framework output, save depth only for depth process block
743       framework_stream_id_set_.insert(stream.id);
744     } else if (stream.stream_type == StreamType::kInput) {
745       process_block_stream_config->streams.push_back(stream);
746     }
747   }
748 
749   // TODO(b/128633958): remove force flag after FLL syncing is verified
750   if (force_internal_stream_ || depth_stream_id_ != -1) {
751     process_block_stream_config->streams.push_back(depth_stream);
752     process_block_stream_config->operation_mode = stream_config.operation_mode;
753     process_block_stream_config->session_params =
754         HalCameraMetadata::Clone(stream_config.session_params.get());
755     process_block_stream_config->stream_config_counter =
756         stream_config.stream_config_counter;
757   }
758   process_block_stream_config->log_id = stream_config.log_id;
759 
760   return OK;
761 }
762 
SetProcessBlock(std::unique_ptr<ProcessBlock> process_block)763 status_t RgbirdResultRequestProcessor::SetProcessBlock(
764     std::unique_ptr<ProcessBlock> process_block) {
765   ATRACE_CALL();
766   if (process_block == nullptr) {
767     ALOGE("%s: process_block is nullptr", __FUNCTION__);
768     return BAD_VALUE;
769   }
770 
771   std::lock_guard<std::mutex> lock(depth_process_block_lock_);
772   if (depth_process_block_ != nullptr) {
773     ALOGE("%s: Already configured.", __FUNCTION__);
774     return ALREADY_EXISTS;
775   }
776 
777   depth_process_block_ = std::move(process_block);
778   return OK;
779 }
780 
ProcessRequest(const CaptureRequest & request)781 status_t RgbirdResultRequestProcessor::ProcessRequest(
782     const CaptureRequest& request) {
783   ATRACE_CALL();
784   std::lock_guard<std::mutex> lock(depth_process_block_lock_);
785   if (depth_process_block_ == nullptr) {
786     ALOGE("%s: depth_process_block_ is null.", __FUNCTION__);
787     return BAD_VALUE;
788   }
789 
790   // Depth Process Block only handles one process block request each time
791   std::vector<ProcessBlockRequest> process_block_requests(1);
792   auto& block_request = process_block_requests[0];
793   block_request.request_id = 0;
794   CaptureRequest& physical_request = block_request.request;
795   physical_request.frame_number = request.frame_number;
796   physical_request.settings = HalCameraMetadata::Clone(request.settings.get());
797   for (auto& metadata : request.input_buffer_metadata) {
798     physical_request.input_buffer_metadata.emplace_back(
799         HalCameraMetadata::Clone(metadata.get()));
800   }
801   physical_request.input_buffers = request.input_buffers;
802   physical_request.output_buffers = request.output_buffers;
803 
804   return depth_process_block_->ProcessRequests(process_block_requests, request);
805 }
806 
Flush()807 status_t RgbirdResultRequestProcessor::Flush() {
808   ATRACE_CALL();
809 
810   std::lock_guard<std::mutex> lock(depth_process_block_lock_);
811   if (depth_process_block_ == nullptr) {
812     ALOGW("%s: depth_process_block_ is null.", __FUNCTION__);
813     return OK;
814   }
815 
816   return depth_process_block_->Flush();
817 }
818 
FlushPendingRequests()819 status_t RgbirdResultRequestProcessor::FlushPendingRequests() {
820   ATRACE_CALL();
821 
822   std::lock_guard<std::mutex> lock(callback_lock_);
823   if (notify_ == nullptr) {
824     ALOGE("%s: notify_ is nullptr. Dropping a message.", __FUNCTION__);
825     return OK;
826   }
827 
828   if (process_capture_result_ == nullptr) {
829     ALOGE("%s: process_capture_result_ is nullptr. Dropping a result.",
830           __FUNCTION__);
831     return OK;
832   }
833 
834   std::lock_guard<std::mutex> requests_lock(depth_requests_mutex_);
835   for (auto& [frame_number, capture_request] : depth_requests_) {
836     // Returns all internal stream buffers
837     for (auto& input_buffer : capture_request->input_buffers) {
838       if (input_buffer.stream_id != kInvalidStreamId) {
839         status_t res =
840             internal_stream_manager_->ReturnStreamBuffer(input_buffer);
841         if (res != OK) {
842           ALOGW("%s: Failed to return internal buffer for depth request %d",
843                 __FUNCTION__, frame_number);
844         }
845       }
846     }
847 
848     // Notify buffer error for the depth stream output buffer
849     const NotifyMessage message = {
850         .type = MessageType::kError,
851         .message.error = {.frame_number = frame_number,
852                           .error_stream_id = depth_stream_id_,
853                           .error_code = ErrorCode::kErrorBuffer}};
854     notify_(message);
855 
856     // Return output buffer for the depth stream
857     auto result = std::make_unique<CaptureResult>();
858     result->frame_number = frame_number;
859     for (auto& output_buffer : capture_request->output_buffers) {
860       if (output_buffer.stream_id == depth_stream_id_) {
861         result->output_buffers.push_back(output_buffer);
862         auto& buffer = result->output_buffers.back();
863         buffer.status = BufferStatus::kError;
864         buffer.acquire_fence = nullptr;
865         buffer.release_fence = nullptr;
866         break;
867       }
868     }
869     process_capture_result_(std::move(result));
870   }
871   depth_requests_.clear();
872   ALOGI("%s: Flushing depth requests done. ", __FUNCTION__);
873   return OK;
874 }
875 
876 }  // namespace google_camera_hal
877 }  // namespace android
878