1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "GCH_RgbirdResultRequestProcessor"
19 #define ATRACE_TAG ATRACE_TAG_CAMERA
20 #include <cutils/properties.h>
21 #include <log/log.h>
22 #include <sync/sync.h>
23 #include <utils/Trace.h>
24 
25 #include <cutils/native_handle.h>
26 #include <inttypes.h>
27 
28 #include "hal_utils.h"
29 #include "rgbird_result_request_processor.h"
30 
31 namespace android {
32 namespace google_camera_hal {
33 
Create(const RgbirdResultRequestProcessorCreateData & create_data)34 std::unique_ptr<RgbirdResultRequestProcessor> RgbirdResultRequestProcessor::Create(
35     const RgbirdResultRequestProcessorCreateData& create_data) {
36   ATRACE_CALL();
37   auto result_processor = std::unique_ptr<RgbirdResultRequestProcessor>(
38       new RgbirdResultRequestProcessor(create_data));
39   if (result_processor == nullptr) {
40     ALOGE("%s: Creating RgbirdResultRequestProcessor failed.", __FUNCTION__);
41     return nullptr;
42   }
43 
44   // TODO(b/128633958): remove this after FLL syncing is verified
45   result_processor->force_internal_stream_ =
46       property_get_bool("persist.vendor.camera.rgbird.forceinternal", false);
47   if (result_processor->force_internal_stream_) {
48     ALOGI("%s: Force creating internal streams for IR pipelines", __FUNCTION__);
49   }
50 
51   // TODO(b/129910835): Change the controlling prop into some deterministic
52   // logic that controls when the front depth autocal will be triggered.
53   result_processor->rgb_ir_auto_cal_enabled_ =
54       property_get_bool("vendor.camera.frontdepth.enableautocal", true);
55   if (result_processor->rgb_ir_auto_cal_enabled_) {
56     ALOGI("%s: autocal is enabled.", __FUNCTION__);
57   }
58 
59   return result_processor;
60 }
61 
RgbirdResultRequestProcessor(const RgbirdResultRequestProcessorCreateData & create_data)62 RgbirdResultRequestProcessor::RgbirdResultRequestProcessor(
63     const RgbirdResultRequestProcessorCreateData& create_data)
64     : kRgbCameraId(create_data.rgb_camera_id),
65       kIr1CameraId(create_data.ir1_camera_id),
66       kIr2CameraId(create_data.ir2_camera_id),
67       rgb_raw_stream_id_(create_data.rgb_raw_stream_id),
68       is_hdrplus_supported_(create_data.is_hdrplus_supported),
69       rgb_internal_yuv_stream_id_(create_data.rgb_internal_yuv_stream_id) {
70 }
71 
SetResultCallback(ProcessCaptureResultFunc process_capture_result,NotifyFunc notify)72 void RgbirdResultRequestProcessor::SetResultCallback(
73     ProcessCaptureResultFunc process_capture_result, NotifyFunc notify) {
74   std::lock_guard<std::mutex> lock(callback_lock_);
75   process_capture_result_ = process_capture_result;
76   notify_ = notify;
77 }
78 
SaveFdForHdrplus(const CaptureRequest & request)79 void RgbirdResultRequestProcessor::SaveFdForHdrplus(
80     const CaptureRequest& request) {
81   // Enable face detect mode for internal use
82   if (request.settings != nullptr) {
83     uint8_t fd_mode;
84     status_t res = hal_utils::GetFdMode(request, &fd_mode);
85     if (res == OK) {
86       current_face_detect_mode_ = fd_mode;
87     }
88   }
89 
90   {
91     std::lock_guard<std::mutex> lock(face_detect_lock_);
92     requested_face_detect_modes_.emplace(request.frame_number,
93                                          current_face_detect_mode_);
94   }
95 }
96 
SaveLsForHdrplus(const CaptureRequest & request)97 void RgbirdResultRequestProcessor::SaveLsForHdrplus(
98     const CaptureRequest& request) {
99   if (request.settings != nullptr) {
100     uint8_t lens_shading_map_mode;
101     status_t res =
102         hal_utils::GetLensShadingMapMode(request, &lens_shading_map_mode);
103     if (res == OK) {
104       current_lens_shading_map_mode_ = lens_shading_map_mode;
105     }
106   }
107 
108   {
109     std::lock_guard<std::mutex> lock(lens_shading_lock_);
110     requested_lens_shading_map_modes_.emplace(request.frame_number,
111                                               current_lens_shading_map_mode_);
112   }
113 }
114 
HandleLsResultForHdrplus(uint32_t frameNumber,HalCameraMetadata * metadata)115 status_t RgbirdResultRequestProcessor::HandleLsResultForHdrplus(
116     uint32_t frameNumber, HalCameraMetadata* metadata) {
117   if (metadata == nullptr) {
118     ALOGE("%s: metadata is nullptr", __FUNCTION__);
119     return BAD_VALUE;
120   }
121   std::lock_guard<std::mutex> lock(lens_shading_lock_);
122   auto iter = requested_lens_shading_map_modes_.find(frameNumber);
123   if (iter == requested_lens_shading_map_modes_.end()) {
124     ALOGW("%s: can't find frame (%d)", __FUNCTION__, frameNumber);
125     return OK;
126   }
127 
128   if (iter->second == ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_OFF) {
129     status_t res = hal_utils::RemoveLsInfoFromResult(metadata);
130     if (res != OK) {
131       ALOGW("%s: RemoveLsInfoFromResult fail", __FUNCTION__);
132     }
133   }
134   requested_lens_shading_map_modes_.erase(iter);
135 
136   return OK;
137 }
138 
IsAutocalRequest(uint32_t frame_number) const139 bool RgbirdResultRequestProcessor::IsAutocalRequest(uint32_t frame_number) const {
140   // TODO(b/129910835): Use the proper logic to control when internal yuv buffer
141   // needs to be passed to the depth process block. Even if the auto cal is
142   // enabled, there is no need to pass the internal yuv buffer for every
143   // request, not even every device session. This is also related to how the
144   // buffer is added into the request. Similar logic exists in realtime request
145   // processor. However, this logic can further filter and determine which
146   // requests contain the internal yuv stream buffers and send them to the depth
147   // process block. Current implementation only treat the kAutocalFrameNumber
148   // request as autocal request. This must be consistent with that of the
149   // rt_request_processor.
150   if (!rgb_ir_auto_cal_enabled_) {
151     return false;
152   }
153 
154   return frame_number == kAutocalFrameNumber;
155 }
156 
TryReturnInternalBufferForDepth(CaptureResult * result,bool * has_internal)157 void RgbirdResultRequestProcessor::TryReturnInternalBufferForDepth(
158     CaptureResult* result, bool* has_internal) {
159   ATRACE_CALL();
160   if (result == nullptr || has_internal == nullptr) {
161     ALOGE("%s: result or has_rgb_raw_output is nullptr", __FUNCTION__);
162     return;
163   }
164 
165   if (internal_stream_manager_ == nullptr) {
166     ALOGE("%s: internal_stream_manager_ nullptr", __FUNCTION__);
167     return;
168   }
169 
170   std::vector<StreamBuffer> modified_output_buffers;
171   for (uint32_t i = 0; i < result->output_buffers.size(); i++) {
172     if (rgb_internal_yuv_stream_id_ == result->output_buffers[i].stream_id &&
173         !IsAutocalRequest(result->frame_number)) {
174       *has_internal = true;
175       status_t res = internal_stream_manager_->ReturnStreamBuffer(
176           result->output_buffers[i]);
177       if (res != OK) {
178         ALOGW("%s: Failed to return RGB internal raw buffer for frame %d",
179               __FUNCTION__, result->frame_number);
180       }
181     } else {
182       modified_output_buffers.push_back(result->output_buffers[i]);
183     }
184   }
185 
186   if (!result->output_buffers.empty()) {
187     result->output_buffers = modified_output_buffers;
188   }
189 }
190 
HandleFdResultForHdrplus(uint32_t frameNumber,HalCameraMetadata * metadata)191 status_t RgbirdResultRequestProcessor::HandleFdResultForHdrplus(
192     uint32_t frameNumber, HalCameraMetadata* metadata) {
193   if (metadata == nullptr) {
194     ALOGE("%s: metadata is nullptr", __FUNCTION__);
195     return BAD_VALUE;
196   }
197   std::lock_guard<std::mutex> lock(face_detect_lock_);
198   auto iter = requested_face_detect_modes_.find(frameNumber);
199   if (iter == requested_face_detect_modes_.end()) {
200     ALOGW("%s: can't find frame (%d)", __FUNCTION__, frameNumber);
201     return OK;
202   }
203 
204   if (iter->second == ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) {
205     status_t res = hal_utils::RemoveFdInfoFromResult(metadata);
206     if (res != OK) {
207       ALOGW("%s: RestoreFdMetadataForHdrplus fail", __FUNCTION__);
208     }
209   }
210   requested_face_detect_modes_.erase(iter);
211 
212   return OK;
213 }
214 
AddPendingRequests(const std::vector<ProcessBlockRequest> &,const CaptureRequest & remaining_session_request)215 status_t RgbirdResultRequestProcessor::AddPendingRequests(
216     const std::vector<ProcessBlockRequest>& /*process_block_requests*/,
217     const CaptureRequest& remaining_session_request) {
218   ATRACE_CALL();
219   std::lock_guard<std::mutex> lock(depth_requests_mutex_);
220   for (auto stream_buffer : remaining_session_request.output_buffers) {
221     if (stream_buffer.acquire_fence != nullptr) {
222       stream_buffer.acquire_fence =
223           native_handle_clone(stream_buffer.acquire_fence);
224       if (stream_buffer.acquire_fence == nullptr) {
225         ALOGE("%s: Cloning acquire_fence of buffer failed", __FUNCTION__);
226         return UNKNOWN_ERROR;
227       }
228     }
229     if (depth_stream_id_ == stream_buffer.stream_id) {
230       ALOGV("%s: request %d has a depth buffer", __FUNCTION__,
231             remaining_session_request.frame_number);
232       auto capture_request = std::make_unique<CaptureRequest>();
233       capture_request->frame_number = remaining_session_request.frame_number;
234       if (remaining_session_request.settings != nullptr) {
235         capture_request->settings =
236             HalCameraMetadata::Clone(remaining_session_request.settings.get());
237       }
238       capture_request->input_buffers.clear();
239       capture_request->output_buffers.push_back(stream_buffer);
240       depth_requests_.emplace(remaining_session_request.frame_number,
241                               std::move(capture_request));
242       break;
243     }
244   }
245 
246   if (is_hdrplus_supported_) {
247     SaveFdForHdrplus(remaining_session_request);
248     SaveLsForHdrplus(remaining_session_request);
249   }
250   return OK;
251 }
252 
ProcessResultForHdrplus(CaptureResult * result,bool * rgb_raw_output)253 void RgbirdResultRequestProcessor::ProcessResultForHdrplus(CaptureResult* result,
254                                                            bool* rgb_raw_output) {
255   ATRACE_CALL();
256   if (result == nullptr || rgb_raw_output == nullptr) {
257     ALOGE("%s: result or rgb_raw_output is nullptr", __FUNCTION__);
258     return;
259   }
260 
261   if (internal_stream_manager_ == nullptr) {
262     ALOGE("%s: internal_stream_manager_ nullptr", __FUNCTION__);
263     return;
264   }
265 
266   // Return filled raw buffer to internal stream manager
267   // And remove raw buffer from result
268   status_t res;
269   std::vector<StreamBuffer> modified_output_buffers;
270   for (uint32_t i = 0; i < result->output_buffers.size(); i++) {
271     if (rgb_raw_stream_id_ == result->output_buffers[i].stream_id) {
272       *rgb_raw_output = true;
273       res = internal_stream_manager_->ReturnFilledBuffer(
274           result->frame_number, result->output_buffers[i]);
275       if (res != OK) {
276         ALOGW("%s: (%d)ReturnStreamBuffer fail", __FUNCTION__,
277               result->frame_number);
278       }
279     } else {
280       modified_output_buffers.push_back(result->output_buffers[i]);
281     }
282   }
283 
284   if (result->output_buffers.size() > 0) {
285     result->output_buffers = modified_output_buffers;
286   }
287 
288   if (result->result_metadata) {
289     res = internal_stream_manager_->ReturnMetadata(
290         rgb_raw_stream_id_, result->frame_number, result->result_metadata.get());
291     if (res != OK) {
292       ALOGW("%s: (%d)ReturnMetadata fail", __FUNCTION__, result->frame_number);
293     }
294 
295     res = HandleFdResultForHdrplus(result->frame_number,
296                                    result->result_metadata.get());
297     if (res != OK) {
298       ALOGE("%s: HandleFdResultForHdrplus(%d) fail", __FUNCTION__,
299             result->frame_number);
300       return;
301     }
302 
303     res = HandleLsResultForHdrplus(result->frame_number,
304                                    result->result_metadata.get());
305     if (res != OK) {
306       ALOGE("%s: HandleLsResultForHdrplus(%d) fail", __FUNCTION__,
307             result->frame_number);
308       return;
309     }
310   }
311 }
312 
ReturnInternalStreams(CaptureResult * result)313 status_t RgbirdResultRequestProcessor::ReturnInternalStreams(
314     CaptureResult* result) {
315   ATRACE_CALL();
316   if (result == nullptr) {
317     ALOGE("%s: block_result is null.", __FUNCTION__);
318     return UNKNOWN_ERROR;
319   }
320 
321   std::vector<StreamBuffer> modified_output_buffers;
322   for (auto& stream_buffer : result->output_buffers) {
323     if (framework_stream_id_set_.find(stream_buffer.stream_id) ==
324         framework_stream_id_set_.end()) {
325       status_t res = internal_stream_manager_->ReturnStreamBuffer(stream_buffer);
326       if (res != OK) {
327         ALOGE("%s: Failed to return stream buffer.", __FUNCTION__);
328         return UNKNOWN_ERROR;
329       }
330     } else {
331       modified_output_buffers.push_back(stream_buffer);
332     }
333   }
334   result->output_buffers = modified_output_buffers;
335   return OK;
336 }
337 
CheckFenceStatus(CaptureRequest * request)338 status_t RgbirdResultRequestProcessor::CheckFenceStatus(CaptureRequest* request) {
339   int fence_status = 0;
340 
341   if (request == nullptr) {
342     ALOGE("%s: request is null.", __FUNCTION__);
343     return UNKNOWN_ERROR;
344   }
345 
346   for (uint32_t i = 0; i < request->output_buffers.size(); i++) {
347     if (request->output_buffers[i].acquire_fence != nullptr) {
348       auto fence =
349           const_cast<native_handle_t*>(request->output_buffers[i].acquire_fence);
350       if (fence->numFds == 1) {
351         fence_status = sync_wait(fence->data[0], kSyncWaitTime);
352       }
353       if (0 != fence_status) {
354         ALOGE("%s: Fence check failed.", __FUNCTION__);
355         return UNKNOWN_ERROR;
356       }
357       native_handle_close(fence);
358       native_handle_delete(fence);
359       request->output_buffers[i].acquire_fence = nullptr;
360     }
361   }
362 
363   return OK;
364 }
365 
IsAutocalMetadataReadyLocked(const HalCameraMetadata & metadata)366 bool RgbirdResultRequestProcessor::IsAutocalMetadataReadyLocked(
367     const HalCameraMetadata& metadata) {
368   camera_metadata_ro_entry entry = {};
369   if (metadata.Get(VendorTagIds::kNonWarpedCropRegion, &entry) != OK) {
370     ALOGV("%s Get kNonWarpedCropRegion, tag fail.", __FUNCTION__);
371     return false;
372   }
373 
374   uint8_t fd_mode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF;
375   if (metadata.Get(ANDROID_STATISTICS_FACE_DETECT_MODE, &entry) != OK) {
376     ALOGV("%s Get ANDROID_STATISTICS_FACE_DETECT_MODE tag fail.", __FUNCTION__);
377     return false;
378   } else {
379     fd_mode = *entry.data.u8;
380   }
381 
382   // If FD mode is off, don't need to check FD related metadata.
383   if (fd_mode != ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) {
384     if (metadata.Get(ANDROID_STATISTICS_FACE_RECTANGLES, &entry) != OK) {
385       ALOGV("%s Get ANDROID_STATISTICS_FACE_RECTANGLES tag fail.", __FUNCTION__);
386       return false;
387     }
388     if (metadata.Get(ANDROID_STATISTICS_FACE_SCORES, &entry) != OK) {
389       ALOGV("%s Get ANDROID_STATISTICS_FACE_SCORES tag fail.", __FUNCTION__);
390       return false;
391     }
392   }
393 
394   return true;
395 }
396 
VerifyAndSubmitDepthRequest(uint32_t frame_number)397 status_t RgbirdResultRequestProcessor::VerifyAndSubmitDepthRequest(
398     uint32_t frame_number) {
399   std::lock_guard<std::mutex> lock(depth_requests_mutex_);
400   if (depth_requests_.find(frame_number) == depth_requests_.end()) {
401     ALOGW("%s: Can not find depth request with frame number %u", __FUNCTION__,
402           frame_number);
403     return NAME_NOT_FOUND;
404   }
405 
406   uint32_t valid_input_buffer_num = 0;
407   auto& depth_request = depth_requests_[frame_number];
408   for (auto& input_buffer : depth_request->input_buffers) {
409     if (input_buffer.stream_id != kInvalidStreamId) {
410       valid_input_buffer_num++;
411     }
412   }
413 
414   if (IsAutocalRequest(frame_number)) {
415     if (valid_input_buffer_num != /*rgb+ir1+ir2*/ 3) {
416       // not all input buffers are ready, early return properly
417       ALOGV("%s: Not all input buffers are ready for frame %u", __FUNCTION__,
418             frame_number);
419       return OK;
420     }
421   } else {
422     // The input buffer for RGB pipeline could be a place holder to be
423     // consistent with the input buffer metadata.
424     if (valid_input_buffer_num != /*ir1+ir2*/ 2) {
425       // not all input buffers are ready, early return properly
426       ALOGV("%s: Not all input buffers are ready for frame %u", __FUNCTION__,
427             frame_number);
428       return OK;
429     }
430   }
431 
432   if (depth_request->input_buffer_metadata.empty()) {
433     // input buffer metadata is not ready(cloned) yet, early return properly
434     ALOGV("%s: Input buffer metadata is not ready for frame %u", __FUNCTION__,
435           frame_number);
436     return OK;
437   }
438 
439   // Check against all metadata needed before move on e.g. check against
440   // cropping info, FD result for internal YUV stream
441   status_t res = OK;
442   if (IsAutocalRequest(frame_number)) {
443     bool is_ready = false;
444     for (auto& metadata : depth_request->input_buffer_metadata) {
445       if (metadata != nullptr) {
446         is_ready = IsAutocalMetadataReadyLocked(*(metadata.get()));
447       }
448     }
449     if (!is_ready) {
450       ALOGV("%s: Not all AutoCal Metadata is ready for frame %u.", __FUNCTION__,
451             frame_number);
452       return OK;
453     }
454   }
455 
456   res = CheckFenceStatus(depth_request.get());
457   if (res != OK) {
458     ALOGE("%s:Fence status wait failed.", __FUNCTION__);
459     return UNKNOWN_ERROR;
460   }
461 
462   res = ProcessRequest(*depth_request.get());
463   if (res != OK) {
464     ALOGE("%s: Failed to submit process request to depth process block.",
465           __FUNCTION__);
466     return UNKNOWN_ERROR;
467   }
468 
469   depth_requests_.erase(frame_number);
470   return OK;
471 }
472 
TrySubmitDepthProcessBlockRequest(const ProcessBlockResult & block_result)473 status_t RgbirdResultRequestProcessor::TrySubmitDepthProcessBlockRequest(
474     const ProcessBlockResult& block_result) {
475   ATRACE_CALL();
476   uint32_t request_id = block_result.request_id;
477   CaptureResult* result = block_result.result.get();
478   uint32_t frame_number = result->frame_number;
479 
480   bool pending_request_updated = false;
481   for (auto& output_buffer : result->output_buffers) {
482     if (request_id == kIr1CameraId || request_id == kIr2CameraId ||
483         (request_id == kRgbCameraId &&
484          rgb_internal_yuv_stream_id_ == output_buffer.stream_id &&
485          IsAutocalRequest(frame_number))) {
486       std::lock_guard<std::mutex> lock(depth_requests_mutex_);
487 
488       // In case depth request is flushed
489       if (depth_requests_.find(frame_number) == depth_requests_.end()) {
490         ALOGV("%s: Can not find depth request with frame number %u",
491               __FUNCTION__, frame_number);
492         status_t res =
493             internal_stream_manager_->ReturnStreamBuffer(output_buffer);
494         if (res != OK) {
495           ALOGW(
496               "%s: Failed to return internal buffer for flushed depth request"
497               " %u",
498               __FUNCTION__, frame_number);
499         }
500         continue;
501       }
502 
503       // If input_buffer_metadata is not empty, the RGB pipeline result metadata
504       // must have been cloned(other entries for IRs set to nullptr). The
505       // yuv_internal_stream buffer has to be inserted into the corresponding
506       // entry in input_buffers. Or if this is not a AutoCal request, the stream
507       // id for the place holder of the RGB input buffer must be invalid. Refer
508       // the logic below for result metadata handling.
509       const auto& metadata_list =
510           depth_requests_[frame_number]->input_buffer_metadata;
511       auto& input_buffers = depth_requests_[frame_number]->input_buffers;
512       if (!metadata_list.empty()) {
513         uint32_t rgb_metadata_index = 0;
514         for (; rgb_metadata_index < metadata_list.size(); rgb_metadata_index++) {
515           // Only the RGB pipeline result metadata is needed and cloned
516           if (metadata_list[rgb_metadata_index] != nullptr) {
517             break;
518           }
519         }
520 
521         if (rgb_metadata_index == metadata_list.size()) {
522           ALOGE("%s: RGB result metadata not found. rgb_metadata_index %u",
523                 __FUNCTION__, rgb_metadata_index);
524           return UNKNOWN_ERROR;
525         }
526 
527         if (input_buffers.size() < kNumOfAutoCalInputBuffers) {
528           input_buffers.resize(kNumOfAutoCalInputBuffers);
529         }
530 
531         if (request_id == kRgbCameraId) {
532           if (input_buffers[rgb_metadata_index].stream_id != kInvalidStreamId) {
533             ALOGE("%s: YUV buffer already exists.", __FUNCTION__);
534             return UNKNOWN_ERROR;
535           }
536           input_buffers[rgb_metadata_index] = output_buffer;
537         } else {
538           for (uint32_t i_buffer = 0; i_buffer < input_buffers.size();
539                i_buffer++) {
540             if (input_buffers[i_buffer].stream_id == kInvalidStreamId &&
541                 rgb_metadata_index != i_buffer) {
542               input_buffers[i_buffer] = output_buffer;
543               break;
544             }
545           }
546         }
547       } else {
548         input_buffers.push_back(output_buffer);
549       }
550       pending_request_updated = true;
551     }
552   }
553 
554   if (result->result_metadata != nullptr && request_id == kRgbCameraId) {
555     std::lock_guard<std::mutex> lock(depth_requests_mutex_);
556 
557     // In case a depth request is flushed
558     if (depth_requests_.find(frame_number) == depth_requests_.end()) {
559       ALOGV("%s No depth request for Autocal", __FUNCTION__);
560       return OK;
561     }
562 
563     // If YUV buffer exists in the input_buffers, the RGB pipeline metadata
564     // needs to be inserted into the corresponding entry in
565     // input_buffer_metadata. Otherwise, insert the RGB pipeline metadata into
566     // the entry that is not reserved for any existing IR input buffer. Refer
567     // above logic for input buffer preparation.
568     auto& input_buffers = depth_requests_[frame_number]->input_buffers;
569     auto& metadata_list = depth_requests_[frame_number]->input_buffer_metadata;
570     metadata_list.resize(kNumOfAutoCalInputBuffers);
571     uint32_t yuv_buffer_index = 0;
572     for (; yuv_buffer_index < input_buffers.size(); yuv_buffer_index++) {
573       if (input_buffers[yuv_buffer_index].stream_id ==
574           rgb_internal_yuv_stream_id_) {
575         break;
576       }
577     }
578 
579     if (yuv_buffer_index >= kNumOfAutoCalInputBuffers) {
580       ALOGE("%s: input_buffers is full and YUV buffer not found.", __FUNCTION__);
581       return UNKNOWN_ERROR;
582     }
583 
584     metadata_list[yuv_buffer_index] =
585         HalCameraMetadata::Clone(result->result_metadata.get());
586     if (metadata_list[yuv_buffer_index] == nullptr) {
587       ALOGE("%s: clone RGB pipeline result metadata failed.", __FUNCTION__);
588       return UNKNOWN_ERROR;
589     }
590     pending_request_updated = true;
591 
592     // If metadata arrives after all IR buffers and there is not RGB buffer
593     if (input_buffers.size() < kNumOfAutoCalInputBuffers) {
594       input_buffers.resize(kNumOfAutoCalInputBuffers);
595     }
596   }
597 
598   if (pending_request_updated) {
599     status_t res = VerifyAndSubmitDepthRequest(frame_number);
600     if (res != OK) {
601       ALOGE("%s: Failed to verify and submit depth request.", __FUNCTION__);
602       return res;
603     }
604   }
605 
606   return OK;
607 }
608 
ProcessResult(ProcessBlockResult block_result)609 void RgbirdResultRequestProcessor::ProcessResult(ProcessBlockResult block_result) {
610   ATRACE_CALL();
611   std::lock_guard<std::mutex> lock(callback_lock_);
612   if (block_result.result == nullptr) {
613     ALOGW("%s: Received a nullptr result.", __FUNCTION__);
614     return;
615   }
616 
617   if (process_capture_result_ == nullptr) {
618     ALOGE("%s: process_capture_result_ is nullptr. Dropping a result.",
619           __FUNCTION__);
620     return;
621   }
622 
623   CaptureResult* result = block_result.result.get();
624 
625   bool has_internal_stream_buffer = false;
626   if (is_hdrplus_supported_) {
627     ProcessResultForHdrplus(result, &has_internal_stream_buffer);
628   } else if (depth_stream_id_ != -1) {
629     TryReturnInternalBufferForDepth(result, &has_internal_stream_buffer);
630   }
631 
632   status_t res = OK;
633   if (result->result_metadata) {
634     res = hal_utils::SetEnableZslMetadata(result->result_metadata.get(), false);
635     if (res != OK) {
636       ALOGW("%s: SetEnableZslMetadata (%d) fail", __FUNCTION__,
637             result->frame_number);
638     }
639   }
640 
641   // Don't send result to framework if only internal raw callback
642   if (has_internal_stream_buffer && result->result_metadata == nullptr &&
643       result->output_buffers.size() == 0 && result->input_buffers.size() == 0) {
644     return;
645   }
646 
647   // TODO(b/128633958): remove the following once FLL syncing is verified
648   {
649     std::lock_guard<std::mutex> lock(depth_requests_mutex_);
650     if (((force_internal_stream_) ||
651          (depth_requests_.find(result->frame_number) == depth_requests_.end())) &&
652         (depth_stream_id_ != -1)) {
653       res = ReturnInternalStreams(result);
654       if (res != OK) {
655         ALOGE("%s: Failed to return internal buffers.", __FUNCTION__);
656         return;
657       }
658     }
659   }
660 
661   // Save necessary data for depth process block request
662   res = TrySubmitDepthProcessBlockRequest(block_result);
663   if (res != OK) {
664     ALOGE("%s: Failed to submit depth process block request.", __FUNCTION__);
665     return;
666   }
667 
668   if (block_result.request_id != kRgbCameraId) {
669     return;
670   }
671 
672   // If internal yuv stream remains in the result output buffer list, it must
673   // be used by some other purposes and will be returned separately. It should
674   // not be returned through the process_capture_result_. So we remove them here.
675   if (!result->output_buffers.empty()) {
676     auto iter = result->output_buffers.begin();
677     while (iter != result->output_buffers.end()) {
678       if (iter->stream_id == rgb_internal_yuv_stream_id_) {
679         result->output_buffers.erase(iter);
680         break;
681       }
682       iter++;
683     }
684   }
685 
686   process_capture_result_(std::move(block_result.result));
687 }
688 
Notify(const ProcessBlockNotifyMessage & block_message)689 void RgbirdResultRequestProcessor::Notify(
690     const ProcessBlockNotifyMessage& block_message) {
691   ATRACE_CALL();
692   std::lock_guard<std::mutex> lock(callback_lock_);
693   if (notify_ == nullptr) {
694     ALOGE("%s: notify_ is nullptr. Dropping a message.", __FUNCTION__);
695     return;
696   }
697 
698   const NotifyMessage& message = block_message.message;
699   // Request ID is set to camera ID by RgbirdRtRequestProcessor.
700   uint32_t camera_id = block_message.request_id;
701   if (message.type == MessageType::kShutter && camera_id != kRgbCameraId) {
702     // Only send out shutters from the lead camera.
703     return;
704   }
705 
706   notify_(block_message.message);
707 }
708 
ConfigureStreams(InternalStreamManager * internal_stream_manager,const StreamConfiguration & stream_config,StreamConfiguration * process_block_stream_config)709 status_t RgbirdResultRequestProcessor::ConfigureStreams(
710     InternalStreamManager* internal_stream_manager,
711     const StreamConfiguration& stream_config,
712     StreamConfiguration* process_block_stream_config) {
713   ATRACE_CALL();
714   if (process_block_stream_config == nullptr) {
715     ALOGE("%s: process_block_stream_config is null.", __FUNCTION__);
716     return BAD_VALUE;
717   }
718 
719   if (internal_stream_manager == nullptr) {
720     ALOGE("%s: internal_stream_manager is null.", __FUNCTION__);
721     return BAD_VALUE;
722   }
723   internal_stream_manager_ = internal_stream_manager;
724 
725   if (is_hdrplus_supported_) {
726     return OK;
727   }
728 
729   process_block_stream_config->streams.clear();
730   Stream depth_stream = {};
731   for (auto& stream : stream_config.streams) {
732     // stream_config passed to this ConfigureStreams must contain only framework
733     // output and internal input streams
734     if (stream.stream_type == StreamType::kOutput) {
735       if (utils::IsDepthStream(stream)) {
736         ALOGI("%s: Depth stream id: %u observed by RgbirdResReqProcessor.",
737               __FUNCTION__, stream.id);
738         depth_stream_id_ = stream.id;
739         depth_stream = stream;
740       }
741       // record all framework output, save depth only for depth process block
742       framework_stream_id_set_.insert(stream.id);
743     } else if (stream.stream_type == StreamType::kInput) {
744       process_block_stream_config->streams.push_back(stream);
745     }
746   }
747 
748   // TODO(b/128633958): remove force flag after FLL syncing is verified
749   if (force_internal_stream_ || depth_stream_id_ != -1) {
750     process_block_stream_config->streams.push_back(depth_stream);
751     process_block_stream_config->operation_mode = stream_config.operation_mode;
752     process_block_stream_config->session_params =
753         HalCameraMetadata::Clone(stream_config.session_params.get());
754     process_block_stream_config->stream_config_counter =
755         stream_config.stream_config_counter;
756   }
757 
758   return OK;
759 }
760 
SetProcessBlock(std::unique_ptr<ProcessBlock> process_block)761 status_t RgbirdResultRequestProcessor::SetProcessBlock(
762     std::unique_ptr<ProcessBlock> process_block) {
763   ATRACE_CALL();
764   if (process_block == nullptr) {
765     ALOGE("%s: process_block is nullptr", __FUNCTION__);
766     return BAD_VALUE;
767   }
768 
769   std::lock_guard<std::mutex> lock(depth_process_block_lock_);
770   if (depth_process_block_ != nullptr) {
771     ALOGE("%s: Already configured.", __FUNCTION__);
772     return ALREADY_EXISTS;
773   }
774 
775   depth_process_block_ = std::move(process_block);
776   return OK;
777 }
778 
ProcessRequest(const CaptureRequest & request)779 status_t RgbirdResultRequestProcessor::ProcessRequest(
780     const CaptureRequest& request) {
781   ATRACE_CALL();
782   std::lock_guard<std::mutex> lock(depth_process_block_lock_);
783   if (depth_process_block_ == nullptr) {
784     ALOGE("%s: depth_process_block_ is null.", __FUNCTION__);
785     return BAD_VALUE;
786   }
787 
788   // Depth Process Block only handles one process block request each time
789   std::vector<ProcessBlockRequest> process_block_requests(1);
790   auto& block_request = process_block_requests[0];
791   block_request.request_id = 0;
792   CaptureRequest& physical_request = block_request.request;
793   physical_request.frame_number = request.frame_number;
794   physical_request.settings = HalCameraMetadata::Clone(request.settings.get());
795   for (auto& metadata : request.input_buffer_metadata) {
796     physical_request.input_buffer_metadata.emplace_back(
797         HalCameraMetadata::Clone(metadata.get()));
798   }
799   physical_request.input_buffers = request.input_buffers;
800   physical_request.output_buffers = request.output_buffers;
801 
802   return depth_process_block_->ProcessRequests(process_block_requests, request);
803 }
804 
Flush()805 status_t RgbirdResultRequestProcessor::Flush() {
806   ATRACE_CALL();
807 
808   std::lock_guard<std::mutex> lock(depth_process_block_lock_);
809   if (depth_process_block_ == nullptr) {
810     ALOGW("%s: depth_process_block_ is null.", __FUNCTION__);
811     return OK;
812   }
813 
814   return depth_process_block_->Flush();
815 }
816 
FlushPendingRequests()817 status_t RgbirdResultRequestProcessor::FlushPendingRequests() {
818   ATRACE_CALL();
819 
820   std::lock_guard<std::mutex> lock(callback_lock_);
821   if (notify_ == nullptr) {
822     ALOGE("%s: notify_ is nullptr. Dropping a message.", __FUNCTION__);
823     return OK;
824   }
825 
826   if (process_capture_result_ == nullptr) {
827     ALOGE("%s: process_capture_result_ is nullptr. Dropping a result.",
828           __FUNCTION__);
829     return OK;
830   }
831 
832   std::lock_guard<std::mutex> requests_lock(depth_requests_mutex_);
833   for (auto& [frame_number, capture_request] : depth_requests_) {
834     // Returns all internal stream buffers
835     for (auto& input_buffer : capture_request->input_buffers) {
836       if (input_buffer.stream_id != kInvalidStreamId) {
837         status_t res =
838             internal_stream_manager_->ReturnStreamBuffer(input_buffer);
839         if (res != OK) {
840           ALOGW("%s: Failed to return internal buffer for depth request %d",
841                 __FUNCTION__, frame_number);
842         }
843       }
844     }
845 
846     // Notify buffer error for the depth stream output buffer
847     const NotifyMessage message = {
848         .type = MessageType::kError,
849         .message.error = {.frame_number = frame_number,
850                           .error_stream_id = depth_stream_id_,
851                           .error_code = ErrorCode::kErrorBuffer}};
852     notify_(message);
853 
854     // Return output buffer for the depth stream
855     auto result = std::make_unique<CaptureResult>();
856     result->frame_number = frame_number;
857     for (auto& output_buffer : capture_request->output_buffers) {
858       if (output_buffer.stream_id == depth_stream_id_) {
859         result->output_buffers.push_back(output_buffer);
860         auto& buffer = result->output_buffers.back();
861         buffer.status = BufferStatus::kError;
862         buffer.acquire_fence = nullptr;
863         buffer.release_fence = nullptr;
864         break;
865       }
866     }
867     process_capture_result_(std::move(result));
868   }
869   depth_requests_.clear();
870   ALOGI("%s: Flushing depth requests done. ", __FUNCTION__);
871   return OK;
872 }
873 
874 }  // namespace google_camera_hal
875 }  // namespace android
876