1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "GCH_DepthProcessBlock"
19 #define ATRACE_TAG ATRACE_TAG_CAMERA
20 #include <cutils/properties.h>
21 #include <hardware/gralloc1.h>
22 #include <log/log.h>
23 #include <sys/mman.h>
24 #include <utils/Trace.h>
25 
26 #include <dlfcn.h>
27 
28 #include "depth_process_block.h"
29 #include "hal_types.h"
30 #include "hal_utils.h"
31 #include "result_processor.h"
32 
33 namespace android {
34 namespace google_camera_hal {
35 
36 #if GCH_HWL_USE_DLOPEN
37 static std::string kDepthGeneratorLib = "/vendor/lib64/libdepthgenerator.so";
38 using android::depth_generator::CreateDepthGenerator_t;
39 #endif
40 const float kSmallOffset = 0.01f;
41 
Create(CameraDeviceSessionHwl * device_session_hwl,HwlRequestBuffersFunc request_stream_buffers,const DepthProcessBlockCreateData & create_data)42 std::unique_ptr<DepthProcessBlock> DepthProcessBlock::Create(
43     CameraDeviceSessionHwl* device_session_hwl,
44     HwlRequestBuffersFunc request_stream_buffers,
45     const DepthProcessBlockCreateData& create_data) {
46   ATRACE_CALL();
47   if (device_session_hwl == nullptr) {
48     ALOGE("%s: device_session_hwl is nullptr", __FUNCTION__);
49     return nullptr;
50   }
51 
52   auto block = std::unique_ptr<DepthProcessBlock>(
53       new DepthProcessBlock(request_stream_buffers, create_data));
54   if (block == nullptr) {
55     ALOGE("%s: Creating DepthProcessBlock failed.", __FUNCTION__);
56     return nullptr;
57   }
58 
59   status_t res = block->InitializeBufferManagementStatus(device_session_hwl);
60   if (res != OK) {
61     ALOGE("%s: Failed to initialize HAL Buffer Management status.",
62           __FUNCTION__);
63     return nullptr;
64   }
65 
66   res = block->CalculateActiveArraySizeRatio(device_session_hwl);
67   if (res != OK) {
68     ALOGE("%s: Calculating active array size ratio failed.", __FUNCTION__);
69     return nullptr;
70   }
71 
72   // TODO(b/128633958): remove this after FLL syncing is verified
73   block->force_internal_stream_ =
74       property_get_bool("persist.vendor.camera.rgbird.forceinternal", false);
75   if (block->force_internal_stream_) {
76     ALOGI("%s: Force creating internal streams for IR pipelines", __FUNCTION__);
77   }
78 
79   block->pipelined_depth_engine_enabled_ = property_get_bool(
80       "persist.vendor.camera.frontdepth.enablepipeline", true);
81 
82   // TODO(b/129910835): Change the controlling prop into some deterministic
83   // logic that controls when the front depth autocal will be triggered.
84   // depth_process_block does not control autocal in current implementation.
85   // Whenever there is a YUV buffer in the process block request, it will
86   // trigger the AutoCal. So the condition is completely controlled by
87   // rt_request_processor and result_request_processor.
88   block->rgb_ir_auto_cal_enabled_ =
89       property_get_bool("vendor.camera.frontdepth.enableautocal", true);
90   block->device_session_hwl_ = device_session_hwl;
91   return block;
92 }
93 
InitializeBufferManagementStatus(CameraDeviceSessionHwl * device_session_hwl)94 status_t DepthProcessBlock::InitializeBufferManagementStatus(
95     CameraDeviceSessionHwl* device_session_hwl) {
96   // Query characteristics to check if buffer management supported
97   std::unique_ptr<google_camera_hal::HalCameraMetadata> characteristics;
98   status_t res = device_session_hwl->GetCameraCharacteristics(&characteristics);
99   if (res != OK) {
100     ALOGE("%s: Get camera characteristics failed: %s(%d)", __FUNCTION__,
101           strerror(-res), res);
102     return res;
103   }
104 
105   camera_metadata_ro_entry entry = {};
106   res = characteristics->Get(ANDROID_INFO_SUPPORTED_BUFFER_MANAGEMENT_VERSION,
107                              &entry);
108   if (res == OK && entry.count > 0) {
109     buffer_management_used_ =
110         (entry.data.u8[0] ==
111          ANDROID_INFO_SUPPORTED_BUFFER_MANAGEMENT_VERSION_HIDL_DEVICE_3_5);
112     session_buffer_management_supported_ =
113         (entry.data.u8[0] ==
114          ANDROID_INFO_SUPPORTED_BUFFER_MANAGEMENT_VERSION_SESSION_CONFIGURABLE);
115   }
116 
117   return OK;
118 }
119 
DepthProcessBlock(HwlRequestBuffersFunc request_stream_buffers,const DepthProcessBlockCreateData & create_data)120 DepthProcessBlock::DepthProcessBlock(
121     HwlRequestBuffersFunc request_stream_buffers,
122     const DepthProcessBlockCreateData& create_data)
123     : request_stream_buffers_(request_stream_buffers),
124       rgb_internal_yuv_stream_id_(create_data.rgb_internal_yuv_stream_id),
125       ir1_internal_raw_stream_id_(create_data.ir1_internal_raw_stream_id),
126       ir2_internal_raw_stream_id_(create_data.ir2_internal_raw_stream_id) {
127 }
128 
~DepthProcessBlock()129 DepthProcessBlock::~DepthProcessBlock() {
130   ATRACE_CALL();
131   depth_generator_ = nullptr;
132 
133   if (depth_generator_lib_handle_ != nullptr) {
134     dlclose(depth_generator_lib_handle_);
135     depth_generator_lib_handle_ = nullptr;
136   }
137 }
138 
SetResultProcessor(std::unique_ptr<ResultProcessor> result_processor)139 status_t DepthProcessBlock::SetResultProcessor(
140     std::unique_ptr<ResultProcessor> result_processor) {
141   ATRACE_CALL();
142   if (result_processor == nullptr) {
143     ALOGE("%s: result_processor is nullptr", __FUNCTION__);
144     return BAD_VALUE;
145   }
146 
147   std::lock_guard<std::mutex> lock(result_processor_lock_);
148   if (result_processor_ != nullptr) {
149     ALOGE("%s: result_processor_ was already set.", __FUNCTION__);
150     return ALREADY_EXISTS;
151   }
152 
153   result_processor_ = std::move(result_processor);
154   return OK;
155 }
156 
GetStreamBufferSize(const Stream & stream,int32_t * buffer_size)157 status_t DepthProcessBlock::GetStreamBufferSize(const Stream& stream,
158                                                 int32_t* buffer_size) {
159   ATRACE_CALL();
160   // TODO(b/130764929): Use actual gralloc buffer stride instead of stream dim
161   switch (stream.format) {
162     case HAL_PIXEL_FORMAT_Y8:
163       *buffer_size = stream.width * stream.height;
164       break;
165     case HAL_PIXEL_FORMAT_Y16:
166       *buffer_size = stream.width * stream.height * 2;
167       break;
168     case HAL_PIXEL_FORMAT_YCBCR_420_888:
169       *buffer_size = static_cast<int32_t>(stream.width * stream.height * 1.5);
170       break;
171     default:
172       ALOGW("%s: Unsupported format:%d", __FUNCTION__, stream.format);
173       *buffer_size = 0;
174       break;
175   }
176 
177   return OK;
178 }
179 
ConfigureStreams(const StreamConfiguration & stream_config,const StreamConfiguration &)180 status_t DepthProcessBlock::ConfigureStreams(
181     const StreamConfiguration& stream_config,
182     const StreamConfiguration& /*overall_config*/) {
183   ATRACE_CALL();
184   std::lock_guard<std::mutex> lock(configure_lock_);
185   if (is_configured_) {
186     ALOGE("%s: Already configured.", __FUNCTION__);
187     return ALREADY_EXISTS;
188   }
189 
190   // TODO(b/128633958): remove this after FLL syncing is verified
191   if (force_internal_stream_) {
192     // Nothing to configure if this is force internal mode
193     ALOGV("%s: Force internal enabled, skip depth block config.", __FUNCTION__);
194     is_configured_ = true;
195     return OK;
196   }
197 
198   uint32_t num_depth_stream = 0;
199   for (auto& stream : stream_config.streams) {
200     if (utils::IsDepthStream(stream)) {
201       num_depth_stream++;
202       // Save depth stream as HAL configured stream
203       depth_stream_.id = stream.id;
204       depth_stream_.override_format = stream.format;
205       depth_stream_.producer_usage = GRALLOC1_PRODUCER_USAGE_CAMERA;
206       depth_stream_.consumer_usage = 0;
207       depth_stream_.max_buffers = kDepthStreamMaxBuffers;
208       depth_stream_.override_data_space = stream.data_space;
209       depth_stream_.is_physical_camera_stream = false;
210       depth_stream_.physical_camera_id = 0;
211     }
212 
213     // Save stream information for mapping purposes
214     depth_io_streams_[stream.id] = stream;
215     int32_t buffer_size = 0;
216     status_t res = GetStreamBufferSize(stream, &buffer_size);
217     if (res != OK) {
218       ALOGE("%s: Failed to get stream buffer size.", __FUNCTION__);
219       return res;
220     }
221     stream_buffer_sizes_[stream.id] = buffer_size;
222   }
223 
224   if (num_depth_stream != 1) {
225     ALOGE(
226         "%s: Depth Process Block can only config 1 depth stream. There are "
227         "%zu streams, including %u depth stream.",
228         __FUNCTION__, stream_config.streams.size(), num_depth_stream);
229     return BAD_VALUE;
230   }
231 
232   if (depth_generator_ == nullptr) {
233     status_t res = LoadDepthGenerator(&depth_generator_);
234     if (res != OK) {
235       ALOGE("%s: Creating DepthGenerator failed.", __FUNCTION__);
236       return NO_INIT;
237     }
238 
239     if (pipelined_depth_engine_enabled_ == true) {
240       auto depth_result_callback =
241           android::depth_generator::DepthResultCallbackFunction(
242               [this](DepthResultStatus result_status, uint32_t frame_number) {
243                 status_t res = ProcessDepthResult(result_status, frame_number);
244                 if (res != OK) {
245                   ALOGE("%s: Failed to process the depth result for frame %d.",
246                         __FUNCTION__, frame_number);
247                 }
248               });
249       ALOGI("%s: Async depth api is used. Callback func is set.", __FUNCTION__);
250       depth_generator_->SetResultCallback(depth_result_callback);
251     } else {
252       ALOGI("%s: Blocking depth api is used.", __FUNCTION__);
253       depth_generator_->SetResultCallback(nullptr);
254     }
255   }
256   if (session_buffer_management_supported_ &&
257       device_session_hwl_->configure_streams_v2()) {
258     hal_buffer_managed_streams_ =
259         device_session_hwl_->GetHalBufferManagedStreams(stream_config);
260   }
261   is_configured_ = true;
262   return OK;
263 }
264 
GetConfiguredHalStreams(std::vector<HalStream> * hal_streams) const265 status_t DepthProcessBlock::GetConfiguredHalStreams(
266     std::vector<HalStream>* hal_streams) const {
267   ATRACE_CALL();
268   std::lock_guard<std::mutex> lock(configure_lock_);
269   if (hal_streams == nullptr) {
270     ALOGE("%s: hal_streams is nullptr.", __FUNCTION__);
271     return BAD_VALUE;
272   }
273 
274   if (!is_configured_) {
275     ALOGE("%s: Not configured yet.", __FUNCTION__);
276     return NO_INIT;
277   }
278 
279   hal_streams->push_back(depth_stream_);
280 
281   return OK;
282 }
283 
SubmitBlockingDepthRequest(const DepthRequestInfo & request_info)284 status_t DepthProcessBlock::SubmitBlockingDepthRequest(
285     const DepthRequestInfo& request_info) {
286   ALOGV("%s: [ud] ExecuteProcessRequest for frame %d", __FUNCTION__,
287         request_info.frame_number);
288 
289   status_t res = depth_generator_->ExecuteProcessRequest(request_info);
290   if (res != OK) {
291     ALOGE("%s: Depth generator fails to process frame %d.", __FUNCTION__,
292           request_info.frame_number);
293     return res;
294   }
295 
296   res = ProcessDepthResult(DepthResultStatus::kOk, request_info.frame_number);
297   if (res != OK) {
298     ALOGE("%s: Failed to process depth result.", __FUNCTION__);
299     return res;
300   }
301 
302   return OK;
303 }
304 
SubmitAsyncDepthRequest(const DepthRequestInfo & request_info)305 status_t DepthProcessBlock::SubmitAsyncDepthRequest(
306     const DepthRequestInfo& request_info) {
307   std::unique_lock<std::mutex> lock(depth_generator_api_lock_);
308   ALOGV("%s: [ud] ExecuteProcessRequest for frame %d", __FUNCTION__,
309         request_info.frame_number);
310   status_t res = depth_generator_->EnqueueProcessRequest(request_info);
311   if (res != OK) {
312     ALOGE("%s: Failed to enqueue depth request.", __FUNCTION__);
313     return res;
314   }
315 
316   return OK;
317 }
318 
ProcessDepthResult(DepthResultStatus result_status,uint32_t frame_number)319 status_t DepthProcessBlock::ProcessDepthResult(DepthResultStatus result_status,
320                                                uint32_t frame_number) {
321   std::unique_lock<std::mutex> lock(depth_generator_api_lock_);
322   ALOGV("%s: [ud] Depth result for frame %u notified.", __FUNCTION__,
323         frame_number);
324 
325   status_t res = UnmapDepthRequestBuffers(frame_number);
326   if (res != OK) {
327     ALOGE("%s: Failed to clean up the depth request info.", __FUNCTION__);
328     return res;
329   }
330 
331   auto capture_result = std::make_unique<CaptureResult>();
332   if (capture_result == nullptr) {
333     ALOGE("%s: Creating capture_result failed.", __FUNCTION__);
334     return NO_MEMORY;
335   }
336 
337   CaptureRequest request;
338   {
339     std::lock_guard<std::mutex> pending_request_lock(pending_requests_mutex_);
340     if (pending_depth_requests_.find(frame_number) ==
341         pending_depth_requests_.end()) {
342       ALOGE("%s: Frame %u does not exist in pending requests list.",
343             __FUNCTION__, frame_number);
344     } else {
345       auto& request = pending_depth_requests_[frame_number].request;
346       capture_result->frame_number = frame_number;
347       capture_result->output_buffers = request.output_buffers;
348 
349       // In case the depth engine fails to process a depth request, mark the
350       // buffer as in error state.
351       if (result_status != DepthResultStatus::kOk) {
352         for (auto& stream_buffer : capture_result->output_buffers) {
353           if (stream_buffer.stream_id == depth_stream_.id) {
354             stream_buffer.status = BufferStatus::kError;
355           }
356         }
357       }
358 
359       capture_result->input_buffers = request.input_buffers;
360       pending_depth_requests_.erase(frame_number);
361     }
362   }
363 
364   ProcessBlockResult block_result = {.request_id = 0,
365                                      .result = std::move(capture_result)};
366   {
367     std::lock_guard<std::mutex> lock(result_processor_lock_);
368     result_processor_->ProcessResult(std::move(block_result));
369   }
370 
371   return OK;
372 }
373 
ProcessRequests(const std::vector<ProcessBlockRequest> & process_block_requests,const CaptureRequest & remaining_session_request)374 status_t DepthProcessBlock::ProcessRequests(
375     const std::vector<ProcessBlockRequest>& process_block_requests,
376     const CaptureRequest& remaining_session_request) {
377   ATRACE_CALL();
378   // TODO(b/128633958): remove this after FLL syncing is verified
379   if (force_internal_stream_) {
380     // Nothing to configure if this is force internal mode
381     ALOGE("%s: Force internal ON, Depth PB should not process request.",
382           __FUNCTION__);
383     return UNKNOWN_ERROR;
384   }
385 
386   std::lock_guard<std::mutex> lock(configure_lock_);
387   if (!is_configured_) {
388     ALOGE("%s: block is not configured.", __FUNCTION__);
389     return NO_INIT;
390   }
391 
392   if (process_block_requests.size() != 1) {
393     ALOGE("%s: Only a single request is supported but there are %zu",
394           __FUNCTION__, process_block_requests.size());
395     return BAD_VALUE;
396   }
397 
398   {
399     std::lock_guard<std::mutex> lock(result_processor_lock_);
400     if (result_processor_ == nullptr) {
401       ALOGE("%s: result processor was not set.", __FUNCTION__);
402       return NO_INIT;
403     }
404 
405     status_t res = result_processor_->AddPendingRequests(
406         process_block_requests, remaining_session_request);
407     if (res != OK) {
408       ALOGE("%s: Adding a pending request to result processor failed: %s(%d)",
409             __FUNCTION__, strerror(-res), res);
410       return res;
411     }
412   }
413 
414   auto& request = process_block_requests[0].request;
415   DepthRequestInfo request_info;
416   request_info.frame_number = request.frame_number;
417   std::unique_ptr<HalCameraMetadata> metadata = nullptr;
418   if (request.settings != nullptr) {
419     metadata = HalCameraMetadata::Clone(request.settings.get());
420   }
421 
422   std::unique_ptr<HalCameraMetadata> color_metadata = nullptr;
423   for (auto& metadata : request.input_buffer_metadata) {
424     if (metadata != nullptr) {
425       color_metadata = HalCameraMetadata::Clone(metadata.get());
426     }
427   }
428 
429   ALOGV("%s: [ud] Prepare depth request info for frame %u .", __FUNCTION__,
430         request.frame_number);
431 
432   status_t res = PrepareDepthRequestInfo(request, &request_info, metadata.get(),
433                                          color_metadata.get());
434   if (res != OK) {
435     ALOGE("%s: Failed to perpare the depth request info.", __FUNCTION__);
436     return res;
437   }
438 
439   if (pipelined_depth_engine_enabled_ == true) {
440     res = SubmitAsyncDepthRequest(request_info);
441     if (res != OK) {
442       ALOGE("%s: Failed to submit asynchronized depth request.", __FUNCTION__);
443     }
444   } else {
445     res = SubmitBlockingDepthRequest(request_info);
446     if (res != OK) {
447       ALOGE("%s: Failed to submit blocking depth request.", __FUNCTION__);
448     }
449   }
450 
451   return OK;
452 }
453 
Flush()454 status_t DepthProcessBlock::Flush() {
455   ATRACE_CALL();
456   std::lock_guard<std::mutex> lock(configure_lock_);
457   if (!is_configured_) {
458     return OK;
459   }
460 
461   // TODO(b/127322570): Implement this method.
462   return OK;
463 }
464 
LoadDepthGenerator(std::unique_ptr<DepthGenerator> * depth_generator)465 status_t DepthProcessBlock::LoadDepthGenerator(
466     std::unique_ptr<DepthGenerator>* depth_generator) {
467   ATRACE_CALL();
468 #if GCH_HWL_USE_DLOPEN
469   CreateDepthGenerator_t create_depth_generator;
470 
471   ALOGI("%s: Loading library: %s", __FUNCTION__, kDepthGeneratorLib.c_str());
472   depth_generator_lib_handle_ =
473       dlopen(kDepthGeneratorLib.c_str(), RTLD_NOW | RTLD_NODELETE);
474   if (depth_generator_lib_handle_ == nullptr) {
475     ALOGE("Depth generator loading %s failed.", kDepthGeneratorLib.c_str());
476     return NO_INIT;
477   }
478 
479   create_depth_generator = (CreateDepthGenerator_t)dlsym(
480       depth_generator_lib_handle_, "CreateDepthGenerator");
481   if (create_depth_generator == nullptr) {
482     ALOGE("%s: dlsym failed (%s).", __FUNCTION__, kDepthGeneratorLib.c_str());
483     dlclose(depth_generator_lib_handle_);
484     depth_generator_lib_handle_ = nullptr;
485     return NO_INIT;
486   }
487 
488   *depth_generator = std::unique_ptr<DepthGenerator>(create_depth_generator());
489   if (*depth_generator == nullptr) {
490     return NO_INIT;
491   }
492 #else
493   if (CreateDepthGenerator == nullptr) {
494     return NO_INIT;
495   }
496   *depth_generator = std::unique_ptr<DepthGenerator>(CreateDepthGenerator());
497 #endif
498 
499   return OK;
500 }
501 
MapBuffersForDepthGenerator(const StreamBuffer & stream_buffer,depth_generator::Buffer * buffer)502 status_t DepthProcessBlock::MapBuffersForDepthGenerator(
503     const StreamBuffer& stream_buffer, depth_generator::Buffer* buffer) {
504   ATRACE_CALL();
505   buffer_handle_t buffer_handle = stream_buffer.buffer;
506   ALOGV("%s: Mapping FD=%d to CPU addr.", __FUNCTION__, buffer_handle->data[0]);
507 
508   int32_t stream_id = stream_buffer.stream_id;
509   if (stream_buffer_sizes_.find(stream_id) == stream_buffer_sizes_.end() ||
510       depth_io_streams_.find(stream_id) == depth_io_streams_.end()) {
511     ALOGE("%s: Stream buffer stream id:%d not found.", __FUNCTION__, stream_id);
512     return UNKNOWN_ERROR;
513   }
514 
515   void* virtual_addr =
516       mmap(NULL, stream_buffer_sizes_[stream_id], (PROT_READ | PROT_WRITE),
517            MAP_SHARED, buffer_handle->data[0], 0);
518 
519   if (virtual_addr == nullptr || virtual_addr == reinterpret_cast<void*>(-1)) {
520     ALOGE("%s: Failed to map the stream buffer to virtual addr.", __FUNCTION__);
521     return UNKNOWN_ERROR;
522   }
523 
524   auto& stream = depth_io_streams_[stream_id];
525   buffer->format = stream.format;
526   buffer->width = stream.width;
527   buffer->height = stream.height;
528   depth_generator::BufferPlane buffer_plane = {};
529   buffer_plane.addr = reinterpret_cast<uint8_t*>(virtual_addr);
530   // TODO(b/130764929): Use actual gralloc buffer stride instead of stream dim
531   buffer_plane.stride = stream.width;
532   buffer_plane.scanline = stream.height;
533   buffer->planes.push_back(buffer_plane);
534 
535   return OK;
536 }
537 
UnmapBuffersForDepthGenerator(const StreamBuffer & stream_buffer,uint8_t * addr)538 status_t DepthProcessBlock::UnmapBuffersForDepthGenerator(
539     const StreamBuffer& stream_buffer, uint8_t* addr) {
540   ATRACE_CALL();
541   if (addr == nullptr) {
542     ALOGE("%s: Addr is null.", __FUNCTION__);
543     return BAD_VALUE;
544   }
545 
546   int32_t stream_id = stream_buffer.stream_id;
547   if (stream_buffer_sizes_.find(stream_id) == stream_buffer_sizes_.end() ||
548       depth_io_streams_.find(stream_id) == depth_io_streams_.end()) {
549     ALOGE("%s: Stream buffer stream id:%d not found.", __FUNCTION__, stream_id);
550     return UNKNOWN_ERROR;
551   }
552 
553   munmap(addr, stream_buffer_sizes_[stream_id]);
554   return OK;
555 }
556 
RequestDepthStreamBuffer(StreamBuffer * incomplete_buffer,uint32_t frame_number)557 status_t DepthProcessBlock::RequestDepthStreamBuffer(
558     StreamBuffer* incomplete_buffer, uint32_t frame_number) {
559   if (request_stream_buffers_ == nullptr) {
560     ALOGE("%s: request_stream_buffers_ is nullptr", __FUNCTION__);
561     return UNKNOWN_ERROR;
562   }
563 
564   std::vector<StreamBuffer> buffers;
565   {
566     status_t res = request_stream_buffers_(
567         incomplete_buffer->stream_id,
568         /* request one depth buffer each time */ 1, &buffers, frame_number);
569     if (res != OK) {
570       ALOGE("%s: Failed to request stream buffers from camera device session.",
571             __FUNCTION__);
572       return UNKNOWN_ERROR;
573     }
574   }
575 
576   *incomplete_buffer = buffers[0];
577   return OK;
578 }
579 
UpdateCropRegion(const CaptureRequest & request,DepthRequestInfo * depth_request_info,HalCameraMetadata * metadata)580 status_t DepthProcessBlock::UpdateCropRegion(const CaptureRequest& request,
581                                              DepthRequestInfo* depth_request_info,
582                                              HalCameraMetadata* metadata) {
583   if (request.settings != nullptr && metadata != nullptr) {
584     camera_metadata_ro_entry_t entry_crop_region_user = {};
585     if (request.settings->Get(ANDROID_SCALER_CROP_REGION,
586                               &entry_crop_region_user) == OK) {
587       const int32_t* crop_region = entry_crop_region_user.data.i32;
588       ALOGV("%s: Depth PB crop region[%d %d %d %d]", __FUNCTION__,
589             crop_region[0], crop_region[1], crop_region[2], crop_region[3]);
590 
591       int32_t resized_crop_region[4] = {};
592       // top
593       resized_crop_region[0] = crop_region[1] / logical_to_ir_ratio_;
594       if (resized_crop_region[0] < 0) {
595         resized_crop_region[0] = 0;
596       }
597       // left
598       resized_crop_region[1] = crop_region[0] / logical_to_ir_ratio_;
599       if (resized_crop_region[1] < 0) {
600         resized_crop_region[1] = 0;
601       }
602       // bottom
603       resized_crop_region[2] =
604           (crop_region[3] / logical_to_ir_ratio_) + resized_crop_region[0];
605       if (resized_crop_region[2] > ir_active_array_height_) {
606         resized_crop_region[2] = ir_active_array_height_;
607       }
608       // right
609       resized_crop_region[3] =
610           (crop_region[2] / logical_to_ir_ratio_) + resized_crop_region[1];
611       if (resized_crop_region[3] > ir_active_array_width_) {
612         resized_crop_region[3] = ir_active_array_width_;
613       }
614       metadata->Set(ANDROID_SCALER_CROP_REGION, resized_crop_region,
615                     sizeof(resized_crop_region) / sizeof(int32_t));
616 
617       depth_request_info->settings = metadata->GetRawCameraMetadata();
618     }
619   }
620   return OK;
621 }
622 
MapDepthRequestBuffers(const CaptureRequest & request,DepthRequestInfo * depth_request_info)623 status_t DepthProcessBlock::MapDepthRequestBuffers(
624     const CaptureRequest& request, DepthRequestInfo* depth_request_info) {
625   status_t res = OK;
626   depth_request_info->ir_buffer.resize(2);
627   for (auto& input_buffer : request.input_buffers) {
628     // If the stream id is invalid. The input buffer is only a place holder
629     // corresponding to the input buffer metadata for the rgb pipeline.
630     if (input_buffer.stream_id == kInvalidStreamId) {
631       ALOGV("%s: Skipping input buffer place holder for frame %u.",
632             __FUNCTION__, depth_request_info->frame_number);
633       continue;
634     }
635 
636     depth_generator::Buffer buffer = {};
637     res = MapBuffersForDepthGenerator(input_buffer, &buffer);
638     if (res != OK) {
639       ALOGE("%s: Mapping buffer for depth generator failed.", __FUNCTION__);
640       return UNKNOWN_ERROR;
641     }
642     const int32_t stream_id = input_buffer.stream_id;
643     if (stream_id == rgb_internal_yuv_stream_id_) {
644       // TODO(b/129910835): Triggering Condition
645       // Adjust the condition according to how rt_request_processor and
646       // result_request_processor handles the triggering condition. If they have
647       // full control of the logic and decide to pass yuv buffer only when
648       // autocal should be triggered, then the logic here can be as simple as
649       // this.
650       depth_request_info->color_buffer.push_back(buffer);
651     } else if (stream_id == ir1_internal_raw_stream_id_) {
652       depth_request_info->ir_buffer[0].push_back(buffer);
653     } else if (stream_id == ir2_internal_raw_stream_id_) {
654       depth_request_info->ir_buffer[1].push_back(buffer);
655     }
656   }
657 
658   res = MapBuffersForDepthGenerator(request.output_buffers[0],
659                                     &depth_request_info->depth_buffer);
660   if (res != OK) {
661     ALOGE("%s: Mapping depth buffer for depth generator failed.", __FUNCTION__);
662     return UNKNOWN_ERROR;
663   }
664 
665   return OK;
666 }
667 
PrepareDepthRequestInfo(const CaptureRequest & request,DepthRequestInfo * depth_request_info,HalCameraMetadata * metadata,const HalCameraMetadata * color_metadata)668 status_t DepthProcessBlock::PrepareDepthRequestInfo(
669     const CaptureRequest& request, DepthRequestInfo* depth_request_info,
670     HalCameraMetadata* metadata, const HalCameraMetadata* color_metadata) {
671   ATRACE_CALL();
672 
673   if (depth_request_info == nullptr) {
674     ALOGE("%s: depth_request_info is nullptr.", __FUNCTION__);
675     return BAD_VALUE;
676   }
677 
678   status_t res = UpdateCropRegion(request, depth_request_info, metadata);
679   if (res != OK) {
680     ALOGE("%s: Failed to update crop region.", __FUNCTION__);
681     return UNKNOWN_ERROR;
682   }
683 
684   if (color_metadata != nullptr) {
685     depth_request_info->color_buffer_metadata =
686         color_metadata->GetRawCameraMetadata();
687   }
688 
689   if (request.input_buffers.size() < 2 || request.input_buffers.size() > 3 ||
690       request.output_buffers.size() != 1) {
691     ALOGE(
692         "%s: Cannot prepare request info, input buffer size is not 2 or 3(is"
693         " %zu) or output buffer size is not 1(is %zu).",
694         __FUNCTION__, request.input_buffers.size(),
695         request.output_buffers.size());
696     return BAD_VALUE;
697   }
698   int32_t stream_id = request.output_buffers[0].stream_id;
699   if (buffer_management_used_ || (hal_buffer_managed_streams_.find(stream_id) !=
700                                   hal_buffer_managed_streams_.end())) {
701     res = RequestDepthStreamBuffer(
702         &(const_cast<CaptureRequest&>(request).output_buffers[0]),
703         request.frame_number);
704     if (res != OK) {
705       ALOGE("%s: Failed to request depth stream buffer.", __FUNCTION__);
706       return UNKNOWN_ERROR;
707     }
708   }
709 
710   res = MapDepthRequestBuffers(request, depth_request_info);
711   if (res != OK) {
712     ALOGE("%s: Failed to map buffers for depth request.", __FUNCTION__);
713     return UNKNOWN_ERROR;
714   }
715 
716   {
717     uint32_t frame_number = request.frame_number;
718     std::lock_guard<std::mutex> lock(pending_requests_mutex_);
719     if (pending_depth_requests_.find(frame_number) !=
720         pending_depth_requests_.end()) {
721       ALOGE("%s: Frame %u already exists in pending requests.", __FUNCTION__,
722             request.frame_number);
723       return UNKNOWN_ERROR;
724     } else {
725       pending_depth_requests_[frame_number] = {};
726       auto& pending_request = pending_depth_requests_[frame_number].request;
727       pending_request.frame_number = frame_number;
728       pending_request.input_buffers = request.input_buffers;
729       pending_request.output_buffers = request.output_buffers;
730       auto& pending_depth_request =
731           pending_depth_requests_[frame_number].depth_request;
732       pending_depth_request = *depth_request_info;
733     }
734   }
735 
736   return OK;
737 }
738 
UnmapDepthRequestBuffers(uint32_t frame_number)739 status_t DepthProcessBlock::UnmapDepthRequestBuffers(uint32_t frame_number) {
740   std::lock_guard<std::mutex> lock(pending_requests_mutex_);
741   if (pending_depth_requests_.find(frame_number) ==
742       pending_depth_requests_.end()) {
743     ALOGE("%s: Can not find frame %u in pending requests list.", __FUNCTION__,
744           frame_number);
745     return BAD_VALUE;
746   }
747 
748   auto& request = pending_depth_requests_[frame_number].request;
749   auto& depth_request_info = pending_depth_requests_[frame_number].depth_request;
750 
751   ATRACE_CALL();
752   if (request.input_buffers.size() < 2 || request.input_buffers.size() > 3 ||
753       request.output_buffers.size() != 1) {
754     ALOGE(
755         "%s: Cannot prepare request info, input buffer size is not 2 or 3(is "
756         "%zu) or output buffer size is not 1(is %zu).",
757         __FUNCTION__, request.input_buffers.size(),
758         request.output_buffers.size());
759     return BAD_VALUE;
760   }
761 
762   status_t res = OK;
763   for (auto& input_buffer : request.input_buffers) {
764     uint8_t* addr = nullptr;
765     int32_t stream_id = input_buffer.stream_id;
766     if (stream_id == kInvalidStreamId) {
767       ALOGV("%s: input buffer place holder found for frame %u", __FUNCTION__,
768             frame_number);
769       continue;
770     }
771 
772     if (stream_id == rgb_internal_yuv_stream_id_) {
773       addr = depth_request_info.color_buffer[0].planes[0].addr;
774     } else if (stream_id == ir1_internal_raw_stream_id_) {
775       addr = depth_request_info.ir_buffer[0][0].planes[0].addr;
776     } else if (stream_id == ir2_internal_raw_stream_id_) {
777       addr = depth_request_info.ir_buffer[1][0].planes[0].addr;
778     }
779 
780     res = UnmapBuffersForDepthGenerator(input_buffer, addr);
781     if (res != OK) {
782       ALOGE("%s: Unmapping input buffer for depth generator failed.",
783             __FUNCTION__);
784       return UNKNOWN_ERROR;
785     }
786   }
787 
788   res = UnmapBuffersForDepthGenerator(
789       request.output_buffers[0], depth_request_info.depth_buffer.planes[0].addr);
790   if (res != OK) {
791     ALOGE("%s: Unmapping depth buffer for depth generator failed.",
792           __FUNCTION__);
793     return UNKNOWN_ERROR;
794   }
795 
796   return OK;
797 }
798 
CalculateActiveArraySizeRatio(CameraDeviceSessionHwl * device_session_hwl)799 status_t DepthProcessBlock::CalculateActiveArraySizeRatio(
800     CameraDeviceSessionHwl* device_session_hwl) {
801   std::unique_ptr<HalCameraMetadata> characteristics;
802   status_t res = device_session_hwl->GetCameraCharacteristics(&characteristics);
803   if (res != OK) {
804     ALOGE("%s: GetCameraCharacteristics failed.", __FUNCTION__);
805     return UNKNOWN_ERROR;
806   }
807 
808   uint32_t active_array_width = 0;
809   uint32_t active_array_height = 0;
810   camera_metadata_ro_entry entry;
811   res = characteristics->Get(
812       ANDROID_SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE, &entry);
813   if (res == OK) {
814     active_array_width = entry.data.i32[2];
815     active_array_height = entry.data.i32[3];
816     ALOGI("%s Active size (%d x %d).", __FUNCTION__, active_array_width,
817           active_array_height);
818   } else {
819     ALOGE("%s Get active size failed: %s (%d).", __FUNCTION__, strerror(-res),
820           res);
821     return UNKNOWN_ERROR;
822   }
823 
824   std::vector<uint32_t> physical_camera_ids =
825       device_session_hwl->GetPhysicalCameraIds();
826   if (physical_camera_ids.size() != 3) {
827     ALOGE("%s: Only support 3 cameras", __FUNCTION__);
828     return UNKNOWN_ERROR;
829   }
830 
831   uint32_t ir_active_array_width = 0;
832   uint32_t ir_active_array_height = 0;
833   std::unique_ptr<HalCameraMetadata> ir_characteristics;
834   for (auto camera_id : physical_camera_ids) {
835     res = device_session_hwl->GetPhysicalCameraCharacteristics(
836         camera_id, &ir_characteristics);
837     if (res != OK) {
838       ALOGE("%s: GetCameraCharacteristics failed.", __FUNCTION__);
839       return UNKNOWN_ERROR;
840     }
841 
842     // assuming both IR camera are of the same size
843     if (hal_utils::IsIrCamera(ir_characteristics.get())) {
844       camera_metadata_ro_entry entry;
845       res = ir_characteristics->Get(
846           ANDROID_SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE, &entry);
847       if (res == OK) {
848         ir_active_array_width = entry.data.i32[2];
849         ir_active_array_height = entry.data.i32[3];
850         ALOGI("%s IR active size (%dx%d).", __FUNCTION__, ir_active_array_width,
851               ir_active_array_height);
852       } else {
853         ALOGE("%s Get ir active size failed: %s (%d).", __FUNCTION__,
854               strerror(-res), res);
855         return UNKNOWN_ERROR;
856       }
857       break;
858     }
859   }
860 
861   if (active_array_width == 0 || active_array_height == 0 ||
862       ir_active_array_width == 0 || ir_active_array_height == 0) {
863     ALOGE(
864         "%s: One dimension of the logical camera active array size or the "
865         "IR camera active array size is 0.",
866         __FUNCTION__);
867     return INVALID_OPERATION;
868   }
869 
870   float logical_aspect_ratio = 1.0;
871   float ir_aspect_ratio = 1.0;
872   if (active_array_width > active_array_height) {
873     logical_aspect_ratio = active_array_width / active_array_height;
874     ir_aspect_ratio = ir_active_array_width / ir_active_array_height;
875   } else {
876     logical_aspect_ratio = active_array_height / active_array_width;
877     ir_aspect_ratio = ir_active_array_height / ir_active_array_width;
878   }
879 
880   ir_active_array_height_ = ir_active_array_height;
881   ir_active_array_width_ = ir_active_array_width;
882 
883   float aspect_ratio_diff = logical_aspect_ratio - ir_aspect_ratio;
884   if (aspect_ratio_diff > kSmallOffset || aspect_ratio_diff < -kSmallOffset) {
885     ALOGE(
886         "%s: Logical camera aspect ratio and IR camera aspect ratio are "
887         "different from each other.",
888         __FUNCTION__);
889     return UNKNOWN_ERROR;
890   }
891 
892   logical_to_ir_ratio_ = float(active_array_height) / ir_active_array_height;
893 
894   ALOGI("%s: logical_to_ir_ratio_ = %f", __FUNCTION__, logical_to_ir_ratio_);
895 
896   return OK;
897 }
898 
899 }  // namespace google_camera_hal
900 }  // namespace android
901