xref: /hardware/google/camera/devices/EmulatedCamera/hwl/EmulatedSensor.cpp

/*
 * Copyright (C) 2012 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//#define LOG_NDEBUG 0
//#define LOG_NNDEBUG 0
#define LOG_TAG "EmulatedSensor"
#define ATRACE_TAG ATRACE_TAG_CAMERA

#ifdef LOG_NNDEBUG
#define ALOGVV(...) ALOGV(__VA_ARGS__)
#else
#define ALOGVV(...) ((void)0)
#endif

#include "EmulatedSensor.h"

#include <cutils/properties.h>
#include <inttypes.h>
#include <libyuv.h>
#include <system/camera_metadata.h>
#include <utils/Log.h>
#include <utils/Trace.h>

#include <cmath>
#include <cstdlib>

#include "utils/ExifUtils.h"
#include "utils/HWLUtils.h"

namespace android {

using google_camera_hal::HalCameraMetadata;
using google_camera_hal::MessageType;
using google_camera_hal::NotifyMessage;

const uint32_t EmulatedSensor::kRegularSceneHandshake = 1; // Scene handshake divider
const uint32_t EmulatedSensor::kReducedSceneHandshake = 2; // Scene handshake divider

// 1 us - 30 sec
const nsecs_t EmulatedSensor::kSupportedExposureTimeRange[2] = {1000LL,
                                                                30000000000LL};

// ~1/30 s - 30 sec
const nsecs_t EmulatedSensor::kSupportedFrameDurationRange[2] = {33331760LL,
                                                                 30000000000LL};

const int32_t EmulatedSensor::kSupportedSensitivityRange[2] = {100, 1600};
const int32_t EmulatedSensor::kDefaultSensitivity = 100;  // ISO
const nsecs_t EmulatedSensor::kDefaultExposureTime = ms2ns(15);
const nsecs_t EmulatedSensor::kDefaultFrameDuration = ms2ns(33);
// Deadline within we should return the results as soon as possible to
// avoid skewing the frame cycle due to external delays.
const nsecs_t EmulatedSensor::kReturnResultThreshod = 3 * kDefaultFrameDuration;

// Sensor defaults
const uint8_t EmulatedSensor::kSupportedColorFilterArrangement =
    ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_RGGB;
const uint32_t EmulatedSensor::kDefaultMaxRawValue = 4000;
const uint32_t EmulatedSensor::kDefaultBlackLevelPattern[4] = {1000, 1000, 1000,
                                                               1000};

const nsecs_t EmulatedSensor::kMinVerticalBlank = 10000L;

// Sensor sensitivity
const float EmulatedSensor::kSaturationVoltage = 0.520f;
const uint32_t EmulatedSensor::kSaturationElectrons = 2000;
const float EmulatedSensor::kVoltsPerLuxSecond = 0.100f;

const float EmulatedSensor::kElectronsPerLuxSecond =
    EmulatedSensor::kSaturationElectrons / EmulatedSensor::kSaturationVoltage *
    EmulatedSensor::kVoltsPerLuxSecond;

const float EmulatedSensor::kReadNoiseStddevBeforeGain = 1.177;  // in electrons
const float EmulatedSensor::kReadNoiseStddevAfterGain =
    2.100;  // in digital counts
const float EmulatedSensor::kReadNoiseVarBeforeGain =
    EmulatedSensor::kReadNoiseStddevBeforeGain *
    EmulatedSensor::kReadNoiseStddevBeforeGain;
const float EmulatedSensor::kReadNoiseVarAfterGain =
    EmulatedSensor::kReadNoiseStddevAfterGain *
    EmulatedSensor::kReadNoiseStddevAfterGain;

const uint32_t EmulatedSensor::kMaxRAWStreams = 1;
const uint32_t EmulatedSensor::kMaxProcessedStreams = 3;
const uint32_t EmulatedSensor::kMaxStallingStreams = 2;
const uint32_t EmulatedSensor::kMaxInputStreams = 1;

const uint32_t EmulatedSensor::kMaxLensShadingMapSize[2]{64, 64};
const int32_t EmulatedSensor::kFixedBitPrecision = 64;  // 6-bit
// In fixed-point math, saturation point of sensor after gain
const int32_t EmulatedSensor::kSaturationPoint = kFixedBitPrecision * 255;
const camera_metadata_rational EmulatedSensor::kNeutralColorPoint[3] = {
    {255, 1}, {255, 1}, {255, 1}};
const float EmulatedSensor::kGreenSplit = 1.f;  // No divergence
// Reduce memory usage by allowing only one buffer in sensor, one in jpeg
// compressor and one pending request to avoid stalls.
const uint8_t EmulatedSensor::kPipelineDepth = 3;

const camera_metadata_rational EmulatedSensor::kDefaultColorTransform[9] = {
    {1, 1}, {0, 1}, {0, 1}, {0, 1}, {1, 1}, {0, 1}, {0, 1}, {0, 1}, {1, 1}};
const float EmulatedSensor::kDefaultColorCorrectionGains[4] = {1.0f, 1.0f, 1.0f,
                                                               1.0f};

const float EmulatedSensor::kDefaultToneMapCurveRed[4] = {.0f, .0f, 1.f, 1.f};
const float EmulatedSensor::kDefaultToneMapCurveGreen[4] = {.0f, .0f, 1.f, 1.f};
const float EmulatedSensor::kDefaultToneMapCurveBlue[4] = {.0f, .0f, 1.f, 1.f};

/** A few utility functions for math, normal distributions */

// Take advantage of IEEE floating-point format to calculate an approximate
// square root. Accurate to within +-3.6%
float sqrtf_approx(float r) {
  // Modifier is based on IEEE floating-point representation; the
  // manipulations boil down to finding approximate log2, dividing by two, and
  // then inverting the log2. A bias is added to make the relative error
  // symmetric about the real answer.
  const int32_t modifier = 0x1FBB4000;

  int32_t r_i = *(int32_t*)(&r);
  r_i = (r_i >> 1) + modifier;

  return *(float*)(&r_i);
}

EmulatedSensor::EmulatedSensor() : Thread(false), got_vsync_(false) {
  gamma_table_.resize(kSaturationPoint + 1);
  for (int32_t i = 0; i <= kSaturationPoint; i++) {
    gamma_table_[i] = ApplysRGBGamma(i, kSaturationPoint);
  }
}

EmulatedSensor::~EmulatedSensor() {
  ShutDown();
}

bool EmulatedSensor::AreCharacteristicsSupported(
    const SensorCharacteristics& characteristics) {
  if ((characteristics.width == 0) || (characteristics.height == 0)) {
    ALOGE("%s: Invalid sensor size %zux%zu", __FUNCTION__,
          characteristics.width, characteristics.height);
    return false;
  }

  if ((characteristics.full_res_width == 0) ||
      (characteristics.full_res_height == 0)) {
    ALOGE("%s: Invalid sensor full res size %zux%zu", __FUNCTION__,
          characteristics.full_res_width, characteristics.full_res_height);
    return false;
  }

  if ((characteristics.exposure_time_range[0] >=
       characteristics.exposure_time_range[1]) ||
      ((characteristics.exposure_time_range[0] < kSupportedExposureTimeRange[0]) ||
       (characteristics.exposure_time_range[1] >
        kSupportedExposureTimeRange[1]))) {
    ALOGE("%s: Unsupported exposure range", __FUNCTION__);
    return false;
  }

  if ((characteristics.frame_duration_range[0] >=
       characteristics.frame_duration_range[1]) ||
      ((characteristics.frame_duration_range[0] <
        kSupportedFrameDurationRange[0]) ||
       (characteristics.frame_duration_range[1] >
        kSupportedFrameDurationRange[1]))) {
    ALOGE("%s: Unsupported frame duration range", __FUNCTION__);
    return false;
  }

  if ((characteristics.sensitivity_range[0] >=
       characteristics.sensitivity_range[1]) ||
      ((characteristics.sensitivity_range[0] < kSupportedSensitivityRange[0]) ||
       (characteristics.sensitivity_range[1] > kSupportedSensitivityRange[1])) ||
      (!((kDefaultSensitivity >= characteristics.sensitivity_range[0]) &&
         (kDefaultSensitivity <= characteristics.sensitivity_range[1])))) {
    ALOGE("%s: Unsupported sensitivity range", __FUNCTION__);
    return false;
  }

  if (characteristics.color_arangement != kSupportedColorFilterArrangement) {
    ALOGE("%s: Unsupported color arrangement!", __FUNCTION__);
    return false;
  }

  for (const auto& blackLevel : characteristics.black_level_pattern) {
    if (blackLevel >= characteristics.max_raw_value) {
      ALOGE("%s: Black level matches or exceeds max RAW value!", __FUNCTION__);
      return false;
    }
  }

  if ((characteristics.frame_duration_range[0] / characteristics.height) == 0) {
    ALOGE("%s: Zero row readout time!", __FUNCTION__);
    return false;
  }

  if (characteristics.max_raw_streams > kMaxRAWStreams) {
    ALOGE("%s: RAW streams maximum %u exceeds supported maximum %u",
          __FUNCTION__, characteristics.max_raw_streams, kMaxRAWStreams);
    return false;
  }

  if (characteristics.max_processed_streams > kMaxProcessedStreams) {
    ALOGE("%s: Processed streams maximum %u exceeds supported maximum %u",
          __FUNCTION__, characteristics.max_processed_streams,
          kMaxProcessedStreams);
    return false;
  }

  if (characteristics.max_stalling_streams > kMaxStallingStreams) {
    ALOGE("%s: Stalling streams maximum %u exceeds supported maximum %u",
          __FUNCTION__, characteristics.max_stalling_streams,
          kMaxStallingStreams);
    return false;
  }

  if (characteristics.max_input_streams > kMaxInputStreams) {
    ALOGE("%s: Input streams maximum %u exceeds supported maximum %u",
          __FUNCTION__, characteristics.max_input_streams, kMaxInputStreams);
    return false;
  }

  if ((characteristics.lens_shading_map_size[0] > kMaxLensShadingMapSize[0]) ||
      (characteristics.lens_shading_map_size[1] > kMaxLensShadingMapSize[1])) {
    ALOGE("%s: Lens shading map [%dx%d] exceeds supprorted maximum [%dx%d]",
          __FUNCTION__, characteristics.lens_shading_map_size[0],
          characteristics.lens_shading_map_size[1], kMaxLensShadingMapSize[0],
          kMaxLensShadingMapSize[1]);
    return false;
  }

  if (characteristics.max_pipeline_depth < kPipelineDepth) {
    ALOGE("%s: Pipeline depth %d smaller than supprorted minimum %d",
          __FUNCTION__, characteristics.max_pipeline_depth, kPipelineDepth);
    return false;
  }

  return true;
}

static void SplitStreamCombination(
    const StreamConfiguration& original_config,
    StreamConfiguration* default_mode_config,
    StreamConfiguration* max_resolution_mode_config,
    StreamConfiguration* input_stream_config) {
  // Go through the streams
  if (default_mode_config == nullptr || max_resolution_mode_config == nullptr ||
      input_stream_config == nullptr) {
    ALOGE("%s: Input stream / output stream configs are nullptr", __FUNCTION__);
    return;
  }
  for (const auto& stream : original_config.streams) {
    if (stream.stream_type == google_camera_hal::StreamType::kInput) {
      input_stream_config->streams.push_back(stream);
      continue;
    }
    if (stream.used_in_default_resolution_mode) {
      default_mode_config->streams.push_back(stream);
    }
    if (stream.used_in_max_resolution_mode) {
      max_resolution_mode_config->streams.push_back(stream);
    }
  }
}

bool EmulatedSensor::IsStreamCombinationSupported(
    uint32_t logical_id, const StreamConfiguration& config,
    StreamConfigurationMap& default_config_map,
    StreamConfigurationMap& max_resolution_config_map,
    const PhysicalStreamConfigurationMap& physical_map,
    const PhysicalStreamConfigurationMap& physical_map_max_resolution,
    const LogicalCharacteristics& sensor_chars) {
  StreamConfiguration default_mode_config, max_resolution_mode_config,
      input_stream_config;
  SplitStreamCombination(config, &default_mode_config,
                         &max_resolution_mode_config, &input_stream_config);

  return IsStreamCombinationSupported(logical_id, default_mode_config,
                                      default_config_map, physical_map,
                                      sensor_chars) &&
         IsStreamCombinationSupported(
             logical_id, max_resolution_mode_config, max_resolution_config_map,
             physical_map_max_resolution, sensor_chars, /*is_max_res*/ true) &&

         (IsStreamCombinationSupported(logical_id, input_stream_config,
                                       default_config_map, physical_map,
                                       sensor_chars) ||
          IsStreamCombinationSupported(
              logical_id, input_stream_config, max_resolution_config_map,
              physical_map_max_resolution, sensor_chars, /*is_max_res*/ true));
}

bool EmulatedSensor::IsStreamCombinationSupported(
    uint32_t logical_id, const StreamConfiguration& config,
    StreamConfigurationMap& config_map,
    const PhysicalStreamConfigurationMap& physical_map,
    const LogicalCharacteristics& sensor_chars, bool is_max_res) {
  uint32_t input_stream_count = 0;
  // Map from physical camera id to number of streams for that physical camera
  std::map<uint32_t, uint32_t> raw_stream_count;
  std::map<uint32_t, uint32_t> processed_stream_count;
  std::map<uint32_t, uint32_t> stalling_stream_count;

  // Only allow the stream configurations specified in
  // dynamicSizeStreamConfigurations.
  for (const auto& stream : config.streams) {
    bool is_dynamic_output =
        (stream.is_physical_camera_stream && stream.group_id != -1);
    if (stream.rotation != google_camera_hal::StreamRotation::kRotation0) {
      ALOGE("%s: Stream rotation: 0x%x not supported!", __FUNCTION__,
            stream.rotation);
      return false;
    }

    if (stream.stream_type == google_camera_hal::StreamType::kInput) {
      if (sensor_chars.at(logical_id).max_input_streams == 0) {
        ALOGE("%s: Input streams are not supported on this device!",
              __FUNCTION__);
        return false;
      }

      auto const& supported_outputs =
          config_map.GetValidOutputFormatsForInput(stream.format);
      if (supported_outputs.empty()) {
        ALOGE("%s: Input stream with format: 0x%x no supported on this device!",
              __FUNCTION__, stream.format);
        return false;
      }

      input_stream_count++;
    } else {
      if (stream.is_physical_camera_stream &&
          physical_map.find(stream.physical_camera_id) == physical_map.end()) {
        ALOGE("%s: Invalid physical camera id %d", __FUNCTION__,
              stream.physical_camera_id);
        return false;
      }

      if (is_dynamic_output) {
        auto dynamic_physical_output_formats =
            physical_map.at(stream.physical_camera_id)
                ->GetDynamicPhysicalStreamOutputFormats();
        if (dynamic_physical_output_formats.find(stream.format) ==
            dynamic_physical_output_formats.end()) {
          ALOGE("%s: Unsupported physical stream format %d", __FUNCTION__,
                stream.format);
          return false;
        }
      }

      switch (stream.format) {
        case HAL_PIXEL_FORMAT_BLOB:
          if ((stream.data_space != HAL_DATASPACE_V0_JFIF) &&
              (stream.data_space != HAL_DATASPACE_UNKNOWN)) {
            ALOGE("%s: Unsupported Blob dataspace 0x%x", __FUNCTION__,
                  stream.data_space);
            return false;
          }
          if (stream.is_physical_camera_stream) {
            stalling_stream_count[stream.physical_camera_id]++;
          } else {
            for (const auto& p : physical_map) {
              stalling_stream_count[p.first]++;
            }
          }
          break;
        case HAL_PIXEL_FORMAT_RAW16: {
          const SensorCharacteristics& sensor_char =
              stream.is_physical_camera_stream
                  ? sensor_chars.at(stream.physical_camera_id)
                  : sensor_chars.at(logical_id);
          auto sensor_height =
              is_max_res ? sensor_char.full_res_height : sensor_char.height;
          auto sensor_width =
              is_max_res ? sensor_char.full_res_width : sensor_char.width;
          if (stream.height != sensor_height || stream.width != sensor_width) {
            ALOGE(
                "%s, RAW16 buffer height %d and width %d must match sensor "
                "height: %zu"
                " and width: %zu",
                __FUNCTION__, stream.height, stream.width, sensor_height,
                sensor_width);
            return false;
          }
          if (stream.is_physical_camera_stream) {
            raw_stream_count[stream.physical_camera_id]++;
          } else {
            for (const auto& p : physical_map) {
              raw_stream_count[p.first]++;
            }
          }
        } break;
        default:
          if (stream.is_physical_camera_stream) {
            processed_stream_count[stream.physical_camera_id]++;
          } else {
            for (const auto& p : physical_map) {
              processed_stream_count[p.first]++;
            }
          }
      }

      auto output_sizes =
          is_dynamic_output
              ? physical_map.at(stream.physical_camera_id)
                    ->GetDynamicPhysicalStreamOutputSizes(stream.format)
              : stream.is_physical_camera_stream
                    ? physical_map.at(stream.physical_camera_id)
                          ->GetOutputSizes(stream.format)
                    : config_map.GetOutputSizes(stream.format);

      auto stream_size = std::make_pair(stream.width, stream.height);
      if (output_sizes.find(stream_size) == output_sizes.end()) {
        ALOGE("%s: Stream with size %dx%d and format 0x%x is not supported!",
              __FUNCTION__, stream.width, stream.height, stream.format);
        return false;
      }
    }
  }

  for (const auto& raw_count : raw_stream_count) {
    unsigned int max_raw_streams =
        sensor_chars.at(raw_count.first).max_raw_streams +
        (is_max_res
             ? 1
             : 0);  // The extra raw stream is allowed for remosaic reprocessing.
    if (raw_count.second > max_raw_streams) {
      ALOGE("%s: RAW streams maximum %u exceeds supported maximum %u",
            __FUNCTION__, raw_count.second, max_raw_streams);
      return false;
    }
  }

  for (const auto& stalling_count : stalling_stream_count) {
    if (stalling_count.second >
        sensor_chars.at(stalling_count.first).max_stalling_streams) {
      ALOGE("%s: Stalling streams maximum %u exceeds supported maximum %u",
            __FUNCTION__, stalling_count.second,
            sensor_chars.at(stalling_count.first).max_stalling_streams);
      return false;
    }
  }

  for (const auto& processed_count : processed_stream_count) {
    if (processed_count.second >
        sensor_chars.at(processed_count.first).max_processed_streams) {
      ALOGE("%s: Processed streams maximum %u exceeds supported maximum %u",
            __FUNCTION__, processed_count.second,
            sensor_chars.at(processed_count.first).max_processed_streams);
      return false;
    }
  }

  if (input_stream_count > sensor_chars.at(logical_id).max_input_streams) {
    ALOGE("%s: Input stream maximum %u exceeds supported maximum %u",
          __FUNCTION__, input_stream_count,
          sensor_chars.at(logical_id).max_input_streams);
    return false;
  }

  return true;
}

status_t EmulatedSensor::StartUp(
    uint32_t logical_camera_id,
    std::unique_ptr<LogicalCharacteristics> logical_chars) {
  if (isRunning()) {
    return OK;
  }

  if (logical_chars.get() == nullptr) {
    return BAD_VALUE;
  }

  chars_ = std::move(logical_chars);
  auto device_chars = chars_->find(logical_camera_id);
  if (device_chars == chars_->end()) {
    ALOGE(
        "%s: Logical camera id: %u absent from logical camera characteristics!",
        __FUNCTION__, logical_camera_id);
    return BAD_VALUE;
  }

  for (const auto& it : *chars_) {
    if (!AreCharacteristicsSupported(it.second)) {
      ALOGE("%s: Sensor characteristics for camera id: %u not supported!",
            __FUNCTION__, it.first);
      return BAD_VALUE;
    }
  }

  logical_camera_id_ = logical_camera_id;
  scene_ = new EmulatedScene(
      device_chars->second.full_res_width, device_chars->second.full_res_height,
      kElectronsPerLuxSecond, device_chars->second.orientation,
      device_chars->second.is_front_facing);
  scene_->InitializeSensorQueue();
  jpeg_compressor_ = std::make_unique<JpegCompressor>();

  auto res = run(LOG_TAG, ANDROID_PRIORITY_URGENT_DISPLAY);
  if (res != OK) {
    ALOGE("Unable to start up sensor capture thread: %d", res);
  }

  return res;
}

status_t EmulatedSensor::ShutDown() {
  int res;
  res = requestExitAndWait();
  if (res != OK) {
    ALOGE("Unable to shut down sensor capture thread: %d", res);
  }
  return res;
}

void EmulatedSensor::SetCurrentRequest(
    std::unique_ptr<LogicalCameraSettings> logical_settings,
    std::unique_ptr<HwlPipelineResult> result,
    std::unique_ptr<Buffers> input_buffers,
    std::unique_ptr<Buffers> output_buffers) {
  Mutex::Autolock lock(control_mutex_);
  current_settings_ = std::move(logical_settings);
  current_result_ = std::move(result);
  current_input_buffers_ = std::move(input_buffers);
  current_output_buffers_ = std::move(output_buffers);
}

bool EmulatedSensor::WaitForVSyncLocked(nsecs_t reltime) {
  got_vsync_ = false;
  while (!got_vsync_) {
    auto res = vsync_.waitRelative(control_mutex_, reltime);
    if (res != OK && res != TIMED_OUT) {
      ALOGE("%s: Error waiting for VSync signal: %d", __FUNCTION__, res);
      return false;
    }
  }

  return got_vsync_;
}

bool EmulatedSensor::WaitForVSync(nsecs_t reltime) {
  Mutex::Autolock lock(control_mutex_);

  return WaitForVSyncLocked(reltime);
}

status_t EmulatedSensor::Flush() {
  Mutex::Autolock lock(control_mutex_);
  auto ret = WaitForVSyncLocked(kSupportedFrameDurationRange[1]);

  // First recreate the jpeg compressor. This will abort any ongoing processing
  // and flush any pending jobs.
  jpeg_compressor_ = std::make_unique<JpegCompressor>();

  // Then return any pending frames here
  if ((current_input_buffers_.get() != nullptr) &&
      (!current_input_buffers_->empty())) {
    current_input_buffers_->clear();
  }
  if ((current_output_buffers_.get() != nullptr) &&
      (!current_output_buffers_->empty())) {
    for (const auto& buffer : *current_output_buffers_) {
      buffer->stream_buffer.status = BufferStatus::kError;
    }

    if ((current_result_.get() != nullptr) &&
        (current_result_->result_metadata.get() != nullptr)) {
      if (current_output_buffers_->at(0)->callback.notify != nullptr) {
        NotifyMessage msg{
            .type = MessageType::kError,
            .message.error = {
                .frame_number = current_output_buffers_->at(0)->frame_number,
                .error_stream_id = -1,
                .error_code = ErrorCode::kErrorResult,
            }};

        current_output_buffers_->at(0)->callback.notify(
            current_result_->pipeline_id, msg);
      }
    }

    current_output_buffers_->clear();
  }

  return ret ? OK : TIMED_OUT;
}

bool EmulatedSensor::threadLoop() {
  ATRACE_CALL();
  /**
   * Sensor capture operation main loop.
   *
   */

  /**
   * Stage 1: Read in latest control parameters
   */
  std::unique_ptr<Buffers> next_buffers;
  std::unique_ptr<Buffers> next_input_buffer;
  std::unique_ptr<HwlPipelineResult> next_result;
  std::unique_ptr<LogicalCameraSettings> settings;
  HwlPipelineCallback callback = {nullptr, nullptr};
  {
    Mutex::Autolock lock(control_mutex_);
    std::swap(settings, current_settings_);
    std::swap(next_buffers, current_output_buffers_);
    std::swap(next_input_buffer, current_input_buffers_);
    std::swap(next_result, current_result_);

    // Signal VSync for start of readout
    ALOGVV("Sensor VSync");
    got_vsync_ = true;
    vsync_.signal();
  }

  auto frame_duration = EmulatedSensor::kSupportedFrameDurationRange[0];
  // Frame duration must always be the same among all physical devices
  if ((settings.get() != nullptr) && (!settings->empty())) {
    frame_duration = settings->begin()->second.frame_duration;
  }

  nsecs_t start_real_time = systemTime();
  // Stagefright cares about system time for timestamps, so base simulated
  // time on that.
  nsecs_t frame_end_real_time = start_real_time + frame_duration;

  /**
   * Stage 2: Capture new image
   */
  next_capture_time_ = frame_end_real_time;

  sensor_binning_factor_info_.clear();

  bool reprocess_request = false;
  if ((next_input_buffer.get() != nullptr) && (!next_input_buffer->empty())) {
    if (next_input_buffer->size() > 1) {
      ALOGW("%s: Reprocess supports only single input!", __FUNCTION__);
    }

      camera_metadata_ro_entry_t entry;
      auto ret =
          next_result->result_metadata->Get(ANDROID_SENSOR_TIMESTAMP, &entry);
      if ((ret == OK) && (entry.count == 1)) {
        next_capture_time_ = entry.data.i64[0];
      } else {
        ALOGW("%s: Reprocess timestamp absent!", __FUNCTION__);
      }

      reprocess_request = true;
  }

  if ((next_buffers != nullptr) && (settings != nullptr)) {
    callback = next_buffers->at(0)->callback;
    if (callback.notify != nullptr) {
      NotifyMessage msg{
          .type = MessageType::kShutter,
          .message.shutter = {
              .frame_number = next_buffers->at(0)->frame_number,
              .timestamp_ns = static_cast<uint64_t>(next_capture_time_)}};
      callback.notify(next_result->pipeline_id, msg);
    }
    auto b = next_buffers->begin();
    while (b != next_buffers->end()) {
      auto device_settings = settings->find((*b)->camera_id);
      if (device_settings == settings->end()) {
        ALOGE("%s: Sensor settings absent for device: %d", __func__,
              (*b)->camera_id);
        b = next_buffers->erase(b);
        continue;
      }

      auto device_chars = chars_->find((*b)->camera_id);
      if (device_chars == chars_->end()) {
        ALOGE("%s: Sensor characteristics absent for device: %d", __func__,
              (*b)->camera_id);
        b = next_buffers->erase(b);
        continue;
      }

      sensor_binning_factor_info_[(*b)->camera_id].quad_bayer_sensor =
          device_chars->second.quad_bayer_sensor;

      ALOGVV("Starting next capture: Exposure: %" PRIu64 " ms, gain: %d",
             ns2ms(device_settings->second.exposure_time),
             device_settings->second.gain);

      scene_->Initialize(device_chars->second.full_res_width,
                         device_chars->second.full_res_height,
                         kElectronsPerLuxSecond);
      scene_->SetExposureDuration((float)device_settings->second.exposure_time /
                                  1e9);
      scene_->SetColorFilterXYZ(device_chars->second.color_filter.rX,
                                device_chars->second.color_filter.rY,
                                device_chars->second.color_filter.rZ,
                                device_chars->second.color_filter.grX,
                                device_chars->second.color_filter.grY,
                                device_chars->second.color_filter.grZ,
                                device_chars->second.color_filter.gbX,
                                device_chars->second.color_filter.gbY,
                                device_chars->second.color_filter.gbZ,
                                device_chars->second.color_filter.bX,
                                device_chars->second.color_filter.bY,
                                device_chars->second.color_filter.bZ);
      scene_->SetTestPattern(device_settings->second.test_pattern_mode ==
                             ANDROID_SENSOR_TEST_PATTERN_MODE_SOLID_COLOR);
      scene_->SetTestPatternData(device_settings->second.test_pattern_data);

      uint32_t handshake_divider =
        (device_settings->second.video_stab == ANDROID_CONTROL_VIDEO_STABILIZATION_MODE_ON) ?
        kReducedSceneHandshake : kRegularSceneHandshake;
      scene_->CalculateScene(next_capture_time_, handshake_divider);

      (*b)->stream_buffer.status = BufferStatus::kOk;
      bool max_res_mode = device_settings->second.sensor_pixel_mode;
      sensor_binning_factor_info_[(*b)->camera_id].max_res_request =
          max_res_mode;
      switch ((*b)->format) {
        case PixelFormat::RAW16:
          sensor_binning_factor_info_[(*b)->camera_id].has_raw_stream = true;
          break;
        default:
          sensor_binning_factor_info_[(*b)->camera_id].has_non_raw_stream = true;
      }

      // TODO: remove hack. Implement RAW -> YUV / JPEG reprocessing http://b/192382904
      bool treat_as_reprocess =
          (device_chars->second.quad_bayer_sensor && reprocess_request &&
           (*next_input_buffer->begin())->format == PixelFormat::RAW16)
              ? false
              : reprocess_request;

      switch ((*b)->format) {
        case PixelFormat::RAW16:
          if (!reprocess_request) {
            uint64_t min_full_res_raw_size =
                2 * device_chars->second.full_res_width *
                device_chars->second.full_res_height;
            uint64_t min_default_raw_size =
                2 * device_chars->second.width * device_chars->second.height;
            bool default_mode_for_qb =
                device_chars->second.quad_bayer_sensor && !max_res_mode;
            size_t buffer_size = (*b)->plane.img.buffer_size;
            if (default_mode_for_qb) {
              if (buffer_size < min_default_raw_size) {
                ALOGE(
                    "%s: Output buffer size too small for RAW capture in "
                    "default "
                    "mode, "
                    "expected %" PRIu64 ", got %zu, for camera id %d",
                    __FUNCTION__, min_default_raw_size, buffer_size,
                    (*b)->camera_id);
                (*b)->stream_buffer.status = BufferStatus::kError;
                break;
              }
            } else if (buffer_size < min_full_res_raw_size) {
              ALOGE(
                  "%s: Output buffer size too small for RAW capture in max res "
                  "mode, "
                  "expected %" PRIu64 ", got %zu, for camera id %d",
                  __FUNCTION__, min_full_res_raw_size, buffer_size,
                  (*b)->camera_id);
              (*b)->stream_buffer.status = BufferStatus::kError;
              break;
            }
            if (default_mode_for_qb) {
              CaptureRawBinned(
                  (*b)->plane.img.img, (*b)->plane.img.stride_in_bytes,
                  device_settings->second.gain, device_chars->second);
            } else {
              CaptureRawFullRes(
                  (*b)->plane.img.img, (*b)->plane.img.stride_in_bytes,
                  device_settings->second.gain, device_chars->second);
            }
          } else {
            if (!device_chars->second.quad_bayer_sensor) {
              ALOGE(
                  "%s: Reprocess requests with output format %x no supported!",
                  __FUNCTION__, (*b)->format);
              (*b)->stream_buffer.status = BufferStatus::kError;
              break;
            }
            // Remosaic the RAW input buffer
            if ((*next_input_buffer->begin())->width != (*b)->width ||
                (*next_input_buffer->begin())->height != (*b)->height) {
              ALOGE(
                  "%s: RAW16 input dimensions %dx%d don't match output buffer "
                  "dimensions %dx%d",
                  __FUNCTION__, (*next_input_buffer->begin())->width,
                  (*next_input_buffer->begin())->height, (*b)->width,
                  (*b)->height);
              (*b)->stream_buffer.status = BufferStatus::kError;
              break;
            }
            ALOGV("%s remosaic Raw16 Image", __FUNCTION__);
            RemosaicRAW16Image(
                (uint16_t*)(*next_input_buffer->begin())->plane.img.img,
                (uint16_t*)(*b)->plane.img.img, (*b)->plane.img.stride_in_bytes,
                device_chars->second);
          }
          break;
        case PixelFormat::RGB_888:
          if (!reprocess_request) {
            CaptureRGB((*b)->plane.img.img, (*b)->width, (*b)->height,
                       (*b)->plane.img.stride_in_bytes, RGBLayout::RGB,
                       device_settings->second.gain, device_chars->second);
          } else {
            ALOGE("%s: Reprocess requests with output format %x no supported!",
                  __FUNCTION__, (*b)->format);
            (*b)->stream_buffer.status = BufferStatus::kError;
          }
          break;
        case PixelFormat::RGBA_8888:
          if (!reprocess_request) {
            CaptureRGB((*b)->plane.img.img, (*b)->width, (*b)->height,
                       (*b)->plane.img.stride_in_bytes, RGBLayout::RGBA,
                       device_settings->second.gain, device_chars->second);
          } else {
            ALOGE("%s: Reprocess requests with output format %x no supported!",
                  __FUNCTION__, (*b)->format);
            (*b)->stream_buffer.status = BufferStatus::kError;
          }
          break;
        case PixelFormat::BLOB:
          if ((*b)->dataSpace == HAL_DATASPACE_V0_JFIF) {
            YUV420Frame yuv_input{
                .width = treat_as_reprocess
                             ? (*next_input_buffer->begin())->width
                             : 0,
                .height = treat_as_reprocess
                              ? (*next_input_buffer->begin())->height
                              : 0,
                .planes = treat_as_reprocess
                              ? (*next_input_buffer->begin())->plane.img_y_crcb
                              : YCbCrPlanes{}};
            auto jpeg_input = std::make_unique<JpegYUV420Input>();
            jpeg_input->width = (*b)->width;
            jpeg_input->height = (*b)->height;
            auto img =
                new uint8_t[(jpeg_input->width * jpeg_input->height * 3) / 2];
            jpeg_input->yuv_planes = {
                .img_y = img,
                .img_cb = img + jpeg_input->width * jpeg_input->height,
                .img_cr = img + (jpeg_input->width * jpeg_input->height * 5) / 4,
                .y_stride = jpeg_input->width,
                .cbcr_stride = jpeg_input->width / 2,
                .cbcr_step = 1};
            jpeg_input->buffer_owner = true;
            YUV420Frame yuv_output{.width = jpeg_input->width,
                                   .height = jpeg_input->height,
                                   .planes = jpeg_input->yuv_planes};

            bool rotate =
                device_settings->second.rotate_and_crop == ANDROID_SCALER_ROTATE_AND_CROP_90;
            ProcessType process_type =
                treat_as_reprocess ? REPROCESS
                                   : (device_settings->second.edge_mode ==
                                      ANDROID_EDGE_MODE_HIGH_QUALITY)
                                         ? HIGH_QUALITY
                                         : REGULAR;
            auto ret = ProcessYUV420(
                yuv_input, yuv_output, device_settings->second.gain,
                process_type, device_settings->second.zoom_ratio,
                rotate, device_chars->second);
            if (ret != 0) {
              (*b)->stream_buffer.status = BufferStatus::kError;
              break;
            }

            auto jpeg_job = std::make_unique<JpegYUV420Job>();
            jpeg_job->exif_utils = std::unique_ptr<ExifUtils>(
                ExifUtils::Create(device_chars->second));
            jpeg_job->input = std::move(jpeg_input);
            // If jpeg compression is successful, then the jpeg compressor
            // must set the corresponding status.
            (*b)->stream_buffer.status = BufferStatus::kError;
            std::swap(jpeg_job->output, *b);
            jpeg_job->result_metadata =
                HalCameraMetadata::Clone(next_result->result_metadata.get());

            Mutex::Autolock lock(control_mutex_);
            jpeg_compressor_->QueueYUV420(std::move(jpeg_job));
          } else {
            ALOGE("%s: Format %x with dataspace %x is TODO", __FUNCTION__,
                  (*b)->format, (*b)->dataSpace);
            (*b)->stream_buffer.status = BufferStatus::kError;
          }
          break;
        case PixelFormat::YCRCB_420_SP:
        case PixelFormat::YCBCR_420_888: {
          YUV420Frame yuv_input{
              .width =
                  treat_as_reprocess ? (*next_input_buffer->begin())->width : 0,
              .height =
                  treat_as_reprocess ? (*next_input_buffer->begin())->height : 0,
              .planes = treat_as_reprocess
                            ? (*next_input_buffer->begin())->plane.img_y_crcb
                            : YCbCrPlanes{}};
          YUV420Frame yuv_output{.width = (*b)->width,
                                 .height = (*b)->height,
                                 .planes = (*b)->plane.img_y_crcb};
          bool rotate =
              device_settings->second.rotate_and_crop == ANDROID_SCALER_ROTATE_AND_CROP_90;
          ProcessType process_type = treat_as_reprocess
                                         ? REPROCESS
                                         : (device_settings->second.edge_mode ==
                                            ANDROID_EDGE_MODE_HIGH_QUALITY)
                                               ? HIGH_QUALITY
                                               : REGULAR;
          auto ret = ProcessYUV420(
              yuv_input, yuv_output, device_settings->second.gain,
              process_type, device_settings->second.zoom_ratio,
              rotate, device_chars->second);
          if (ret != 0) {
            (*b)->stream_buffer.status = BufferStatus::kError;
          }
        } break;
        case PixelFormat::Y16:
          if (!reprocess_request) {
            if ((*b)->dataSpace == HAL_DATASPACE_DEPTH) {
              CaptureDepth((*b)->plane.img.img, device_settings->second.gain,
                           (*b)->width, (*b)->height,
                           (*b)->plane.img.stride_in_bytes,
                           device_chars->second);
            } else {
              ALOGE("%s: Format %x with dataspace %x is TODO", __FUNCTION__,
                    (*b)->format, (*b)->dataSpace);
              (*b)->stream_buffer.status = BufferStatus::kError;
            }
          } else {
            ALOGE("%s: Reprocess requests with output format %x no supported!",
                  __FUNCTION__, (*b)->format);
            (*b)->stream_buffer.status = BufferStatus::kError;
          }
          break;
        case PixelFormat::YCBCR_P010:
            if (!reprocess_request) {
              bool rotate = device_settings->second.rotate_and_crop ==
                            ANDROID_SCALER_ROTATE_AND_CROP_90;
              CaptureYUV420((*b)->plane.img_y_crcb, (*b)->width, (*b)->height,
                            device_settings->second.gain,
                            device_settings->second.zoom_ratio, rotate,
                            device_chars->second);
            } else {
              ALOGE(
                  "%s: Reprocess requests with output format %x no supported!",
                  __FUNCTION__, (*b)->format);
              (*b)->stream_buffer.status = BufferStatus::kError;
            }
            break;
        default:
          ALOGE("%s: Unknown format %x, no output", __FUNCTION__, (*b)->format);
          (*b)->stream_buffer.status = BufferStatus::kError;
          break;
      }

      b = next_buffers->erase(b);
    }
  }

  if (reprocess_request) {
    auto input_buffer = next_input_buffer->begin();
    while (input_buffer != next_input_buffer->end()) {
      (*input_buffer++)->stream_buffer.status = BufferStatus::kOk;
    }
    next_input_buffer->clear();
  }

  nsecs_t work_done_real_time = systemTime();
  // Returning the results at this point is not entirely correct from timing
  // perspective. Under ideal conditions where 'ReturnResults' completes
  // in less than 'time_accuracy' we need to return the results after the
  // frame cycle expires. However under real conditions various system
  // components like SurfaceFlinger, Encoder, LMK etc. could be consuming most
  // of the resources and the duration of "ReturnResults" can get comparable to
  // 'kDefaultFrameDuration'. This will skew the frame cycle and can result in
  // potential frame drops. To avoid this scenario when we are running under
  // tight deadlines (less than 'kReturnResultThreshod') try to return the
  // results immediately. In all other cases with more relaxed deadlines
  // the occasional bump during 'ReturnResults' should not have any
  // noticeable effect.
  if ((work_done_real_time + kReturnResultThreshod) > frame_end_real_time) {
    ReturnResults(callback, std::move(settings), std::move(next_result),
                  reprocess_request);
  }

  work_done_real_time = systemTime();
  ALOGVV("Sensor vertical blanking interval");
  const nsecs_t time_accuracy = 2e6;  // 2 ms of imprecision is ok
  if (work_done_real_time < frame_end_real_time - time_accuracy) {
    timespec t;
    t.tv_sec = (frame_end_real_time - work_done_real_time) / 1000000000L;
    t.tv_nsec = (frame_end_real_time - work_done_real_time) % 1000000000L;

    int ret;
    do {
      ret = nanosleep(&t, &t);
    } while (ret != 0);
  }
  nsecs_t end_real_time __unused = systemTime();
  ALOGVV("Frame cycle took %" PRIu64 "  ms, target %" PRIu64 " ms",
         ns2ms(end_real_time - start_real_time), ns2ms(frame_duration));

  ReturnResults(callback, std::move(settings), std::move(next_result),
                reprocess_request);

  return true;
};

void EmulatedSensor::ReturnResults(
    HwlPipelineCallback callback,
    std::unique_ptr<LogicalCameraSettings> settings,
    std::unique_ptr<HwlPipelineResult> result, bool reprocess_request) {
  if ((callback.process_pipeline_result != nullptr) &&
      (result.get() != nullptr) && (result->result_metadata.get() != nullptr)) {
    auto logical_settings = settings->find(logical_camera_id_);
    if (logical_settings == settings->end()) {
      ALOGE("%s: Logical camera id: %u not found in settings!", __FUNCTION__,
            logical_camera_id_);
      return;
    }
    auto device_chars = chars_->find(logical_camera_id_);
    if (device_chars == chars_->end()) {
      ALOGE("%s: Sensor characteristics absent for device: %d", __func__,
            logical_camera_id_);
      return;
    }
    result->result_metadata->Set(ANDROID_SENSOR_TIMESTAMP, &next_capture_time_,
                                 1);
    uint8_t raw_binned_factor_used = false;
    if (sensor_binning_factor_info_.find(logical_camera_id_) !=
        sensor_binning_factor_info_.end()) {
      auto& info = sensor_binning_factor_info_[logical_camera_id_];
      // Logical stream was included in the request
      if (!reprocess_request && info.quad_bayer_sensor && info.max_res_request &&
          info.has_raw_stream && !info.has_non_raw_stream) {
        raw_binned_factor_used = true;
      }
      result->result_metadata->Set(ANDROID_SENSOR_RAW_BINNING_FACTOR_USED,
                                   &raw_binned_factor_used, 1);
    }
    if (logical_settings->second.lens_shading_map_mode ==
        ANDROID_STATISTICS_LENS_SHADING_MAP_MODE_ON) {
      if ((device_chars->second.lens_shading_map_size[0] > 0) &&
          (device_chars->second.lens_shading_map_size[1] > 0)) {
        // Perfect lens, no actual shading needed.
        std::vector<float> lens_shading_map(
            device_chars->second.lens_shading_map_size[0] *
                device_chars->second.lens_shading_map_size[1] * 4,
            1.f);

        result->result_metadata->Set(ANDROID_STATISTICS_LENS_SHADING_MAP,
                                     lens_shading_map.data(),
                                     lens_shading_map.size());
      }
    }
    if (logical_settings->second.report_video_stab) {
      result->result_metadata->Set(ANDROID_CONTROL_VIDEO_STABILIZATION_MODE,
                                   &logical_settings->second.video_stab, 1);
    }
    if (logical_settings->second.report_edge_mode) {
      result->result_metadata->Set(ANDROID_EDGE_MODE,
                                   &logical_settings->second.edge_mode, 1);
    }
    if (logical_settings->second.report_neutral_color_point) {
      result->result_metadata->Set(ANDROID_SENSOR_NEUTRAL_COLOR_POINT,
                                   kNeutralColorPoint,
                                   ARRAY_SIZE(kNeutralColorPoint));
    }
    if (logical_settings->second.report_green_split) {
      result->result_metadata->Set(ANDROID_SENSOR_GREEN_SPLIT, &kGreenSplit, 1);
    }
    if (logical_settings->second.report_noise_profile) {
      CalculateAndAppendNoiseProfile(
          logical_settings->second.gain,
          GetBaseGainFactor(device_chars->second.max_raw_value),
          result->result_metadata.get());
    }
    if (logical_settings->second.report_rotate_and_crop) {
      result->result_metadata->Set(ANDROID_SCALER_ROTATE_AND_CROP,
          &logical_settings->second.rotate_and_crop, 1);
    }

    if (!result->physical_camera_results.empty()) {
      for (auto& it : result->physical_camera_results) {
        auto physical_settings = settings->find(it.first);
        if (physical_settings == settings->end()) {
          ALOGE("%s: Physical settings for camera id: %u are absent!",
                __FUNCTION__, it.first);
          continue;
        }
        uint8_t raw_binned_factor_used = false;
        if (sensor_binning_factor_info_.find(it.first) !=
            sensor_binning_factor_info_.end()) {
          auto& info = sensor_binning_factor_info_[it.first];
          // physical stream was included in the request
          if (!reprocess_request && info.quad_bayer_sensor &&
              info.max_res_request && info.has_raw_stream &&
              !info.has_non_raw_stream) {
            raw_binned_factor_used = true;
          }
          it.second->Set(ANDROID_SENSOR_RAW_BINNING_FACTOR_USED,
                         &raw_binned_factor_used, 1);
        }
        // Sensor timestamp for all physical devices must be the same.
        it.second->Set(ANDROID_SENSOR_TIMESTAMP, &next_capture_time_, 1);
        if (physical_settings->second.report_neutral_color_point) {
          it.second->Set(ANDROID_SENSOR_NEUTRAL_COLOR_POINT, kNeutralColorPoint,
                         ARRAY_SIZE(kNeutralColorPoint));
        }
        if (physical_settings->second.report_green_split) {
          it.second->Set(ANDROID_SENSOR_GREEN_SPLIT, &kGreenSplit, 1);
        }
        if (physical_settings->second.report_noise_profile) {
          auto device_chars = chars_->find(it.first);
          if (device_chars == chars_->end()) {
            ALOGE("%s: Sensor characteristics absent for device: %d", __func__,
                  it.first);
          }
          CalculateAndAppendNoiseProfile(
              physical_settings->second.gain,
              GetBaseGainFactor(device_chars->second.max_raw_value),
              it.second.get());
        }
      }
    }

    callback.process_pipeline_result(std::move(result));
  }
}

void EmulatedSensor::CalculateAndAppendNoiseProfile(
    float gain /*in ISO*/, float base_gain_factor,
    HalCameraMetadata* result /*out*/) {
  if (result != nullptr) {
    float total_gain = gain / 100.0 * base_gain_factor;
    float noise_var_gain = total_gain * total_gain;
    float read_noise_var =
        kReadNoiseVarBeforeGain * noise_var_gain + kReadNoiseVarAfterGain;
    // Noise profile is the same across all 4 CFA channels
    double noise_profile[2 * 4] = {
        noise_var_gain, read_noise_var, noise_var_gain, read_noise_var,
        noise_var_gain, read_noise_var, noise_var_gain, read_noise_var};
    result->Set(ANDROID_SENSOR_NOISE_PROFILE, noise_profile,
                ARRAY_SIZE(noise_profile));
  }
}

EmulatedScene::ColorChannels EmulatedSensor::GetQuadBayerColor(uint32_t x,
                                                               uint32_t y) {
  // Row within larger set of quad bayer filter
  uint32_t row_mod = y % 4;
  // Column within larger set of quad bayer filter
  uint32_t col_mod = x % 4;

  // Row is within the left quadrants of a quad bayer sensor
  if (row_mod < 2) {
    if (col_mod < 2) {
      return EmulatedScene::ColorChannels::R;
    }
    return EmulatedScene::ColorChannels::Gr;
  } else {
    if (col_mod < 2) {
      return EmulatedScene::ColorChannels::Gb;
    }
    return EmulatedScene::ColorChannels::B;
  }
}

void EmulatedSensor::RemosaicQuadBayerBlock(uint16_t* img_in, uint16_t* img_out,
                                            int xstart, int ystart,
                                            int row_stride_in_bytes) {
  uint32_t quad_block_copy_idx_map[16] = {0, 2, 1, 3, 8,  10, 6,  11,
                                          4, 9, 5, 7, 12, 14, 13, 15};
  uint16_t quad_block_copy[16];
  uint32_t i = 0;
  for (uint32_t row = 0; row < 4; row++) {
    uint16_t* quad_bayer_row =
        img_in + (ystart + row) * (row_stride_in_bytes / 2) + xstart;
    for (uint32_t j = 0; j < 4; j++, i++) {
      quad_block_copy[i] = quad_bayer_row[j];
    }
  }

  for (uint32_t row = 0; row < 4; row++) {
    uint16_t* regular_bayer_row =
        img_out + (ystart + row) * (row_stride_in_bytes / 2) + xstart;
    for (uint32_t j = 0; j < 4; j++, i++) {
      uint32_t idx = quad_block_copy_idx_map[row + 4 * j];
      regular_bayer_row[j] = quad_block_copy[idx];
    }
  }
}

status_t EmulatedSensor::RemosaicRAW16Image(uint16_t* img_in, uint16_t* img_out,
                                            size_t row_stride_in_bytes,
                                            const SensorCharacteristics& chars) {
  if (chars.full_res_width % 2 != 0 || chars.full_res_height % 2 != 0) {
    ALOGE(
        "%s RAW16 Image with quad CFA, height %zu and width %zu, not multiples "
        "of 4",
        __FUNCTION__, chars.full_res_height, chars.full_res_width);
    return BAD_VALUE;
  }
  for (uint32_t i = 0; i < chars.full_res_width; i += 4) {
    for (uint32_t j = 0; j < chars.full_res_height; j += 4) {
      RemosaicQuadBayerBlock(img_in, img_out, i, j, row_stride_in_bytes);
    }
  }
  return OK;
}

void EmulatedSensor::CaptureRawBinned(uint8_t* img, size_t row_stride_in_bytes,
                                      uint32_t gain,
                                      const SensorCharacteristics& chars) {
  ATRACE_CALL();
  // inc = how many pixels to skip while reading every next pixel
  float total_gain = gain / 100.0 * GetBaseGainFactor(chars.max_raw_value);
  float noise_var_gain = total_gain * total_gain;
  float read_noise_var =
      kReadNoiseVarBeforeGain * noise_var_gain + kReadNoiseVarAfterGain;
  int bayer_select[4] = {EmulatedScene::R, EmulatedScene::Gr, EmulatedScene::Gb,
                         EmulatedScene::B};
  scene_->SetReadoutPixel(0, 0);
  for (unsigned int out_y = 0; out_y < chars.height; out_y++) {
    // Stride still stays width since the buffer is binned size.
    int* bayer_row = bayer_select + (out_y & 0x1) * 2;
    uint16_t* px = (uint16_t*)img + out_y * (row_stride_in_bytes / 2);
    for (unsigned int out_x = 0; out_x < chars.width; out_x++) {
      int color_idx = bayer_row[out_x & 0x1];
      uint16_t raw_count = 0;
      // Color  filter will be the same for each quad.
      uint32_t electron_count = 0;
      int x, y;
      float norm_x = (float)out_x / chars.width;
      float norm_y = (float)out_y / chars.height;
      x = static_cast<int>(chars.full_res_width * norm_x);
      y = static_cast<int>(chars.full_res_height * norm_y);

      x = std::min(std::max(x, 0), (int)chars.full_res_width - 1);
      y = std::min(std::max(y, 0), (int)chars.full_res_height - 1);

      scene_->SetReadoutPixel(x, y);

      const uint32_t* pixel = scene_->GetPixelElectrons();
      electron_count = pixel[color_idx];
      // TODO: Better pixel saturation curve?
      electron_count = (electron_count < kSaturationElectrons)
                           ? electron_count
                           : kSaturationElectrons;

      // TODO: Better A/D saturation curve?
      raw_count = electron_count * total_gain;
      raw_count =
          (raw_count < chars.max_raw_value) ? raw_count : chars.max_raw_value;

      // Calculate noise value
      // TODO: Use more-correct Gaussian instead of uniform noise
      float photon_noise_var = electron_count * noise_var_gain;
      float noise_stddev = sqrtf_approx(read_noise_var + photon_noise_var);
      // Scaled to roughly match gaussian/uniform noise stddev
      float noise_sample = rand_r(&rand_seed_) * (2.5 / (1.0 + RAND_MAX)) - 1.25;

      raw_count += chars.black_level_pattern[color_idx];
      raw_count += noise_stddev * noise_sample;
      *px++ = raw_count;
    }
  }
  ALOGVV("Binned RAW sensor image captured");
}

void EmulatedSensor::CaptureRawFullRes(uint8_t* img, size_t row_stride_in_bytes,
                                       uint32_t gain,
                                       const SensorCharacteristics& chars) {
  ATRACE_CALL();
  float total_gain = gain / 100.0 * GetBaseGainFactor(chars.max_raw_value);
  float noise_var_gain = total_gain * total_gain;
  float read_noise_var =
      kReadNoiseVarBeforeGain * noise_var_gain + kReadNoiseVarAfterGain;

  scene_->SetReadoutPixel(0, 0);
  // RGGB
  int bayer_select[4] = {EmulatedScene::R, EmulatedScene::Gr, EmulatedScene::Gb,
                         EmulatedScene::B};

  for (unsigned int y = 0; y < chars.full_res_height; y++) {
    int* bayer_row = bayer_select + (y & 0x1) * 2;
    uint16_t* px = (uint16_t*)img + y * (row_stride_in_bytes / 2);
    for (unsigned int x = 0; x < chars.full_res_width; x++) {
      int color_idx = chars.quad_bayer_sensor ? GetQuadBayerColor(x, y)
                                              : bayer_row[x & 0x1];
      uint32_t electron_count;
      electron_count = scene_->GetPixelElectrons()[color_idx];

      // TODO: Better pixel saturation curve?
      electron_count = (electron_count < kSaturationElectrons)
                           ? electron_count
                           : kSaturationElectrons;

      // TODO: Better A/D saturation curve?
      uint16_t raw_count = electron_count * total_gain;
      raw_count =
          (raw_count < chars.max_raw_value) ? raw_count : chars.max_raw_value;

      // Calculate noise value
      // TODO: Use more-correct Gaussian instead of uniform noise
      float photon_noise_var = electron_count * noise_var_gain;
      float noise_stddev = sqrtf_approx(read_noise_var + photon_noise_var);
      // Scaled to roughly match gaussian/uniform noise stddev
      float noise_sample = rand_r(&rand_seed_) * (2.5 / (1.0 + RAND_MAX)) - 1.25;

      raw_count += chars.black_level_pattern[color_idx];
      raw_count += noise_stddev * noise_sample;

      *px++ = raw_count;
    }
    // TODO: Handle this better
    // simulatedTime += mRowReadoutTime;
  }
  ALOGVV("Raw sensor image captured");
}

void EmulatedSensor::CaptureRGB(uint8_t* img, uint32_t width, uint32_t height,
                                uint32_t stride, RGBLayout layout, uint32_t gain,
                                const SensorCharacteristics& chars) {
  ATRACE_CALL();
  float total_gain = gain / 100.0 * GetBaseGainFactor(chars.max_raw_value);
  // In fixed-point math, calculate total scaling from electrons to 8bpp
  int scale64x = 64 * total_gain * 255 / chars.max_raw_value;
  uint32_t inc_h = ceil((float)chars.full_res_width / width);
  uint32_t inc_v = ceil((float)chars.full_res_height / height);

  for (unsigned int y = 0, outy = 0; y < chars.full_res_height;
       y += inc_v, outy++) {
    scene_->SetReadoutPixel(0, y);
    uint8_t* px = img + outy * stride;
    for (unsigned int x = 0; x < chars.full_res_width; x += inc_h) {
      uint32_t r_count, g_count, b_count;
      // TODO: Perfect demosaicing is a cheat
      const uint32_t* pixel = scene_->GetPixelElectrons();
      r_count = pixel[EmulatedScene::R] * scale64x;
      g_count = pixel[EmulatedScene::Gr] * scale64x;
      b_count = pixel[EmulatedScene::B] * scale64x;

      uint8_t r = r_count < 255 * 64 ? r_count / 64 : 255;
      uint8_t g = g_count < 255 * 64 ? g_count / 64 : 255;
      uint8_t b = b_count < 255 * 64 ? b_count / 64 : 255;
      switch (layout) {
        case RGB:
          *px++ = r;
          *px++ = g;
          *px++ = b;
          break;
        case RGBA:
          *px++ = r;
          *px++ = g;
          *px++ = b;
          *px++ = 255;
          break;
        case ARGB:
          *px++ = 255;
          *px++ = r;
          *px++ = g;
          *px++ = b;
          break;
        default:
          ALOGE("%s: RGB layout: %d not supported", __FUNCTION__, layout);
          return;
      }
      for (unsigned int j = 1; j < inc_h; j++) scene_->GetPixelElectrons();
    }
  }
  ALOGVV("RGB sensor image captured");
}

void EmulatedSensor::CaptureYUV420(YCbCrPlanes yuv_layout, uint32_t width,
                                   uint32_t height, uint32_t gain,
                                   float zoom_ratio, bool rotate,
                                   const SensorCharacteristics& chars) {
  ATRACE_CALL();
  float total_gain = gain / 100.0 * GetBaseGainFactor(chars.max_raw_value);
  // Using fixed-point math with 6 bits of fractional precision.
  // In fixed-point math, calculate total scaling from electrons to 8bpp
  const int scale64x =
      kFixedBitPrecision * total_gain * 255 / chars.max_raw_value;
  // Fixed-point coefficients for RGB-YUV transform
  // Based on JFIF RGB->YUV transform.
  // Cb/Cr offset scaled by 64x twice since they're applied post-multiply
  const int rgb_to_y[] = {19, 37, 7};
  const int rgb_to_cb[] = {-10, -21, 32, 524288};
  const int rgb_to_cr[] = {32, -26, -5, 524288};
  // Scale back to 8bpp non-fixed-point
  const int scale_out = 64;
  const int scale_out_sq = scale_out * scale_out;  // after multiplies

  // inc = how many pixels to skip while reading every next pixel
  const float aspect_ratio = static_cast<float>(width) / height;

  // precalculate normalized coordinates and dimensions
  const float norm_left_top = 0.5f - 0.5f / zoom_ratio;
  const float norm_rot_top = norm_left_top;
  const float norm_width = 1 / zoom_ratio;
  const float norm_rot_width = norm_width / aspect_ratio;
  const float norm_rot_height = norm_width;
  const float norm_rot_left =
      norm_left_top + (norm_width + norm_rot_width) * 0.5f;

  for (unsigned int out_y = 0; out_y < height; out_y++) {
    uint8_t* px_y = yuv_layout.img_y + out_y * yuv_layout.y_stride;
    uint8_t* px_cb = yuv_layout.img_cb + (out_y / 2) * yuv_layout.cbcr_stride;
    uint8_t* px_cr = yuv_layout.img_cr + (out_y / 2) * yuv_layout.cbcr_stride;

    for (unsigned int out_x = 0; out_x < width; out_x++) {
      int x, y;
      float norm_x = out_x / (width * zoom_ratio);
      float norm_y = out_y / (height * zoom_ratio);
      if (rotate) {
        x = static_cast<int>(chars.full_res_width *
                             (norm_rot_left - norm_y * norm_rot_width));
        y = static_cast<int>(chars.full_res_height *
                             (norm_rot_top + norm_x * norm_rot_height));
      } else {
        x = static_cast<int>(chars.full_res_width * (norm_left_top + norm_x));
        y = static_cast<int>(chars.full_res_height * (norm_left_top + norm_y));
      }
      x = std::min(std::max(x, 0), (int)chars.full_res_width - 1);
      y = std::min(std::max(y, 0), (int)chars.full_res_height - 1);
      scene_->SetReadoutPixel(x, y);

      int32_t r_count, g_count, b_count;
      // TODO: Perfect demosaicing is a cheat
      const uint32_t* pixel = rotate ? scene_->GetPixelElectronsColumn()
                                     : scene_->GetPixelElectrons();
      r_count = pixel[EmulatedScene::R] * scale64x;
      r_count = r_count < kSaturationPoint ? r_count : kSaturationPoint;
      g_count = pixel[EmulatedScene::Gr] * scale64x;
      g_count = g_count < kSaturationPoint ? g_count : kSaturationPoint;
      b_count = pixel[EmulatedScene::B] * scale64x;
      b_count = b_count < kSaturationPoint ? b_count : kSaturationPoint;

      // Gamma correction
      r_count = gamma_table_[r_count];
      g_count = gamma_table_[g_count];
      b_count = gamma_table_[b_count];

      uint8_t y8 = (rgb_to_y[0] * r_count + rgb_to_y[1] * g_count +
                    rgb_to_y[2] * b_count) /
                   scale_out_sq;
      if (yuv_layout.bytesPerPixel == 1) {
        *px_y = y8;
      } else if (yuv_layout.bytesPerPixel == 2) {
        *(reinterpret_cast<uint16_t*>(px_y)) = htole16(y8 << 8);
      } else {
        ALOGE("%s: Unsupported bytes per pixel value: %zu", __func__,
              yuv_layout.bytesPerPixel);
        return;
      }
      px_y += yuv_layout.bytesPerPixel;

      if (out_y % 2 == 0 && out_x % 2 == 0) {
        uint8_t cb8 = (rgb_to_cb[0] * r_count + rgb_to_cb[1] * g_count +
                       rgb_to_cb[2] * b_count + rgb_to_cb[3]) /
                      scale_out_sq;
        uint8_t cr8 = (rgb_to_cr[0] * r_count + rgb_to_cr[1] * g_count +
                       rgb_to_cr[2] * b_count + rgb_to_cr[3]) /
                      scale_out_sq;
        if (yuv_layout.bytesPerPixel == 1) {
          *px_cb = cb8;
          *px_cr = cr8;
        } else if (yuv_layout.bytesPerPixel == 2) {
          *(reinterpret_cast<uint16_t*>(px_cb)) = htole16(cb8 << 8);
          *(reinterpret_cast<uint16_t*>(px_cr)) = htole16(cr8 << 8);
        } else {
          ALOGE("%s: Unsupported bytes per pixel value: %zu", __func__,
                yuv_layout.bytesPerPixel);
          return;
        }
        px_cr += yuv_layout.cbcr_step;
        px_cb += yuv_layout.cbcr_step;
      }
    }
  }
  ALOGVV("YUV420 sensor image captured");
}

void EmulatedSensor::CaptureDepth(uint8_t* img, uint32_t gain, uint32_t width,
                                  uint32_t height, uint32_t stride,
                                  const SensorCharacteristics& chars) {
  ATRACE_CALL();
  float total_gain = gain / 100.0 * GetBaseGainFactor(chars.max_raw_value);
  // In fixed-point math, calculate scaling factor to 13bpp millimeters
  int scale64x = 64 * total_gain * 8191 / chars.max_raw_value;
  uint32_t inc_h = ceil((float)chars.full_res_width / width);
  uint32_t inc_v = ceil((float)chars.full_res_height / height);

  for (unsigned int y = 0, out_y = 0; y < chars.full_res_height;
       y += inc_v, out_y++) {
    scene_->SetReadoutPixel(0, y);
    uint16_t* px = (uint16_t*)(img + (out_y * stride));
    for (unsigned int x = 0; x < chars.full_res_width; x += inc_h) {
      uint32_t depth_count;
      // TODO: Make up real depth scene instead of using green channel
      // as depth
      const uint32_t* pixel = scene_->GetPixelElectrons();
      depth_count = pixel[EmulatedScene::Gr] * scale64x;

      *px++ = depth_count < 8191 * 64 ? depth_count / 64 : 0;
      for (unsigned int j = 1; j < inc_h; j++) scene_->GetPixelElectrons();
    }
    // TODO: Handle this better
    // simulatedTime += mRowReadoutTime;
  }
  ALOGVV("Depth sensor image captured");
}

status_t EmulatedSensor::ProcessYUV420(const YUV420Frame& input,
                                       const YUV420Frame& output, uint32_t gain,
                                       ProcessType process_type, float zoom_ratio,
                                       bool rotate_and_crop,
                                       const SensorCharacteristics& chars) {
  ATRACE_CALL();
  size_t input_width, input_height;
  YCbCrPlanes input_planes, output_planes;
  std::vector<uint8_t> temp_yuv, temp_output_uv, temp_input_uv;

  // Overwrite HIGH_QUALITY to REGULAR for Emulator if property
  // ro.boot.qemu.camera_hq_edge_processing is false;
  if (process_type == HIGH_QUALITY &&
      !property_get_bool("ro.boot.qemu.camera_hq_edge_processing", true)) {
    process_type = REGULAR;
  }

  switch (process_type) {
    case HIGH_QUALITY:
      CaptureYUV420(output.planes, output.width, output.height, gain, zoom_ratio,
                    rotate_and_crop, chars);
      return OK;
    case REPROCESS:
      input_width = input.width;
      input_height = input.height;
      input_planes = input.planes;

      // libyuv only supports planar YUV420 during scaling.
      // Split the input U/V plane in separate planes if needed.
      if (input_planes.cbcr_step == 2) {
        temp_input_uv.resize(input_width * input_height / 2);
        auto temp_uv_buffer = temp_input_uv.data();
        input_planes.img_cb = temp_uv_buffer;
        input_planes.img_cr = temp_uv_buffer + (input_width * input_height) / 4;
        input_planes.cbcr_stride = input_width / 2;
        if (input.planes.img_cb < input.planes.img_cr) {
          libyuv::SplitUVPlane(input.planes.img_cb, input.planes.cbcr_stride,
                               input_planes.img_cb, input_planes.cbcr_stride,
                               input_planes.img_cr, input_planes.cbcr_stride,
                               input_width / 2, input_height / 2);
        } else {
          libyuv::SplitUVPlane(input.planes.img_cr, input.planes.cbcr_stride,
                               input_planes.img_cr, input_planes.cbcr_stride,
                               input_planes.img_cb, input_planes.cbcr_stride,
                               input_width / 2, input_height / 2);
        }
      }
      break;
    case REGULAR:
    default:
      // Generate the smallest possible frame with the expected AR and
      // then scale using libyuv.
      float aspect_ratio = static_cast<float>(output.width) / output.height;
      zoom_ratio = std::max(1.f, zoom_ratio);
      input_width = EmulatedScene::kSceneWidth * aspect_ratio;
      input_height = EmulatedScene::kSceneHeight;
      temp_yuv.reserve((input_width * input_height * 3) / 2);
      auto temp_yuv_buffer = temp_yuv.data();
      input_planes = {
          .img_y = temp_yuv_buffer,
          .img_cb = temp_yuv_buffer + input_width * input_height,
          .img_cr = temp_yuv_buffer + (input_width * input_height * 5) / 4,
          .y_stride = static_cast<uint32_t>(input_width),
          .cbcr_stride = static_cast<uint32_t>(input_width) / 2,
          .cbcr_step = 1};
      CaptureYUV420(input_planes, input_width, input_height, gain, zoom_ratio,
                    rotate_and_crop, chars);
  }

  output_planes = output.planes;
  // libyuv only supports planar YUV420 during scaling.
  // Treat the output UV space as planar first and then
  // interleave in the second step.
  if (output_planes.cbcr_step == 2) {
    temp_output_uv.resize(output.width * output.height / 2);
    auto temp_uv_buffer = temp_output_uv.data();
    output_planes.img_cb = temp_uv_buffer;
    output_planes.img_cr = temp_uv_buffer + output.width * output.height / 4;
    output_planes.cbcr_stride = output.width / 2;
  }

  auto ret = I420Scale(
      input_planes.img_y, input_planes.y_stride, input_planes.img_cb,
      input_planes.cbcr_stride, input_planes.img_cr, input_planes.cbcr_stride,
      input_width, input_height, output_planes.img_y, output_planes.y_stride,
      output_planes.img_cb, output_planes.cbcr_stride, output_planes.img_cr,
      output_planes.cbcr_stride, output.width, output.height,
      libyuv::kFilterNone);
  if (ret != 0) {
    ALOGE("%s: Failed during YUV scaling: %d", __FUNCTION__, ret);
    return ret;
  }

  // Merge U/V Planes for the interleaved case
  if (output_planes.cbcr_step == 2) {
    if (output.planes.img_cb < output.planes.img_cr) {
      libyuv::MergeUVPlane(output_planes.img_cb, output_planes.cbcr_stride,
                           output_planes.img_cr, output_planes.cbcr_stride,
                           output.planes.img_cb, output.planes.cbcr_stride,
                           output.width / 2, output.height / 2);
    } else {
      libyuv::MergeUVPlane(output_planes.img_cr, output_planes.cbcr_stride,
                           output_planes.img_cb, output_planes.cbcr_stride,
                           output.planes.img_cr, output.planes.cbcr_stride,
                           output.width / 2, output.height / 2);
    }
  }

  return ret;
}

int32_t EmulatedSensor::ApplysRGBGamma(int32_t value, int32_t saturation) {
  float n_value = (static_cast<float>(value) / saturation);
  n_value = (n_value <= 0.0031308f)
                ? n_value * 12.92f
                : 1.055f * pow(n_value, 0.4166667f) - 0.055f;
  return n_value * saturation;
}

}  // namespace android