1# Copyright 2019 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verifies per_frame_control."""
15
16
17import logging
18import os.path
19import matplotlib
20from matplotlib import pylab
21from mobly import test_runner
22import numpy as np
23
24import its_base_test
25import camera_properties_utils
26import capture_request_utils
27import image_processing_utils
28import its_session_utils
29
30_AE_STATE_CONVERGED = 2
31_AE_STATE_FLASH_REQUIRED = 4
32_DELTA_GAIN_THRESH = 3  # >3% gain change --> luma change in same dir.
33_DELTA_LUMA_THRESH = 3  # 3% frame-to-frame noise test_burst_sameness_manual.
34_DELTA_NO_GAIN_THRESH = 1  # <1% gain change --> min luma change.
35_NAME = os.path.splitext(os.path.basename(__file__))[0]
36_NS_TO_MS = 1.0E-6
37_NUM_CAPS = 1
38_NUM_FRAMES = 30
39_PATCH_H = 0.1  # Center 10%.
40_PATCH_W = 0.1
41_PATCH_X = 0.5 - _PATCH_W/2
42_PATCH_Y = 0.5 - _PATCH_H/2
43_RAW_NIBBLE_SIZE = 6  # Used to increase NUM_CAPS & decrease NUM_FRAMES for RAW.
44_RAW_GR_CH = 1
45_VALID_LUMA_MIN = 0.1
46_VALID_LUMA_MAX = 0.9
47_YUV_Y_CH = 0
48
49
50def _check_delta_luma_vs_delta_gain(fmt, j, lumas, total_gains):
51  """Determine if luma and gain move together for current frame."""
52  delta_gain = total_gains[j] - total_gains[j-1]
53  delta_luma = lumas[j] - lumas[j-1]
54  delta_gain_rel = delta_gain / total_gains[j-1] * 100  # %
55  delta_luma_rel = delta_luma / lumas[j-1] * 100  # %
56  # luma and total_gain should change in same direction
57  if abs(delta_gain_rel) > _DELTA_GAIN_THRESH:
58    logging.debug('frame %d: %.2f%% delta gain, %.2f%% delta luma',
59                  j, delta_gain_rel, delta_luma_rel)
60    if delta_gain * delta_luma < 0.0:
61      return (f"{fmt['format']}: frame {j}: gain {total_gains[j-1]:.1f} "
62              f'-> {total_gains[j]:.1f} ({delta_gain_rel:.1f}%), '
63              f'luma {lumas[j-1]} -> {lumas[j]} ({delta_luma_rel:.2f}%) '
64              f'GAIN/LUMA OPPOSITE DIR')
65  elif abs(delta_gain_rel) < _DELTA_NO_GAIN_THRESH:
66    logging.debug('frame %d: <|%.1f%%| delta gain, %.2f%% delta luma', j,
67                  _DELTA_NO_GAIN_THRESH, delta_luma_rel)
68    if abs(delta_luma_rel) > _DELTA_LUMA_THRESH:
69      return (f"{fmt['format']}: frame {j}: gain {total_gains[j-1]:.1f} "
70              f'-> {total_gains[j]:.1f} ({delta_gain_rel:.1f}%), '
71              f'luma {lumas[j-1]} -> {lumas[j]} ({delta_luma_rel:.2f}%), '
72              f'<|{_DELTA_NO_GAIN_THRESH:.1f}%| GAIN, '
73              f'>|{_DELTA_LUMA_THRESH:.1f}%| LUMA DELTA')
74  else:
75    logging.debug('frame %d: %.1f%% delta gain, %.2f%% delta luma',
76                  j, delta_gain_rel, delta_luma_rel)
77    return None
78
79
80def _determine_test_formats(cam, props, raw_avlb, debug):
81  """Determines the capture formats to test.
82
83  Args:
84    cam: Camera capture object.
85    props: Camera properties dict.
86    raw_avlb: Boolean for if RAW captures are available.
87    debug: Boolean for whether in debug mode.
88  Returns:
89    fmts: List of formats.
90  """
91  largest_yuv = capture_request_utils.get_largest_yuv_format(props)
92  match_ar = (largest_yuv['width'], largest_yuv['height'])
93  fmt = capture_request_utils.get_smallest_yuv_format(
94      props, match_ar=match_ar)
95  if raw_avlb and debug:
96    return (cam.CAP_RAW, fmt)
97  else:
98    return (fmt,)
99
100
101def _tabulate_frame_data(metadata, luma, raw_cap, debug):
102  """Puts relevant frame data into a dictionary."""
103  ae_state = metadata['android.control.aeState']
104  iso = metadata['android.sensor.sensitivity']
105  isp_gain = metadata['android.control.postRawSensitivityBoost'] / 100
106  exp_time = metadata['android.sensor.exposureTime'] * _NS_TO_MS
107  total_gain = iso * exp_time
108  if not raw_cap:
109    total_gain *= isp_gain
110  awb_state = metadata['android.control.awbState']
111  frame = {
112      'awb_gains': metadata['android.colorCorrection.gains'],
113      'ccm': metadata['android.colorCorrection.transform'],
114      'fd': metadata['android.lens.focusDistance'],
115  }
116
117  # Convert CCM from rational to float, as numpy arrays.
118  awb_ccm = np.array(capture_request_utils.rational_to_float(
119      frame['ccm'])).reshape(3, 3)
120
121  logging.debug('AE: %d ISO: %d ISP_sen: %d exp: %4fms tot_gain: %f luma: %f',
122                ae_state, iso, isp_gain, exp_time, total_gain, luma)
123  logging.debug('fd: %f', frame['fd'])
124  logging.debug('AWB state: %d, AWB gains: %s\n AWB matrix: %s', awb_state,
125                str(frame['awb_gains']), str(awb_ccm))
126  if debug:
127    logging.debug('Tonemap curve: %s', str(metadata['android.tonemap.curve']))
128
129  return frame, ae_state, total_gain
130
131
132def _compute_frame_luma(cap, props, raw_cap):
133  """Determines the luma for the center patch of the frame.
134
135  RAW captures use GR plane, YUV captures use Y plane.
136
137  Args:
138    cap: Camera capture object.
139    props: Camera properties dict.
140    raw_cap: Boolean for capture is RAW or YUV.
141  Returns:
142    luma: Luma value for center patch of image.
143  """
144  if raw_cap:
145    plane = image_processing_utils.convert_capture_to_planes(
146        cap, props=props)[_RAW_GR_CH]
147  else:
148    plane = image_processing_utils.convert_capture_to_planes(cap)[_YUV_Y_CH]
149
150  patch = image_processing_utils.get_image_patch(
151      plane, _PATCH_X, _PATCH_Y, _PATCH_W, _PATCH_H)
152  return image_processing_utils.compute_image_means(patch)[0]
153
154
155def _plot_data(lumas, gains, fmt, log_path):
156  """Plots lumas and gains data for this test.
157
158  Args:
159    lumas: List of luma data from captures.
160    gains: List of gain data from captures.
161    fmt: String to identify 'YUV' or 'RAW' plots.
162    log_path: Location to store data.
163  """
164  norm_gains = [x / max(gains) * max(lumas) for x in gains]
165
166  pylab.figure(fmt)
167  pylab.plot(range(len(lumas)), lumas, '-g.', label='Center patch brightness')
168  pylab.plot(range(len(gains)), norm_gains, '-r.',
169             label='Metadata AE setting product')
170  pylab.title(_NAME + ' ' + fmt)
171  pylab.xlabel('frame index')
172
173  # expand y axis for low delta results
174  ymin = min(norm_gains + lumas)
175  ymax = max(norm_gains + lumas)
176  yavg = (ymax + ymin) / 2.0
177  if ymax - ymin < 3 * _DELTA_LUMA_THRESH/100:
178    ymin = round(yavg - 1.5 * _DELTA_LUMA_THRESH/100, 3)
179    ymax = round(yavg + 1.5 * _DELTA_LUMA_THRESH/100, 3)
180    pylab.ylim(ymin, ymax)
181  pylab.legend()
182  matplotlib.pyplot.savefig(
183      '%s_plot_%s.png' % (os.path.join(log_path, _NAME), fmt))
184
185
186def _is_awb_af_stable(cap_info, i):
187  """Determines if Auto White Balance and Auto Focus are stable."""
188  awb_gains_i_1 = cap_info[i-1]['awb_gains']
189  awb_gains_i = cap_info[i]['awb_gains']
190
191  return (np.allclose(awb_gains_i_1, awb_gains_i, rtol=0.01) and
192          cap_info[i-1]['ccm'] == cap_info[i]['ccm'] and
193          np.isclose(cap_info[i-1]['fd'], cap_info[i]['fd'], rtol=0.01))
194
195
196class AutoPerFrameControlTest(its_base_test.ItsBaseTest):
197  """Tests PER_FRAME_CONTROL properties for auto capture requests.
198
199  Takes a sequence of images with auto capture request.
200  Determines if luma and gain settings move in same direction for large setting
201  changes.
202  Small settings changes should result in small changes in luma.
203  Threshold for checking is DELTA_GAIN_THRESH. Theshold where not change is
204  expected is DELTA_NO_GAIN_THRESH.
205
206  While not included in this test, if camera debug is required:
207    MANUAL_POSTPROCESSING capability is implied since
208    camera_properties_utils.read_3a is valid for test.
209
210    debug can also be performed with a defined tonemap curve:
211      req['android.tonemap.mode'] = 0
212      gamma = sum([[i/63.0,math.pow(i/63.0,1/2.2)] for i in xrange(64)],[])
213      req['android.tonemap.curve'] = {'red': gamma, 'green': gamma,
214                                      'blue': gamma}
215  """
216
217  def test_auto_per_frame_control(self):
218    logging.debug('Starting %s', _NAME)
219    with its_session_utils.ItsSession(
220        device_id=self.dut.serial,
221        camera_id=self.camera_id,
222        hidden_physical_id=self.hidden_physical_id) as cam:
223      props = cam.get_camera_properties()
224      props = cam.override_with_hidden_physical_camera_props(props)
225      log_path = self.log_path
226
227      # Check SKIP conditions.
228      camera_properties_utils.skip_unless(
229          camera_properties_utils.per_frame_control(props) and
230          camera_properties_utils.read_3a(props))
231
232      # Load chart for scene.
233      its_session_utils.load_scene(
234          cam, props, self.scene, self.tablet, self.chart_distance)
235
236      debug = self.debug_mode
237      raw_avlb = camera_properties_utils.raw16(props)
238      fmts = _determine_test_formats(cam, props, raw_avlb, debug)
239
240      failed = []
241      for i, fmt in enumerate(fmts):
242        logging.debug('fmt: %s', str(fmt['format']))
243        cam.do_3a()
244        req = capture_request_utils.auto_capture_request()
245        cap_info = {}
246        ae_states = []
247        lumas = []
248        total_gains = []
249        num_caps = _NUM_CAPS
250        num_frames = _NUM_FRAMES
251        raw_cap = i == 0 and raw_avlb and debug
252        # Break up caps if RAW to reduce bandwidth requirements.
253        if raw_cap:
254          num_caps = _NUM_CAPS * _RAW_NIBBLE_SIZE
255          num_frames = _NUM_FRAMES // _RAW_NIBBLE_SIZE
256
257        # Capture frames and tabulate info.
258        for j in range(num_caps):
259          caps = cam.do_capture([req] * num_frames, fmt)
260          for k, cap in enumerate(caps):
261            idx = k + j * num_frames
262            logging.debug('=========== frame %d ==========', idx)
263            luma = _compute_frame_luma(cap, props, raw_cap)
264            frame, ae_state, total_gain = _tabulate_frame_data(
265                cap['metadata'], luma, raw_cap, debug)
266            cap_info[idx] = frame
267            ae_states.append(ae_state)
268            lumas.append(luma)
269            total_gains.append(total_gain)
270
271             # Save image.
272            img = image_processing_utils.convert_capture_to_rgb_image(
273                cap, props=props)
274            image_processing_utils.write_image(img, '%s_frame_%s_%d.jpg' % (
275                os.path.join(log_path, _NAME), fmt['format'], idx))
276
277        _plot_data(lumas, total_gains, fmt['format'], log_path)
278
279        # Check correct behavior
280        logging.debug('fmt: %s', str(fmt['format']))
281        for j in range(1, num_caps * num_frames):
282          if _is_awb_af_stable(cap_info, j):
283            error_msg = _check_delta_luma_vs_delta_gain(
284                fmt, j, lumas, total_gains)
285            if error_msg:
286              failed.append(error_msg)
287          else:
288            logging.debug('frame %d -> %d: AWB/AF changed', j-1, j)
289
290        for j, luma in enumerate(lumas):
291          if ((ae_states[j] == _AE_STATE_CONVERGED or
292               ae_states[j] == _AE_STATE_FLASH_REQUIRED) and
293              (_VALID_LUMA_MIN > luma or luma > _VALID_LUMA_MAX)):
294            failed.append(
295                f"{fmt['format']}: frame {j} AE converged luma {luma}. "
296                f'Valid range: ({_VALID_LUMA_MIN}, {_VALID_LUMA_MAX})'
297            )
298      if failed:
299        logging.error('Error summary')
300        for fail in failed:
301          logging.error('%s', fail)
302        raise AssertionError
303
304if __name__ == '__main__':
305  test_runner.main()
306