1# Copyright 2015 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verifies android.edge.mode param behavior for reprocessing reqs."""
15
16
17import logging
18import os
19import matplotlib
20from matplotlib import pylab
21from mobly import test_runner
22import numpy as np
23
24import its_base_test
25import camera_properties_utils
26import capture_request_utils
27import image_processing_utils
28import its_session_utils
29import opencv_processing_utils
30
31EDGE_MODES = {'OFF': 0, 'FAST': 1, 'HQ': 2, 'ZSL': 3}
32EDGE_MODES_VALUES = list(EDGE_MODES.values())
33NAME = os.path.splitext(os.path.basename(__file__))[0]
34NUM_SAMPLES = 4
35PLOT_COLORS = {'yuv': 'r', 'private': 'g', 'none': 'b'}
36SHARPNESS_RTOL = 0.15
37
38
39def check_edge_modes(sharpness):
40  """Check that the sharpness for the different edge modes is correct."""
41  logging.debug('Verify HQ is sharper than OFF')
42  if sharpness[EDGE_MODES['HQ']] < sharpness[EDGE_MODES['OFF']]:
43    raise AssertionError(f"HQ < OFF! HQ: {sharpness[EDGE_MODES['HQ']]:.5f}, "
44                         f"OFF: {sharpness[EDGE_MODES['OFF']]:.5f}")
45
46  logging.debug('Verify ZSL is similar to OFF')
47  e_msg = 'ZSL: %.5f, OFF: %.5f, RTOL: %.2f' % (
48      sharpness[EDGE_MODES['ZSL']], sharpness[EDGE_MODES['OFF']],
49      SHARPNESS_RTOL)
50  assert np.isclose(sharpness[EDGE_MODES['ZSL']], sharpness[EDGE_MODES['OFF']],
51                    SHARPNESS_RTOL), e_msg
52
53  logging.debug('Verify OFF is not sharper than FAST')
54  e_msg = 'FAST: %.5f, OFF: %.5f, RTOL: %.2f' % (
55      sharpness[EDGE_MODES['FAST']], sharpness[EDGE_MODES['OFF']],
56      SHARPNESS_RTOL)
57  assert (sharpness[EDGE_MODES['FAST']] >
58          sharpness[EDGE_MODES['OFF']] * (1.0-SHARPNESS_RTOL)), e_msg
59
60  logging.debug('Verify FAST is not sharper than HQ')
61  e_msg = 'FAST: %.5f, HQ: %.5f, RTOL: %.2f' % (
62      sharpness[EDGE_MODES['FAST']], sharpness[EDGE_MODES['HQ']],
63      SHARPNESS_RTOL)
64  assert (sharpness[EDGE_MODES['HQ']] >
65          sharpness[EDGE_MODES['FAST']] * (1.0-SHARPNESS_RTOL)), e_msg
66
67
68def do_capture_and_determine_sharpness(
69    cam, edge_mode, sensitivity, exp, fd, out_surface, chart, log_path,
70    reprocess_format=None):
71  """Return sharpness of the output images and the capture result metadata.
72
73   Processes a capture request with a given edge mode, sensitivity, exposure
74   time, focus distance, output surface parameter, and reprocess format
75   (None for a regular request.)
76
77  Args:
78    cam: An open device session.
79    edge_mode: Edge mode for the request as defined in android.edge.mode
80    sensitivity: Sensitivity for the request as defined in
81                 android.sensor.sensitivity
82    exp: Exposure time for the request as defined in
83        android.sensor.exposureTime.
84    fd: Focus distance for the request as defined in
85        android.lens.focusDistance
86    out_surface: Specifications of the output image format and size.
87    chart: object containing chart information
88    log_path: location to save files
89    reprocess_format: (Optional) The reprocessing format. If not None,
90                      reprocessing will be enabled.
91
92  Returns:
93    Object containing reported edge mode and the sharpness of the output
94    image, keyed by the following strings:
95        'edge_mode'
96        'sharpness'
97  """
98
99  req = capture_request_utils.manual_capture_request(sensitivity, exp)
100  req['android.lens.focusDistance'] = fd
101  req['android.edge.mode'] = edge_mode
102  if reprocess_format:
103    req['android.reprocess.effectiveExposureFactor'] = 1.0
104
105  sharpness_list = []
106  caps = cam.do_capture([req]*NUM_SAMPLES, [out_surface], reprocess_format)
107  for n in range(NUM_SAMPLES):
108    y, _, _ = image_processing_utils.convert_capture_to_planes(caps[n])
109    chart.img = image_processing_utils.normalize_img(
110        image_processing_utils.get_image_patch(
111            y, chart.xnorm, chart.ynorm, chart.wnorm, chart.hnorm))
112    if n == 0:
113      image_processing_utils.write_image(
114          chart.img, '%s_reprocess_fmt_%s_edge=%d.jpg' % (
115              os.path.join(log_path, NAME), reprocess_format, edge_mode))
116      edge_mode_res = caps[n]['metadata']['android.edge.mode']
117    sharpness_list.append(
118        image_processing_utils.compute_image_sharpness(chart.img))
119  logging.debug('Sharpness list for edge mode %d: %s',
120                edge_mode, str(sharpness_list))
121  return {'edge_mode': edge_mode_res, 'sharpness': np.mean(sharpness_list)}
122
123
124class ReprocessEdgeEnhancementTest(its_base_test.ItsBaseTest):
125  """Test android.edge.mode param applied when set for reprocessing requests.
126
127  Capture non-reprocess images for each edge mode and calculate their
128  sharpness as a baseline.
129
130  Capture reprocessed images for each supported reprocess format and edge_mode
131  mode. Calculate the sharpness of reprocessed images and compare them against
132  the sharpess of non-reprocess images.
133  """
134
135  def test_reprocess_edge_enhancement(self):
136    logging.debug('Starting %s', NAME)
137    logging.debug('Edge modes: %s', str(EDGE_MODES))
138    with its_session_utils.ItsSession(
139        device_id=self.dut.serial,
140        camera_id=self.camera_id,
141        hidden_physical_id=self.hidden_physical_id) as cam:
142      chart_loc_arg = self.chart_loc_arg
143      props = cam.get_camera_properties()
144      props = cam.override_with_hidden_physical_camera_props(props)
145      log_path = self.log_path
146
147      # Check skip conditions
148      camera_properties_utils.skip_unless(
149          camera_properties_utils.read_3a(props) and
150          camera_properties_utils.per_frame_control(props) and
151          camera_properties_utils.edge_mode(props, 0) and
152          (camera_properties_utils.yuv_reprocess(props) or
153           camera_properties_utils.private_reprocess(props)))
154
155      # Load chart for scene
156      its_session_utils.load_scene(
157          cam, props, self.scene, self.tablet, self.chart_distance)
158
159      # Initialize chart class and locate chart in scene
160      chart = opencv_processing_utils.Chart(
161          cam, props, self.log_path, chart_loc=chart_loc_arg)
162
163      # If reprocessing is supported, ZSL edge mode must be avaiable.
164      assert camera_properties_utils.edge_mode(
165          props, EDGE_MODES['ZSL']), 'ZSL android.edge.mode not available!'
166
167      reprocess_formats = []
168      if camera_properties_utils.yuv_reprocess(props):
169        reprocess_formats.append('yuv')
170      if camera_properties_utils.private_reprocess(props):
171        reprocess_formats.append('private')
172
173      size = capture_request_utils.get_available_output_sizes('jpg', props)[0]
174      logging.debug('image W: %d, H: %d', size[0], size[1])
175      out_surface = {'width': size[0], 'height': size[1], 'format': 'jpg'}
176
177      # Get proper sensitivity, exposure time, and focus distance.
178      mono_camera = camera_properties_utils.mono_camera(props)
179      s, e, _, _, fd = cam.do_3a(get_results=True, mono_camera=mono_camera)
180
181      # Initialize plot
182      pylab.figure('reprocess_result')
183      pylab.title(NAME)
184      pylab.xlabel('Edge Enhance Mode')
185      pylab.ylabel('Sharpness')
186      pylab.xticks(EDGE_MODES_VALUES)
187
188      # Get the sharpness for each edge mode for regular requests
189      sharpness_regular = []
190      edge_mode_reported_regular = []
191      for edge_mode in EDGE_MODES.values():
192        # Skip unavailable modes
193        if not camera_properties_utils.edge_mode(props, edge_mode):
194          edge_mode_reported_regular.append(edge_mode)
195          sharpness_regular.append(0)
196          continue
197        ret = do_capture_and_determine_sharpness(
198            cam, edge_mode, s, e, fd, out_surface, chart, log_path)
199        edge_mode_reported_regular.append(ret['edge_mode'])
200        sharpness_regular.append(ret['sharpness'])
201
202      pylab.plot(EDGE_MODES_VALUES, sharpness_regular,
203                 '-'+PLOT_COLORS['none']+'o', label='None')
204      logging.debug('Sharpness for edge modes with regular request: %s',
205                    str(sharpness_regular))
206
207      # Get sharpness for each edge mode and reprocess format
208      sharpnesses_reprocess = []
209      edge_mode_reported_reprocess = []
210
211      for reprocess_format in reprocess_formats:
212        # List of sharpness
213        sharpnesses = []
214        edge_mode_reported = []
215        for edge_mode in range(4):
216          # Skip unavailable modes
217          if not camera_properties_utils.edge_mode(props, edge_mode):
218            edge_mode_reported.append(edge_mode)
219            sharpnesses.append(0)
220            continue
221
222          ret = do_capture_and_determine_sharpness(
223              cam, edge_mode, s, e, fd, out_surface, chart, log_path,
224              reprocess_format)
225          edge_mode_reported.append(ret['edge_mode'])
226          sharpnesses.append(ret['sharpness'])
227
228        sharpnesses_reprocess.append(sharpnesses)
229        edge_mode_reported_reprocess.append(edge_mode_reported)
230
231        # Add to plot and log results
232        pylab.plot(EDGE_MODES_VALUES, sharpnesses,
233                   '-'+PLOT_COLORS[reprocess_format]+'o',
234                   label=reprocess_format)
235        logging.debug('Sharpness for edge modes w/ %s reprocess fmt: %s',
236                      reprocess_format, str(sharpnesses))
237      # Finalize plot
238      pylab.legend(numpoints=1, fancybox=True)
239      matplotlib.pyplot.savefig('%s_plot.png' %
240                                os.path.join(log_path, NAME))
241      logging.debug('Check regular requests')
242      check_edge_modes(sharpness_regular)
243
244      for reprocess_format in range(len(reprocess_formats)):
245        logging.debug('Check reprocess format: %s', reprocess_format)
246        check_edge_modes(sharpnesses_reprocess[reprocess_format])
247
248        hq_div_off_reprocess = (
249            sharpnesses_reprocess[reprocess_format][EDGE_MODES['HQ']] /
250            sharpnesses_reprocess[reprocess_format][EDGE_MODES['OFF']])
251        hq_div_off_regular = (
252            sharpness_regular[EDGE_MODES['HQ']] /
253            sharpness_regular[EDGE_MODES['OFF']])
254        e_msg = 'HQ/OFF_reprocess: %.4f, HQ/OFF_reg: %.4f, RTOL: %.2f' % (
255            hq_div_off_reprocess, hq_div_off_regular, SHARPNESS_RTOL)
256        logging.debug('Verify reprocess HQ ~= reg HQ relative to OFF')
257        assert np.isclose(hq_div_off_reprocess, hq_div_off_regular,
258                          SHARPNESS_RTOL), e_msg
259
260if __name__ == '__main__':
261  test_runner.main()
262
263