1# Copyright 2024 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verify that lens intrinsics changes when OIS is triggered."""
15
16import logging
17import math
18import numpy as np
19import os
20
21from mobly import test_runner
22from matplotlib import pylab
23import matplotlib.pyplot
24
25import its_base_test
26import camera_properties_utils
27import its_session_utils
28import preview_processing_utils
29import sensor_fusion_utils
30
31_INTRINSICS_SAMPLES = 'android.statistics.lensIntrinsicsSamples'
32_NAME = os.path.splitext(os.path.basename(__file__))[0]
33_MIN_PHONE_MOVEMENT_ANGLE = 5  # degrees
34_PRINCIPAL_POINT_THRESH = 1  # Threshold for principal point changes in pixels.
35_START_FRAME = 30  # give 3A some frames to warm up
36_VIDEO_DELAY_TIME = 5.5  # seconds
37
38
39def calculate_principal_point(f_x, f_y, c_x, c_y, s):
40  """Calculates the principal point of a camera given its intrinsic parameters.
41
42  Args:
43    f_x: Horizontal focal length.
44    f_y: Vertical focal length.
45    c_x: X coordinate of the optical axis.
46    c_y: Y coordinate of the optical axis.
47    s: Skew parameter.
48
49  Returns:
50    A numpy array containing the principal point coordinates (px, py).
51  """
52
53  # Create the camera calibration matrix
54  transform_k = np.array([[f_x, s, c_x],
55                          [0, f_y, c_y],
56                          [0, 0, 1]])
57
58  # The Principal point is the intersection of the optical axis with the
59  # image plane. Since the optical axis passes through the camera center
60  # (defined by K), the principal point coordinates are simply the
61  # projection of the camera center onto the image plane.
62  principal_point = np.dot(transform_k, np.array([0, 0, 1]))
63
64  # Normalize by the homogeneous coordinate
65  px = principal_point[0] / principal_point[2]
66  py = principal_point[1] / principal_point[2]
67
68  return px, py
69
70
71def plot_principal_points(principal_points_dist, start_frame,
72                          video_quality, plot_name_stem):
73  """Plot principal points values vs Camera frames.
74
75  Args:
76    principal_points_dist: array of principal point distances in pixels/frame
77    start_frame: int value of start frame
78    video_quality: str for video quality identifier
79    plot_name_stem: str; name of the plot
80  """
81
82  pylab.figure(video_quality)
83  frames = range(start_frame, len(principal_points_dist)+start_frame)
84  pylab.title(f'Lens Intrinsics vs frame {video_quality}')
85  pylab.plot(frames, principal_points_dist, '-ro', label='dist')
86  pylab.xlabel('Frame #')
87  pylab.ylabel('Principal points in pixels')
88  matplotlib.pyplot.savefig(f'{plot_name_stem}.png')
89  pylab.close(video_quality)
90
91
92def verify_lens_intrinsics(recording_obj, gyro_events, test_name, log_path):
93  """Verify principal points changes due to OIS changes.
94
95  Args:
96    recording_obj: Camcorder recording object.
97    gyro_events: Gyroscope events collected while recording.
98    test_name: Name of the test
99    log_path: Path for the log file
100
101  Returns:
102    A dictionary containing the maximum gyro angle, the maximum changes of
103    principal point, and a failure message if principal point doesn't change
104    due to OIS changes triggered by device motion.
105  """
106
107  file_name = recording_obj['recordedOutputPath'].split('/')[-1]
108  logging.debug('recorded file name: %s', file_name)
109  video_size = recording_obj['videoSize']
110  logging.debug('video size: %s', video_size)
111
112  capture_results = recording_obj['captureMetadata']
113  file_name_stem = f'{os.path.join(log_path, test_name)}_{video_size}'
114
115  # Extract principal points from capture result
116  principal_points = []
117  for capture_result in capture_results:
118    if capture_result.get('android.lens.intrinsicCalibration'):
119      intrinsic_cal = capture_result['android.lens.intrinsicCalibration']
120      logging.debug('Intrinsic Calibration: %s', str(intrinsic_cal))
121      principal_points.append(calculate_principal_point(*intrinsic_cal[:5]))
122
123  if not principal_points:
124    logging.debug('Lens Intrinsic are not reported in Capture Results.')
125    return {'gyro': None, 'max_pp_diff': None,
126            'failure': None, 'skip': True}
127
128  # Calculate variations in principal points
129  first_point = principal_points[0]
130  principal_points_diff = [math.dist(first_point, x) for x in principal_points]
131
132  plot_principal_points(principal_points_diff,
133                        _START_FRAME,
134                        video_size,
135                        file_name_stem)
136
137  max_pp_diff = max(principal_points_diff)
138
139  # Extract gyro rotations
140  sensor_fusion_utils.plot_gyro_events(
141      gyro_events, f'{test_name}_{video_size}', log_path)
142  gyro_rots = sensor_fusion_utils.conv_acceleration_to_movement(
143      gyro_events, _VIDEO_DELAY_TIME)
144  max_gyro_angle = sensor_fusion_utils.calc_max_rotation_angle(
145      gyro_rots, 'Gyro')
146  logging.debug(
147      'Max deflection (degrees) %s: gyro: %.3f',
148      video_size, max_gyro_angle)
149
150  # Assert phone is moved enough during test
151  if max_gyro_angle < _MIN_PHONE_MOVEMENT_ANGLE:
152    raise AssertionError(
153        f'Phone not moved enough! Movement: {max_gyro_angle}, '
154        f'THRESH: {_MIN_PHONE_MOVEMENT_ANGLE} degrees')
155
156  failure_msg = None
157  if(max_pp_diff > _PRINCIPAL_POINT_THRESH):
158    logging.debug('Principal point diff: x = %.2f', max_pp_diff)
159  else:
160    failure_msg = (
161        'Change in principal point not enough with respect to OIS changes. '
162        f'video_size: {video_size}, '
163        f'Max Principal Point deflection (pixels):  {max_pp_diff:.3f}, '
164        f'Max gyro angle: {max_gyro_angle:.3f}, '
165        f'THRESHOLD : {_PRINCIPAL_POINT_THRESH}.')
166
167  return {'gyro': max_gyro_angle, 'max_pp_diff': max_pp_diff,
168          'failure': failure_msg, 'skip': False}
169
170
171def verify_lens_intrinsics_sample(recording_obj):
172  """Verify principal points changes in intrinsics samples.
173
174  Validate if principal points changes in at least one intrinsics samples.
175  Validate if timestamp changes in each intrinsics samples.
176
177  Args:
178    recording_obj: Camcorder recording object.
179
180  Returns:
181    a failure message if principal point doesn't change.
182    a failure message if timestamps doesn't change
183    None: either test passes or capture results doesn't include
184          intrinsics samples
185  """
186
187  file_name = recording_obj['recordedOutputPath'].split('/')[-1]
188  logging.debug('recorded file name: %s', file_name)
189  video_size = recording_obj['videoSize']
190  logging.debug('video size: %s', video_size)
191
192  capture_results = recording_obj['captureMetadata']
193
194  # Extract Lens Intrinsics Samples from capture result
195  intrinsics_samples_list = []
196  for capture_result in capture_results:
197    if _INTRINSICS_SAMPLES in capture_result:
198      samples = capture_result[_INTRINSICS_SAMPLES]
199      intrinsics_samples_list.append(samples)
200
201  if not intrinsics_samples_list:
202    logging.debug('Lens Intrinsic Samples are not reported')
203    # Don't change print to logging. Used for KPI.
204    print(f'{_NAME}_samples_principal_points_diff_detected: false')
205    return {'failure': None, 'skip': True}
206
207  failure_msg = ''
208
209  max_samples_pp_diffs = []
210  max_samples_timestamp_diffs = []
211  for samples in intrinsics_samples_list:
212    pp_diffs = []
213    timestamp_diffs = []
214
215    # Evaluate intrinsics samples
216    first_sample = samples[0]
217    first_instrinsics = first_sample['lensIntrinsics']
218    first_ts = first_sample['timestamp']
219    first_point = calculate_principal_point(*first_instrinsics[:5])
220
221    for sample in samples:
222      samples_intrinsics = sample['lensIntrinsics']
223      timestamp = sample['timestamp']
224      principal_point = calculate_principal_point(*samples_intrinsics[:5])
225      distance = math.dist(first_point, principal_point)
226      pp_diffs.append(distance)
227      timestamp_diffs.append(timestamp-first_ts)
228
229    max_samples_pp_diffs.append(max(pp_diffs))
230    max_samples_timestamp_diffs.append(max(timestamp_diffs))
231
232  if any(value != 0 for value in max_samples_pp_diffs):
233    # Don't change print to logging. Used for KPI.
234    print(f'{_NAME}_samples_principal_points_diff_detected: true')
235    logging.debug('Principal points variations found in at lease one sample')
236  else:
237    # Don't change print to logging. Used for KPI.
238    print(f'{_NAME}_samples_principal_points_diff_detected: false')
239    failure_msg = failure_msg + (
240        'No variation of principal points found in any samples.\n\n'
241    )
242  if all(diff > 0 for diff in max_samples_timestamp_diffs[1:]):
243    logging.debug('Timestamps variations found in all samples')
244  else:
245    failure_msg = failure_msg + 'Timestamps in samples did not change. \n\n'
246
247  failure_msg = None if failure_msg else failure_msg
248
249  return {'failure': failure_msg, 'skip': False}
250
251
252class LensIntrinsicCalibrationTest(its_base_test.ItsBaseTest):
253  """Tests if lens intrinsics changes when OIS is triggered.
254
255  Camera is moved in sensor fusion rig on an angle of 15 degrees.
256  Speed is set to mimic hand movement (and not be too fast).
257  Preview is recorded after rotation rig starts moving, and the
258  gyroscope data is dumped.
259
260  Camera movement is extracted from angle of deflection in gyroscope
261  movement. Test is a PASS if principal point in lens intrinsics
262  changes upon camera movement.
263  """
264
265  def test_lens_intrinsic_calibration(self):
266    rot_rig = {}
267    log_path = self.log_path
268
269    with its_session_utils.ItsSession(
270        device_id=self.dut.serial,
271        camera_id=self.camera_id,
272        hidden_physical_id=self.hidden_physical_id) as cam:
273
274      props = cam.get_camera_properties()
275      props = cam.override_with_hidden_physical_camera_props(props)
276
277      # Check if OIS supported
278      camera_properties_utils.skip_unless(
279          camera_properties_utils.optical_stabilization_supported(props))
280
281      # Initialize rotation rig
282      rot_rig['cntl'] = self.rotator_cntl
283      rot_rig['ch'] = self.rotator_ch
284      if rot_rig['cntl'].lower() != 'arduino':
285        raise AssertionError(
286            f'You must use the arduino controller for {_NAME}.')
287
288      preview_size = preview_processing_utils.get_max_preview_test_size(
289          cam, self.camera_id)
290      logging.debug('preview_test_size: %s', preview_size)
291
292      recording_obj = preview_processing_utils.collect_data(
293          cam, self.tablet_device, preview_size, False,
294          rot_rig=rot_rig, ois=True)
295
296      # Get gyro events
297      logging.debug('Reading out inertial sensor events')
298      gyro_events = cam.get_sensor_events()['gyro']
299      logging.debug('Number of gyro samples %d', len(gyro_events))
300
301      # Grab the video from the save location on DUT
302      self.dut.adb.pull([recording_obj['recordedOutputPath'], log_path])
303
304      intrinsic_result = verify_lens_intrinsics(
305          recording_obj, gyro_events, _NAME, log_path)
306
307      # Don't change print to logging. Used for KPI.
308      print(f'{_NAME}_max_principal_point_diff: ',
309            intrinsic_result['max_pp_diff'])
310      # Assert PASS/FAIL criteria
311      if intrinsic_result['failure']:
312        first_api_level = its_session_utils.get_first_api_level(self.dut.serial)
313        failure_msg = intrinsic_result['failure']
314        if first_api_level >= its_session_utils.ANDROID15_API_LEVEL:
315          raise AssertionError(failure_msg)
316        else:
317          raise AssertionError(f'{its_session_utils.NOT_YET_MANDATED_MESSAGE}'
318                               f'\n\n{failure_msg}')
319
320      samples_results = verify_lens_intrinsics_sample(recording_obj)
321      if samples_results['failure']:
322        raise AssertionError(samples_results['failure'])
323
324      camera_properties_utils.skip_unless(
325          not (intrinsic_result['skip'] and samples_results['skip']),
326          'Lens intrinsic and samples are not available in results.')
327
328
329if __name__ == '__main__':
330  test_runner.main()
331
332