1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Verify video is stable during phone movement.""" 15 16import logging 17import os 18import threading 19import time 20 21from mobly import test_runner 22 23import its_base_test 24import camera_properties_utils 25import image_processing_utils 26import its_session_utils 27import sensor_fusion_utils 28import video_processing_utils 29 30_ASPECT_RATIO_16_9 = 16/9 # determine if video fmt > 16:9 31_IMG_FORMAT = 'png' 32_MIN_PHONE_MOVEMENT_ANGLE = 5 # degrees 33_NAME = os.path.splitext(os.path.basename(__file__))[0] 34_NUM_ROTATIONS = 24 35_START_FRAME = 30 # give 3A 1s to warm up 36_VIDEO_DELAY_TIME = 5.5 # seconds 37_VIDEO_DURATION = 5.5 # seconds 38_VIDEO_QUALITIES_TESTED = ('CIF:3', '480P:4', '720P:5', '1080P:6', 'QVGA:7', 39 'VGA:9') 40_VIDEO_STABILIZATION_FACTOR = 0.7 # 70% of gyro movement allowed 41_VIDEO_STABILIZATION_MODE = 1 42_SIZE_TO_PROFILE = {'176x144': 'QCIF:2', '352x288': 'CIF:3', 43 '320x240': 'QVGA:7'} 44 45 46def _collect_data(cam, tablet_device, video_profile, video_quality, rot_rig): 47 """Capture a new set of data from the device. 48 49 Captures camera frames while the user is moving the device in the prescribed 50 manner. 51 52 Args: 53 cam: camera object 54 tablet_device: boolean; based on config.yml 55 video_profile: str; number of video profile 56 video_quality: str; key string for video quality. ie. 1080P 57 rot_rig: dict with 'cntl' and 'ch' defined 58 59 Returns: 60 recording object 61 """ 62 logging.debug('Starting sensor event collection') 63 props = cam.get_camera_properties() 64 props = cam.override_with_hidden_physical_camera_props(props) 65 66 serial_port = None 67 if rot_rig['cntl'].lower() == sensor_fusion_utils.ARDUINO_STRING.lower(): 68 # identify port 69 serial_port = sensor_fusion_utils.serial_port_def( 70 sensor_fusion_utils.ARDUINO_STRING) 71 # send test cmd to Arduino until cmd returns properly 72 sensor_fusion_utils.establish_serial_comm(serial_port) 73 # Start camera vibration 74 if tablet_device: 75 servo_speed = sensor_fusion_utils.ARDUINO_SERVO_SPEED_STABILIZATION_TABLET 76 else: 77 servo_speed = sensor_fusion_utils.ARDUINO_SERVO_SPEED_STABILIZATION 78 79 p = threading.Thread( 80 target=sensor_fusion_utils.rotation_rig, 81 args=( 82 rot_rig['cntl'], 83 rot_rig['ch'], 84 _NUM_ROTATIONS, 85 sensor_fusion_utils.ARDUINO_ANGLES_STABILIZATION, 86 servo_speed, 87 sensor_fusion_utils.ARDUINO_MOVE_TIME_STABILIZATION, 88 serial_port, 89 ), 90 ) 91 p.start() 92 93 cam.start_sensor_events() 94 95 # Record video and return recording object 96 time.sleep(_VIDEO_DELAY_TIME) # allow time for rig to start moving 97 recording_obj = cam.do_basic_recording( 98 video_profile, video_quality, _VIDEO_DURATION, _VIDEO_STABILIZATION_MODE) 99 logging.debug('Recorded output path: %s', recording_obj['recordedOutputPath']) 100 logging.debug('Tested quality: %s', recording_obj['quality']) 101 102 # Wait for vibration to stop 103 p.join() 104 105 return recording_obj 106 107 108class VideoStabilizationTest(its_base_test.ItsBaseTest): 109 """Tests if video is stabilized. 110 111 Camera is moved in sensor fusion rig on an arc of 15 degrees. 112 Speed is set to mimic hand movement (and not be too fast.) 113 Video is captured after rotation rig starts moving, and the 114 gyroscope data is dumped. 115 116 Video is processed to dump all of the frames to PNG files. 117 Camera movement is extracted from frames by determining max 118 angle of deflection in video movement vs max angle of deflection 119 in gyroscope movement. Test is a PASS if rotation is reduced in video. 120 """ 121 122 def test_video_stabilization(self): 123 rot_rig = {} 124 log_path = self.log_path 125 126 with its_session_utils.ItsSession( 127 device_id=self.dut.serial, 128 camera_id=self.camera_id, 129 hidden_physical_id=self.hidden_physical_id) as cam: 130 props = cam.get_camera_properties() 131 props = cam.override_with_hidden_physical_camera_props(props) 132 first_api_level = its_session_utils.get_first_api_level(self.dut.serial) 133 supported_stabilization_modes = props[ 134 'android.control.availableVideoStabilizationModes'] 135 136 camera_properties_utils.skip_unless( 137 first_api_level >= its_session_utils.ANDROID13_API_LEVEL and 138 _VIDEO_STABILIZATION_MODE in supported_stabilization_modes) 139 140 # Log ffmpeg version being used 141 video_processing_utils.log_ffmpeg_version() 142 143 # Raise error if not FRONT or REAR facing camera 144 facing = props['android.lens.facing'] 145 camera_properties_utils.check_front_or_rear_camera(props) 146 147 # Initialize rotation rig 148 rot_rig['cntl'] = self.rotator_cntl 149 rot_rig['ch'] = self.rotator_ch 150 if rot_rig['cntl'].lower() != 'arduino': 151 raise AssertionError(f'You must use an arduino controller for {_NAME}.') 152 153 # Create list of video qualities to test 154 excluded_sizes = video_processing_utils.LOW_RESOLUTION_SIZES 155 excluded_qualities = [ 156 _SIZE_TO_PROFILE[s] for s in excluded_sizes if s in _SIZE_TO_PROFILE 157 ] 158 supported_video_qualities = cam.get_supported_video_qualities( 159 self.camera_id) 160 logging.debug('Supported video qualities: %s', supported_video_qualities) 161 tested_video_qualities = list(set(_VIDEO_QUALITIES_TESTED) & 162 set(supported_video_qualities) - 163 set(excluded_qualities)) 164 165 # Raise error if no video qualities to test 166 if not tested_video_qualities: 167 raise AssertionError( 168 f'QUALITY_LOW not supported: {supported_video_qualities}') 169 else: 170 logging.debug('video qualities tested: %s', str(tested_video_qualities)) 171 172 max_cam_gyro_angles = {} 173 174 for video_tested in tested_video_qualities: 175 video_profile = video_tested.split(':')[1] 176 video_quality = video_tested.split(':')[0] 177 178 # Record video 179 recording_obj = _collect_data( 180 cam, self.tablet_device, video_profile, video_quality, rot_rig) 181 182 # Grab the video from the save location on DUT 183 self.dut.adb.pull([recording_obj['recordedOutputPath'], log_path]) 184 file_name = recording_obj['recordedOutputPath'].split('/')[-1] 185 logging.debug('file_name: %s', file_name) 186 187 # Get gyro events 188 logging.debug('Reading out inertial sensor events') 189 gyro_events = cam.get_sensor_events()['gyro'] 190 logging.debug('Number of gyro samples %d', len(gyro_events)) 191 192 # Extract all frames from video 193 file_list = video_processing_utils.extract_all_frames_from_video( 194 log_path, file_name, _IMG_FORMAT) 195 frames = [] 196 logging.debug('Number of frames %d', len(file_list)) 197 for file in file_list: 198 img = image_processing_utils.convert_image_to_numpy_array( 199 os.path.join(log_path, file)) 200 frames.append(img/255) 201 frame_shape = frames[0].shape 202 logging.debug('Frame size %d x %d', frame_shape[1], frame_shape[0]) 203 204 # Extract camera rotations 205 file_name_stem = f'{os.path.join(log_path, _NAME)}_{video_quality}' 206 cam_rots = sensor_fusion_utils.get_cam_rotations( 207 frames[_START_FRAME:], facing, frame_shape[0], 208 file_name_stem, _START_FRAME, stabilized_video=True) 209 sensor_fusion_utils.plot_camera_rotations( 210 cam_rots, _START_FRAME, video_quality, file_name_stem) 211 max_camera_angle = sensor_fusion_utils.calc_max_rotation_angle( 212 cam_rots, 'Camera') 213 214 # Extract gyro rotations 215 sensor_fusion_utils.plot_gyro_events( 216 gyro_events, f'{_NAME}_{video_quality}', log_path) 217 gyro_rots = sensor_fusion_utils.conv_acceleration_to_movement( 218 gyro_events, _VIDEO_DELAY_TIME) 219 max_gyro_angle = sensor_fusion_utils.calc_max_rotation_angle( 220 gyro_rots, 'Gyro') 221 logging.debug( 222 'Max deflection (degrees) %s: video: %.3f, gyro: %.3f, ratio: %.4f', 223 video_quality, max_camera_angle, max_gyro_angle, 224 max_camera_angle / max_gyro_angle) 225 max_cam_gyro_angles[video_quality] = {'gyro': max_gyro_angle, 226 'cam': max_camera_angle, 227 'frame_shape': frame_shape} 228 229 # Assert phone is moved enough during test 230 if max_gyro_angle < _MIN_PHONE_MOVEMENT_ANGLE: 231 raise AssertionError( 232 f'Phone not moved enough! Movement: {max_gyro_angle}, ' 233 f'THRESH: {_MIN_PHONE_MOVEMENT_ANGLE} degrees') 234 235 # Assert PASS/FAIL criteria 236 test_failures = [] 237 for video_quality, max_angles in max_cam_gyro_angles.items(): 238 aspect_ratio = (max_angles['frame_shape'][1] / 239 max_angles['frame_shape'][0]) 240 if aspect_ratio > _ASPECT_RATIO_16_9: 241 video_stabilization_factor = _VIDEO_STABILIZATION_FACTOR * 1.1 242 else: 243 video_stabilization_factor = _VIDEO_STABILIZATION_FACTOR 244 if max_angles['cam'] >= max_angles['gyro']*video_stabilization_factor: 245 test_failures.append( 246 f'{video_quality} video not stabilized enough! ' 247 f"Max video angle: {max_angles['cam']:.3f}, " 248 f"Max gyro angle: {max_angles['gyro']:.3f}, " 249 f"ratio: {max_angles['cam']/max_angles['gyro']:.3f} " 250 f'THRESH: {video_stabilization_factor}.') 251 else: # remove frames if PASS 252 its_session_utils.remove_tmp_files( 253 log_path, f'*_{video_quality}_*_stabilized_frame_*.png' 254 ) 255 if test_failures: 256 raise AssertionError(test_failures) 257 258 259if __name__ == '__main__': 260 test_runner.main() 261