1# Copyright 2014 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import os 17import os.path 18import tempfile 19import subprocess 20import time 21import sys 22 23import its.caps 24import its.device 25from its.device import ItsSession 26 27CHART_DELAY = 1 # seconds 28FACING_EXTERNAL = 2 29SKIP_RET_CODE = 101 # note this must be same as tests/scene*/test_* 30 31 32def skip_sensor_fusion(): 33 """Determine if sensor fusion test is skipped for this camera.""" 34 35 skip_code = SKIP_RET_CODE 36 with ItsSession() as cam: 37 props = cam.get_camera_properties() 38 if (its.caps.sensor_fusion(props) and its.caps.manual_sensor(props) and 39 props['android.lens.facing'] is not FACING_EXTERNAL): 40 skip_code = None 41 return skip_code 42 43 44def main(): 45 """Run all the automated tests, saving intermediate files, and producing 46 a summary/report of the results. 47 48 Script should be run from the top-level CameraITS directory. 49 50 Command line Arguments: 51 camera: the camera(s) to be tested. Use comma to separate multiple 52 camera Ids. Ex: "camera=0,1" or "camera=1" 53 scenes: the test scene(s) to be executed. Use comma to separate multiple 54 scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion" 55 (sceneX can be abbreviated by X where X is a integer) 56 chart: [Experimental] another android device served as test chart 57 display. When this argument presents, change of test scene will 58 be handled automatically. Note that this argument requires 59 special physical/hardware setup to work and may not work on 60 all android devices. 61 """ 62 63 # Not yet mandated tests 64 NOT_YET_MANDATED = { 65 "scene0": [ 66 "test_jitter", 67 "test_burst_capture" 68 ], 69 "scene1": [ 70 "test_ae_af", 71 "test_ae_precapture_trigger", 72 "test_crop_region_raw", 73 "test_ev_compensation_advanced", 74 "test_ev_compensation_basic", 75 "test_yuv_plus_jpeg" 76 ], 77 "scene2": [ 78 "test_num_faces", 79 ], 80 "scene3": [ 81 "test_3a_consistency", 82 "test_lens_movement_reporting", 83 "test_lens_position" 84 ], 85 "scene4": [], 86 "scene5": [], 87 "sensor_fusion": [] 88 } 89 90 all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5", 91 "sensor_fusion"] 92 93 auto_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4"] 94 95 scene_req = { 96 "scene0": None, 97 "scene1": "A grey card covering at least the middle 30% of the scene", 98 "scene2": "A picture containing human faces", 99 "scene3": "The ISO 12233 chart", 100 "scene4": "A specific test page of a circle covering at least the " 101 "middle 50% of the scene. See CameraITS.pdf section 2.3.4 " 102 "for more details", 103 "scene5": "Capture images with a diffuser attached to the camera. See " 104 "CameraITS.pdf section 2.3.4 for more details", 105 "sensor_fusion": "Rotating checkboard pattern. See " 106 "sensor_fusion/SensorFusion.pdf for detailed " 107 "instructions.\nNote that this test will be skipped " 108 "on devices not supporting REALTIME camera timestamp." 109 } 110 scene_extra_args = { 111 "scene5": ["doAF=False"] 112 } 113 114 camera_ids = [] 115 scenes = [] 116 chart_host_id = None 117 result_device_id = None 118 rot_rig_id = None 119 120 for s in sys.argv[1:]: 121 if s[:7] == "camera=" and len(s) > 7: 122 camera_ids = s[7:].split(',') 123 elif s[:7] == "scenes=" and len(s) > 7: 124 scenes = s[7:].split(',') 125 elif s[:6] == 'chart=' and len(s) > 6: 126 chart_host_id = s[6:] 127 elif s[:7] == 'result=' and len(s) > 7: 128 result_device_id = s[7:] 129 elif s[:8] == 'rot_rig=' and len(s) > 8: 130 rot_rig_id = s[8:] # valid values: 'default' or '$VID:$PID:$CH' 131 # The default '$VID:$PID:$CH' is '04d8:fc73:1' 132 133 auto_scene_switch = chart_host_id is not None 134 merge_result_switch = result_device_id is not None 135 136 # Run through all scenes if user does not supply one 137 possible_scenes = auto_scenes if auto_scene_switch else all_scenes 138 if not scenes: 139 scenes = possible_scenes 140 else: 141 # Validate user input scene names 142 valid_scenes = True 143 temp_scenes = [] 144 for s in scenes: 145 if s in possible_scenes: 146 temp_scenes.append(s) 147 else: 148 try: 149 # Try replace "X" to "sceneX" 150 scene_str = "scene" + s 151 if scene_str not in possible_scenes: 152 valid_scenes = False 153 break 154 temp_scenes.append(scene_str) 155 except ValueError: 156 valid_scenes = False 157 break 158 159 if not valid_scenes: 160 print "Unknown scene specifiied:", s 161 assert False 162 scenes = temp_scenes 163 164 # Initialize test results 165 results = {} 166 result_key = ItsSession.RESULT_KEY 167 for s in all_scenes: 168 results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED} 169 170 # Make output directories to hold the generated files. 171 topdir = tempfile.mkdtemp() 172 subprocess.call(['chmod', 'g+rx', topdir]) 173 print "Saving output files to:", topdir, "\n" 174 175 device_id = its.device.get_device_id() 176 device_id_arg = "device=" + device_id 177 print "Testing device " + device_id 178 179 # Sanity Check for devices 180 device_bfp = its.device.get_device_fingerprint(device_id) 181 assert device_bfp is not None 182 183 if auto_scene_switch: 184 chart_host_bfp = its.device.get_device_fingerprint(chart_host_id) 185 assert chart_host_bfp is not None 186 187 if merge_result_switch: 188 result_device_bfp = its.device.get_device_fingerprint(result_device_id) 189 assert_err_msg = ('Cannot merge result to a different build, from ' 190 '%s to %s' % (device_bfp, result_device_bfp)) 191 assert device_bfp == result_device_bfp, assert_err_msg 192 193 # user doesn't specify camera id, run through all cameras 194 if not camera_ids: 195 camera_ids_path = os.path.join(topdir, "camera_ids.txt") 196 out_arg = "out=" + camera_ids_path 197 cmd = ['python', 198 os.path.join(os.getcwd(), "tools/get_camera_ids.py"), out_arg, 199 device_id_arg] 200 cam_code = subprocess.call(cmd, cwd=topdir) 201 assert cam_code == 0 202 with open(camera_ids_path, "r") as f: 203 for line in f: 204 camera_ids.append(line.replace('\n', '')) 205 206 print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes) 207 208 if auto_scene_switch: 209 # merge_result only supports run_parallel_tests 210 if merge_result_switch and camera_ids[0] == '1': 211 print 'Skip chart screen' 212 time.sleep(1) 213 else: 214 print 'Waking up chart screen: ', chart_host_id 215 screen_id_arg = ('screen=%s' % chart_host_id) 216 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 217 'wake_up_screen.py'), screen_id_arg] 218 wake_code = subprocess.call(cmd) 219 assert wake_code == 0 220 221 for camera_id in camera_ids: 222 # Loop capturing images until user confirm test scene is correct 223 camera_id_arg = "camera=" + camera_id 224 print "Preparing to run ITS on camera", camera_id 225 226 os.mkdir(os.path.join(topdir, camera_id)) 227 for d in scenes: 228 os.mkdir(os.path.join(topdir, camera_id, d)) 229 230 for scene in scenes: 231 skip_code = None 232 tests = [(s[:-3], os.path.join("tests", scene, s)) 233 for s in os.listdir(os.path.join("tests", scene)) 234 if s[-3:] == ".py" and s[:4] == "test"] 235 tests.sort() 236 237 summary = "Cam" + camera_id + " " + scene + "\n" 238 numpass = 0 239 numskip = 0 240 num_not_mandated_fail = 0 241 numfail = 0 242 validate_switch = True 243 if scene_req[scene] is not None: 244 out_path = os.path.join(topdir, camera_id, scene+".jpg") 245 out_arg = "out=" + out_path 246 if scene == 'sensor_fusion': 247 skip_code = skip_sensor_fusion() 248 if rot_rig_id or skip_code == SKIP_RET_CODE: 249 validate_switch = False 250 if scene == 'scene5': 251 validate_switch = False 252 cmd = None 253 if auto_scene_switch: 254 if (not merge_result_switch or 255 (merge_result_switch and camera_ids[0] == '0')): 256 scene_arg = 'scene=' + scene 257 cmd = ['python', 258 os.path.join(os.getcwd(), 'tools/load_scene.py'), 259 scene_arg, screen_id_arg] 260 else: 261 time.sleep(CHART_DELAY) 262 else: 263 # Skip scene validation under certain conditions 264 if validate_switch and not merge_result_switch: 265 scene_arg = 'scene=' + scene_req[scene] 266 extra_args = scene_extra_args.get(scene, []) 267 cmd = ['python', 268 os.path.join(os.getcwd(), 269 'tools/validate_scene.py'), 270 camera_id_arg, out_arg, 271 scene_arg, device_id_arg] + extra_args 272 if cmd is not None: 273 valid_scene_code = subprocess.call(cmd, cwd=topdir) 274 assert valid_scene_code == 0 275 print "Start running ITS on camera %s, %s" % (camera_id, scene) 276 # Run each test, capturing stdout and stderr. 277 for (testname, testpath) in tests: 278 if auto_scene_switch: 279 if merge_result_switch and camera_ids[0] == '0': 280 # Send an input event to keep the screen not dimmed. 281 # Since we are not using camera of chart screen, FOCUS event 282 # should does nothing but keep the screen from dimming. 283 # The "sleep after x minutes of inactivity" display setting 284 # determines how long this command can keep screen bright. 285 # Setting it to something like 30 minutes should be enough. 286 cmd = ('adb -s %s shell input keyevent FOCUS' 287 % chart_host_id) 288 subprocess.call(cmd.split()) 289 t0 = time.time() 290 outdir = os.path.join(topdir, camera_id, scene) 291 outpath = os.path.join(outdir, testname+'_stdout.txt') 292 errpath = os.path.join(outdir, testname+'_stderr.txt') 293 if scene == 'sensor_fusion': 294 if skip_code is not SKIP_RET_CODE: 295 if rot_rig_id: 296 print 'Rotating phone w/ rig %s' % rot_rig_id 297 rig = ('python tools/rotation_rig.py rotator=%s' % 298 rot_rig_id) 299 subprocess.Popen(rig.split()) 300 else: 301 print 'Rotate phone 15s as shown in SensorFusion.pdf' 302 else: 303 test_code = skip_code 304 if skip_code is not SKIP_RET_CODE: 305 cmd = ['python', os.path.join(os.getcwd(), testpath)] 306 cmd += sys.argv[1:] + [camera_id_arg] 307 with open(outpath, 'w') as fout, open(errpath, 'w') as ferr: 308 test_code = subprocess.call( 309 cmd, stderr=ferr, stdout=fout, cwd=outdir) 310 t1 = time.time() 311 312 test_failed = False 313 if test_code == 0: 314 retstr = "PASS " 315 numpass += 1 316 elif test_code == SKIP_RET_CODE: 317 retstr = "SKIP " 318 numskip += 1 319 elif test_code != 0 and testname in NOT_YET_MANDATED[scene]: 320 retstr = "FAIL*" 321 num_not_mandated_fail += 1 322 else: 323 retstr = "FAIL " 324 numfail += 1 325 test_failed = True 326 327 msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0) 328 print msg 329 msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0) 330 if test_failed: 331 summary += msg_short + "\n" 332 333 if numskip > 0: 334 skipstr = ", %d test%s skipped" % ( 335 numskip, "s" if numskip > 1 else "") 336 else: 337 skipstr = "" 338 339 test_result = "\n%d / %d tests passed (%.1f%%)%s" % ( 340 numpass + num_not_mandated_fail, len(tests) - numskip, 341 100.0 * float(numpass + num_not_mandated_fail) / 342 (len(tests) - numskip) 343 if len(tests) != numskip else 100.0, skipstr) 344 print test_result 345 346 if num_not_mandated_fail > 0: 347 msg = "(*) tests are not yet mandated" 348 print msg 349 350 summary_path = os.path.join(topdir, camera_id, scene, "summary.txt") 351 with open(summary_path, "w") as f: 352 f.write(summary) 353 354 passed = numfail == 0 355 results[scene][result_key] = (ItsSession.RESULT_PASS if passed 356 else ItsSession.RESULT_FAIL) 357 results[scene][ItsSession.SUMMARY_KEY] = summary_path 358 359 print "Reporting ITS result to CtsVerifier" 360 if merge_result_switch: 361 # results are modified by report_result 362 results_backup = copy.deepcopy(results) 363 its.device.report_result(result_device_id, camera_id, results_backup) 364 365 its.device.report_result(device_id, camera_id, results) 366 367 if auto_scene_switch: 368 if merge_result_switch: 369 print 'Skip shutting down chart screen' 370 else: 371 print 'Shutting down chart screen: ', chart_host_id 372 screen_id_arg = ('screen=%s' % chart_host_id) 373 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 374 'turn_off_screen.py'), screen_id_arg] 375 screen_off_code = subprocess.call(cmd) 376 assert screen_off_code == 0 377 378 print 'Shutting down DUT screen: ', device_id 379 screen_id_arg = ('screen=%s' % device_id) 380 cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 381 'turn_off_screen.py'), screen_id_arg] 382 screen_off_code = subprocess.call(cmd) 383 assert screen_off_code == 0 384 385 print "ITS tests finished. Please go back to CtsVerifier and proceed" 386 387if __name__ == '__main__': 388 main() 389