# Copyright 2014 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import os import os.path import tempfile import subprocess import time import sys import its.caps import its.device from its.device import ItsSession CHART_DELAY = 1 # seconds FACING_EXTERNAL = 2 SKIP_RET_CODE = 101 # note this must be same as tests/scene*/test_* def skip_sensor_fusion(): """Determine if sensor fusion test is skipped for this camera.""" skip_code = SKIP_RET_CODE with ItsSession() as cam: props = cam.get_camera_properties() if (its.caps.sensor_fusion(props) and its.caps.manual_sensor(props) and props['android.lens.facing'] is not FACING_EXTERNAL): skip_code = None return skip_code def main(): """Run all the automated tests, saving intermediate files, and producing a summary/report of the results. Script should be run from the top-level CameraITS directory. Command line Arguments: camera: the camera(s) to be tested. Use comma to separate multiple camera Ids. Ex: "camera=0,1" or "camera=1" scenes: the test scene(s) to be executed. Use comma to separate multiple scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion" (sceneX can be abbreviated by X where X is a integer) chart: [Experimental] another android device served as test chart display. When this argument presents, change of test scene will be handled automatically. Note that this argument requires special physical/hardware setup to work and may not work on all android devices. """ # Not yet mandated tests NOT_YET_MANDATED = { "scene0": [ "test_jitter", "test_burst_capture" ], "scene1": [ "test_ae_af", "test_ae_precapture_trigger", "test_crop_region_raw", "test_ev_compensation_advanced", "test_ev_compensation_basic", "test_yuv_plus_jpeg" ], "scene2": [ "test_num_faces", ], "scene3": [ "test_3a_consistency", "test_lens_movement_reporting", "test_lens_position" ], "scene4": [], "scene5": [], "sensor_fusion": [] } all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5", "sensor_fusion"] auto_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4"] scene_req = { "scene0": None, "scene1": "A grey card covering at least the middle 30% of the scene", "scene2": "A picture containing human faces", "scene3": "The ISO 12233 chart", "scene4": "A specific test page of a circle covering at least the " "middle 50% of the scene. See CameraITS.pdf section 2.3.4 " "for more details", "scene5": "Capture images with a diffuser attached to the camera. See " "CameraITS.pdf section 2.3.4 for more details", "sensor_fusion": "Rotating checkboard pattern. See " "sensor_fusion/SensorFusion.pdf for detailed " "instructions.\nNote that this test will be skipped " "on devices not supporting REALTIME camera timestamp." } scene_extra_args = { "scene5": ["doAF=False"] } camera_ids = [] scenes = [] chart_host_id = None result_device_id = None rot_rig_id = None for s in sys.argv[1:]: if s[:7] == "camera=" and len(s) > 7: camera_ids = s[7:].split(',') elif s[:7] == "scenes=" and len(s) > 7: scenes = s[7:].split(',') elif s[:6] == 'chart=' and len(s) > 6: chart_host_id = s[6:] elif s[:7] == 'result=' and len(s) > 7: result_device_id = s[7:] elif s[:8] == 'rot_rig=' and len(s) > 8: rot_rig_id = s[8:] # valid values: 'default' or '$VID:$PID:$CH' # The default '$VID:$PID:$CH' is '04d8:fc73:1' auto_scene_switch = chart_host_id is not None merge_result_switch = result_device_id is not None # Run through all scenes if user does not supply one possible_scenes = auto_scenes if auto_scene_switch else all_scenes if not scenes: scenes = possible_scenes else: # Validate user input scene names valid_scenes = True temp_scenes = [] for s in scenes: if s in possible_scenes: temp_scenes.append(s) else: try: # Try replace "X" to "sceneX" scene_str = "scene" + s if scene_str not in possible_scenes: valid_scenes = False break temp_scenes.append(scene_str) except ValueError: valid_scenes = False break if not valid_scenes: print "Unknown scene specifiied:", s assert False scenes = temp_scenes # Initialize test results results = {} result_key = ItsSession.RESULT_KEY for s in all_scenes: results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED} # Make output directories to hold the generated files. topdir = tempfile.mkdtemp() subprocess.call(['chmod', 'g+rx', topdir]) print "Saving output files to:", topdir, "\n" device_id = its.device.get_device_id() device_id_arg = "device=" + device_id print "Testing device " + device_id # Sanity Check for devices device_bfp = its.device.get_device_fingerprint(device_id) assert device_bfp is not None if auto_scene_switch: chart_host_bfp = its.device.get_device_fingerprint(chart_host_id) assert chart_host_bfp is not None if merge_result_switch: result_device_bfp = its.device.get_device_fingerprint(result_device_id) assert_err_msg = ('Cannot merge result to a different build, from ' '%s to %s' % (device_bfp, result_device_bfp)) assert device_bfp == result_device_bfp, assert_err_msg # user doesn't specify camera id, run through all cameras if not camera_ids: camera_ids_path = os.path.join(topdir, "camera_ids.txt") out_arg = "out=" + camera_ids_path cmd = ['python', os.path.join(os.getcwd(), "tools/get_camera_ids.py"), out_arg, device_id_arg] cam_code = subprocess.call(cmd, cwd=topdir) assert cam_code == 0 with open(camera_ids_path, "r") as f: for line in f: camera_ids.append(line.replace('\n', '')) print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes) if auto_scene_switch: # merge_result only supports run_parallel_tests if merge_result_switch and camera_ids[0] == '1': print 'Skip chart screen' time.sleep(1) else: print 'Waking up chart screen: ', chart_host_id screen_id_arg = ('screen=%s' % chart_host_id) cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 'wake_up_screen.py'), screen_id_arg] wake_code = subprocess.call(cmd) assert wake_code == 0 for camera_id in camera_ids: # Loop capturing images until user confirm test scene is correct camera_id_arg = "camera=" + camera_id print "Preparing to run ITS on camera", camera_id os.mkdir(os.path.join(topdir, camera_id)) for d in scenes: os.mkdir(os.path.join(topdir, camera_id, d)) for scene in scenes: skip_code = None tests = [(s[:-3], os.path.join("tests", scene, s)) for s in os.listdir(os.path.join("tests", scene)) if s[-3:] == ".py" and s[:4] == "test"] tests.sort() summary = "Cam" + camera_id + " " + scene + "\n" numpass = 0 numskip = 0 num_not_mandated_fail = 0 numfail = 0 validate_switch = True if scene_req[scene] is not None: out_path = os.path.join(topdir, camera_id, scene+".jpg") out_arg = "out=" + out_path if scene == 'sensor_fusion': skip_code = skip_sensor_fusion() if rot_rig_id or skip_code == SKIP_RET_CODE: validate_switch = False if scene == 'scene5': validate_switch = False cmd = None if auto_scene_switch: if (not merge_result_switch or (merge_result_switch and camera_ids[0] == '0')): scene_arg = 'scene=' + scene cmd = ['python', os.path.join(os.getcwd(), 'tools/load_scene.py'), scene_arg, screen_id_arg] else: time.sleep(CHART_DELAY) else: # Skip scene validation under certain conditions if validate_switch and not merge_result_switch: scene_arg = 'scene=' + scene_req[scene] extra_args = scene_extra_args.get(scene, []) cmd = ['python', os.path.join(os.getcwd(), 'tools/validate_scene.py'), camera_id_arg, out_arg, scene_arg, device_id_arg] + extra_args if cmd is not None: valid_scene_code = subprocess.call(cmd, cwd=topdir) assert valid_scene_code == 0 print "Start running ITS on camera %s, %s" % (camera_id, scene) # Run each test, capturing stdout and stderr. for (testname, testpath) in tests: if auto_scene_switch: if merge_result_switch and camera_ids[0] == '0': # Send an input event to keep the screen not dimmed. # Since we are not using camera of chart screen, FOCUS event # should does nothing but keep the screen from dimming. # The "sleep after x minutes of inactivity" display setting # determines how long this command can keep screen bright. # Setting it to something like 30 minutes should be enough. cmd = ('adb -s %s shell input keyevent FOCUS' % chart_host_id) subprocess.call(cmd.split()) t0 = time.time() outdir = os.path.join(topdir, camera_id, scene) outpath = os.path.join(outdir, testname+'_stdout.txt') errpath = os.path.join(outdir, testname+'_stderr.txt') if scene == 'sensor_fusion': if skip_code is not SKIP_RET_CODE: if rot_rig_id: print 'Rotating phone w/ rig %s' % rot_rig_id rig = ('python tools/rotation_rig.py rotator=%s' % rot_rig_id) subprocess.Popen(rig.split()) else: print 'Rotate phone 15s as shown in SensorFusion.pdf' else: test_code = skip_code if skip_code is not SKIP_RET_CODE: cmd = ['python', os.path.join(os.getcwd(), testpath)] cmd += sys.argv[1:] + [camera_id_arg] with open(outpath, 'w') as fout, open(errpath, 'w') as ferr: test_code = subprocess.call( cmd, stderr=ferr, stdout=fout, cwd=outdir) t1 = time.time() test_failed = False if test_code == 0: retstr = "PASS " numpass += 1 elif test_code == SKIP_RET_CODE: retstr = "SKIP " numskip += 1 elif test_code != 0 and testname in NOT_YET_MANDATED[scene]: retstr = "FAIL*" num_not_mandated_fail += 1 else: retstr = "FAIL " numfail += 1 test_failed = True msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0) print msg msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0) if test_failed: summary += msg_short + "\n" if numskip > 0: skipstr = ", %d test%s skipped" % ( numskip, "s" if numskip > 1 else "") else: skipstr = "" test_result = "\n%d / %d tests passed (%.1f%%)%s" % ( numpass + num_not_mandated_fail, len(tests) - numskip, 100.0 * float(numpass + num_not_mandated_fail) / (len(tests) - numskip) if len(tests) != numskip else 100.0, skipstr) print test_result if num_not_mandated_fail > 0: msg = "(*) tests are not yet mandated" print msg summary_path = os.path.join(topdir, camera_id, scene, "summary.txt") with open(summary_path, "w") as f: f.write(summary) passed = numfail == 0 results[scene][result_key] = (ItsSession.RESULT_PASS if passed else ItsSession.RESULT_FAIL) results[scene][ItsSession.SUMMARY_KEY] = summary_path print "Reporting ITS result to CtsVerifier" if merge_result_switch: # results are modified by report_result results_backup = copy.deepcopy(results) its.device.report_result(result_device_id, camera_id, results_backup) its.device.report_result(device_id, camera_id, results) if auto_scene_switch: if merge_result_switch: print 'Skip shutting down chart screen' else: print 'Shutting down chart screen: ', chart_host_id screen_id_arg = ('screen=%s' % chart_host_id) cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 'turn_off_screen.py'), screen_id_arg] screen_off_code = subprocess.call(cmd) assert screen_off_code == 0 print 'Shutting down DUT screen: ', device_id screen_id_arg = ('screen=%s' % device_id) cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools', 'turn_off_screen.py'), screen_id_arg] screen_off_code = subprocess.call(cmd) assert screen_off_code == 0 print "ITS tests finished. Please go back to CtsVerifier and proceed" if __name__ == '__main__': main()