1# Copyright 2014 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import copy
16import os
17import os.path
18import tempfile
19import subprocess
20import time
21import sys
22
23import its.caps
24import its.device
25from its.device import ItsSession
26
27CHART_DELAY = 1  # seconds
28FACING_EXTERNAL = 2
29SKIP_RET_CODE = 101  # note this must be same as tests/scene*/test_*
30
31
32def skip_sensor_fusion():
33    """Determine if sensor fusion test is skipped for this camera."""
34
35    skip_code = SKIP_RET_CODE
36    with ItsSession() as cam:
37        props = cam.get_camera_properties()
38        if (its.caps.sensor_fusion(props) and its.caps.manual_sensor(props) and
39                props['android.lens.facing'] is not FACING_EXTERNAL):
40            skip_code = None
41    return skip_code
42
43
44def main():
45    """Run all the automated tests, saving intermediate files, and producing
46    a summary/report of the results.
47
48    Script should be run from the top-level CameraITS directory.
49
50    Command line Arguments:
51        camera: the camera(s) to be tested. Use comma to separate multiple
52                camera Ids. Ex: "camera=0,1" or "camera=1"
53        scenes: the test scene(s) to be executed. Use comma to separate multiple
54                scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion"
55                (sceneX can be abbreviated by X where X is a integer)
56        chart: [Experimental] another android device served as test chart
57               display. When this argument presents, change of test scene will
58               be handled automatically. Note that this argument requires
59               special physical/hardware setup to work and may not work on
60               all android devices.
61    """
62
63    # Not yet mandated tests
64    NOT_YET_MANDATED = {
65        "scene0": [
66            "test_jitter",
67            "test_burst_capture"
68            ],
69        "scene1": [
70            "test_ae_af",
71            "test_ae_precapture_trigger",
72            "test_crop_region_raw",
73            "test_ev_compensation_advanced",
74            "test_ev_compensation_basic",
75            "test_yuv_plus_jpeg"
76            ],
77        "scene2": [
78            "test_num_faces",
79            ],
80        "scene3": [
81            "test_3a_consistency",
82            "test_lens_movement_reporting",
83            "test_lens_position"
84            ],
85        "scene4": [],
86        "scene5": [],
87        "sensor_fusion": []
88    }
89
90    all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5",
91                  "sensor_fusion"]
92
93    auto_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4"]
94
95    scene_req = {
96        "scene0": None,
97        "scene1": "A grey card covering at least the middle 30% of the scene",
98        "scene2": "A picture containing human faces",
99        "scene3": "The ISO 12233 chart",
100        "scene4": "A specific test page of a circle covering at least the "
101                  "middle 50% of the scene. See CameraITS.pdf section 2.3.4 "
102                  "for more details",
103        "scene5": "Capture images with a diffuser attached to the camera. See "
104                  "CameraITS.pdf section 2.3.4 for more details",
105        "sensor_fusion": "Rotating checkboard pattern. See "
106                         "sensor_fusion/SensorFusion.pdf for detailed "
107                         "instructions.\nNote that this test will be skipped "
108                         "on devices not supporting REALTIME camera timestamp."
109    }
110    scene_extra_args = {
111        "scene5": ["doAF=False"]
112    }
113
114    camera_ids = []
115    scenes = []
116    chart_host_id = None
117    result_device_id = None
118    rot_rig_id = None
119
120    for s in sys.argv[1:]:
121        if s[:7] == "camera=" and len(s) > 7:
122            camera_ids = s[7:].split(',')
123        elif s[:7] == "scenes=" and len(s) > 7:
124            scenes = s[7:].split(',')
125        elif s[:6] == 'chart=' and len(s) > 6:
126            chart_host_id = s[6:]
127        elif s[:7] == 'result=' and len(s) > 7:
128            result_device_id = s[7:]
129        elif s[:8] == 'rot_rig=' and len(s) > 8:
130            rot_rig_id = s[8:]  # valid values: 'default' or '$VID:$PID:$CH'
131            # The default '$VID:$PID:$CH' is '04d8:fc73:1'
132
133    auto_scene_switch = chart_host_id is not None
134    merge_result_switch = result_device_id is not None
135
136    # Run through all scenes if user does not supply one
137    possible_scenes = auto_scenes if auto_scene_switch else all_scenes
138    if not scenes:
139        scenes = possible_scenes
140    else:
141        # Validate user input scene names
142        valid_scenes = True
143        temp_scenes = []
144        for s in scenes:
145            if s in possible_scenes:
146                temp_scenes.append(s)
147            else:
148                try:
149                    # Try replace "X" to "sceneX"
150                    scene_str = "scene" + s
151                    if scene_str not in possible_scenes:
152                        valid_scenes = False
153                        break
154                    temp_scenes.append(scene_str)
155                except ValueError:
156                    valid_scenes = False
157                    break
158
159        if not valid_scenes:
160            print "Unknown scene specifiied:", s
161            assert False
162        scenes = temp_scenes
163
164    # Initialize test results
165    results = {}
166    result_key = ItsSession.RESULT_KEY
167    for s in all_scenes:
168        results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED}
169
170    # Make output directories to hold the generated files.
171    topdir = tempfile.mkdtemp()
172    subprocess.call(['chmod', 'g+rx', topdir])
173    print "Saving output files to:", topdir, "\n"
174
175    device_id = its.device.get_device_id()
176    device_id_arg = "device=" + device_id
177    print "Testing device " + device_id
178
179    # Sanity Check for devices
180    device_bfp = its.device.get_device_fingerprint(device_id)
181    assert device_bfp is not None
182
183    if auto_scene_switch:
184        chart_host_bfp = its.device.get_device_fingerprint(chart_host_id)
185        assert chart_host_bfp is not None
186
187    if merge_result_switch:
188        result_device_bfp = its.device.get_device_fingerprint(result_device_id)
189        assert_err_msg = ('Cannot merge result to a different build, from '
190                          '%s to %s' % (device_bfp, result_device_bfp))
191        assert device_bfp == result_device_bfp, assert_err_msg
192
193    # user doesn't specify camera id, run through all cameras
194    if not camera_ids:
195        camera_ids_path = os.path.join(topdir, "camera_ids.txt")
196        out_arg = "out=" + camera_ids_path
197        cmd = ['python',
198               os.path.join(os.getcwd(), "tools/get_camera_ids.py"), out_arg,
199               device_id_arg]
200        cam_code = subprocess.call(cmd, cwd=topdir)
201        assert cam_code == 0
202        with open(camera_ids_path, "r") as f:
203            for line in f:
204                camera_ids.append(line.replace('\n', ''))
205
206    print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes)
207
208    if auto_scene_switch:
209        # merge_result only supports run_parallel_tests
210        if merge_result_switch and camera_ids[0] == '1':
211            print 'Skip chart screen'
212            time.sleep(1)
213        else:
214            print 'Waking up chart screen: ', chart_host_id
215            screen_id_arg = ('screen=%s' % chart_host_id)
216            cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools',
217                                          'wake_up_screen.py'), screen_id_arg]
218            wake_code = subprocess.call(cmd)
219            assert wake_code == 0
220
221    for camera_id in camera_ids:
222        # Loop capturing images until user confirm test scene is correct
223        camera_id_arg = "camera=" + camera_id
224        print "Preparing to run ITS on camera", camera_id
225
226        os.mkdir(os.path.join(topdir, camera_id))
227        for d in scenes:
228            os.mkdir(os.path.join(topdir, camera_id, d))
229
230        for scene in scenes:
231            skip_code = None
232            tests = [(s[:-3], os.path.join("tests", scene, s))
233                     for s in os.listdir(os.path.join("tests", scene))
234                     if s[-3:] == ".py" and s[:4] == "test"]
235            tests.sort()
236
237            summary = "Cam" + camera_id + " " + scene + "\n"
238            numpass = 0
239            numskip = 0
240            num_not_mandated_fail = 0
241            numfail = 0
242            validate_switch = True
243            if scene_req[scene] is not None:
244                out_path = os.path.join(topdir, camera_id, scene+".jpg")
245                out_arg = "out=" + out_path
246                if scene == 'sensor_fusion':
247                    skip_code = skip_sensor_fusion()
248                    if rot_rig_id or skip_code == SKIP_RET_CODE:
249                        validate_switch = False
250                if scene == 'scene5':
251                    validate_switch = False
252                cmd = None
253                if auto_scene_switch:
254                    if (not merge_result_switch or
255                            (merge_result_switch and camera_ids[0] == '0')):
256                        scene_arg = 'scene=' + scene
257                        cmd = ['python',
258                               os.path.join(os.getcwd(), 'tools/load_scene.py'),
259                               scene_arg, screen_id_arg]
260                    else:
261                        time.sleep(CHART_DELAY)
262                else:
263                    # Skip scene validation under certain conditions
264                    if validate_switch and not merge_result_switch:
265                        scene_arg = 'scene=' + scene_req[scene]
266                        extra_args = scene_extra_args.get(scene, [])
267                        cmd = ['python',
268                               os.path.join(os.getcwd(),
269                                            'tools/validate_scene.py'),
270                               camera_id_arg, out_arg,
271                               scene_arg, device_id_arg] + extra_args
272                if cmd is not None:
273                    valid_scene_code = subprocess.call(cmd, cwd=topdir)
274                    assert valid_scene_code == 0
275            print "Start running ITS on camera %s, %s" % (camera_id, scene)
276            # Run each test, capturing stdout and stderr.
277            for (testname, testpath) in tests:
278                if auto_scene_switch:
279                    if merge_result_switch and camera_ids[0] == '0':
280                        # Send an input event to keep the screen not dimmed.
281                        # Since we are not using camera of chart screen, FOCUS event
282                        # should does nothing but keep the screen from dimming.
283                        # The "sleep after x minutes of inactivity" display setting
284                        # determines how long this command can keep screen bright.
285                        # Setting it to something like 30 minutes should be enough.
286                        cmd = ('adb -s %s shell input keyevent FOCUS'
287                               % chart_host_id)
288                        subprocess.call(cmd.split())
289                t0 = time.time()
290                outdir = os.path.join(topdir, camera_id, scene)
291                outpath = os.path.join(outdir, testname+'_stdout.txt')
292                errpath = os.path.join(outdir, testname+'_stderr.txt')
293                if scene == 'sensor_fusion':
294                    if skip_code is not SKIP_RET_CODE:
295                        if rot_rig_id:
296                            print 'Rotating phone w/ rig %s' % rot_rig_id
297                            rig = ('python tools/rotation_rig.py rotator=%s' %
298                                   rot_rig_id)
299                            subprocess.Popen(rig.split())
300                        else:
301                            print 'Rotate phone 15s as shown in SensorFusion.pdf'
302                    else:
303                        test_code = skip_code
304                if skip_code is not SKIP_RET_CODE:
305                    cmd = ['python', os.path.join(os.getcwd(), testpath)]
306                    cmd += sys.argv[1:] + [camera_id_arg]
307                    with open(outpath, 'w') as fout, open(errpath, 'w') as ferr:
308                        test_code = subprocess.call(
309                            cmd, stderr=ferr, stdout=fout, cwd=outdir)
310                t1 = time.time()
311
312                test_failed = False
313                if test_code == 0:
314                    retstr = "PASS "
315                    numpass += 1
316                elif test_code == SKIP_RET_CODE:
317                    retstr = "SKIP "
318                    numskip += 1
319                elif test_code != 0 and testname in NOT_YET_MANDATED[scene]:
320                    retstr = "FAIL*"
321                    num_not_mandated_fail += 1
322                else:
323                    retstr = "FAIL "
324                    numfail += 1
325                    test_failed = True
326
327                msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
328                print msg
329                msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0)
330                if test_failed:
331                    summary += msg_short + "\n"
332
333            if numskip > 0:
334                skipstr = ", %d test%s skipped" % (
335                    numskip, "s" if numskip > 1 else "")
336            else:
337                skipstr = ""
338
339            test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
340                numpass + num_not_mandated_fail, len(tests) - numskip,
341                100.0 * float(numpass + num_not_mandated_fail) /
342                (len(tests) - numskip)
343                if len(tests) != numskip else 100.0, skipstr)
344            print test_result
345
346            if num_not_mandated_fail > 0:
347                msg = "(*) tests are not yet mandated"
348                print msg
349
350            summary_path = os.path.join(topdir, camera_id, scene, "summary.txt")
351            with open(summary_path, "w") as f:
352                f.write(summary)
353
354            passed = numfail == 0
355            results[scene][result_key] = (ItsSession.RESULT_PASS if passed
356                                          else ItsSession.RESULT_FAIL)
357            results[scene][ItsSession.SUMMARY_KEY] = summary_path
358
359        print "Reporting ITS result to CtsVerifier"
360        if merge_result_switch:
361            # results are modified by report_result
362            results_backup = copy.deepcopy(results)
363            its.device.report_result(result_device_id, camera_id, results_backup)
364
365        its.device.report_result(device_id, camera_id, results)
366
367    if auto_scene_switch:
368        if merge_result_switch:
369            print 'Skip shutting down chart screen'
370        else:
371            print 'Shutting down chart screen: ', chart_host_id
372            screen_id_arg = ('screen=%s' % chart_host_id)
373            cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools',
374                                          'turn_off_screen.py'), screen_id_arg]
375            screen_off_code = subprocess.call(cmd)
376            assert screen_off_code == 0
377
378            print 'Shutting down DUT screen: ', device_id
379            screen_id_arg = ('screen=%s' % device_id)
380            cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools',
381                                          'turn_off_screen.py'), screen_id_arg]
382            screen_off_code = subprocess.call(cmd)
383            assert screen_off_code == 0
384
385    print "ITS tests finished. Please go back to CtsVerifier and proceed"
386
387if __name__ == '__main__':
388    main()
389