1# Copyright 2014 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import os
16import os.path
17import tempfile
18import subprocess
19import time
20import sys
21import textwrap
22import its.device
23
24def main():
25    """Run all the automated tests, saving intermediate files, and producing
26    a summary/report of the results.
27
28    Script should be run from the top-level CameraITS directory.
29    """
30
31    SKIP_RET_CODE = 101
32
33    # Not yet mandated tests
34    NOT_YET_MANDATED = {
35        "scene0":[
36            "test_jitter"
37        ],
38        "scene1":[
39            "test_ae_precapture_trigger",
40            "test_crop_region_raw",
41            "test_ev_compensation_advanced",
42            "test_ev_compensation_basic",
43            "test_yuv_plus_jpeg"
44        ],
45        "scene2":[],
46        "scene3":[],
47        "scene4":[],
48        "scene5":[],
49        "sensor_fusion":[]
50    }
51
52    # Get all the scene0 and scene1 tests, which can be run using the same
53    # physical setup.
54    scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5"]
55
56    scene_req = {
57        "scene0" : None,
58        "scene1" : "A grey card covering at least the middle 30% of the scene",
59        "scene2" : "A picture containing human faces",
60        "scene3" : "A chart containing sharp edges like ISO 12233",
61        "scene4" : "A specific test page of a circle covering at least the "
62                   "middle 50% of the scene. See CameraITS.pdf section 2.3.4 "
63                   "for more details",
64        "scene5" : "Capture images with a diffuser attached to the camera. See "
65                   "CameraITS.pdf section 2.3.4 for more details",
66        "sensor_fusion" : "Rotating checkboard pattern. See "
67                          "sensor_fusion/SensorFusion.pdf for detailed "
68                          "instructions. Note that this test will be skipped "
69                          "on devices not supporting REALTIME camera timestamp."
70                          "If that is the case, no scene setup is required and "
71                          "you can just answer Y when being asked if the scene "
72                          "is okay"
73    }
74    scene_extra_args = {
75        "scene5" : ["doAF=False"]
76    }
77    tests = []
78    for d in scenes:
79        tests += [(d,s[:-3],os.path.join("tests", d, s))
80                  for s in os.listdir(os.path.join("tests",d))
81                  if s[-3:] == ".py"]
82    tests.sort()
83
84    # Make output directories to hold the generated files.
85    topdir = tempfile.mkdtemp()
86    print "Saving output files to:", topdir, "\n"
87
88    device_id = its.device.get_device_id()
89    device_id_arg = "device=" + device_id
90    print "Testing device " + device_id
91
92    camera_ids = []
93    for s in sys.argv[1:]:
94        if s[:7] == "camera=" and len(s) > 7:
95            camera_ids.append(s[7:])
96
97    # user doesn't specify camera id, run through all cameras
98    if not camera_ids:
99        camera_ids_path = os.path.join(topdir, "camera_ids.txt")
100        out_arg = "out=" + camera_ids_path
101        cmd = ['python',
102               os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg,
103               device_id_arg]
104        retcode = subprocess.call(cmd,cwd=topdir)
105        assert(retcode == 0)
106        with open(camera_ids_path, "r") as f:
107            for line in f:
108                camera_ids.append(line.replace('\n', ''))
109
110    print "Running ITS on the following cameras:", camera_ids
111
112    for camera_id in camera_ids:
113        # Loop capturing images until user confirm test scene is correct
114        camera_id_arg = "camera=" + camera_id
115        print "Preparing to run ITS on camera", camera_id
116
117        os.mkdir(os.path.join(topdir, camera_id))
118        for d in scenes:
119            os.mkdir(os.path.join(topdir, camera_id, d))
120
121        print "Start running ITS on camera: ", camera_id
122        # Run each test, capturing stdout and stderr.
123        summary = "ITS test result summary for camera " + camera_id + "\n"
124        numpass = 0
125        numskip = 0
126        numnotmandatedfail = 0
127        numfail = 0
128
129        prev_scene = ""
130        for (scene,testname,testpath) in tests:
131            if scene != prev_scene and scene_req[scene] != None:
132                out_path = os.path.join(topdir, camera_id, scene+".jpg")
133                out_arg = "out=" + out_path
134                scene_arg = "scene=" + scene_req[scene]
135                extra_args = scene_extra_args.get(scene, [])
136                cmd = ['python',
137                        os.path.join(os.getcwd(),"tools/validate_scene.py"),
138                        camera_id_arg, out_arg, scene_arg, device_id_arg] + \
139                        extra_args
140                retcode = subprocess.call(cmd,cwd=topdir)
141                assert(retcode == 0)
142                print "Start running tests for", scene
143            prev_scene = scene
144            cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
145                  sys.argv[1:] + [camera_id_arg]
146            outdir = os.path.join(topdir,camera_id,scene)
147            outpath = os.path.join(outdir,testname+"_stdout.txt")
148            errpath = os.path.join(outdir,testname+"_stderr.txt")
149            t0 = time.time()
150            with open(outpath,"w") as fout, open(errpath,"w") as ferr:
151                retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
152            t1 = time.time()
153
154            if retcode == 0:
155                retstr = "PASS "
156                numpass += 1
157            elif retcode == SKIP_RET_CODE:
158                retstr = "SKIP "
159                numskip += 1
160            elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
161                retstr = "FAIL*"
162                numnotmandatedfail += 1
163            else:
164                retstr = "FAIL "
165                numfail += 1
166
167            msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
168            print msg
169            summary += msg + "\n"
170            if retcode != 0 and retcode != SKIP_RET_CODE:
171                # Dump the stderr if the test fails
172                with open (errpath, "r") as error_file:
173                    errors = error_file.read()
174                    summary += errors + "\n"
175
176        if numskip > 0:
177            skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "")
178        else:
179            skipstr = ""
180
181        test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
182                numpass + numnotmandatedfail, len(tests) - numskip,
183                100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip)
184                    if len(tests) != numskip else 100.0,
185                skipstr)
186        print test_result
187        summary += test_result + "\n"
188
189        if numnotmandatedfail > 0:
190            msg = "(*) tests are not yet mandated"
191            print msg
192            summary += msg + "\n"
193
194        result = numfail == 0
195        print "Reporting ITS result to CtsVerifier"
196        summary_path = os.path.join(topdir, camera_id, "summary.txt")
197        with open(summary_path, "w") as f:
198            f.write(summary)
199        its.device.report_result(device_id, camera_id, result, summary_path)
200
201    print "ITS tests finished. Please go back to CtsVerifier and proceed"
202
203if __name__ == '__main__':
204    main()
205