1#!/usr/bin/python3
2#
3# Copyright 2017 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# run_code_generation.py:
8#   Runs ANGLE format table and other script code generation scripts.
9
10import hashlib
11import json
12import os
13import subprocess
14import sys
15import platform
16
17script_dir = sys.path[0]
18root_dir = os.path.abspath(os.path.join(script_dir, '..'))
19
20hash_dir = 'code_generation_hashes'
21
22# auto_script is a standard way for scripts to return their inputs and outputs.
23
24
25def get_child_script_dirname(script):
26    # All script names are relative to ANGLE's root
27    return os.path.dirname(os.path.abspath(os.path.join(root_dir, script)))
28
29
30# Replace all backslashes with forward slashes to be platform independent
31def clean_path_slashes(path):
32    return path.replace("\\", "/")
33
34
35# Takes a script file name which is relative to the code generation script's directory and
36# changes it to be relative to the angle root directory
37def rebase_script_path(script_path, relative_path):
38    return os.path.relpath(os.path.join(os.path.dirname(script_path), relative_path), root_dir)
39
40
41# Check if we need a module from vpython
42def get_executable_name(first_line):
43    binary = os.path.basename(first_line.strip().replace(' ', '/'))
44    if platform.system() == 'Windows':
45        if binary == 'python2':
46            return 'python.bat'
47        else:
48            return binary + '.bat'
49    else:
50        return binary
51
52
53def grab_from_script(script, param):
54    res = ''
55    f = open(os.path.basename(script), 'r')
56    exe = get_executable_name(f.readline())
57    try:
58        res = subprocess.check_output([exe, script, param]).decode().strip()
59    except Exception:
60        print('Error grabbing script output: %s, executable %s' % (script, exe))
61        raise
62    f.close()
63    if res == '':
64        return []
65    return [clean_path_slashes(rebase_script_path(script, name)) for name in res.split(',')]
66
67
68def auto_script(script):
69    # Set the CWD to the script directory.
70    os.chdir(get_child_script_dirname(script))
71    base_script = os.path.basename(script)
72    info = {
73        'inputs': grab_from_script(base_script, 'inputs'),
74        'outputs': grab_from_script(base_script, 'outputs')
75    }
76    # Reset the CWD to the root ANGLE directory.
77    os.chdir(root_dir)
78    return info
79
80
81generators = {
82    'ANGLE format':
83        'src/libANGLE/renderer/gen_angle_format_table.py',
84    'ANGLE load functions table':
85        'src/libANGLE/renderer/gen_load_functions_table.py',
86    'ANGLE load texture border functions table':
87        'src/libANGLE/renderer/gen_load_texture_border_functions_table.py',
88    'ANGLE shader preprocessor':
89        'src/compiler/preprocessor/generate_parser.py',
90    'ANGLE shader translator':
91        'src/compiler/translator/generate_parser.py',
92    'D3D11 blit shader selection':
93        'src/libANGLE/renderer/d3d/d3d11/gen_blit11helper.py',
94    'D3D11 format':
95        'src/libANGLE/renderer/d3d/d3d11/gen_texture_format_table.py',
96    'DXGI format':
97        'src/libANGLE/renderer/gen_dxgi_format_table.py',
98    'DXGI format support':
99        'src/libANGLE/renderer/gen_dxgi_support_tables.py',
100    'Emulated HLSL functions':
101        'src/compiler/translator/gen_emulated_builtin_function_tables.py',
102    'GL copy conversion table':
103        'src/libANGLE/gen_copy_conversion_table.py',
104    'GL CTS (dEQP) build files':
105        'scripts/gen_vk_gl_cts_build.py',
106    'GL/EGL/WGL loader':
107        'scripts/generate_loader.py',
108    'GL/EGL entry points':
109        'scripts/generate_entry_points.py',
110    'GLenum value to string map':
111        'scripts/gen_gl_enum_utils.py',
112    'GL format map':
113        'src/libANGLE/gen_format_map.py',
114    'Metal format table':
115        'src/libANGLE/renderer/metal/gen_mtl_format_table.py',
116    'Metal default shaders':
117        'src/libANGLE/renderer/metal/shaders/gen_mtl_internal_shaders.py',
118    'OpenGL dispatch table':
119        'src/libANGLE/renderer/gl/generate_gl_dispatch_table.py',
120    'overlay fonts':
121        'src/libANGLE/gen_overlay_fonts.py',
122    'overlay widgets':
123        'src/libANGLE/gen_overlay_widgets.py',
124    'packed enum':
125        'src/common/gen_packed_gl_enums.py',
126    'proc table':
127        'scripts/gen_proc_table.py',
128    'restricted traces':
129        'src/tests/restricted_traces/gen_restricted_traces.py',
130    'SPIR-V helpers':
131        'src/common/spirv/gen_spirv_builder_and_parser.py',
132    'Static builtins':
133        'src/compiler/translator/gen_builtin_symbols.py',
134    'Test spec JSON':
135        'infra/specs/generate_test_spec_json.py',
136    'uniform type':
137        'src/common/gen_uniform_type_table.py',
138    'Vulkan format':
139        'src/libANGLE/renderer/vulkan/gen_vk_format_table.py',
140    'Vulkan internal shader programs':
141        'src/libANGLE/renderer/vulkan/gen_vk_internal_shaders.py',
142    'Vulkan mandatory format support table':
143        'src/libANGLE/renderer/vulkan/gen_vk_mandatory_format_support_table.py',
144}
145
146
147def md5(fname):
148    hash_md5 = hashlib.md5()
149    with open(fname, "r") as f:
150        for chunk in iter(lambda: f.read(4096), ""):
151            hash_md5.update(chunk.encode())
152    return hash_md5.hexdigest()
153
154
155def get_hash_file_name(name):
156    return name.replace(' ', '_').replace('/', '_') + '.json'
157
158
159def any_hash_dirty(name, filenames, new_hashes, old_hashes):
160    found_dirty_hash = False
161
162    for fname in filenames:
163        if not os.path.isfile(fname):
164            print('File not found: "%s". Code gen dirty for %s' % (fname, name))
165            found_dirty_hash = True
166        else:
167            new_hashes[fname] = md5(fname)
168            if (not fname in old_hashes) or (old_hashes[fname] != new_hashes[fname]):
169                print('Hash for "%s" dirty for %s generator.' % (fname, name))
170                found_dirty_hash = True
171    return found_dirty_hash
172
173
174def any_old_hash_missing(all_new_hashes, all_old_hashes):
175    result = False
176    for file, old_hashes in all_old_hashes.items():
177        if file not in all_new_hashes:
178            print('"%s" does not exist. Code gen dirty.' % file)
179            result = True
180        else:
181            for name, _ in old_hashes.items():
182                if name not in all_new_hashes[file]:
183                    print('Hash for %s is missing from "%s". Code gen is dirty.' % (name, file))
184                    result = True
185    return result
186
187
188def update_output_hashes(script, outputs, new_hashes):
189    for output in outputs:
190        if not os.path.isfile(output):
191            print('Output is missing from %s: %s' % (script, output))
192            sys.exit(1)
193        new_hashes[output] = md5(output)
194
195
196def load_hashes():
197    hashes = {}
198    for file in os.listdir(hash_dir):
199        hash_fname = os.path.join(hash_dir, file)
200        with open(hash_fname) as hash_file:
201            try:
202                hashes[file] = json.load(hash_file)
203            except ValueError:
204                raise Exception("Could not decode JSON from %s" % file)
205    return hashes
206
207
208def main():
209    os.chdir(script_dir)
210
211    all_old_hashes = load_hashes()
212    all_new_hashes = {}
213    any_dirty = False
214
215    verify_only = False
216    if len(sys.argv) > 1 and sys.argv[1] == '--verify-no-dirty':
217        verify_only = True
218
219    for name, script in sorted(generators.items()):
220        info = auto_script(script)
221        fname = get_hash_file_name(name)
222        filenames = info['inputs'] + info['outputs'] + [script]
223        new_hashes = {}
224        if fname not in all_old_hashes:
225            all_old_hashes[fname] = {}
226        if any_hash_dirty(name, filenames, new_hashes, all_old_hashes[fname]):
227            any_dirty = True
228
229            if not verify_only:
230                print('Running ' + name + ' code generator')
231
232                # Set the CWD to the script directory.
233                os.chdir(get_child_script_dirname(script))
234
235                f = open(os.path.basename(script), "r")
236                if subprocess.call([get_executable_name(f.readline()),
237                                    os.path.basename(script)]) != 0:
238                    sys.exit(1)
239                f.close()
240
241        # Update the hash dictionary.
242        all_new_hashes[fname] = new_hashes
243
244    if any_old_hash_missing(all_new_hashes, all_old_hashes):
245        any_dirty = True
246
247    if verify_only:
248        sys.exit(any_dirty)
249
250    if any_dirty:
251        args = ['git.bat'] if os.name == 'nt' else ['git']
252        args += ['cl', 'format']
253        print('Calling git cl format')
254        if subprocess.call(args) != 0:
255            sys.exit(1)
256
257        # Update the output hashes again since they can be formatted.
258        for name, script in sorted(generators.items()):
259            info = auto_script(script)
260            fname = get_hash_file_name(name)
261            update_output_hashes(name, info['outputs'], all_new_hashes[fname])
262
263        os.chdir(script_dir)
264
265        for fname, new_hashes in all_new_hashes.items():
266            hash_fname = os.path.join(hash_dir, fname)
267            json.dump(
268                new_hashes,
269                open(hash_fname, "w"),
270                indent=2,
271                sort_keys=True,
272                separators=(',', ':\n    '))
273
274
275if __name__ == '__main__':
276    sys.exit(main())
277