1import argparse
2import base64
3import datetime
4import enum
5import glob
6import hashlib
7import hmac
8import json
9import os
10import requests
11import sys
12import tempfile
13import time
14import yaml
15import shutil
16import xml.etree.ElementTree as ET
17
18from email.utils import formatdate
19from pathlib import Path
20from PIL import Image
21from urllib import parse
22
23import dump_trace_images
24
25TRACES_DB_PATH = "./traces-db/"
26RESULTS_PATH = "./results/"
27MINIO_HOST = "minio-packet.freedesktop.org"
28DASHBOARD_URL = "https://tracie.freedesktop.org/dashboard"
29
30minio_credentials = None
31
32def replay(trace_path, device_name):
33    success = dump_trace_images.dump_from_trace(trace_path, [], device_name)
34
35    if not success:
36        print("[check_image] Trace %s couldn't be replayed. See above logs for more information." % (str(trace_path)))
37        return None, None, None
38    else:
39        base_path = trace_path.parent
40        file_name = trace_path.name
41        files = glob.glob(str(base_path / "test" / device_name / (file_name + "-*" + ".png")))
42        assert(files)
43        image_file = files[0]
44        files = glob.glob(str(base_path / "test" / device_name / (file_name + ".log")))
45        assert(files)
46        log_file = files[0]
47        return hashlib.md5(Image.open(image_file).tobytes()).hexdigest(), image_file, log_file
48
49def gitlab_ensure_trace(project_url, trace):
50    trace_path = TRACES_DB_PATH + trace['path']
51    if project_url is None:
52        if not os.path.exists(trace_path):
53            print("{} missing".format(trace_path))
54            sys.exit(1)
55        return
56
57    os.makedirs(os.path.dirname(trace_path), exist_ok=True)
58
59    if os.path.exists(trace_path):
60        return
61
62    print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True)
63    download_time = time.time()
64    r = requests.get(project_url + trace['path'])
65    open(trace_path, "wb").write(r.content)
66    print("took %ds." % (time.time() - download_time), flush=True)
67
68def sign_with_hmac(key, message):
69    key = key.encode("UTF-8")
70    message = message.encode("UTF-8")
71
72    signature = hmac.new(key, message, hashlib.sha1).digest()
73
74    return base64.encodebytes(signature).strip().decode()
75
76def ensure_minio_credentials():
77    global minio_credentials
78
79    if minio_credentials is None:
80        minio_credentials = {}
81
82    params = {'Action': 'AssumeRoleWithWebIdentity',
83              'Version': '2011-06-15',
84              'RoleArn': 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole',
85              'RoleSessionName': '%s:%s' % (os.environ['CI_PROJECT_PATH'], os.environ['CI_JOB_ID']),
86              'DurationSeconds': 900,
87              'WebIdentityToken': os.environ['CI_JOB_JWT']}
88    r = requests.post('https://%s' % (MINIO_HOST), params=params)
89    if r.status_code >= 400:
90        print(r.text)
91    r.raise_for_status()
92
93    root = ET.fromstring(r.text)
94    for attr in root.iter():
95        if attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}AccessKeyId':
96            minio_credentials['AccessKeyId'] = attr.text
97        elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SecretAccessKey':
98            minio_credentials['SecretAccessKey'] = attr.text
99        elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SessionToken':
100            minio_credentials['SessionToken'] = attr.text
101
102def upload_to_minio(file_name, resource, content_type):
103    ensure_minio_credentials()
104
105    minio_key = minio_credentials['AccessKeyId']
106    minio_secret = minio_credentials['SecretAccessKey']
107    minio_token = minio_credentials['SessionToken']
108
109    date = formatdate(timeval=None, localtime=False, usegmt=True)
110    url = 'https://%s%s' % (MINIO_HOST, resource)
111    to_sign = "PUT\n\n%s\n%s\nx-amz-security-token:%s\n%s" % (content_type, date, minio_token, resource)
112    signature = sign_with_hmac(minio_secret, to_sign)
113
114    with open(file_name, 'rb') as data:
115        headers = {'Host': MINIO_HOST,
116                   'Date': date,
117                   'Content-Type': content_type,
118                   'Authorization': 'AWS %s:%s' % (minio_key, signature),
119                   'x-amz-security-token': minio_token}
120        print("Uploading artifact to %s" % url);
121        r = requests.put(url, headers=headers, data=data)
122        if r.status_code >= 400:
123            print(r.text)
124        r.raise_for_status()
125
126def upload_artifact(file_name, key, content_type):
127    resource = '/artifacts/%s/%s/%s/%s' % (os.environ['CI_PROJECT_PATH'],
128                                           os.environ['CI_PIPELINE_ID'],
129                                           os.environ['CI_JOB_ID'],
130                                           key)
131    upload_to_minio(file_name, resource, content_type)
132
133def ensure_reference_image(file_name, checksum):
134    resource = '/mesa-tracie-results/%s/%s.png' % (os.environ['CI_PROJECT_PATH'], checksum)
135    url = 'https://%s%s' % (MINIO_HOST, resource)
136    r = requests.head(url, allow_redirects=True)
137    if r.status_code == 200:
138        return
139    upload_to_minio(file_name, resource, 'image/png')
140
141def image_diff_url(trace_path):
142    return "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL,
143                                      os.environ.get('CI_PROJECT_PATH'),
144                                      os.environ.get('CI_JOB_ID'),
145                                      trace_path)
146
147def gitlab_check_trace(project_url, device_name, trace, expectation):
148    gitlab_ensure_trace(project_url, trace)
149
150    result = {}
151    result[trace['path']] = {}
152    result[trace['path']]['expected'] = expectation['checksum']
153
154    trace_path = Path(TRACES_DB_PATH + trace['path'])
155    checksum, image_file, log_file = replay(trace_path, device_name)
156    if checksum is None:
157        result[trace['path']]['actual'] = 'error'
158        return False, result
159    elif checksum == expectation['checksum']:
160        print("[check_image] Images match for %s" % (trace['path']))
161        ok = True
162    else:
163        print("[check_image] Images differ for %s (expected: %s, actual: %s)" %
164                (trace['path'], expectation['checksum'], checksum))
165        print("[check_image] For more information see "
166                "https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md")
167        print("[check_image] %s" % image_diff_url(trace['path']))
168        ok = False
169
170    trace_dir = os.path.split(trace['path'])[0]
171    dir_in_results = os.path.join(trace_dir, "test", device_name)
172    results_path = os.path.join(RESULTS_PATH, dir_in_results)
173    os.makedirs(results_path, exist_ok=True)
174    shutil.move(log_file, os.path.join(results_path, os.path.split(log_file)[1]))
175    if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1':
176        if ok:
177            if os.environ['CI_PROJECT_PATH'] == 'mesa/mesa':
178                ensure_reference_image(image_file, checksum)
179        else:
180            upload_artifact(image_file, 'traces/%s.png' % checksum, 'image/png')
181    if not ok or os.environ.get('TRACIE_STORE_IMAGES', '0') == '1':
182        image_name = os.path.split(image_file)[1]
183        shutil.move(image_file, os.path.join(results_path, image_name))
184        result[trace['path']]['image'] = os.path.join(dir_in_results, image_name)
185
186    result[trace['path']]['actual'] = checksum
187
188    return ok, result
189
190def write_junit_xml(junit_xml_path, traces_filename, device_name, results):
191    tests = len(results)
192    failures = sum(1 for r in results.values() if r["actual"] != r["expected"])
193
194    try:
195        testsuites = ET.parse(junit_xml_path).getroot()
196    except:
197        test_name = os.environ.get('CI_PROJECT_PATH') + "/" + \
198                    os.environ.get('CI_PIPELINE_ID') + "/" + \
199                    os.environ.get('CI_JOB_ID')
200        testsuites = ET.Element('testsuites', name=test_name)
201
202    testsuites.set('tests', str(int(testsuites.get('tests', 0)) + tests))
203    testsuites.set('failures', str(int(testsuites.get('failures', 0)) + failures))
204
205    testsuite_name = os.path.basename(traces_filename) + ":" + device_name
206
207    testsuite = ET.SubElement(testsuites, 'testsuite',
208                              name=testsuite_name,
209                              tests=str(tests), failures=str(failures))
210
211    for (path, result) in results.items():
212        testcase = ET.SubElement(testsuite, 'testcase', name=path,
213                                 classname=testsuite_name)
214        if result["actual"] != result["expected"]:
215            failure = ET.SubElement(testcase, 'failure')
216            failure.text = \
217                ("Images differ (expected: %s, actual: %s).\n" + \
218                 "To view the image differences visit:\n%s") % \
219                (result["expected"], result["actual"], image_diff_url(path))
220
221    ET.ElementTree(testsuites).write(junit_xml_path)
222
223def run(filename, device_name):
224
225    with open(filename, 'r') as f:
226        y = yaml.safe_load(f)
227
228    if "traces-db" in y:
229        project_url = y["traces-db"]["download-url"]
230    else:
231        project_url = None
232
233    traces = y['traces'] or []
234    all_ok = True
235    results = {}
236    for trace in traces:
237        for expectation in trace['expectations']:
238            if expectation['device'] == device_name:
239                ok, result = gitlab_check_trace(project_url,
240                                                device_name, trace,
241                                                expectation)
242                all_ok = all_ok and ok
243                results.update(result)
244
245    os.makedirs(RESULTS_PATH, exist_ok=True)
246    with open(os.path.join(RESULTS_PATH, 'results.yml'), 'w') as f:
247        yaml.safe_dump(results, f, default_flow_style=False)
248
249    junit_xml_path = os.path.join(RESULTS_PATH, "junit.xml")
250    write_junit_xml(junit_xml_path, filename, device_name, results)
251
252    if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1':
253        upload_artifact(os.path.join(RESULTS_PATH, 'results.yml'), 'traces/results.yml', 'text/yaml')
254        upload_artifact(junit_xml_path, 'traces/junit.xml', 'text/xml')
255
256    return all_ok
257
258def main(args):
259    parser = argparse.ArgumentParser()
260    parser.add_argument('--file', required=True,
261                        help='the name of the traces.yml file listing traces and their checksums for each device')
262    parser.add_argument('--device-name', required=True,
263                        help="the name of the graphics device used to replay traces")
264
265    args = parser.parse_args(args)
266    return run(args.file, args.device_name)
267
268if __name__ == "__main__":
269    all_ok = main(sys.argv[1:])
270    sys.exit(0 if all_ok else 1)
271