1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import sys
25import shutil
26import tarfile
27import hashlib
28import argparse
29import subprocess
30import ssl
31import stat
32
33sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
34
35from build.common import *
36
37EXTERNAL_DIR	= os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
38
39def computeChecksum (data):
40	return hashlib.sha256(data).hexdigest()
41
42def onReadonlyRemoveError (func, path, exc_info):
43	os.chmod(path, stat.S_IWRITE)
44	os.unlink(path)
45
46class Source:
47	def __init__(self, baseDir, extractDir):
48		self.baseDir		= baseDir
49		self.extractDir		= extractDir
50
51	def clean (self):
52		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
53		# Remove read-only first
54		readonlydir = os.path.join(fullDstPath, ".git", "objects", "pack")
55		if os.path.exists(readonlydir):
56			shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError )
57		if os.path.exists(fullDstPath):
58			shutil.rmtree(fullDstPath, ignore_errors=False)
59
60class SourcePackage (Source):
61	def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
62		Source.__init__(self, baseDir, extractDir)
63		self.url			= url
64		self.filename		= filename
65		self.checksum		= checksum
66		self.archiveDir		= "packages"
67		self.postExtract	= postExtract
68
69	def clean (self):
70		Source.clean(self)
71		self.removeArchives()
72
73	def update (self, cmdProtocol = None, force = False):
74		if not self.isArchiveUpToDate():
75			self.fetchAndVerifyArchive()
76
77		if self.getExtractedChecksum() != self.checksum:
78			Source.clean(self)
79			self.extract()
80			self.storeExtractedChecksum(self.checksum)
81
82	def removeArchives (self):
83		archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
84		if os.path.exists(archiveDir):
85			shutil.rmtree(archiveDir, ignore_errors=False)
86
87	def isArchiveUpToDate (self):
88		archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
89		if os.path.exists(archiveFile):
90			return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
91		else:
92			return False
93
94	def getExtractedChecksumFilePath (self):
95		return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
96
97	def getExtractedChecksum (self):
98		extractedChecksumFile = self.getExtractedChecksumFilePath()
99
100		if os.path.exists(extractedChecksumFile):
101			return readFile(extractedChecksumFile)
102		else:
103			return None
104
105	def storeExtractedChecksum (self, checksum):
106		checksum_bytes = checksum.encode("utf-8")
107		writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
108
109	def connectToUrl (self, url):
110		result = None
111
112		if sys.version_info < (3, 0):
113			from urllib2 import urlopen
114		else:
115			from urllib.request import urlopen
116
117		if args.insecure:
118			print("Ignoring certificate checks")
119			ssl_context = ssl._create_unverified_context()
120			result = urlopen(url, context=ssl_context)
121		else:
122			result = urlopen(url)
123
124		return result
125
126	def fetchAndVerifyArchive (self):
127		print("Fetching %s" % self.url)
128
129		req			= self.connectToUrl(self.url)
130		data		= req.read()
131		checksum	= computeChecksum(data)
132		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
133
134		if checksum != self.checksum:
135			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
136
137		if not os.path.exists(os.path.dirname(dstPath)):
138			os.mkdir(os.path.dirname(dstPath))
139
140		writeBinaryFile(dstPath, data)
141
142	def extract (self):
143		print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
144
145		srcPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
146		tmpPath	= os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
147		dstPath	= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
148		archive	= tarfile.open(srcPath)
149
150		if os.path.exists(tmpPath):
151			shutil.rmtree(tmpPath, ignore_errors=False)
152
153		os.mkdir(tmpPath)
154
155		archive.extractall(tmpPath)
156		archive.close()
157
158		extractedEntries = os.listdir(tmpPath)
159		if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
160			raise Exception("%s doesn't contain single top-level directory" % self.filename)
161
162		topLevelPath = os.path.join(tmpPath, extractedEntries[0])
163
164		if not os.path.exists(dstPath):
165			os.mkdir(dstPath)
166
167		for entry in os.listdir(topLevelPath):
168			if os.path.exists(os.path.join(dstPath, entry)):
169				raise Exception("%s exists already" % entry)
170
171			shutil.move(os.path.join(topLevelPath, entry), dstPath)
172
173		shutil.rmtree(tmpPath, ignore_errors=True)
174
175		if self.postExtract != None:
176			self.postExtract(dstPath)
177
178class SourceFile (Source):
179	def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
180		Source.__init__(self, baseDir, extractDir)
181		self.url			= url
182		self.filename		= filename
183		self.checksum		= checksum
184
185	def update (self, cmdProtocol = None, force = False):
186		if not self.isFileUpToDate():
187			Source.clean(self)
188			self.fetchAndVerifyFile()
189
190	def isFileUpToDate (self):
191		file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
192		if os.path.exists(file):
193			data = readFile(file)
194			return computeChecksum(data.encode('utf-8')) == self.checksum
195		else:
196			return False
197
198	def connectToUrl (self, url):
199		result = None
200
201		if sys.version_info < (3, 0):
202			from urllib2 import urlopen
203		else:
204			from urllib.request import urlopen
205
206		if args.insecure:
207			print("Ignoring certificate checks")
208			ssl_context = ssl._create_unverified_context()
209			result = urlopen(url, context=ssl_context)
210		else:
211			result = urlopen(url)
212
213		return result
214
215	def fetchAndVerifyFile (self):
216		print("Fetching %s" % self.url)
217
218		req			= self.connectToUrl(self.url)
219		data		= req.read()
220		checksum	= computeChecksum(data)
221		dstPath		= os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
222
223		if checksum != self.checksum:
224			raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
225
226		if not os.path.exists(os.path.dirname(dstPath)):
227			os.mkdir(os.path.dirname(dstPath))
228
229		writeBinaryFile(dstPath, data)
230
231class GitRepo (Source):
232	def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = []):
233		Source.__init__(self, baseDir, extractDir)
234		self.httpsUrl	= httpsUrl
235		self.sshUrl		= sshUrl
236		self.revision	= revision
237		self.removeTags	= removeTags
238
239	def detectProtocol(self, cmdProtocol = None):
240		# reuse parent repo protocol
241		proc = subprocess.Popen(['git', 'ls-remote', '--get-url', 'origin'], stdout=subprocess.PIPE, universal_newlines=True)
242		(stdout, stderr) = proc.communicate()
243
244		if proc.returncode != 0:
245			raise Exception("Failed to execute 'git ls-remote origin', got %d" % proc.returncode)
246		if (stdout[:3] == 'ssh') or (stdout[:3] == 'git'):
247			protocol = 'ssh'
248		else:
249			# remote 'origin' doesn't exist, assume 'https' as checkout protocol
250			protocol = 'https'
251		return protocol
252
253	def selectUrl(self, cmdProtocol = None):
254		try:
255			if cmdProtocol == None:
256				protocol = self.detectProtocol(cmdProtocol)
257			else:
258				protocol = cmdProtocol
259		except:
260			# fallback to https on any issues
261			protocol = 'https'
262
263		if protocol == 'ssh':
264			if self.sshUrl != None:
265				url = self.sshUrl
266			else:
267				assert self.httpsUrl != None
268				url = self.httpsUrl
269		else:
270			assert protocol == 'https'
271			url = self.httpsUrl
272
273		assert url != None
274		return url
275
276	def update (self, cmdProtocol = None, force = False):
277		fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
278
279		url = self.selectUrl(cmdProtocol)
280		if not os.path.exists(os.path.join(fullDstPath, '.git')):
281			execute(["git", "clone", "--no-checkout", url, fullDstPath])
282
283		pushWorkingDir(fullDstPath)
284		try:
285			for tag in self.removeTags:
286				proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
287				(stdout, stderr) = proc.communicate()
288				if proc.returncode == 0:
289					execute(["git", "tag", "-d",tag])
290			force_arg = ['--force'] if force else []
291			execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
292			execute(["git", "checkout"] + force_arg + [self.revision])
293		finally:
294			popWorkingDir()
295
296def postExtractLibpng (path):
297	shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
298				os.path.join(path, "pnglibconf.h"))
299
300PACKAGES = [
301	SourcePackage(
302		"http://zlib.net/zlib-1.2.11.tar.gz",
303		"zlib-1.2.11.tar.gz",
304		"c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
305		"zlib"),
306	SourcePackage(
307		"http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
308		"libpng-1.6.27.tar.gz",
309		"c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
310		"libpng",
311		postExtract = postExtractLibpng),
312	SourceFile(
313		"https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
314		"renderdoc_app.h",
315		"e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
316		"renderdoc"),
317	GitRepo(
318		"https://github.com/KhronosGroup/SPIRV-Tools.git",
319		None,
320		"f11f7434815838bbad349124767b258ce7df41f0",
321		"spirv-tools"),
322	GitRepo(
323		"https://github.com/KhronosGroup/glslang.git",
324		None,
325		"5c4f421121c4d24aad23a507e630dc5dc6c92c7c",
326		"glslang",
327		removeTags = ["master-tot"]),
328	GitRepo(
329		"https://github.com/KhronosGroup/SPIRV-Headers.git",
330		None,
331		"faa570afbc91ac73d594d787486bcf8f2df1ace0",
332		"spirv-headers"),
333	GitRepo(
334		"https://github.com/google/amber.git",
335		None,
336		"dabae26164714abf951c6815a2b4513260f7c6a4",
337		"amber"),
338]
339
340def parseArgs ():
341	versionsForInsecure = ((2,7,9), (3,4,3))
342	versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
343
344	parser = argparse.ArgumentParser(description = "Fetch external sources")
345	parser.add_argument('--clean', dest='clean', action='store_true', default=False,
346						help='Remove sources instead of fetching')
347	parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
348						help="Disable certificate check for external sources."
349						" Minimum python version required " + versionsForInsecureStr)
350	parser.add_argument('--protocol', dest='protocol', default=None, choices=['ssh', 'https'],
351						help="Select protocol to checkout git repositories.")
352	parser.add_argument('--force', dest='force', action='store_true', default=False,
353						help="Pass --force to git fetch and checkout commands")
354
355	args = parser.parse_args()
356
357	if args.insecure:
358		for versionItem in versionsForInsecure:
359			if (sys.version_info.major == versionItem[0]):
360				if sys.version_info < versionItem:
361					parser.error("For --insecure minimum required python version is " +
362								versionsForInsecureStr)
363				break;
364
365	return args
366
367if __name__ == "__main__":
368	args = parseArgs()
369
370	for pkg in PACKAGES:
371		if args.clean:
372			pkg.clean()
373		else:
374			pkg.update(args.protocol, args.force)
375