1# -*- coding: utf-8 -*-
2
3import os
4import re
5import sys
6import copy
7import zlib
8import time
9import shlex
10import shutil
11import fnmatch
12import tarfile
13import argparse
14import platform
15import datetime
16import tempfile
17import posixpath
18import subprocess
19
20from build.common import *
21from build.config import *
22from build.build import *
23
24def die (msg):
25	print msg
26	sys.exit(-1)
27
28def removeLeadingPath (path, basePath):
29	# Both inputs must be normalized already
30	assert os.path.normpath(path) == path
31	assert os.path.normpath(basePath) == basePath
32	return path[len(basePath) + 1:]
33
34def findFile (candidates):
35	for file in candidates:
36		if os.path.exists(file):
37			return file
38	return None
39
40def getFileList (basePath):
41	allFiles	= []
42	basePath	= os.path.normpath(basePath)
43	for root, dirs, files in os.walk(basePath):
44		for file in files:
45			relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
46			allFiles.append(relPath)
47	return allFiles
48
49def toDatetime (dateTuple):
50	Y, M, D = dateTuple
51	return datetime.datetime(Y, M, D)
52
53class PackageBuildInfo:
54	def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
55		self.releaseConfig	= releaseConfig
56		self.srcBasePath	= srcBasePath
57		self.dstBasePath	= dstBasePath
58		self.tmpBasePath	= tmpBasePath
59
60	def getReleaseConfig (self):
61		return self.releaseConfig
62
63	def getReleaseVersion (self):
64		return self.releaseConfig.getVersion()
65
66	def getReleaseId (self):
67		# Release id is crc32(releaseConfig + release)
68		return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
69
70	def getSrcBasePath (self):
71		return self.srcBasePath
72
73	def getTmpBasePath (self):
74		return self.tmpBasePath
75
76class DstFile (object):
77	def __init__ (self, dstFile):
78		self.dstFile = dstFile
79
80	def makeDir (self):
81		dirName = os.path.dirname(self.dstFile)
82		if not os.path.exists(dirName):
83			os.makedirs(dirName)
84
85	def make (self, packageBuildInfo):
86		assert False # Should not be called
87
88class CopyFile (DstFile):
89	def __init__ (self, srcFile, dstFile):
90		super(CopyFile, self).__init__(dstFile)
91		self.srcFile = srcFile
92
93	def make (self, packageBuildInfo):
94		self.makeDir()
95		if os.path.exists(self.dstFile):
96			die("%s already exists" % self.dstFile)
97		shutil.copyfile(self.srcFile, self.dstFile)
98
99class GenInfoFile (DstFile):
100	def __init__ (self, srcFile, dstFile):
101		super(GenInfoFile, self).__init__(dstFile)
102		self.srcFile = srcFile
103
104	def make (self, packageBuildInfo):
105		self.makeDir()
106		print "    GenInfoFile: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
107		src			= readFile(self.srcFile)
108
109		for define, value in [("DEQP_RELEASE_NAME", 	"\"%s\""	% packageBuildInfo.getReleaseVersion()),
110							  ("DEQP_RELEASE_ID",		"0x%08x"	% packageBuildInfo.getReleaseId())]:
111			src = re.sub('(#\s*define\s+%s\s+)[^\n\r]+' % re.escape(define), r'\1 %s' % value, src)
112
113		writeFile(self.dstFile, src)
114
115class GenCMake (DstFile):
116	def __init__ (self, srcFile, dstFile, replaceVars):
117		super(GenCMake, self).__init__(dstFile)
118		self.srcFile		= srcFile
119		self.replaceVars	= replaceVars
120
121	def make (self, packageBuildInfo):
122		self.makeDir()
123		print "    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
124		src = readFile(self.srcFile)
125		for var, value in self.replaceVars:
126			src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
127						 'set(%s "%s"' % (var, value), src)
128		writeFile(self.dstFile, src)
129
130def createFileTargets (srcBasePath, dstBasePath, files, filters):
131	usedFiles	= set() # Files that are already included by other filters
132	targets		= []
133
134	for isMatch, createFileObj in filters:
135		# Build list of files that match filter
136		matchingFiles = []
137		for file in files:
138			if not file in usedFiles and isMatch(file):
139				matchingFiles.append(file)
140
141		# Build file objects, add to used set
142		for file in matchingFiles:
143			usedFiles.add(file)
144			targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
145
146	return targets
147
148# Generates multiple file targets based on filters
149class FileTargetGroup:
150	def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
151		self.srcBasePath	= srcBasePath
152		self.dstBasePath	= dstBasePath
153		self.filters		= filters
154		self.getSrcBasePath	= srcBasePathFunc
155
156	def make (self, packageBuildInfo):
157		fullSrcPath		= os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
158		fullDstPath		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
159
160		allFiles		= getFileList(fullSrcPath)
161		targets		 	= createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
162
163		# Make all file targets
164		for file in targets:
165			file.make(packageBuildInfo)
166
167# Single file target
168class SingleFileTarget:
169	def __init__ (self, srcFile, dstFile, makeTarget):
170		self.srcFile	= srcFile
171		self.dstFile	= dstFile
172		self.makeTarget	= makeTarget
173
174	def make (self, packageBuildInfo):
175		fullSrcPath		= os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
176		fullDstPath		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
177
178		target = self.makeTarget(fullSrcPath, fullDstPath)
179		target.make(packageBuildInfo)
180
181class BuildTarget:
182	def __init__ (self, baseConfig, generator, targets = None):
183		self.baseConfig	= baseConfig
184		self.generator	= generator
185		self.targets	= targets
186
187	def make (self, packageBuildInfo):
188		print "    Building %s" % self.baseConfig.getBuildDir()
189
190		# Create config with full build dir path
191		config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
192							 self.baseConfig.getBuildType(),
193							 self.baseConfig.getArgs(),
194							 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
195
196		assert not os.path.exists(config.getBuildDir())
197		build(config, self.generator, self.targets)
198
199class BuildAndroidTarget:
200	def __init__ (self, dstFile):
201		self.dstFile = dstFile
202
203	def make (self, packageBuildInfo):
204		print "    Building Android binary"
205
206		buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
207
208		assert not os.path.exists(buildRoot)
209		os.makedirs(buildRoot)
210
211		# Execute build script
212		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
213		execute([
214				"python",
215				"-B", # no .py[co]
216				scriptPath,
217				"--build-root=%s" % buildRoot,
218			])
219
220		srcFile		= os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
221		dstFile		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
222
223		CopyFile(srcFile, dstFile).make(packageBuildInfo)
224
225class FetchExternalSourcesTarget:
226	def __init__ (self):
227		pass
228
229	def make (self, packageBuildInfo):
230		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
231		execute([
232				"python",
233				"-B", # no .py[co]
234				scriptPath,
235			])
236
237class RemoveSourcesTarget:
238	def __init__ (self):
239		pass
240
241	def make (self, packageBuildInfo):
242		shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
243
244class Module:
245	def __init__ (self, name, targets):
246		self.name		= name
247		self.targets	= targets
248
249	def make (self, packageBuildInfo):
250		for target in self.targets:
251			target.make(packageBuildInfo)
252
253class ReleaseConfig:
254	def __init__ (self, name, version, modules, sources = True):
255		self.name			= name
256		self.version		= version
257		self.modules		= modules
258		self.sources		= sources
259
260	def getName (self):
261		return self.name
262
263	def getVersion (self):
264		return self.version
265
266	def getModules (self):
267		return self.modules
268
269	def packageWithSources (self):
270		return self.sources
271
272def matchIncludeExclude (includePatterns, excludePatterns, filename):
273	components = os.path.normpath(filename).split(os.sep)
274	for pattern in excludePatterns:
275		for component in components:
276			if fnmatch.fnmatch(component, pattern):
277				return False
278
279	for pattern in includePatterns:
280		for component in components:
281			if fnmatch.fnmatch(component, pattern):
282				return True
283
284	return False
285
286def copyFileFilter (includePatterns, excludePatterns=[]):
287	return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
288			lambda s, d: CopyFile(s, d))
289
290def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
291	return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
292
293def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
294	return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
295
296def makeFileCopy (srcFile, dstFile):
297	return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
298
299def getReleaseFileName (configName, releaseName):
300	today = datetime.date.today()
301	return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
302
303def getTempDir ():
304	dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
305	if not os.path.exists(dirName):
306		os.makedirs(dirName)
307	return dirName
308
309def makeRelease (releaseConfig):
310	releaseName			= getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
311	tmpPath				= getTempDir()
312	srcBasePath			= DEQP_DIR
313	dstBasePath			= os.path.join(tmpPath, releaseName)
314	tmpBasePath			= os.path.join(tmpPath, releaseName + "-tmp")
315	packageBuildInfo	= PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
316	dstArchiveName		= releaseName + ".tar.bz2"
317
318	print "Creating release %s to %s" % (releaseName, tmpPath)
319
320	# Remove old temporary dirs
321	for path in [dstBasePath, tmpBasePath]:
322		if os.path.exists(path):
323			shutil.rmtree(path, ignore_errors=False)
324
325	# Make all modules
326	for module in releaseConfig.getModules():
327		print "  Processing module %s" % module.name
328		module.make(packageBuildInfo)
329
330	# Remove sources?
331	if not releaseConfig.packageWithSources():
332		shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
333
334	# Create archive
335	print "Creating %s" % dstArchiveName
336	archive	= tarfile.open(dstArchiveName, 'w:bz2')
337	archive.add(dstBasePath, arcname=releaseName)
338	archive.close()
339
340	# Remove tmp dirs
341	for path in [dstBasePath, tmpBasePath]:
342		if os.path.exists(path):
343			shutil.rmtree(path, ignore_errors=False)
344
345	print "Done!"
346
347# Module declarations
348
349SRC_FILE_PATTERNS	= ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
350TARGET_PATTERNS		= ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
351
352BASE = Module("Base", [
353	makeFileCopy		("LICENSE",									"src/LICENSE"),
354	makeFileCopy		("CMakeLists.txt",							"src/CMakeLists.txt"),
355	makeFileCopyGroup	("targets",									"src/targets",							TARGET_PATTERNS),
356	makeFileCopyGroup	("execserver",								"src/execserver",						SRC_FILE_PATTERNS),
357	makeFileCopyGroup	("executor",								"src/executor",							SRC_FILE_PATTERNS),
358	makeFileCopy		("modules/CMakeLists.txt", 					"src/modules/CMakeLists.txt"),
359	makeFileCopyGroup	("external", 								"src/external",							["CMakeLists.txt", "*.py"]),
360
361	# Stylesheet for displaying test logs on browser
362	makeFileCopyGroup	("doc/testlog-stylesheet",					"doc/testlog-stylesheet",				["*"]),
363
364	# Non-optional parts of framework
365	# \note qpInfo.c must be processed!
366	FileTargetGroup		("framework", "src/framework", [
367			# If file is qpInfo.c use GenInfoFile
368			(lambda f: f[-8:] == "qpInfo.c", lambda s, d: GenInfoFile(s, d)),
369			# Otherwise just CopyFile targets
370			copyFileFilter(SRC_FILE_PATTERNS, ["imagedifftester", "randomshaders", "simplereference", "referencerenderer"])
371		]),
372
373	# Main modules CMakeLists.txt
374
375	# android sources
376	makeFileCopyGroup	("android/package/src",						"src/android/package/src",				SRC_FILE_PATTERNS),
377	makeFileCopy		("android/package/AndroidManifest.xml",		"src/android/package/AndroidManifest.xml"),
378	makeFileCopyGroup	("android/package/res",						"src/android/package/res",				["*.png", "*.xml"]),
379	makeFileCopyGroup	("android/scripts",							"src/android/scripts", [
380		"common.py",
381		"build.py",
382		"resources.py",
383		"install.py",
384		"launch.py",
385		"debug.py"
386		]),
387])
388
389DOCUMENTATION = Module("Documentation", [
390	makeFileCopyGroup	("doc/pdf",									"doc",									["*.pdf"]),
391	makeFileCopyGroup	("doc",										"doc",									["porting_layer_changes_*.txt"]),
392])
393
394GLSHARED = Module("Shared GL Tests", [
395	# Optional framework components
396	makeFileCopyGroup	("framework/randomshaders",					"src/framework/randomshaders",			SRC_FILE_PATTERNS),
397	makeFileCopyGroup	("framework/opengl/simplereference",		"src/framework/opengl/simplereference",	SRC_FILE_PATTERNS),
398	makeFileCopyGroup	("framework/referencerenderer",				"src/framework/referencerenderer",		SRC_FILE_PATTERNS),
399
400	makeFileCopyGroup	("modules/glshared",						"src/modules/glshared",					SRC_FILE_PATTERNS),
401])
402
403GLES2 = Module("GLES2", [
404	makeFileCopyGroup	("modules/gles2",							"src/modules/gles2",					SRC_FILE_PATTERNS),
405	makeFileCopyGroup	("data/gles2",								"src/data/gles2", 						["*.*"]),
406	makeFileCopyGroup	("doc/testspecs/GLES2",						"doc/testspecs/GLES2",					["*.txt"])
407])
408
409GLES3 = Module("GLES3", [
410	makeFileCopyGroup	("modules/gles3",							"src/modules/gles3",					SRC_FILE_PATTERNS),
411	makeFileCopyGroup	("data/gles3",								"src/data/gles3", 						["*.*"]),
412	makeFileCopyGroup	("doc/testspecs/GLES3",						"doc/testspecs/GLES3",					["*.txt"])
413])
414
415GLES31 = Module("GLES31", [
416	makeFileCopyGroup	("modules/gles31",							"src/modules/gles31",					SRC_FILE_PATTERNS),
417	makeFileCopyGroup	("data/gles31",								"src/data/gles31", 						["*.*"]),
418	makeFileCopyGroup	("doc/testspecs/GLES31",					"doc/testspecs/GLES31",					["*.txt"])
419])
420
421EGL = Module("EGL", [
422	makeFileCopyGroup	("modules/egl",								"src/modules/egl",						SRC_FILE_PATTERNS)
423])
424
425INTERNAL = Module("Internal", [
426	makeFileCopyGroup	("modules/internal",						"src/modules/internal",					SRC_FILE_PATTERNS),
427	makeFileCopyGroup	("data/internal",							"src/data/internal", 					["*.*"]),
428])
429
430EXTERNAL_SRCS = Module("External sources", [
431	FetchExternalSourcesTarget()
432])
433
434ANDROID_BINARIES = Module("Android Binaries", [
435	BuildAndroidTarget	("bin/android/dEQP.apk"),
436	makeFileCopyGroup	("targets/android",							"bin/android",							["*.bat", "*.sh"]),
437])
438
439COMMON_BUILD_ARGS	= ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
440NULL_X32_CONFIG		= BuildConfig('null-x32',	'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
441NULL_X64_CONFIG		= BuildConfig('null-x64',	'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
442GLX_X32_CONFIG		= BuildConfig('glx-x32',	'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
443GLX_X64_CONFIG		= BuildConfig('glx-x64',	'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
444
445EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
446
447LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
448	BuildTarget			(NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
449	makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver",		"bin/linux32",					["*"],	EXCLUDE_BUILD_FILES),
450	makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor",		"bin/linux32",					["*"],	EXCLUDE_BUILD_FILES),
451])
452
453LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
454	BuildTarget			(NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
455	makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver",		"bin/linux64",					["*"],	EXCLUDE_BUILD_FILES),
456	makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor",		"bin/linux64",					["*"],	EXCLUDE_BUILD_FILES),
457])
458
459# Special module to remove src dir, for example after binary build
460REMOVE_SOURCES = Module("Remove sources from package", [
461	RemoveSourcesTarget()
462])
463
464# Release configuration
465
466ALL_MODULES		= [
467	BASE,
468	DOCUMENTATION,
469	GLSHARED,
470	GLES2,
471	GLES3,
472	GLES31,
473	EGL,
474	INTERNAL,
475	EXTERNAL_SRCS,
476]
477
478ALL_BINARIES	= [
479	LINUX_X64_COMMON_BINARIES,
480	ANDROID_BINARIES,
481]
482
483RELEASE_CONFIGS	= {
484	"src":		ALL_MODULES,
485	"src-bin":	ALL_MODULES + ALL_BINARIES,
486	"bin":		ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
487}
488
489def parseArgs ():
490	parser = argparse.ArgumentParser(description = "Build release package")
491	parser.add_argument("-c",
492						"--config",
493						dest="config",
494						choices=RELEASE_CONFIGS.keys(),
495						required=True,
496						help="Release configuration")
497	parser.add_argument("-n",
498						"--name",
499						dest="name",
500						required=True,
501						help="Package-specific name")
502	parser.add_argument("-v",
503						"--version",
504						dest="version",
505						required=True,
506						help="Version code")
507	return parser.parse_args()
508
509if __name__ == "__main__":
510	args	= parseArgs()
511	config	= ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
512	makeRelease(config)
513