1# -*- coding: utf-8 -*-
2
3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
23import os
24import re
25import sys
26import copy
27import zlib
28import time
29import shlex
30import shutil
31import fnmatch
32import tarfile
33import argparse
34import platform
35import datetime
36import tempfile
37import posixpath
38import subprocess
39
40from build.common import *
41from build.config import *
42from build.build import *
43
44def die (msg):
45	print msg
46	sys.exit(-1)
47
48def removeLeadingPath (path, basePath):
49	# Both inputs must be normalized already
50	assert os.path.normpath(path) == path
51	assert os.path.normpath(basePath) == basePath
52	return path[len(basePath) + 1:]
53
54def findFile (candidates):
55	for file in candidates:
56		if os.path.exists(file):
57			return file
58	return None
59
60def getFileList (basePath):
61	allFiles	= []
62	basePath	= os.path.normpath(basePath)
63	for root, dirs, files in os.walk(basePath):
64		for file in files:
65			relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
66			allFiles.append(relPath)
67	return allFiles
68
69def toDatetime (dateTuple):
70	Y, M, D = dateTuple
71	return datetime.datetime(Y, M, D)
72
73class PackageBuildInfo:
74	def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
75		self.releaseConfig	= releaseConfig
76		self.srcBasePath	= srcBasePath
77		self.dstBasePath	= dstBasePath
78		self.tmpBasePath	= tmpBasePath
79
80	def getReleaseConfig (self):
81		return self.releaseConfig
82
83	def getReleaseVersion (self):
84		return self.releaseConfig.getVersion()
85
86	def getReleaseId (self):
87		# Release id is crc32(releaseConfig + release)
88		return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
89
90	def getSrcBasePath (self):
91		return self.srcBasePath
92
93	def getTmpBasePath (self):
94		return self.tmpBasePath
95
96class DstFile (object):
97	def __init__ (self, dstFile):
98		self.dstFile = dstFile
99
100	def makeDir (self):
101		dirName = os.path.dirname(self.dstFile)
102		if not os.path.exists(dirName):
103			os.makedirs(dirName)
104
105	def make (self, packageBuildInfo):
106		assert False # Should not be called
107
108class CopyFile (DstFile):
109	def __init__ (self, srcFile, dstFile):
110		super(CopyFile, self).__init__(dstFile)
111		self.srcFile = srcFile
112
113	def make (self, packageBuildInfo):
114		self.makeDir()
115		if os.path.exists(self.dstFile):
116			die("%s already exists" % self.dstFile)
117		shutil.copyfile(self.srcFile, self.dstFile)
118
119class GenReleaseInfoFileTarget (DstFile):
120	def __init__ (self, dstFile):
121		super(GenReleaseInfoFileTarget, self).__init__(dstFile)
122
123	def make (self, packageBuildInfo):
124		self.makeDir()
125
126		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
127		execute([
128				"python",
129				"-B", # no .py[co]
130				scriptPath,
131				"--name=%s" % packageBuildInfo.getReleaseVersion(),
132				"--id=0x%08x" % packageBuildInfo.getReleaseId(),
133				"--out=%s" % self.dstFile
134			])
135
136class GenCMake (DstFile):
137	def __init__ (self, srcFile, dstFile, replaceVars):
138		super(GenCMake, self).__init__(dstFile)
139		self.srcFile		= srcFile
140		self.replaceVars	= replaceVars
141
142	def make (self, packageBuildInfo):
143		self.makeDir()
144		print "    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
145		src = readFile(self.srcFile)
146		for var, value in self.replaceVars:
147			src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
148						 'set(%s "%s"' % (var, value), src)
149		writeFile(self.dstFile, src)
150
151def createFileTargets (srcBasePath, dstBasePath, files, filters):
152	usedFiles	= set() # Files that are already included by other filters
153	targets		= []
154
155	for isMatch, createFileObj in filters:
156		# Build list of files that match filter
157		matchingFiles = []
158		for file in files:
159			if not file in usedFiles and isMatch(file):
160				matchingFiles.append(file)
161
162		# Build file objects, add to used set
163		for file in matchingFiles:
164			usedFiles.add(file)
165			targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
166
167	return targets
168
169# Generates multiple file targets based on filters
170class FileTargetGroup:
171	def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
172		self.srcBasePath	= srcBasePath
173		self.dstBasePath	= dstBasePath
174		self.filters		= filters
175		self.getSrcBasePath	= srcBasePathFunc
176
177	def make (self, packageBuildInfo):
178		fullSrcPath		= os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
179		fullDstPath		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
180
181		allFiles		= getFileList(fullSrcPath)
182		targets		 	= createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
183
184		# Make all file targets
185		for file in targets:
186			file.make(packageBuildInfo)
187
188# Single file target
189class SingleFileTarget:
190	def __init__ (self, srcFile, dstFile, makeTarget):
191		self.srcFile	= srcFile
192		self.dstFile	= dstFile
193		self.makeTarget	= makeTarget
194
195	def make (self, packageBuildInfo):
196		fullSrcPath		= os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
197		fullDstPath		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
198
199		target = self.makeTarget(fullSrcPath, fullDstPath)
200		target.make(packageBuildInfo)
201
202class BuildTarget:
203	def __init__ (self, baseConfig, generator, targets = None):
204		self.baseConfig	= baseConfig
205		self.generator	= generator
206		self.targets	= targets
207
208	def make (self, packageBuildInfo):
209		print "    Building %s" % self.baseConfig.getBuildDir()
210
211		# Create config with full build dir path
212		config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
213							 self.baseConfig.getBuildType(),
214							 self.baseConfig.getArgs(),
215							 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
216
217		assert not os.path.exists(config.getBuildDir())
218		build(config, self.generator, self.targets)
219
220class BuildAndroidTarget:
221	def __init__ (self, dstFile):
222		self.dstFile = dstFile
223
224	def make (self, packageBuildInfo):
225		print "    Building Android binary"
226
227		buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
228
229		assert not os.path.exists(buildRoot)
230		os.makedirs(buildRoot)
231
232		# Execute build script
233		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
234		execute([
235				"python",
236				"-B", # no .py[co]
237				scriptPath,
238				"--build-root=%s" % buildRoot,
239			])
240
241		srcFile		= os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
242		dstFile		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
243
244		CopyFile(srcFile, dstFile).make(packageBuildInfo)
245
246class FetchExternalSourcesTarget:
247	def __init__ (self):
248		pass
249
250	def make (self, packageBuildInfo):
251		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
252		execute([
253				"python",
254				"-B", # no .py[co]
255				scriptPath,
256			])
257
258class RemoveSourcesTarget:
259	def __init__ (self):
260		pass
261
262	def make (self, packageBuildInfo):
263		shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
264
265class Module:
266	def __init__ (self, name, targets):
267		self.name		= name
268		self.targets	= targets
269
270	def make (self, packageBuildInfo):
271		for target in self.targets:
272			target.make(packageBuildInfo)
273
274class ReleaseConfig:
275	def __init__ (self, name, version, modules, sources = True):
276		self.name			= name
277		self.version		= version
278		self.modules		= modules
279		self.sources		= sources
280
281	def getName (self):
282		return self.name
283
284	def getVersion (self):
285		return self.version
286
287	def getModules (self):
288		return self.modules
289
290	def packageWithSources (self):
291		return self.sources
292
293def matchIncludeExclude (includePatterns, excludePatterns, filename):
294	components = os.path.normpath(filename).split(os.sep)
295	for pattern in excludePatterns:
296		for component in components:
297			if fnmatch.fnmatch(component, pattern):
298				return False
299
300	for pattern in includePatterns:
301		for component in components:
302			if fnmatch.fnmatch(component, pattern):
303				return True
304
305	return False
306
307def copyFileFilter (includePatterns, excludePatterns=[]):
308	return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
309			lambda s, d: CopyFile(s, d))
310
311def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
312	return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
313
314def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
315	return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
316
317def makeFileCopy (srcFile, dstFile):
318	return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
319
320def getReleaseFileName (configName, releaseName):
321	today = datetime.date.today()
322	return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
323
324def getTempDir ():
325	dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
326	if not os.path.exists(dirName):
327		os.makedirs(dirName)
328	return dirName
329
330def makeRelease (releaseConfig):
331	releaseName			= getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
332	tmpPath				= getTempDir()
333	srcBasePath			= DEQP_DIR
334	dstBasePath			= os.path.join(tmpPath, releaseName)
335	tmpBasePath			= os.path.join(tmpPath, releaseName + "-tmp")
336	packageBuildInfo	= PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
337	dstArchiveName		= releaseName + ".tar.bz2"
338
339	print "Creating release %s to %s" % (releaseName, tmpPath)
340
341	# Remove old temporary dirs
342	for path in [dstBasePath, tmpBasePath]:
343		if os.path.exists(path):
344			shutil.rmtree(path, ignore_errors=False)
345
346	# Make all modules
347	for module in releaseConfig.getModules():
348		print "  Processing module %s" % module.name
349		module.make(packageBuildInfo)
350
351	# Remove sources?
352	if not releaseConfig.packageWithSources():
353		shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
354
355	# Create archive
356	print "Creating %s" % dstArchiveName
357	archive	= tarfile.open(dstArchiveName, 'w:bz2')
358	archive.add(dstBasePath, arcname=releaseName)
359	archive.close()
360
361	# Remove tmp dirs
362	for path in [dstBasePath, tmpBasePath]:
363		if os.path.exists(path):
364			shutil.rmtree(path, ignore_errors=False)
365
366	print "Done!"
367
368# Module declarations
369
370SRC_FILE_PATTERNS	= ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
371TARGET_PATTERNS		= ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
372
373BASE = Module("Base", [
374	makeFileCopy		("LICENSE",									"src/LICENSE"),
375	makeFileCopy		("CMakeLists.txt",							"src/CMakeLists.txt"),
376	makeFileCopyGroup	("targets",									"src/targets",							TARGET_PATTERNS),
377	makeFileCopyGroup	("execserver",								"src/execserver",						SRC_FILE_PATTERNS),
378	makeFileCopyGroup	("executor",								"src/executor",							SRC_FILE_PATTERNS),
379	makeFileCopy		("modules/CMakeLists.txt", 					"src/modules/CMakeLists.txt"),
380	makeFileCopyGroup	("external", 								"src/external",							["CMakeLists.txt", "*.py"]),
381
382	# Stylesheet for displaying test logs on browser
383	makeFileCopyGroup	("doc/testlog-stylesheet",					"doc/testlog-stylesheet",				["*"]),
384
385	# Non-optional parts of framework
386	makeFileCopy		("framework/CMakeLists.txt", 				"src/framework/CMakeLists.txt"),
387	makeFileCopyGroup	("framework/delibs",						"src/framework/delibs",					SRC_FILE_PATTERNS),
388	makeFileCopyGroup	("framework/common",						"src/framework/common",					SRC_FILE_PATTERNS),
389	makeFileCopyGroup	("framework/qphelper",						"src/framework/qphelper",				SRC_FILE_PATTERNS),
390	makeFileCopyGroup	("framework/platform",						"src/framework/platform",				SRC_FILE_PATTERNS),
391	makeFileCopyGroup	("framework/opengl",						"src/framework/opengl",					SRC_FILE_PATTERNS, ["simplereference"]),
392	makeFileCopyGroup	("framework/egl",							"src/framework/egl",					SRC_FILE_PATTERNS),
393
394	# android sources
395	makeFileCopyGroup	("android/package/src",						"src/android/package/src",				SRC_FILE_PATTERNS),
396	makeFileCopy		("android/package/AndroidManifest.xml",		"src/android/package/AndroidManifest.xml"),
397	makeFileCopyGroup	("android/package/res",						"src/android/package/res",				["*.png", "*.xml"]),
398	makeFileCopyGroup	("android/scripts",							"src/android/scripts", [
399		"common.py",
400		"build.py",
401		"resources.py",
402		"install.py",
403		"launch.py",
404		"debug.py"
405		]),
406
407	# Release info
408	GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
409])
410
411DOCUMENTATION = Module("Documentation", [
412	makeFileCopyGroup	("doc/pdf",									"doc",									["*.pdf"]),
413	makeFileCopyGroup	("doc",										"doc",									["porting_layer_changes_*.txt"]),
414])
415
416GLSHARED = Module("Shared GL Tests", [
417	# Optional framework components
418	makeFileCopyGroup	("framework/randomshaders",					"src/framework/randomshaders",			SRC_FILE_PATTERNS),
419	makeFileCopyGroup	("framework/opengl/simplereference",		"src/framework/opengl/simplereference",	SRC_FILE_PATTERNS),
420	makeFileCopyGroup	("framework/referencerenderer",				"src/framework/referencerenderer",		SRC_FILE_PATTERNS),
421
422	makeFileCopyGroup	("modules/glshared",						"src/modules/glshared",					SRC_FILE_PATTERNS),
423])
424
425GLES2 = Module("GLES2", [
426	makeFileCopyGroup	("modules/gles2",							"src/modules/gles2",					SRC_FILE_PATTERNS),
427	makeFileCopyGroup	("data/gles2",								"src/data/gles2", 						["*.*"]),
428	makeFileCopyGroup	("doc/testspecs/GLES2",						"doc/testspecs/GLES2",					["*.txt"])
429])
430
431GLES3 = Module("GLES3", [
432	makeFileCopyGroup	("modules/gles3",							"src/modules/gles3",					SRC_FILE_PATTERNS),
433	makeFileCopyGroup	("data/gles3",								"src/data/gles3", 						["*.*"]),
434	makeFileCopyGroup	("doc/testspecs/GLES3",						"doc/testspecs/GLES3",					["*.txt"])
435])
436
437GLES31 = Module("GLES31", [
438	makeFileCopyGroup	("modules/gles31",							"src/modules/gles31",					SRC_FILE_PATTERNS),
439	makeFileCopyGroup	("data/gles31",								"src/data/gles31", 						["*.*"]),
440	makeFileCopyGroup	("doc/testspecs/GLES31",					"doc/testspecs/GLES31",					["*.txt"])
441])
442
443EGL = Module("EGL", [
444	makeFileCopyGroup	("modules/egl",								"src/modules/egl",						SRC_FILE_PATTERNS)
445])
446
447INTERNAL = Module("Internal", [
448	makeFileCopyGroup	("modules/internal",						"src/modules/internal",					SRC_FILE_PATTERNS),
449	makeFileCopyGroup	("data/internal",							"src/data/internal", 					["*.*"]),
450])
451
452EXTERNAL_SRCS = Module("External sources", [
453	FetchExternalSourcesTarget()
454])
455
456ANDROID_BINARIES = Module("Android Binaries", [
457	BuildAndroidTarget	("bin/android/dEQP.apk"),
458	makeFileCopyGroup	("targets/android",							"bin/android",							["*.bat", "*.sh"]),
459])
460
461COMMON_BUILD_ARGS	= ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
462NULL_X32_CONFIG		= BuildConfig('null-x32',	'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
463NULL_X64_CONFIG		= BuildConfig('null-x64',	'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
464GLX_X32_CONFIG		= BuildConfig('glx-x32',	'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
465GLX_X64_CONFIG		= BuildConfig('glx-x64',	'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
466
467EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
468
469LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
470	BuildTarget			(NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
471	makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver",		"bin/linux32",					["*"],	EXCLUDE_BUILD_FILES),
472	makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor",		"bin/linux32",					["*"],	EXCLUDE_BUILD_FILES),
473])
474
475LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
476	BuildTarget			(NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
477	makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver",		"bin/linux64",					["*"],	EXCLUDE_BUILD_FILES),
478	makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor",		"bin/linux64",					["*"],	EXCLUDE_BUILD_FILES),
479])
480
481# Special module to remove src dir, for example after binary build
482REMOVE_SOURCES = Module("Remove sources from package", [
483	RemoveSourcesTarget()
484])
485
486# Release configuration
487
488ALL_MODULES		= [
489	BASE,
490	DOCUMENTATION,
491	GLSHARED,
492	GLES2,
493	GLES3,
494	GLES31,
495	EGL,
496	INTERNAL,
497	EXTERNAL_SRCS,
498]
499
500ALL_BINARIES	= [
501	LINUX_X64_COMMON_BINARIES,
502	ANDROID_BINARIES,
503]
504
505RELEASE_CONFIGS	= {
506	"src":		ALL_MODULES,
507	"src-bin":	ALL_MODULES + ALL_BINARIES,
508	"bin":		ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
509}
510
511def parseArgs ():
512	parser = argparse.ArgumentParser(description = "Build release package")
513	parser.add_argument("-c",
514						"--config",
515						dest="config",
516						choices=RELEASE_CONFIGS.keys(),
517						required=True,
518						help="Release configuration")
519	parser.add_argument("-n",
520						"--name",
521						dest="name",
522						required=True,
523						help="Package-specific name")
524	parser.add_argument("-v",
525						"--version",
526						dest="version",
527						required=True,
528						help="Version code")
529	return parser.parse_args()
530
531if __name__ == "__main__":
532	args	= parseArgs()
533	config	= ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
534	makeRelease(config)
535