summaryrefslogtreecommitdiff
path: root/utils/common/deploy.py
diff options
context:
space:
mode:
Diffstat (limited to 'utils/common/deploy.py')
-rwxr-xr-xutils/common/deploy.py677
1 files changed, 0 insertions, 677 deletions
diff --git a/utils/common/deploy.py b/utils/common/deploy.py
deleted file mode 100755
index 04eef0b7d5..0000000000
--- a/utils/common/deploy.py
+++ /dev/null
@@ -1,677 +0,0 @@
1#!/usr/bin/python
2# __________ __ ___.
3# Open \______ \ ____ ____ | | _\_ |__ _______ ___
4# Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5# Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6# Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7# \/ \/ \/ \/ \/
8# $Id$
9#
10# Copyright (c) 2009 Dominik Riebeling
11#
12# All files in this archive are subject to the GNU General Public License.
13# See the file COPYING in the source tree root for full license agreement.
14#
15# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16# KIND, either express or implied.
17#
18#
19# Automate building releases for deployment.
20# Run from any folder to build
21# - trunk
22# - any tag (using the -t option)
23# - any local folder (using the -p option)
24# Will build a binary archive (tar.bz2 / zip) and source archive.
25# The source archive won't be built for local builds. Trunk and
26# tag builds will retrieve the sources directly from svn and build
27# below the systems temporary folder.
28#
29# If the required Qt installation isn't in PATH use --qmake option.
30# Tested on Linux and MinGW / W32
31#
32# requires upx.exe in PATH on Windows.
33#
34
35import re
36import os
37import sys
38import tarfile
39import zipfile
40import shutil
41import subprocess
42import getopt
43import time
44import hashlib
45import tempfile
46from datetime import datetime
47import multiprocessing
48import gitscraper
49
50CPUS = multiprocessing.cpu_count()
51print("Info: %s cores found." % CPUS)
52
53# == Global stuff ==
54# DLL files to ignore when searching for required DLL files.
55SYSTEMDLLS = [
56 'advapi32.dll',
57 'comdlg32.dll',
58 'crypt32.dll',
59 'd3d9.dll',
60 'dwmapi.dll',
61 'dxva2.dll',
62 'evr.dll',
63 'gdi32.dll',
64 'imm32.dll',
65 'imm32.dll',
66 'iphlpapi.dll',
67 'kernel32.dll',
68 'mf.dll',
69 'mfplat.dll',
70 'msvcrt.dll',
71 'msvcrt.dll',
72 'netapi32.dll',
73 'ole32.dll',
74 'oleaut32.dll',
75 'setupapi.dll',
76 'shell32.dll',
77 'user32.dll',
78 'userenv.dll',
79 'uxtheme.dll',
80 'version.dll',
81 'winmm.dll',
82 'winspool.drv',
83 'ws2_32.dll',
84 'wtsapi32.dll'
85 ]
86
87gitrepo = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
88
89
90# == Functions ==
91def usage(myself):
92 print("Usage: %s [options]" % myself)
93 print(" -q, --qmake=<qmake> path to qmake")
94 print(" -p, --project=<pro> path to .pro file for building with local tree")
95 print(" -t, --tag=<tag> use specified tag from svn")
96 print(" -a, --add=<file> add file to build folder before building")
97 print(" -s, --source-only only create source archive")
98 print(" -b, --binary-only only create binary archive")
99 if nsisscript != "":
100 print(" -n, --makensis=<file> path to makensis for building Windows setup program.")
101 if sys.platform != "darwin":
102 print(" -d, --dynamic link dynamically instead of static")
103 if sys.platform != "win32":
104 print(" -x, --cross= prefix to cross compile for win32")
105 print(" -k, --keep-temp keep temporary folder on build failure")
106 print(" -h, --help this help")
107 print(" If neither a project file nor tag is specified trunk will get downloaded")
108 print(" from svn.")
109
110
111def which(executable):
112 path = os.environ.get("PATH", "").split(os.pathsep)
113 for p in path:
114 fullpath = p + "/" + executable
115 if os.path.exists(fullpath):
116 return fullpath
117 print("which: could not find " + executable)
118 return ""
119
120
121def getsources(treehash, filelist, dest):
122 '''Get the files listed in filelist from svnsrv and put it at dest.'''
123 gitscraper.scrape_files(gitrepo, treehash, filelist, dest)
124 return 0
125
126
127def getfolderrev(svnsrv):
128 '''Get the most recent revision for svnsrv'''
129 client = pysvn.Client()
130 entries = client.info2(svnsrv, recurse=False)
131 return entries[0][1].rev.number
132
133
134def findversion(versionfile):
135 '''figure most recent program version from version.h,
136 returns version string.'''
137 h = open(versionfile, "r")
138 c = h.read()
139 h.close()
140 version = dict()
141 for v in ['MAJOR', 'MINOR', 'MICRO']:
142 r = re.compile("#define +VERSION_" + v + " +([0-9a-z]+)")
143 m = re.search(r, c)
144 version[v] = m.group(1)
145 return "%s.%s.%s" % (version['MAJOR'], version['MINOR'], version['MICRO'])
146
147
148def findqt(cross=""):
149 '''Search for Qt4 installation. Return path to qmake.'''
150 print("Searching for Qt")
151 bins = [cross + "qmake", cross + "qmake-qt4"]
152 for binary in bins:
153 try:
154 q = which(binary)
155 if len(q) > 0:
156 result = checkqt(q)
157 if not result == "":
158 return result
159 except:
160 print(sys.exc_info()[1])
161
162 return ""
163
164
165def checkqt(qmakebin):
166 '''Check if given path to qmake exists and is a suitable version.'''
167 result = ""
168 # check if binary exists
169 if not os.path.exists(qmakebin):
170 print("Specified qmake path does not exist!")
171 return result
172 # check version
173 output = subprocess.Popen(
174 [qmakebin, "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
175 cmdout = output.communicate()
176 # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
177 for ou in cmdout:
178 r = re.compile(b'Qt[^0-9]+([0-9\.]+[a-z]*)')
179 m = re.search(r, ou)
180 if m is not None:
181 print("Qt found: %s" % m.group(1).decode())
182 s = re.compile(b'[45]\..*')
183 n = re.search(s, m.group(1))
184 if n is not None:
185 result = qmakebin
186 return result
187
188
189def qmake(qmake, projfile, platform=sys.platform, wd=".", static=True, cross=""):
190 print("Running qmake in %s..." % wd)
191 command = [qmake, "-config", "release", "-config", "noccache"]
192 if static == True:
193 command.extend(["-config", "-static"])
194 # special spec required?
195 if len(qmakespec[platform]) > 0:
196 command.extend(["-spec", qmakespec[platform]])
197 # cross compiling prefix set?
198 if len(cross) > 0:
199 command.extend(["-config", "cross"])
200 command.append(projfile)
201 output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
202 output.communicate()
203 if not output.returncode == 0:
204 print("qmake returned an error!")
205 return -1
206 return 0
207
208
209def build(wd=".", platform=sys.platform, cross=""):
210 # make
211 print("Building ...")
212 # use the current platforms make here, cross compiling uses the native make.
213 command = [make[sys.platform]]
214 if CPUS > 1:
215 command.append("-j")
216 command.append(str(CPUS))
217 output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
218 while True:
219 c = output.stdout.readline()
220 sys.stdout.write(".")
221 sys.stdout.flush()
222 if not output.poll() == None:
223 sys.stdout.write("\n")
224 sys.stdout.flush()
225 if not output.returncode == 0:
226 print("Build failed!")
227 return -1
228 break
229 if platform != "darwin":
230 # strip. OS X handles this via macdeployqt.
231 print("Stripping binary.")
232 output = subprocess.Popen([cross + "strip", progexe[platform]], \
233 stdout=subprocess.PIPE, cwd=wd)
234 output.communicate()
235 if not output.returncode == 0:
236 print("Stripping failed!")
237 return -1
238 return 0
239
240
241def upxfile(wd=".", platform=sys.platform):
242 # run upx on binary
243 print("UPX'ing binary ...")
244 output = subprocess.Popen(["upx", progexe[platform]], \
245 stdout=subprocess.PIPE, cwd=wd)
246 output.communicate()
247 if not output.returncode == 0:
248 print("UPX'ing failed!")
249 return -1
250 return 0
251
252
253def runnsis(versionstring, nsis, script, srcfolder):
254 # run script through nsis to create installer.
255 print("Running NSIS ...")
256
257 # Assume the generated installer gets placed in the same folder the nsi
258 # script lives in. This seems to be a valid assumption unless the nsi
259 # script specifies a path. NSIS expects files relative to source folder so
260 # copy progexe. Additional files are injected into the nsis script.
261
262 # FIXME: instead of copying binaries around copy the NSI file and inject
263 # the correct paths.
264 # Only win32 supported as target platform so hard coded.
265 b = srcfolder + "/" + os.path.dirname(script) + "/" \
266 + os.path.dirname(progexe["win32"])
267 if not os.path.exists(b):
268 os.mkdir(b)
269 shutil.copy(srcfolder + "/" + progexe["win32"], b)
270 output = subprocess.Popen([nsis, srcfolder + "/" + script], \
271 stdout=subprocess.PIPE)
272 output.communicate()
273 if not output.returncode == 0:
274 print("NSIS failed!")
275 return -1
276 setupfile = program + "-" + versionstring + "-setup.exe"
277 # find output filename in nsis script file
278 nsissetup = ""
279 for line in open(srcfolder + "/" + script):
280 if re.match(r'^[^;]*OutFile\s+', line) != None:
281 nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip()
282 if nsissetup == "":
283 print("Could not retrieve output file name!")
284 return -1
285 shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, \
286 setupfile)
287 return 0
288
289
290def nsisfileinject(nsis, outscript, filelist):
291 '''Inject files in filelist into NSIS script file after the File line
292 containing the main binary. This assumes that the main binary is present
293 in the NSIS script and that all additiona files (dlls etc) to get placed
294 into $INSTDIR.'''
295 output = open(outscript, "w")
296 for line in open(nsis, "r"):
297 output.write(line)
298 # inject files after the progexe binary.
299 # Match the basename only to avoid path mismatches.
300 if re.match(r'^\s*File\s*.*' + os.path.basename(progexe["win32"]),
301 line, re.IGNORECASE):
302 for f in filelist:
303 injection = " File /oname=$INSTDIR\\" + os.path.basename(f) \
304 + " " + os.path.normcase(f) + "\n"
305 output.write(injection)
306 output.write(" ; end of injected files\n")
307 output.close()
308
309
310def finddlls(program, extrapaths=[], cross=""):
311 '''Check program for required DLLs. Find all required DLLs except ignored
312 ones and return a list of DLL filenames (including path).'''
313 # ask objdump about dependencies.
314 output = subprocess.Popen([cross + "objdump", "-x", program], \
315 stdout=subprocess.PIPE)
316 cmdout = output.communicate()
317
318 # create list of used DLLs. Store as lower case as W32 is case-insensitive.
319 dlls = []
320 for line in cmdout[0].decode().split('\n'):
321 if re.match(r'\s*DLL Name', line) != None:
322 dll = re.sub(r'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r'\1', line)
323 dlls.append(dll.lower())
324
325 # find DLLs in extrapaths and PATH environment variable.
326 dllpaths = []
327 for file in dlls:
328 if file in SYSTEMDLLS:
329 print("System DLL: " + file)
330 continue
331 dllpath = ""
332 for path in extrapaths:
333 if os.path.exists(path + "/" + file):
334 dllpath = re.sub(r"\\", r"/", path + "/" + file)
335 print(file + ": found at " + dllpath)
336 dllpaths.append(dllpath)
337 break
338 if dllpath == "":
339 try:
340 dllpath = re.sub(r"\\", r"/", which(file))
341 print(file + ": found at " + dllpath)
342 dllpaths.append(dllpath)
343 except:
344 print("MISSING DLL: " + file)
345 return dllpaths
346
347
348def zipball(programfiles, versionstring, buildfolder, platform=sys.platform):
349 '''package created binary'''
350 print("Creating binary zipball.")
351 archivebase = program + "-" + versionstring
352 outfolder = buildfolder + "/" + archivebase
353 archivename = archivebase + ".zip"
354 # create output folder
355 os.mkdir(outfolder)
356 # move program files to output folder
357 for f in programfiles:
358 if re.match(r'^(/|[a-zA-Z]:)', f) != None:
359 shutil.copy(f, outfolder)
360 else:
361 shutil.copy(buildfolder + "/" + f, outfolder)
362 # create zipball from output folder
363 zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
364 for root, dirs, files in os.walk(outfolder):
365 for name in files:
366 physname = os.path.normpath(os.path.join(root, name))
367 filename = os.path.relpath(physname, buildfolder)
368 zf.write(physname, filename)
369 zf.close()
370 # remove output folder
371 shutil.rmtree(outfolder)
372 return archivename
373
374
375def tarball(programfiles, versionstring, buildfolder):
376 '''package created binary'''
377 print("Creating binary tarball.")
378 archivebase = program + "-" + versionstring
379 outfolder = buildfolder + "/" + archivebase
380 archivename = archivebase + ".tar.bz2"
381 # create output folder
382 os.mkdir(outfolder)
383 # move program files to output folder
384 for f in programfiles:
385 shutil.copy(buildfolder + "/" + f, outfolder)
386 # create tarball from output folder
387 tf = tarfile.open(archivename, mode='w:bz2')
388 tf.add(outfolder, archivebase)
389 tf.close()
390 # remove output folder
391 shutil.rmtree(outfolder)
392 return archivename
393
394
395def macdeploy(versionstring, buildfolder, platform=sys.platform):
396 '''package created binary to dmg'''
397 dmgfile = program + "-" + versionstring + ".dmg"
398 appbundle = buildfolder + "/" + progexe[platform]
399
400 # workaround to Qt issues when building out-of-tree. Copy files into bundle.
401 sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
402 print(sourcebase)
403 for src in bundlecopy:
404 shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
405 # end of Qt workaround
406
407 output = subprocess.Popen(["macdeployqt", progexe[platform], "-dmg"], \
408 stdout=subprocess.PIPE, cwd=buildfolder)
409 output.communicate()
410 if not output.returncode == 0:
411 print("macdeployqt failed!")
412 return -1
413 # copy dmg to output folder
414 shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
415 return dmgfile
416
417
418def filehashes(filename):
419 '''Calculate md5 and sha1 hashes for a given file.'''
420 if not os.path.exists(filename):
421 return ["", ""]
422 m = hashlib.md5()
423 s = hashlib.sha1()
424 f = open(filename, 'rb')
425 while True:
426 d = f.read(65536)
427 if d == b"":
428 break
429 m.update(d)
430 s.update(d)
431 return [m.hexdigest(), s.hexdigest()]
432
433
434def filestats(filename):
435 if not os.path.exists(filename):
436 return
437 st = os.stat(filename)
438 print("%s\n%s" % (filename, "-" * len(filename)))
439 print("Size: %i bytes" % st.st_size)
440 h = filehashes(filename)
441 print("md5sum: %s" % h[0])
442 print("sha1sum: %s" % h[1])
443 print("%s\n" % ("-" * len(filename)))
444
445
446def tempclean(workfolder, nopro):
447 if nopro == True:
448 print("Cleaning up working folder %s" % workfolder)
449 shutil.rmtree(workfolder)
450 else:
451 print("Project file specified or cleanup disabled!")
452 print("Temporary files kept at %s" % workfolder)
453
454
455def deploy():
456 startup = time.time()
457
458 try:
459 opts, args = getopt.getopt(
460 sys.argv[1:], "q:p:t:a:n:sbdkx:i:h",
461 ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only",
462 "binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"])
463 except getopt.GetoptError as err:
464 print(str(err))
465 usage(sys.argv[0])
466 sys.exit(1)
467 qt = ""
468 proj = ""
469 svnbase = svnserver + "trunk/"
470 tag = ""
471 addfiles = []
472 cleanup = True
473 binary = True
474 source = True
475 keeptemp = False
476 makensis = ""
477 cross = ""
478 buildid = None
479 platform = sys.platform
480 treehash = gitscraper.get_refs(gitrepo)['refs/remotes/origin/HEAD']
481 if sys.platform != "darwin":
482 static = True
483 else:
484 static = False
485 for o, a in opts:
486 if o in ("-q", "--qmake"):
487 qt = a
488 if o in ("-p", "--project"):
489 proj = a
490 cleanup = False
491 if o in ("-a", "--add"):
492 addfiles.append(a)
493 if o in ("-n", "--makensis"):
494 makensis = a
495 if o in ("-s", "--source-only"):
496 binary = False
497 if o in ("-b", "--binary-only"):
498 source = False
499 if o in ("-d", "--dynamic") and sys.platform != "darwin":
500 static = False
501 if o in ("-k", "--keep-temp"):
502 keeptemp = True
503 if o in ("-t", "--tree"):
504 treehash = a
505 if o in ("-x", "--cross") and sys.platform != "win32":
506 cross = a
507 platform = "win32"
508 if o in ("-i", "--buildid"):
509 buildid = a
510 if o in ("-h", "--help"):
511 usage(sys.argv[0])
512 sys.exit(0)
513
514 if source == False and binary == False:
515 print("Building build neither source nor binary means nothing to do. Exiting.")
516 sys.exit(1)
517
518 print("Building " + progexe[platform] + " for " + platform)
519 # search for qmake
520 if qt == "":
521 qm = findqt(cross)
522 else:
523 qm = checkqt(qt)
524 if qm == "":
525 print("ERROR: No suitable Qt installation found.")
526 sys.exit(1)
527
528 # create working folder. Use current directory if -p option used.
529 if proj == "":
530 w = tempfile.mkdtemp()
531 # make sure the path doesn't contain backslashes to prevent issues
532 # later when running on windows.
533 workfolder = re.sub(r'\\', '/', w)
534 revision = gitscraper.describe_treehash(gitrepo, treehash)
535 # try to find a version number from describe output.
536 # WARNING: this is broken and just a temporary workaround!
537 v = re.findall(b'([\d\.a-f]+)', revision)
538 if v:
539 if v[-1].decode().find('.') >= 0:
540 revision = "v" + v[-1].decode()
541 else:
542 revision = v[-1].decode()
543 if buildid == None:
544 versionextra = ""
545 else:
546 versionextra = "-" + buildid
547 sourcefolder = workfolder + "/" + program + "-" + str(revision) + versionextra + "/"
548 archivename = program + "-" + str(revision) + versionextra + "-src.tar.bz2"
549 ver = str(revision)
550 os.mkdir(sourcefolder)
551 print("Version: %s" % revision)
552 else:
553 workfolder = "."
554 sourcefolder = "."
555 archivename = ""
556 # check if project file explicitly given. If yes, don't get sources from svn
557 if proj == "":
558 proj = sourcefolder + project
559 # get sources and pack source tarball
560 if getsources(treehash, svnpaths, sourcefolder) != 0:
561 tempclean(workfolder, cleanup and not keeptemp)
562 sys.exit(1)
563
564 # replace version strings.
565 print("Updating version information in sources")
566 for f in regreplace:
567 infile = open(sourcefolder + "/" + f, "r")
568 incontents = infile.readlines()
569 infile.close()
570
571 outfile = open(sourcefolder + "/" + f, "w")
572 for line in incontents:
573 newline = line
574 for r in regreplace[f]:
575 # replacements made on the replacement string:
576 # %REVISION% is replaced with the revision number
577 replacement = re.sub("%REVISION%", str(revision), r[1])
578 newline = re.sub(r[0], replacement, newline)
579 # %BUILD% is replaced with buildid as passed on the command line
580 if buildid != None:
581 replacement = re.sub("%BUILDID%", "-" + str(buildid), replacement)
582 else:
583 replacement = re.sub("%BUILDID%", "", replacement)
584 newline = re.sub(r[0], replacement, newline)
585 outfile.write(newline)
586 outfile.close()
587
588 if source == True:
589 print("Creating source tarball %s\n" % archivename)
590 tf = tarfile.open(archivename, mode='w:bz2')
591 tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
592 tf.close()
593 if binary == False:
594 shutil.rmtree(workfolder)
595 sys.exit(0)
596 else:
597 # figure version from sources. Need to take path to project file into account.
598 versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
599 ver = findversion(versionfile) + "-dev" + datetime.now().strftime('%Y%m%d%H%M%S')
600 # append buildid if any.
601 if buildid != None:
602 ver += "-" + buildid
603
604 # check project file
605 if not os.path.exists(proj):
606 print("ERROR: path to project file wrong.")
607 sys.exit(1)
608
609 # copy specified (--add) files to working folder
610 for f in addfiles:
611 shutil.copy(f, sourcefolder)
612 buildstart = time.time()
613 header = "Building %s %s" % (program, ver)
614 print(header)
615 print(len(header) * "=")
616
617 # build it.
618 if not qmake(qm, proj, platform, sourcefolder, static, cross) == 0:
619 tempclean(workfolder, cleanup and not keeptemp)
620 sys.exit(1)
621 if not build(sourcefolder, platform, cross) == 0:
622 tempclean(workfolder, cleanup and not keeptemp)
623 sys.exit(1)
624 buildtime = time.time() - buildstart
625 progfiles = programfiles
626 progfiles.append(progexe[platform])
627 if platform == "win32":
628 if useupx == True:
629 if not upxfile(sourcefolder, platform) == 0:
630 tempclean(workfolder, cleanup and not keeptemp)
631 sys.exit(1)
632 dllfiles = finddlls(sourcefolder + "/" + progexe[platform], \
633 [os.path.dirname(qm)], cross)
634 if len(dllfiles) > 0:
635 progfiles.extend(dllfiles)
636 archive = zipball(progfiles, ver, sourcefolder, platform)
637 # only when running native right now.
638 if nsisscript != "" and makensis != "":
639 nsisfileinject(sourcefolder + "/" + nsisscript, sourcefolder \
640 + "/" + nsisscript + ".tmp", dllfiles)
641 runnsis(ver, makensis, nsisscript + ".tmp", sourcefolder)
642 elif platform == "darwin":
643 archive = macdeploy(ver, sourcefolder, platform)
644 else:
645 if platform in ['linux', 'linux2']:
646 for p in progfiles:
647 prog = sourcefolder + "/" + p
648 output = subprocess.Popen(
649 ["file", prog], stdout=subprocess.PIPE)
650 res = output.communicate()
651 if re.findall("ELF 64-bit", res[0]):
652 ver += "-64bit"
653 break
654
655 archive = tarball(progfiles, ver, sourcefolder)
656
657 # remove temporary files
658 tempclean(workfolder, cleanup)
659
660 # display summary
661 headline = "Build Summary for %s" % program
662 print("\n%s\n%s" % (headline, "=" * len(headline)))
663 if archivename != "":
664 filestats(archivename)
665 filestats(archive)
666 duration = time.time() - startup
667 durmins = (int)(duration / 60)
668 dursecs = (int)(duration % 60)
669 buildmins = (int)(buildtime / 60)
670 buildsecs = (int)(buildtime % 60)
671 print("Overall time %smin %ssec, building took %smin %ssec." % \
672 (durmins, dursecs, buildmins, buildsecs))
673
674
675if __name__ == "__main__":
676 print("You cannot run this module directly!")
677 print("Set required environment and call deploy().")
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-05 13:26:13 +0000