diff options
author | Dominik Riebeling <Dominik.Riebeling@gmail.com> | 2009-10-04 16:45:59 +0000 |
---|---|---|
committer | Dominik Riebeling <Dominik.Riebeling@gmail.com> | 2009-10-04 16:45:59 +0000 |
commit | 45c411e6c8b73f0b3ab4bc69eb6da15901ced59c (patch) | |
tree | e207579cafcdf43277832c6d789a214a3575605f | |
parent | f65170f1dfca13e1f0ecb76e5e2b0c842da9064d (diff) | |
download | rockbox-45c411e6c8b73f0b3ab4bc69eb6da15901ced59c.tar.gz rockbox-45c411e6c8b73f0b3ab4bc69eb6da15901ced59c.zip |
Rework rbutil deployment script.
- When building from trunk or a tag, retrieve the sources to build from svn.
- Create a source archive as well.
- Default to building from trunk instead of using a local source copy.
- Disable creation of a source tarball when building from a local source tree.
- Build in the systems temporary folder when building with sources from svn.
- Display a short summary of the files created on success.
- Disable the use of ccache when building.
git-svn-id: svn://svn.rockbox.org/rockbox/trunk@22923 a1c6a512-1295-4272-9138-f99709370657
-rwxr-xr-x | rbutil/rbutilqt/deploy-release.py | 247 |
1 files changed, 191 insertions, 56 deletions
diff --git a/rbutil/rbutilqt/deploy-release.py b/rbutil/rbutilqt/deploy-release.py index a76ce8fd55..ee9591c50a 100755 --- a/rbutil/rbutilqt/deploy-release.py +++ b/rbutil/rbutilqt/deploy-release.py | |||
@@ -17,11 +17,20 @@ | |||
17 | # | 17 | # |
18 | # | 18 | # |
19 | # Automate building releases for deployment. | 19 | # Automate building releases for deployment. |
20 | # Run from source folder. Error checking / handling rather is limited. | 20 | # Run from any folder to build |
21 | # - trunk | ||
22 | # - any tag (using the -t option) | ||
23 | # - any local folder (using the -p option) | ||
24 | # Will build a binary archive (tar.bz2 / zip) and source archive. | ||
25 | # The source archive won't be built for local builds. Trunk and | ||
26 | # tag builds will retrieve the sources directly from svn and build | ||
27 | # below the systems temporary folder. | ||
28 | # | ||
21 | # If the required Qt installation isn't in PATH use --qmake option. | 29 | # If the required Qt installation isn't in PATH use --qmake option. |
22 | # Tested on Linux and MinGW / W32 | 30 | # Tested on Linux and MinGW / W32 |
23 | # | 31 | # |
24 | # requires python which package (http://code.google.com/p/which/) | 32 | # requires python which package (http://code.google.com/p/which/) |
33 | # requires pysvn package. | ||
25 | # requires upx.exe in PATH on Windows. | 34 | # requires upx.exe in PATH on Windows. |
26 | # | 35 | # |
27 | 36 | ||
@@ -35,12 +44,15 @@ import subprocess | |||
35 | import getopt | 44 | import getopt |
36 | import which | 45 | import which |
37 | import time | 46 | import time |
47 | import hashlib | ||
48 | import pysvn | ||
49 | import tempfile | ||
38 | 50 | ||
39 | # == Global stuff == | 51 | # == Global stuff == |
40 | # Windows nees some special treatment. Differentiate between program name | 52 | # Windows nees some special treatment. Differentiate between program name |
41 | # and executable filename. | 53 | # and executable filename. |
42 | program = "rbutilqt" | 54 | program = "rbutilqt" |
43 | project = "rbutilqt.pro" | 55 | project = "rbutil/rbutilqt/rbutilqt.pro" |
44 | if sys.platform == "win32": | 56 | if sys.platform == "win32": |
45 | progexe = "Release/rbutilqt.exe" | 57 | progexe = "Release/rbutilqt.exe" |
46 | else: | 58 | else: |
@@ -48,13 +60,57 @@ else: | |||
48 | 60 | ||
49 | programfiles = [ progexe ] | 61 | programfiles = [ progexe ] |
50 | 62 | ||
63 | svnserver = "svn://svn.rockbox.org/rockbox/" | ||
64 | # Paths and files to retrieve from svn when creating a tarball. | ||
65 | # This is a mixed list, holding both paths and filenames. | ||
66 | svnpaths = [ "rbutil/", | ||
67 | "tools/ucl", | ||
68 | "tools/rbspeex", | ||
69 | "apps/codecs/libspeex", | ||
70 | "tools/iriver.c", | ||
71 | "tools/Makefile", | ||
72 | "tools/mkboot.h", | ||
73 | "tools/voicefont.c", | ||
74 | "tools/VOICE_PAUSE.wav", | ||
75 | "tools/wavtrim.h", | ||
76 | "tools/iriver.h", | ||
77 | "tools/mkboot.c", | ||
78 | "tools/voicefont.h", | ||
79 | "tools/wavtrim.c", | ||
80 | "tools/sapi_voice.vbs" ] | ||
51 | 81 | ||
52 | # == Functions == | 82 | # == Functions == |
53 | def usage(myself): | 83 | def usage(myself): |
54 | print "Usage: %s [options]" % myself | 84 | print "Usage: %s [options]" % myself |
55 | print " -q, --qmake=<qmake> path to qmake" | 85 | print " -q, --qmake=<qmake> path to qmake" |
56 | print " -p, --project=<pro> path to .pro file for building out-of-tree" | 86 | print " -p, --project=<pro> path to .pro file for building with local tree" |
87 | print " -t, --tag=<tag> use specified tag from svn" | ||
57 | print " -h, --help this help" | 88 | print " -h, --help this help" |
89 | print " If neither a project file nor tag is specified trunk will get downloaded" | ||
90 | print " from svn." | ||
91 | |||
92 | def getsources(svnsrv, filelist, dest): | ||
93 | '''Get the files listed in filelist from svnsrv and put it at dest.''' | ||
94 | client = pysvn.Client() | ||
95 | print "Checking out sources from %s, please wait." % svnsrv | ||
96 | |||
97 | for elem in filelist: | ||
98 | url = re.subn('/$', '', svnsrv + elem)[0] | ||
99 | destpath = re.subn('/$', '', dest + elem)[0] | ||
100 | try: | ||
101 | client.export(url, destpath) | ||
102 | except: | ||
103 | print "SVN client error: %s" % sys.exc_value | ||
104 | return -1 | ||
105 | print "Checkout finished." | ||
106 | return 0 | ||
107 | |||
108 | |||
109 | def gettrunkrev(svnsrv): | ||
110 | '''Get the revision of trunk for svnsrv''' | ||
111 | client = pysvn.Client() | ||
112 | entries = client.info2(svnsrv, recurse=False) | ||
113 | return entries[0][1].rev.number | ||
58 | 114 | ||
59 | 115 | ||
60 | def findversion(versionfile): | 116 | def findversion(versionfile): |
@@ -113,20 +169,11 @@ def checkqt(qmakebin): | |||
113 | return result | 169 | return result |
114 | 170 | ||
115 | 171 | ||
116 | def removedir(folder): | 172 | def qmake(qmake="qmake", projfile=project, wd="."): |
117 | # remove output folder | 173 | print "Running qmake in %s..." % wd |
118 | for root, dirs, files in os.walk(folder, topdown=False): | 174 | output = subprocess.Popen([qmake, "-config", "static", |
119 | for name in files: | 175 | "-config", "release", "-config", "noccache", projfile], |
120 | os.remove(os.path.join(root, name)) | 176 | stdout=subprocess.PIPE, cwd=wd) |
121 | for name in dirs: | ||
122 | os.rmdir(os.path.join(root, name)) | ||
123 | os.rmdir(folder) | ||
124 | |||
125 | |||
126 | def qmake(qmake="qmake", projfile=project): | ||
127 | print "Running qmake ..." | ||
128 | output = subprocess.Popen([qmake, "-config", "static", "-config", "release", projfile], | ||
129 | stdout=subprocess.PIPE) | ||
130 | output.communicate() | 177 | output.communicate() |
131 | if not output.returncode == 0: | 178 | if not output.returncode == 0: |
132 | print "qmake returned an error!" | 179 | print "qmake returned an error!" |
@@ -134,17 +181,17 @@ def qmake(qmake="qmake", projfile=project): | |||
134 | return 0 | 181 | return 0 |
135 | 182 | ||
136 | 183 | ||
137 | def build(): | 184 | def build(wd="."): |
138 | # make | 185 | # make |
139 | print "Building ..." | 186 | print "Building ..." |
140 | output = subprocess.Popen(["make"], stdout=subprocess.PIPE) | 187 | output = subprocess.Popen(["make"], stdout=subprocess.PIPE, cwd=wd) |
141 | output.communicate() | 188 | output.communicate() |
142 | if not output.returncode == 0: | 189 | if not output.returncode == 0: |
143 | print "Build failed!" | 190 | print "Build failed!" |
144 | return -1 | 191 | return -1 |
145 | # strip | 192 | # strip |
146 | print "Stripping binary." | 193 | print "Stripping binary." |
147 | output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE) | 194 | output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd) |
148 | output.communicate() | 195 | output.communicate() |
149 | if not output.returncode == 0: | 196 | if not output.returncode == 0: |
150 | print "Stripping failed!" | 197 | print "Stripping failed!" |
@@ -152,10 +199,10 @@ def build(): | |||
152 | return 0 | 199 | return 0 |
153 | 200 | ||
154 | 201 | ||
155 | def upxfile(): | 202 | def upxfile(wd="."): |
156 | # run upx on binary | 203 | # run upx on binary |
157 | print "UPX'ing binary ..." | 204 | print "UPX'ing binary ..." |
158 | output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE) | 205 | output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd) |
159 | output.communicate() | 206 | output.communicate() |
160 | if not output.returncode == 0: | 207 | if not output.returncode == 0: |
161 | print "UPX'ing failed!" | 208 | print "UPX'ing failed!" |
@@ -163,72 +210,110 @@ def upxfile(): | |||
163 | return 0 | 210 | return 0 |
164 | 211 | ||
165 | 212 | ||
166 | def zipball(versionstring): | 213 | def zipball(versionstring, buildfolder): |
167 | '''package created binary''' | 214 | '''package created binary''' |
168 | print "Creating binary zipball." | 215 | print "Creating binary zipball." |
169 | outfolder = program + "-v" + versionstring | 216 | archivebase = program + "-" + versionstring |
170 | archivename = outfolder + ".zip" | 217 | outfolder = buildfolder + "/" + archivebase |
218 | archivename = archivebase + ".zip" | ||
171 | # create output folder | 219 | # create output folder |
172 | os.mkdir(outfolder) | 220 | os.mkdir(outfolder) |
173 | # move program files to output folder | 221 | # move program files to output folder |
174 | for f in programfiles: | 222 | for f in programfiles: |
175 | shutil.copy(f, outfolder) | 223 | shutil.copy(buildfolder + "/" + f, outfolder) |
176 | # create zipball from output folder | 224 | # create zipball from output folder |
177 | zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED) | 225 | zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED) |
178 | for root, dirs, files in os.walk(outfolder): | 226 | for root, dirs, files in os.walk(outfolder): |
179 | for name in files: | 227 | for name in files: |
180 | zf.write(os.path.join(root, name)) | 228 | physname = os.path.join(root, name) |
229 | filename = re.sub("^" + buildfolder, "", physname) | ||
230 | zf.write(physname, filename) | ||
181 | for name in dirs: | 231 | for name in dirs: |
182 | zf.write(os.path.join(root, name)) | 232 | physname = os.path.join(root, name) |
233 | filename = re.sub("^" + buildfolder, "", physname) | ||
234 | zf.write(physname, filename) | ||
183 | zf.close() | 235 | zf.close() |
184 | # remove output folder | 236 | # remove output folder |
185 | removedir(outfolder) | 237 | shutil.rmtree(outfolder) |
186 | st = os.stat(archivename) | ||
187 | print "done: %s, %i bytes" % (archivename, st.st_size) | ||
188 | return archivename | 238 | return archivename |
189 | 239 | ||
190 | 240 | ||
191 | def tarball(versionstring): | 241 | def tarball(versionstring, buildfolder): |
192 | '''package created binary''' | 242 | '''package created binary''' |
193 | print "Creating binary tarball." | 243 | print "Creating binary tarball." |
194 | outfolder = program + "-v" + versionstring | 244 | archivebase = program + "-" + versionstring |
195 | archivename = outfolder + ".tar.bz2" | 245 | outfolder = buildfolder + "/" + archivebase |
246 | archivename = archivebase + ".tar.bz2" | ||
196 | # create output folder | 247 | # create output folder |
197 | os.mkdir(outfolder) | 248 | os.mkdir(outfolder) |
198 | # move program files to output folder | 249 | # move program files to output folder |
199 | for f in programfiles: | 250 | for f in programfiles: |
200 | shutil.copy(f, outfolder) | 251 | shutil.copy(buildfolder + "/" + f, outfolder) |
201 | # create tarball from output folder | 252 | # create tarball from output folder |
202 | tf = tarfile.open(archivename, mode='w:bz2') | 253 | tf = tarfile.open(archivename, mode='w:bz2') |
203 | tf.add(outfolder) | 254 | tf.add(outfolder, archivebase) |
204 | tf.close() | 255 | tf.close() |
205 | # remove output folder | 256 | # remove output folder |
206 | removedir(outfolder) | 257 | shutil.rmtree(outfolder) |
207 | st = os.stat(archivename) | ||
208 | print "done: %s, %i bytes" % (archivename, st.st_size) | ||
209 | return archivename | 258 | return archivename |
210 | 259 | ||
211 | 260 | ||
261 | def filehashes(filename): | ||
262 | '''Calculate md5 and sha1 hashes for a given file.''' | ||
263 | if not os.path.exists(filename): | ||
264 | return ["", ""] | ||
265 | m = hashlib.md5() | ||
266 | s = hashlib.sha1() | ||
267 | f = open(filename, 'rb') | ||
268 | while True: | ||
269 | d = f.read(65536) | ||
270 | if d == "": | ||
271 | break | ||
272 | m.update(d) | ||
273 | s.update(d) | ||
274 | return [m.hexdigest(), s.hexdigest()] | ||
275 | |||
276 | |||
277 | def filestats(filename): | ||
278 | if not os.path.exists(filename): | ||
279 | return | ||
280 | st = os.stat(filename) | ||
281 | print filename, "\n", "-" * len(filename) | ||
282 | print "Size: %i bytes" % st.st_size | ||
283 | h = filehashes(filename) | ||
284 | print "md5sum: %s" % h[0] | ||
285 | print "sha1sum: %s" % h[1] | ||
286 | print "-" * len(filename), "\n" | ||
287 | |||
288 | |||
212 | def main(): | 289 | def main(): |
213 | startup = time.time() | 290 | startup = time.time() |
214 | try: | 291 | try: |
215 | opts, args = getopt.getopt(sys.argv[1:], "q:p:h", ["qmake=", "project=", "help"]) | 292 | opts, args = getopt.getopt(sys.argv[1:], "q:p:t:h", |
293 | ["qmake=", "project=", "tag=", "help"]) | ||
216 | except getopt.GetoptError, err: | 294 | except getopt.GetoptError, err: |
217 | print str(err) | 295 | print str(err) |
218 | usage(sys.argv[0]) | 296 | usage(sys.argv[0]) |
219 | sys.exit(1) | 297 | sys.exit(1) |
220 | qt = "" | 298 | qt = "" |
221 | proj = project | 299 | proj = "" |
300 | svnbase = svnserver + "trunk/" | ||
301 | tag = "" | ||
302 | cleanup = True | ||
222 | for o, a in opts: | 303 | for o, a in opts: |
223 | if o in ("-q", "--qmake"): | 304 | if o in ("-q", "--qmake"): |
224 | qt = a | 305 | qt = a |
225 | if o in ("-p", "--project"): | 306 | if o in ("-p", "--project"): |
226 | proj = a | 307 | proj = a |
308 | cleanup = False | ||
309 | if o in ("-t", "--tag"): | ||
310 | tag = a | ||
311 | svnbase = svnserver + "tags/" + tag + "/" | ||
227 | if o in ("-h", "--help"): | 312 | if o in ("-h", "--help"): |
228 | usage(sys.argv[0]) | 313 | usage(sys.argv[0]) |
229 | sys.exit(0) | 314 | sys.exit(0) |
230 | 315 | ||
231 | # qmake path | 316 | # search for qmake |
232 | if qt == "": | 317 | if qt == "": |
233 | qm = findqt() | 318 | qm = findqt() |
234 | else: | 319 | else: |
@@ -236,38 +321,88 @@ def main(): | |||
236 | if qm == "": | 321 | if qm == "": |
237 | print "ERROR: No suitable Qt installation found." | 322 | print "ERROR: No suitable Qt installation found." |
238 | sys.exit(1) | 323 | sys.exit(1) |
324 | |||
325 | # create working folder. Use current directory if -p option used. | ||
326 | if proj == "": | ||
327 | w = tempfile.mkdtemp() | ||
328 | # make sure the path doesn't contain backslashes to prevent issues | ||
329 | # later when running on windows. | ||
330 | workfolder = re.sub(r'\\', '/', w) | ||
331 | if not tag == "": | ||
332 | sourcefolder = workfolder + "/" + tag + "/" | ||
333 | archivename = tag + "-src.tar.bz2" | ||
334 | # get numeric version part from tag | ||
335 | ver = "v" + re.sub('^[^\d]+', '', tag) | ||
336 | else: | ||
337 | trunk = gettrunkrev(svnbase) | ||
338 | sourcefolder = workfolder + "/rbutil-r" + str(trunk) + "/" | ||
339 | archivename = "rbutil-r" + str(trunk) + "-src.tar.bz2" | ||
340 | ver = "r" + str(trunk) | ||
341 | os.mkdir(sourcefolder) | ||
342 | else: | ||
343 | workfolder = "." | ||
344 | sourcefolder = "." | ||
345 | archivename = "" | ||
346 | # check if project file explicitly given. If yes, don't get sources from svn | ||
347 | if proj == "": | ||
348 | proj = sourcefolder + project | ||
349 | # get sources and pack source tarball | ||
350 | if not getsources(svnbase, svnpaths, sourcefolder) == 0: | ||
351 | sys.exit(1) | ||
352 | |||
353 | tf = tarfile.open(archivename, mode='w:bz2') | ||
354 | tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0])) | ||
355 | tf.close() | ||
356 | else: | ||
357 | # figure version from sources. Need to take path to project file into account. | ||
358 | versionfile = re.subn('[\w\.]+$', "version.h", proj)[0] | ||
359 | ver = findversion(versionfile) | ||
360 | |||
239 | # check project file | 361 | # check project file |
240 | if not os.path.exists(proj): | 362 | if not os.path.exists(proj): |
241 | print "ERROR: path to project file wrong. You need to specify the path " \ | 363 | print "ERROR: path to project file wrong." |
242 | "when building out-of-tree." | ||
243 | sys.exit(1) | 364 | sys.exit(1) |
244 | 365 | ||
245 | # figure version from sources. Need to take path to project file into account. | 366 | buildstart = time.time() |
246 | versionfile = re.subn('[\w\.]+$', "version.h", proj)[0] | ||
247 | ver = findversion(versionfile) | ||
248 | header = "Building %s %s" % (program, ver) | 367 | header = "Building %s %s" % (program, ver) |
249 | print header | 368 | print header |
250 | print len(header) * "=" | 369 | print len(header) * "=" |
251 | 370 | ||
252 | # build it. | 371 | # build it. |
253 | if not qmake(qm, proj) == 0: | 372 | if not qmake(qm, proj, sourcefolder) == 0: |
254 | os.exit(1) | 373 | sys.exit(1) |
255 | if not build() == 0: | 374 | if not build(sourcefolder) == 0: |
256 | sys.exit(1) | 375 | sys.exit(1) |
257 | if sys.platform == "win32": | 376 | if sys.platform == "win32": |
258 | if not upxfile() == 0: | 377 | if not upxfile(sourcefolder) == 0: |
259 | sys.exit(1) | 378 | sys.exit(1) |
260 | zipball(ver) | 379 | archive = zipball(ver, sourcefolder) |
261 | else: | 380 | else: |
262 | tarball(ver) | 381 | archive = tarball(ver, sourcefolder) |
263 | print "done." | 382 | |
383 | # remove temporary files | ||
384 | print "Cleaning up working folder %s" % workfolder | ||
385 | if cleanup == True: | ||
386 | shutil.rmtree(workfolder) | ||
387 | else: | ||
388 | print "Project file specified, not cleaning up!" | ||
389 | |||
390 | # display summary | ||
391 | headline = "Build Summary for %s" % program | ||
392 | print "\n", headline, "\n", "=" * len(headline) | ||
393 | if not archivename == "": | ||
394 | filestats(archivename) | ||
395 | filestats(archive) | ||
264 | duration = time.time() - startup | 396 | duration = time.time() - startup |
397 | building = time.time() - buildstart | ||
265 | durmins = (int)(duration / 60) | 398 | durmins = (int)(duration / 60) |
266 | dursecs = (int)(duration % 60) | 399 | dursecs = (int)(duration % 60) |
267 | print "Building took %smin %ssec." % (durmins, dursecs) | 400 | buildmins = (int)(building / 60) |
401 | buildsecs = (int)(building % 60) | ||
402 | print "Overall time %smin %ssec, building took %smin %ssec." % \ | ||
403 | (durmins, dursecs, buildmins, buildsecs) | ||
268 | 404 | ||
269 | 405 | ||
270 | if __name__ == "__main__": | 406 | if __name__ == "__main__": |
271 | main() | 407 | main() |
272 | 408 | ||
273 | |||