From 98efb153a47ff7037bf555168b11ebac0dffaff3 Mon Sep 17 00:00:00 2001 From: Luca Falavigna Date: Sat, 22 Jan 2011 10:53:58 +0000 Subject: [PATCH] make-changelog: refactorize code to improve performances Signed-off-by: Luca Falavigna --- dak/make_changelog.py | 103 +++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/dak/make_changelog.py b/dak/make_changelog.py index 8039bfab..6d33e052 100755 --- a/dak/make_changelog.py +++ b/dak/make_changelog.py @@ -54,11 +54,11 @@ import sys import apt_pkg from commands import getstatusoutput from glob import glob -from re import split from shutil import rmtree from daklib.dbconn import * from daklib import utils from daklib.config import Config +from daklib.regexes import re_no_epoch ################################################################################ @@ -166,6 +166,9 @@ def export_files(session, pool, clpool, temppath): """ sources = {} + unpack = {} + files = ('changelog', 'copyright', 'NEWS.Debian', 'README.Debian') + stats = {'unpack': 0, 'created': 0, 'removed': 0, 'errors': 0, 'files': 0} query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, f.filename FROM source s JOIN newest_source n ON n.source = s.source AND n.version = s.version @@ -177,64 +180,80 @@ def export_files(session, pool, clpool, temppath): for p in session.execute(query): if not sources.has_key(p[0]): sources[p[0]] = {} - sources[p[0]][p[1]] = (p[2], p[3]) - - tempdir = utils.temp_dirname(parent=temppath) - os.rmdir(tempdir) + sources[p[0]][p[1]] = (re_no_epoch.sub('', p[2]), p[3]) for p in sources.keys(): for s in sources[p].keys(): - files = (('changelog', True), - ('copyright', True), - ('NEWS.Debian', False), - ('README.Debian', False)) - path = os.path.join(clpool, sources[p][s][1].split('/')[0], \ - split('(^lib\S|^\S)', p)[1], p) + path = os.path.join(clpool, '/'.join(sources[p][s][1].split('/')[:-1])) if not os.path.exists(path): os.makedirs(path) + if not os.path.exists(os.path.join(path, \ + '%s_%s.changelog' % (p, sources[p][s][0]))): + if not unpack.has_key(os.path.join(pool, sources[p][s][1])): + unpack[os.path.join(pool, sources[p][s][1])] = (path, set()) + unpack[os.path.join(pool, sources[p][s][1])][1].add(s) + else: + for file in glob('%s/%s_%s*' % (path, p, sources[p][s][0])): + link = '%s%s' % (s, file.split('%s_%s' \ + % (p, sources[p][s][0]))[1]) + try: + os.unlink(os.path.join(path, link)) + except OSError: + pass + os.link(os.path.join(path, file), os.path.join(path, link)) + + tempdir = utils.temp_dirname(parent=temppath) + os.rmdir(tempdir) + + for p in unpack.keys(): + package = os.path.splitext(os.path.basename(p))[0].split('_') + cmd = 'dpkg-source --no-check --no-copy -x %s %s' % (p, tempdir) + (result, output) = getstatusoutput(cmd) + if not result: + stats['unpack'] += 1 for file in files: - for f in glob(os.path.join(path, s + '.*')): - os.unlink(f) - try: - for file in files: - t = os.path.join(path, '%s_%s.*%s' % (p, sources[p][s][0], file[0])) - if file[1] and not glob(t): - raise OSError - else: - for f in glob(t): - os.link(f, os.path.join(path, '%s.%s' % \ - (s, os.path.basename(f).split('%s_%s.' \ - % (p, sources[p][s][0]))[1]))) - except OSError: - cmd = 'dpkg-source --no-check --no-copy -x %s %s' \ - % (os.path.join(pool, sources[p][s][1]), tempdir) - (result, output) = getstatusoutput(cmd) - if not result: - for file in files: + for f in glob(os.path.join(tempdir, 'debian', '*%s*' % file)): + for s in unpack[p][1]: + suite = os.path.join(unpack[p][0], '%s.%s' \ + % (s, os.path.basename(f))) + version = os.path.join(unpack[p][0], '%s_%s.%s' % \ + (package[0], package[1], os.path.basename(f))) + if not os.path.exists(version): + os.link(f, version) + stats['created'] += 1 try: - for f in glob(os.path.join(tempdir, 'debian', '*' + file[0])): - for dest in os.path.join(path, '%s_%s.%s' \ - % (p, sources[p][s][0], os.path.basename(f))), \ - os.path.join(path, '%s.%s' % (s, os.path.basename(f))): - if not os.path.exists(dest): - os.link(f, dest) - except: - print 'make-changelog: unable to extract %s for %s_%s' \ - % (os.path.basename(f), p, sources[p][s][0]) - else: - print 'make-changelog: unable to unpack %s_%s: %s' % (p, sources[p][s][0], output) - - rmtree(tempdir) + os.unlink(suite) + except OSError: + pass + os.link(version, suite) + stats['created'] += 1 + else: + print 'make-changelog: unable to unpack %s_%s: %s' \ + % (package[0], package[1], output) + stats['errors'] += 1 + + rmtree(tempdir) for root, dirs, files in os.walk(clpool): if len(files): if root.split('/')[-1] not in sources.keys(): if os.path.exists(root): rmtree(root) + stats['removed'] += 1 for file in files: if os.path.exists(os.path.join(root, file)): if os.stat(os.path.join(root, file)).st_nlink == 1: os.unlink(os.path.join(root, file)) + stats['removed'] += 1 + + for root, dirs, files in os.walk(clpool): + stats['files'] += len(files) + print 'make-changelog: file exporting finished' + print ' * New packages unpacked: %d' % stats['unpack'] + print ' * New files created: %d' % stats['created'] + print ' * New files removed: %d' % stats['removed'] + print ' * Unpack errors: %d' % stats['errors'] + print ' * Files available into changelog pool: %d' % stats['files'] def main(): Cnf = utils.get_conf() -- 2.39.5