From 23a6a05608604aa34382b41f70f8252c6a52a7cb Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Sun, 11 Jul 2010 23:30:51 +0200 Subject: [PATCH] generate-packages-sources finalize the first step. there needs to be a rework of the apt-trees part into an aptconfig class. Much to inflexible the way it is now, but we can start testing this out in real life now. Signed-off-by: Joerg Jaspert --- dak/generate_packages_sources.py | 282 +++++++++++++++++++++++++++++-- daklib/dbconn.py | 252 --------------------------- 2 files changed, 272 insertions(+), 262 deletions(-) diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py index 366d722b..6e91aeda 100755 --- a/dak/generate_packages_sources.py +++ b/dak/generate_packages_sources.py @@ -27,20 +27,20 @@ import os import os.path -import stat import sys -from datetime import datetime import apt_pkg +from tempfile import mkstemp, mkdtemp +import commands +from multiprocessing import Pool from daklib import daklog from daklib.dbconn import * from daklib.config import Config -from daklib.threadpool import ThreadPool ################################################################################ -Options = None -Logger = None +Options = None #: Commandline arguments parsed into this +Logger = None #: Our logging object ################################################################################ @@ -62,6 +62,265 @@ SUITE can be a space seperated list, e.g. ################################################################################ +def generate_packages_sources(arch, suite, tmppath): + """ + Generate Packages/Sources files with apt-ftparchive for the given suite/arch + + @type suite: string + @param suite: Suite name + + @type arch: string + @param arch: Architecture name + + @type tmppath: string + @param tmppath: The temporary path to work ing + """ + + DAILY_APT_CONF=""" +Dir +{ + ArchiveDir "/srv/ftp-master.debian.org/ftp/"; + OverrideDir "/srv/ftp-master.debian.org/scripts/override/"; + CacheDir "/srv/ftp-master.debian.org/database/"; +}; + +Default +{ + Packages::Compress "bzip2 gzip"; + Sources::Compress "bzip2 gzip"; + Contents::Compress "gzip"; + DeLinkLimit 0; + MaxContentsChange 25000; + FileMode 0664; +} + +TreeDefault +{ + Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top"; +}; + +""" + + apt_trees={} + apt_trees["di"]={} + apt_trees["testing"]=""" +tree "dists/testing" +{ + FakeDI "dists/unstable"; + FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; +}; +""" + apt_trees["di"]["testing"]=""" +tree "dists/testing/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; + +tree "dists/testing/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; +""" + + apt_trees["unstable"]=""" +tree "dists/unstable" +{ + FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.sid.$(SECTION)"; + ExtraOverride "override.sid.extra.$(SECTION)"; + SrcOverride "override.sid.$(SECTION).src"; +}; +""" + apt_trees["di"]["unstable"]=""" +tree "dists/unstable/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; + +tree "dists/unstable/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; +""" + + apt_trees["experimental"]=""" +tree "dists/experimental" +{ + FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.sid.$(SECTION)"; + SrcOverride "override.sid.$(SECTION).src"; +}; +""" + apt_trees["di"]["experimental"]=""" +tree "dists/experimental/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; + +tree "dists/experimental/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + %(contentsline)s +}; +""" + + apt_trees["testing-proposed-updates"]=""" +tree "dists/testing-proposed-updates" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; + Contents " "; +}; +""" + apt_trees["di"]["testing-proposed-updates"]=""" +tree "dists/testing-proposed-updates/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents " "; +}; +""" + + apt_trees["proposed-updates"]=""" +tree "dists/proposed-updates" +{ + FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.lenny.$(SECTION)"; + ExtraOverride "override.lenny.extra.$(SECTION)"; + SrcOverride "override.lenny.$(SECTION).src"; + Contents " "; +}; +""" + apt_trees["di"]["proposed-updates"]=""" +tree "dists/proposed-updates/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.lenny.main.$(SECTION)"; + SrcOverride "override.lenny.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents " "; +}; +""" + + cnf = Config() + try: + # Write apt.conf + (ac_fd, ac_name) = mkstemp(dir=tmppath) + os.write(ac_fd, DAILY_APT_CONF) + # here we want to generate the tree entries + os.write(ac_fd, apt_trees[suite] % {'arch': arch}) + # this special casing needs to go away, but this whole thing may just want an + # aptconfig class anyways + if arch != 'source': + if arch == 'hurd-i386' and suite == 'experimental': + pass + else: + if arch == "amd64": + os.write(ac_fd, apt_trees["di"][suite] % + {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'}) + else: + os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''}) + os.close(ac_fd) + + print "Going to run apt-ftparchive for %s/%s" % (arch, suite) + # Run apt-ftparchive generate + # We dont want to add a -q or -qq here, this output should go into our logs, sometimes + # it has errormessages we like to see + os.environ['GZIP'] = '--rsyncable' + os.chdir(tmppath) + (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name)) + sn="a-f %s,%s: " % (suite, arch) + print sn + output.replace('\n', '\n%s' % (sn)) + + # Clean up any left behind files + finally: + if ac_fd: + try: + os.close(ac_fd) + except OSError: + pass + + if ac_name: + try: + os.unlink(ac_name) + except OSError: + pass + +def sname(arch): + return arch.arch_string + +######################################################################## +######################################################################## + def main (): global Options, Logger @@ -98,19 +357,22 @@ def main (): else: suites=session.query(Suite).filter(Suite.untouchable == False).all() - threadpool = ThreadPool() - startdir = os.getcwd() os.chdir(cnf["Dir::TempPath"]) + # Setup a multiprocessing Pool. As many workers as we have CPU cores. + pool = Pool() + # For each given suite, each architecture, run one apt-ftparchive for s in suites: arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session) - Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, arch_list)]) + Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))]) for a in arch_list: - threadpool.queueTask(s.generate_packages_sources, (a.arch_string)) + pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"])) - threadpool.joinAll() + # No more work will be added to our pool, close it and then wait for all to finish + pool.close() + pool.join() os.chdir(startdir) # this script doesn't change the database session.close() diff --git a/daklib/dbconn.py b/daklib/dbconn.py index b987a372..a1bff894 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -2459,202 +2459,6 @@ SUITE_FIELDS = [ ('SuiteName', 'suite_name'), ('OverrideSuite', 'overridesuite'), ('ChangelogBase', 'changelogbase')] - -DAILY_APT_CONF=""" -Dir -{ - ArchiveDir "/srv/ftp-master.debian.org/ftp/"; - OverrideDir "/srv/ftp-master.debian.org/scripts/override/"; - CacheDir "/srv/ftp-master.debian.org/database/"; -}; - -Default -{ - Packages::Compress "bzip2 gzip"; - Sources::Compress "bzip2 gzip"; - Contents::Compress "gzip"; - DeLinkLimit 0; - MaxContentsChange 25000; - FileMode 0664; -} - -TreeDefault -{ - Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top"; -}; - -""" - -apt_trees={} -apt_trees["di"]={} -apt_trees["testing"]=""" -tree "dists/testing" -{ - FakeDI "dists/unstable"; - FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "%(arch)s"; - BinOverride "override.squeeze.$(SECTION)"; - ExtraOverride "override.squeeze.extra.$(SECTION)"; - SrcOverride "override.squeeze.$(SECTION).src"; -}; -""" -apt_trees["di"]["testing"]=""" -tree "dists/testing/main" -{ - FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.squeeze.main.$(SECTION)"; - SrcOverride "override.squeeze.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - %(contentsline)s -}; - -tree "dists/testing/non-free" -{ - FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.squeeze.main.$(SECTION)"; - SrcOverride "override.squeeze.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - %(contentsline)s -}; -""" - -apt_trees["unstable"]=""" -tree "dists/unstable" -{ - FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "%(arch)s"; - BinOverride "override.sid.$(SECTION)"; - ExtraOverride "override.sid.extra.$(SECTION)"; - SrcOverride "override.sid.$(SECTION).src"; -}; -""" -apt_trees["di"]["unstable"]=""" -tree "dists/unstable/main" -{ - FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.sid.main.$(SECTION)"; - SrcOverride "override.sid.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - %(contentsline)s -}; - -tree "dists/unstable/non-free" -{ - FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.sid.main.$(SECTION)"; - SrcOverride "override.sid.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - %(contentsline)s -}; -""" - -apt_trees["experimental"]=""" -tree "dists/experimental" -{ - FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "%(arch)s"; - BinOverride "override.sid.$(SECTION)"; - SrcOverride "override.sid.$(SECTION).src"; -}; -""" -apt_trees["di"]["experimental"]=""" -tree "dists/experimental/main" -{ - FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.sid.main.$(SECTION)"; - SrcOverride "override.sid.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - %(contentsline)s -}; - -tree "dists/experimental/non-free" -{ - FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.sid.main.$(SECTION)"; - SrcOverride "override.sid.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - %(contentsline)s -}; -""" - -apt_trees["testing-proposed-updates"]=""" -tree "dists/testing-proposed-updates" -{ - FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "%(arch)s"; - BinOverride "override.squeeze.$(SECTION)"; - ExtraOverride "override.squeeze.extra.$(SECTION)"; - SrcOverride "override.squeeze.$(SECTION).src"; - Contents " "; -}; -""" -apt_trees["di"]["testing-proposed-updates"]=""" -tree "dists/testing-proposed-updates/main" -{ - FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.squeeze.main.$(SECTION)"; - SrcOverride "override.squeeze.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - Contents " "; -}; -""" - -apt_trees["proposed-updates"]=""" -tree "dists/proposed-updates" -{ - FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list"; - SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list"; - Sections "main contrib non-free"; - Architectures "%(arch)s"; - BinOverride "override.lenny.$(SECTION)"; - ExtraOverride "override.lenny.extra.$(SECTION)"; - SrcOverride "override.lenny.$(SECTION).src"; - Contents " "; -}; -""" -apt_trees["di"]["proposed-updates"]=""" -tree "dists/proposed-updates/main" -{ - FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; - Sections "debian-installer"; - Architectures "%(arch)s"; - BinOverride "override.lenny.main.$(SECTION)"; - SrcOverride "override.lenny.main.src"; - BinCacheDB "packages-debian-installer-$(ARCH).db"; - Packages::Extensions ".udeb"; - Contents " "; -}; -""" - class Suite(object): def __init__(self, *args, **kwargs): pass @@ -2683,62 +2487,6 @@ class Suite(object): return "\n".join(ret) - def generate_packages_sources(self, arch): - """ - Generate Packages/Sources files with apt-ftparchive for the given suite/arch - - @type suite: string - @param suite: Suite name - - @type arch: string - @param arch: Architecture name - """ - - cnf = Config() - try: - # Write apt.conf - (ac_fd, ac_name) = mkstemp(dir=cnf["Dir::TempPath"]) - os.write(ac_fd, DAILY_APT_CONF) - # here we want to generate the tree entries - os.write(ac_fd, apt_trees[self.suite_name] % {'arch': arch}) - # this special casing needs to go away, but this whole thing may just want an - # aptconfig class anyways - if arch != 'source': - if arch == 'hurd-i386' and self.suite_name == 'experimental': - pass - else: - if arch == "amd64": - os.write(ac_fd, apt_trees["di"][self.suite_name] % - {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'}) - else: - os.write(ac_fd, apt_trees["di"][self.suite_name] % {'arch': arch, 'contentsline': ''}) - os.close(ac_fd) - - # Run apt-ftparchive generate - # We might want to add a -q or -qq here - os.environ['GZIP'] = '--rsyncable' - os.chdir(cnf["Dir::TempPath"]) - (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name)) - sn="a-f %s,%s: " % (self.suite_name, arch) - print sn + output.replace('\n', '\n%s' % (sn)) - - # Clean up any left behind files - finally: - if ac_fd: - try: - os.close(ac_fd) - except OSError: - pass - - if ac_name: - try: - os.unlink(ac_name) - except OSError: - pass - - - - __all__.append('Suite') @session_wrapper -- 2.39.5