From f6a7dfc1e5cac520c07a0ca8bf498695d0df2356 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 6 Jul 2010 22:08:35 +0200 Subject: [PATCH] generate-packages-sources generate packages sources files, threaded, one thread per suite/arch Signed-off-by: Joerg Jaspert --- dak/dak.py | 2 + dak/generate_packages_sources.py | 27 ++-- daklib/dbconn.py | 230 ++++++++++++++++++++++++++++++- 3 files changed, 246 insertions(+), 13 deletions(-) diff --git a/dak/dak.py b/dak/dak.py index 4ad59579..bd5108fa 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -81,6 +81,8 @@ def init(): "Generate file lists for apt-ftparchive"), ("generate-releases", "Generate Release files"), + ("generate-packages-sources", + "Generate Packages/Sources files"), ("contents", "Generate content files"), ("generate-index-diffs", diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py index d51779f9..79db33ba 100755 --- a/dak/generate_packages_sources.py +++ b/dak/generate_packages_sources.py @@ -35,6 +35,7 @@ import apt_pkg from daklib import daklog from daklib.dbconn import * from daklib.config import Config +from daklib.threadpool import ThreadPool ################################################################################ @@ -80,7 +81,7 @@ def main (): if Options["Help"]: usage() -# Logger = daklog.Logger(cnf, 'generate-packages-sources') + Logger = daklog.Logger(cnf, 'generate-packages-sources') session = DBConn().session() @@ -93,19 +94,23 @@ def main (): suites.append(suite) else: print "cannot find suite %s" % s -# Logger.log(['cannot find suite %s' % s]) + Logger.log(['cannot find suite %s' % s]) else: - suites=session.query(Suite).filter(Suite.untouchable == 'false').all() + suites=session.query(Suite).filter(Suite.untouchable == False).all() + threadpool = ThreadPool() + # For each given suite, each architecture, run one apt-ftparchive for s in suites: - print "Working on: %s" % (s.suite_name) - sys.exit(0) - # For each given suite, look up object and call generate-filelist - for s in suites: - Logger.log(['generating filelist for %s' % (s.suite_name)]) - s.generate_filelist(Logger) - -# Logger.close() + arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=False, session=session) + for a in arch_list: + Logger.log(['generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string)]) + print 'generating output for Suite %s, Architecture %s' % (s.suite_name, a.arch_string) + threadpool.queueTask(s.generate_packages_sources, (s.suite_name, a.arch_string)) + + threadpool.joinAll() + # this script doesn't change the database + session.close() + Logger.close() ####################################################################################### diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 13d324ca..97c2bc38 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -2459,6 +2459,190 @@ SUITE_FIELDS = [ ('SuiteName', 'suite_name'), ('ChangelogBase', 'changelogbase')] +DAILY_APT_CONF=""" +Dir +{ + ArchiveDir "%(archivepath)s"; + OverrideDir "/srv/ftp-master.debian.org/scripts/override/"; + CacheDir "/srv/ftp-master.debian.org/database/"; +}; + +Default +{ + Packages::Compress ". bzip2 gzip"; + Sources::Compress ". bzip2 gzip"; + Contents::Compress "gzip"; + DeLinkLimit 0; + MaxContentsChange 25000; + FileMode 0664; +} + +TreeDefault +{ + Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top"; +}; + +""" + +apt_trees={} +apt_trees["testing"]=""" +tree "dists/testing" +{ + FakeDI "dists/unstable"; + FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; +}; +tree "dists/testing/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents "$(DIST)/../Contents-udeb"; +}; + +tree "dists/testing/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents "$(DIST)/../Contents-udeb-nf"; +}; +""" + +apt_trees["unstable"]=""" +tree "dists/unstable" +{ + FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.sid.$(SECTION)"; + ExtraOverride "override.sid.extra.$(SECTION)"; + SrcOverride "override.sid.$(SECTION).src"; +}; +tree "dists/unstable/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents "$(DIST)/../Contents-udeb"; +}; + +tree "dists/unstable/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents "$(DIST)/../Contents-udeb-nf"; +}; +""" + +apt_trees["experimental"]=""" +tree "dists/experimental" +{ + FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.sid.$(SECTION)"; + SrcOverride "override.sid.$(SECTION).src"; +}; +tree "dists/experimental/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents "$(DIST)/../Contents-udeb"; +}; + +tree "dists/experimental/non-free" +{ + FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.sid.main.$(SECTION)"; + SrcOverride "override.sid.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents "$(DIST)/../Contents-udeb-nf"; +}; +""" + +apt_trees["testing-proposed-updates"]=""" +tree "dists/testing-proposed-updates" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.$(SECTION)"; + ExtraOverride "override.squeeze.extra.$(SECTION)"; + SrcOverride "override.squeeze.$(SECTION).src"; + Contents " "; +}; +tree "dists/testing-proposed-updates/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.squeeze.main.$(SECTION)"; + SrcOverride "override.squeeze.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents " "; +}; +""" + +apt_trees["proposed-updates"]=""" +tree "dists/proposed-updates" +{ + FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list"; + SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list"; + Sections "main contrib non-free"; + Architectures "amd64"; + BinOverride "override.lenny.$(SECTION)"; + ExtraOverride "override.lenny.extra.$(SECTION)"; + SrcOverride "override.lenny.$(SECTION).src"; + Contents " "; +}; +tree "dists/proposed-updates/main" +{ + FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "%(arch)s"; + BinOverride "override.lenny.main.$(SECTION)"; + SrcOverride "override.lenny.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents " "; +}; +""" + class Suite(object): def __init__(self, *args, **kwargs): pass @@ -2487,8 +2671,50 @@ class Suite(object): return "\n".join(ret) - def generate_packages_sources(self, Logger): - """ Generate Packages/Sources files for the given suite" + def generate_packages_sources(self, suite, arch): + """ + Generate Packages/Sources files with apt-ftparchive for the given suite/arch + + @type suite: string + @param suite: Suite name + + @type arch: string + @param arch: Architecture name + """ + + tempdir = None + startdir = os.getcwd() + + try: + # Write apt.conf + (ac_fd, ac_name) = mkstemp() + os.write(ac_fd, DAILY_APT_CONF % {'archivepath': self.path}) + # here we want to generate the tree entries + os.write(ac_fd, apt_trees[suite] % {'arch': arch}) + os.close(ac_fd) + + # Run apt-ftparchive generate + os.chdir(os.path.dirname(ac_name)) + # We might want to add a -q or -qq here + os.system('apt-ftparchive generate %s' % os.path.basename(ac_name)) + + # Clean up any left behind files + finally: + os.chdir(startdir) + if ac_fd: + try: + os.close(ac_fd) + except OSError: + pass + + if ac_name: + try: + os.unlink(ac_name) + except OSError: + pass + + + __all__.append('Suite') -- 2.39.5