generate-packages-sources

finalize the first step. there needs to be a rework of the apt-trees part
into an aptconfig class. Much to inflexible the way it is now, but we can
start testing this out in real life now.

Signed-off-by: Joerg Jaspert <joerg@debian.org>

diff --git a/dak/generate_packages_sources.py b/dak/generate_packages_sources.py
index 366d722b3351ebf2ea6ce737a02e855a366be390..6e91aedae5470474a5130c3bfe59d002816d4682 100755 (executable)
--- a/dak/generate_packages_sources.py
+++ b/dak/generate_packages_sources.py
@@ -27,20 +27,20 @@
 
 import os
 import os.path
-import stat
 import sys
-from datetime import datetime
 import apt_pkg
+from tempfile import mkstemp, mkdtemp
+import commands
+from multiprocessing import Pool
 
 from daklib import daklog
 from daklib.dbconn import *
 from daklib.config import Config
-from daklib.threadpool import ThreadPool
 
 ################################################################################
 
-Options = None
-Logger = None
+Options = None                 #: Commandline arguments parsed into this
+Logger = None                  #: Our logging object
 
 ################################################################################
 
@@ -62,6 +62,265 @@ SUITE can be a space seperated list, e.g.
 
 ################################################################################
 
+def generate_packages_sources(arch, suite, tmppath):
+    """
+    Generate Packages/Sources files with apt-ftparchive for the given suite/arch
+
+    @type suite: string
+    @param suite: Suite name
+
+    @type arch: string
+    @param arch: Architecture name
+
+    @type tmppath: string
+    @param tmppath: The temporary path to work ing
+    """
+
+    DAILY_APT_CONF="""
+Dir
+{
+   ArchiveDir "/srv/ftp-master.debian.org/ftp/";
+   OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
+   CacheDir "/srv/ftp-master.debian.org/database/";
+};
+
+Default
+{
+   Packages::Compress "bzip2 gzip";
+   Sources::Compress "bzip2 gzip";
+   Contents::Compress "gzip";
+   DeLinkLimit 0;
+   MaxContentsChange 25000;
+   FileMode 0664;
+}
+
+TreeDefault
+{
+   Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
+};
+
+"""
+
+    apt_trees={}
+    apt_trees["di"]={}
+    apt_trees["testing"]="""
+tree "dists/testing"
+{
+   FakeDI "dists/unstable";
+   FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.$(SECTION)";
+   ExtraOverride "override.squeeze.extra.$(SECTION)";
+   SrcOverride "override.squeeze.$(SECTION).src";
+};
+"""
+    apt_trees["di"]["testing"]="""
+tree "dists/testing/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.main.$(SECTION)";
+   SrcOverride "override.squeeze.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   %(contentsline)s
+};
+
+tree "dists/testing/non-free"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.main.$(SECTION)";
+   SrcOverride "override.squeeze.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   %(contentsline)s
+};
+"""
+
+    apt_trees["unstable"]="""
+tree "dists/unstable"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.$(SECTION)";
+   ExtraOverride "override.sid.extra.$(SECTION)";
+   SrcOverride "override.sid.$(SECTION).src";
+};
+"""
+    apt_trees["di"]["unstable"]="""
+tree "dists/unstable/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   %(contentsline)s
+};
+
+tree "dists/unstable/non-free"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   %(contentsline)s
+};
+"""
+
+    apt_trees["experimental"]="""
+tree "dists/experimental"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.$(SECTION)";
+   SrcOverride "override.sid.$(SECTION).src";
+};
+"""
+    apt_trees["di"]["experimental"]="""
+tree "dists/experimental/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   %(contentsline)s
+};
+
+tree "dists/experimental/non-free"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.sid.main.$(SECTION)";
+   SrcOverride "override.sid.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   %(contentsline)s
+};
+"""
+
+    apt_trees["testing-proposed-updates"]="""
+tree "dists/testing-proposed-updates"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.$(SECTION)";
+   ExtraOverride "override.squeeze.extra.$(SECTION)";
+   SrcOverride "override.squeeze.$(SECTION).src";
+   Contents " ";
+};
+"""
+    apt_trees["di"]["testing-proposed-updates"]="""
+tree "dists/testing-proposed-updates/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.squeeze.main.$(SECTION)";
+   SrcOverride "override.squeeze.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents " ";
+};
+"""
+
+    apt_trees["proposed-updates"]="""
+tree "dists/proposed-updates"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
+   SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
+   Sections "main contrib non-free";
+   Architectures "%(arch)s";
+   BinOverride "override.lenny.$(SECTION)";
+   ExtraOverride "override.lenny.extra.$(SECTION)";
+   SrcOverride "override.lenny.$(SECTION).src";
+   Contents " ";
+};
+"""
+    apt_trees["di"]["proposed-updates"]="""
+tree "dists/proposed-updates/main"
+{
+   FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
+   Sections "debian-installer";
+   Architectures "%(arch)s";
+   BinOverride "override.lenny.main.$(SECTION)";
+   SrcOverride "override.lenny.main.src";
+   BinCacheDB "packages-debian-installer-$(ARCH).db";
+   Packages::Extensions ".udeb";
+   Contents " ";
+};
+"""
+
+    cnf = Config()
+    try:
+        # Write apt.conf
+        (ac_fd, ac_name) = mkstemp(dir=tmppath)
+        os.write(ac_fd, DAILY_APT_CONF)
+        # here we want to generate the tree entries
+        os.write(ac_fd, apt_trees[suite] % {'arch': arch})
+        # this special casing needs to go away, but this whole thing may just want an
+        # aptconfig class anyways
+        if arch != 'source':
+            if arch == 'hurd-i386' and suite == 'experimental':
+                pass
+            else:
+                if arch == "amd64":
+                    os.write(ac_fd, apt_trees["di"][suite] %
+                             {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
+                else:
+                    os.write(ac_fd, apt_trees["di"][suite] % {'arch': arch, 'contentsline': ''})
+        os.close(ac_fd)
+
+        print "Going to run apt-ftparchive for %s/%s" % (arch, suite)
+        # Run apt-ftparchive generate
+        # We dont want to add a -q or -qq here, this output should go into our logs, sometimes
+        # it has errormessages we like to see
+        os.environ['GZIP'] = '--rsyncable'
+        os.chdir(tmppath)
+        (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
+        sn="a-f %s,%s: " % (suite, arch)
+        print sn + output.replace('\n', '\n%s' % (sn))
+
+    # Clean up any left behind files
+    finally:
+        if ac_fd:
+            try:
+                os.close(ac_fd)
+            except OSError:
+                pass
+
+        if ac_name:
+            try:
+                os.unlink(ac_name)
+            except OSError:
+                pass
+
+def sname(arch):
+    return arch.arch_string
+
+########################################################################
+########################################################################
+
 def main ():
     global Options, Logger
 
@@ -98,19 +357,22 @@ def main ():
     else:
         suites=session.query(Suite).filter(Suite.untouchable == False).all()
 
-    threadpool = ThreadPool()
-
     startdir = os.getcwd()
     os.chdir(cnf["Dir::TempPath"])
 
+    # Setup a multiprocessing Pool. As many workers as we have CPU cores.
+    pool = Pool()
+
     # For each given suite, each architecture, run one apt-ftparchive
     for s in suites:
         arch_list=get_suite_architectures(s.suite_name, skipsrc=False, skipall=True, session=session)
-        Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, arch_list)])
+        Logger.log(['generating output for Suite %s, Architectures %s' % (s.suite_name, map(sname, arch_list))])
         for a in arch_list:
-            threadpool.queueTask(s.generate_packages_sources, (a.arch_string))
+            pool.apply_async(generate_packages_sources, (a.arch_string, s.suite_name, cnf["Dir::TempPath"]))
 
-    threadpool.joinAll()
+    # No more work will be added to our pool, close it and then wait for all to finish
+    pool.close()
+    pool.join()
     os.chdir(startdir)
     # this script doesn't change the database
     session.close()

diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index b987a372fcd1b51c1388a99ae5f916ec5b567dac..a1bff894aadd88576b24cd8f0c2aa7d84da0527d 100755 (executable)
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -2459,202 +2459,6 @@ SUITE_FIELDS = [ ('SuiteName', 'suite_name'),
                  ('OverrideSuite', 'overridesuite'),
                  ('ChangelogBase', 'changelogbase')]
 
-
-DAILY_APT_CONF="""
-Dir
-{
-   ArchiveDir "/srv/ftp-master.debian.org/ftp/";
-   OverrideDir "/srv/ftp-master.debian.org/scripts/override/";
-   CacheDir "/srv/ftp-master.debian.org/database/";
-};
-
-Default
-{
-   Packages::Compress "bzip2 gzip";
-   Sources::Compress "bzip2 gzip";
-   Contents::Compress "gzip";
-   DeLinkLimit 0;
-   MaxContentsChange 25000;
-   FileMode 0664;
-}
-
-TreeDefault
-{
-   Contents::Header "/srv/ftp-master.debian.org/dak/config/debian/Contents.top";
-};
-
-"""
-
-apt_trees={}
-apt_trees["di"]={}
-apt_trees["testing"]="""
-tree "dists/testing"
-{
-   FakeDI "dists/unstable";
-   FileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_binary-$(ARCH).list";
-   SourceFileList "/srv/ftp-master.debian.org/database/dists/testing_$(SECTION)_source.list";
-   Sections "main contrib non-free";
-   Architectures "%(arch)s";
-   BinOverride "override.squeeze.$(SECTION)";
-   ExtraOverride "override.squeeze.extra.$(SECTION)";
-   SrcOverride "override.squeeze.$(SECTION).src";
-};
-"""
-apt_trees["di"]["testing"]="""
-tree "dists/testing/main"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.squeeze.main.$(SECTION)";
-   SrcOverride "override.squeeze.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   %(contentsline)s
-};
-
-tree "dists/testing/non-free"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/testing_non-free_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.squeeze.main.$(SECTION)";
-   SrcOverride "override.squeeze.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   %(contentsline)s
-};
-"""
-
-apt_trees["unstable"]="""
-tree "dists/unstable"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_binary-$(ARCH).list";
-   SourceFileList "/srv/ftp-master.debian.org/database/dists/unstable_$(SECTION)_source.list";
-   Sections "main contrib non-free";
-   Architectures "%(arch)s";
-   BinOverride "override.sid.$(SECTION)";
-   ExtraOverride "override.sid.extra.$(SECTION)";
-   SrcOverride "override.sid.$(SECTION).src";
-};
-"""
-apt_trees["di"]["unstable"]="""
-tree "dists/unstable/main"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/unstable_main_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.sid.main.$(SECTION)";
-   SrcOverride "override.sid.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   %(contentsline)s
-};
-
-tree "dists/unstable/non-free"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/unstable_non-free_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.sid.main.$(SECTION)";
-   SrcOverride "override.sid.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   %(contentsline)s
-};
-"""
-
-apt_trees["experimental"]="""
-tree "dists/experimental"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_binary-$(ARCH).list";
-   SourceFileList "/srv/ftp-master.debian.org/database/dists/experimental_$(SECTION)_source.list";
-   Sections "main contrib non-free";
-   Architectures "%(arch)s";
-   BinOverride "override.sid.$(SECTION)";
-   SrcOverride "override.sid.$(SECTION).src";
-};
-"""
-apt_trees["di"]["experimental"]="""
-tree "dists/experimental/main"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/experimental_main_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.sid.main.$(SECTION)";
-   SrcOverride "override.sid.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   %(contentsline)s
-};
-
-tree "dists/experimental/non-free"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/experimental_non-free_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.sid.main.$(SECTION)";
-   SrcOverride "override.sid.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   %(contentsline)s
-};
-"""
-
-apt_trees["testing-proposed-updates"]="""
-tree "dists/testing-proposed-updates"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_binary-$(ARCH).list";
-   SourceFileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list";
-   Sections "main contrib non-free";
-   Architectures "%(arch)s";
-   BinOverride "override.squeeze.$(SECTION)";
-   ExtraOverride "override.squeeze.extra.$(SECTION)";
-   SrcOverride "override.squeeze.$(SECTION).src";
-   Contents " ";
-};
-"""
-apt_trees["di"]["testing-proposed-updates"]="""
-tree "dists/testing-proposed-updates/main"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.squeeze.main.$(SECTION)";
-   SrcOverride "override.squeeze.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   Contents " ";
-};
-"""
-
-apt_trees["proposed-updates"]="""
-tree "dists/proposed-updates"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_binary-$(ARCH).list";
-   SourceFileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_$(SECTION)_source.list";
-   Sections "main contrib non-free";
-   Architectures "%(arch)s";
-   BinOverride "override.lenny.$(SECTION)";
-   ExtraOverride "override.lenny.extra.$(SECTION)";
-   SrcOverride "override.lenny.$(SECTION).src";
-   Contents " ";
-};
-"""
-apt_trees["di"]["proposed-updates"]="""
-tree "dists/proposed-updates/main"
-{
-   FileList "/srv/ftp-master.debian.org/database/dists/proposed-updates_main_$(SECTION)_binary-$(ARCH).list";
-   Sections "debian-installer";
-   Architectures "%(arch)s";
-   BinOverride "override.lenny.main.$(SECTION)";
-   SrcOverride "override.lenny.main.src";
-   BinCacheDB "packages-debian-installer-$(ARCH).db";
-   Packages::Extensions ".udeb";
-   Contents " ";
-};
-"""
-
 class Suite(object):
     def __init__(self, *args, **kwargs):
         pass
@@ -2683,62 +2487,6 @@ class Suite(object):
 
         return "\n".join(ret)
 
-    def generate_packages_sources(self, arch):
-        """
-        Generate Packages/Sources files with apt-ftparchive for the given suite/arch
-
-        @type suite: string
-        @param suite: Suite name
-
-        @type arch: string
-        @param arch: Architecture name
-        """
-
-        cnf = Config()
-        try:
-            # Write apt.conf
-            (ac_fd, ac_name) = mkstemp(dir=cnf["Dir::TempPath"])
-            os.write(ac_fd, DAILY_APT_CONF)
-            # here we want to generate the tree entries
-            os.write(ac_fd, apt_trees[self.suite_name] % {'arch': arch})
-            # this special casing needs to go away, but this whole thing may just want an
-            # aptconfig class anyways
-            if arch != 'source':
-                if arch == 'hurd-i386' and self.suite_name == 'experimental':
-                    pass
-                else:
-                    if arch == "amd64":
-                        os.write(ac_fd, apt_trees["di"][self.suite_name] %
-                                 {'arch': arch, 'contentsline': 'Contents "$(DIST)/../Contents-udeb";'})
-                    else:
-                        os.write(ac_fd, apt_trees["di"][self.suite_name] % {'arch': arch, 'contentsline': ''})
-            os.close(ac_fd)
-
-            # Run apt-ftparchive generate
-            # We might want to add a -q or -qq here
-            os.environ['GZIP'] = '--rsyncable'
-            os.chdir(cnf["Dir::TempPath"])
-            (result, output) = commands.getstatusoutput('apt-ftparchive generate %s' % os.path.basename(ac_name))
-            sn="a-f %s,%s: " % (self.suite_name, arch)
-            print sn + output.replace('\n', '\n%s' % (sn))
-
-        # Clean up any left behind files
-        finally:
-            if ac_fd:
-                try:
-                    os.close(ac_fd)
-                except OSError:
-                    pass
-
-            if ac_name:
-                try:
-                    os.unlink(ac_name)
-                except OSError:
-                    pass
-
-
-
-
 __all__.append('Suite')
 
 @session_wrapper