miscellaneous fixups from testing

Signed-off-by: Mark Hymers <mhy@debian.org>

diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py
index 7ff9a0f34bef9ae21bd5aa9953d5f8ede986d461..fab21cbf4492a7d3bf4447e0dbf28be6c793e911 100755 (executable)
--- a/dak/process_unchecked.py
+++ b/dak/process_unchecked.py
@@ -52,7 +52,7 @@ from debian_bundle import deb822
 from daklib.dbconn import *
 from daklib.binary import Binary
 from daklib import daklog
-from daklib import queue
+from daklib.queue import *
 from daklib import utils
 from daklib.textutils import fix_maintainer
 from daklib.dak_exceptions import *
@@ -433,10 +433,16 @@ def acknowledge_new(u, summary, short_summary):
 # the .changes file.
 
 def process_it(changes_file):
+    global Logger
+
     cnf = Config()
 
+    holding = Holding()
+
     u = Upload()
     u.pkg.changes_file = changes_file
+    u.pkg.directory = os.getcwd()
+    u.logger = Logger
 
     # Some defaults in case we can't fully process the .changes file
     u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"]
@@ -466,18 +472,19 @@ def process_it(changes_file):
 
             # Absolutize the filename to avoid the requirement of being in the
             # same directory as the .changes file.
-            copy_to_holding(os.path.abspath(changes_file))
+            holding.copy_to_holding(os.path.abspath(changes_file))
 
             # Relativize the filename so we use the copy in holding
             # rather than the original...
             changespath = os.path.basename(u.pkg.changes_file)
 
-        changes["fingerprint"] = utils.check_signature(changespath, reject)
+        (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changespath)
 
-        if changes["fingerprint"]:
+        if u.pkg.changes["fingerprint"]:
             valid_changes_p = u.load_changes(changespath)
         else:
             valid_changes_p = False
+           u.rejects.extend(rejects)
 
         if valid_changes_p:
             while u.reprocess:

diff --git a/daklib/changes.py b/daklib/changes.py
index 5cd26e2a69e071689e5adba343f0590a1973b3f1..1bb90753601bd9f3b0e220363b39e7aeac6932a9 100755 (executable)
--- a/daklib/changes.py
+++ b/daklib/changes.py
@@ -150,7 +150,6 @@ class Changes(object):
         or the text of a warning if there are
         """
 
-        conf = Config()
         summary = ""
 
         # Abandon the check if it's a non-sourceful upload
@@ -166,7 +165,7 @@ class Changes(object):
                                                                                             entry["override section"])
 
                 if entry["priority"] != "-":
-                    if entry["priority"] != entry["override_priority"]:
+                    if entry["priority"] != entry["override priority"]:
                         summary += "%s: package says priority is %s, override says %s.\n" % (name,
                                                                                              entry["priority"],
                                                                                              entry["override priority"])

diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 22fa262cae20cd68f7a68a78b717d8752b6b5d8a..f32a94bb91d550fa82886943a80c83d620ec613e 100755 (executable)
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -417,6 +417,9 @@ def get_or_set_contents_file_id(filename, session=None):
             session.add(cf)
             if privatetrans:
                 session.commit()
+                session.close()
+            else:
+                session.flush()
             return cf.cafilename_id
         else:
             return q.one().cafilename_id
@@ -491,7 +494,7 @@ class ContentFilepath(object):
 
 __all__.append('ContentFilepath')
 
-def get_or_set_contents_path_id(filepath, session):
+def get_or_set_contents_path_id(filepath, session=None):
     """
     Returns database id for given path.
 
@@ -520,6 +523,9 @@ def get_or_set_contents_path_id(filepath, session):
             session.add(cf)
             if privatetrans:
                 session.commit()
+                session.close()
+            else:
+                session.flush()
             return cf.cafilepath_id
         else:
             return q.one().cafilepath_id
@@ -566,27 +572,40 @@ def insert_content_paths(binary_id, fullpaths, session=None):
         privatetrans = True
 
     try:
+        # Insert paths
+        pathcache = {}
         for fullpath in fullpaths:
+            # Get the necessary IDs ...
             (path, file) = os.path.split(fullpath)
 
-            # Get the necessary IDs ...
+            filepath_id = get_or_set_contents_path_id(path, session)
+            filename_id = get_or_set_contents_file_id(file, session)
+
+            pathcache[fullpath] = (filepath_id, filename_id)
+
+        for fullpath, dat in pathcache.items():
             ca = ContentAssociation()
             ca.binary_id = binary_id
-            ca.filename_id = get_or_set_contents_file_id(file)
-            ca.filepath_id = get_or_set_contents_path_id(path)
+            ca.filepath_id = dat[0]
+            ca.filename_id = dat[1]
             session.add(ca)
 
         # Only commit if we set up the session ourself
         if privatetrans:
             session.commit()
+            session.close()
+        else:
+            session.flush()
 
         return True
+
     except:
         traceback.print_exc()
 
         # Only rollback if we set up the session ourself
         if privatetrans:
             session.rollback()
+            session.close()
 
         return False
 
@@ -1156,23 +1175,32 @@ def insert_pending_content_paths(package, fullpaths, session=None):
         q.delete()
 
         # Insert paths
+        pathcache = {}
         for fullpath in fullpaths:
             (path, file) = os.path.split(fullpath)
 
             if path.startswith( "./" ):
                 path = path[2:]
 
+            filepath_id = get_or_set_contents_path_id(path, session)
+            filename_id = get_or_set_contents_file_id(file, session)
+
+            pathcache[fullpath] = (filepath_id, filename_id)
+
+        for fullpath, dat in pathcache.items():
             pca = PendingContentAssociation()
             pca.package = package['Package']
             pca.version = package['Version']
-            pca.filename_id = get_or_set_contents_file_id(file, session)
-            pca.filepath_id = get_or_set_contents_path_id(path, session)
+            pca.filepath_id = dat[0]
+            pca.filename_id = dat[1]
             pca.architecture = arch_id
             session.add(pca)
 
         # Only commit if we set up the session ourself
         if privatetrans:
             session.commit()
+        else:
+            session.flush()
 
         return True
     except:
@@ -1319,7 +1347,7 @@ class Queue(object):
                 dest = os.path.join(dest_dir, file_entry)
 
                 # TODO: Move into database as above
-                if Cnf.FindB("Dinstall::SecurityQueueBuild"):
+                if conf.FindB("Dinstall::SecurityQueueBuild"):
                     # Copy it since the original won't be readable by www-data
                     utils.copy(src, dest)
                 else:

diff --git a/daklib/holding.py b/daklib/holding.py
index d77eecde10d1bcdb61d4a9be8d35febd54b3d772..0c472d120f6b07b256cfc1af065aca7a753526dd 100755 (executable)
--- a/daklib/holding.py
+++ b/daklib/holding.py
@@ -28,6 +28,7 @@ Simple singleton class for storing info about Holding directory
 
 import os
 from errno import ENOENT, EEXIST, EACCES
+import shutil
 
 from singleton import Singleton
 from config import Config
@@ -74,7 +75,7 @@ class Holding(Singleton):
 
         return None
 
-    def clean(self, filename):
+    def clean(self):
         cwd = os.getcwd()
         os.chdir(self.holding_dir)
         for f in self.in_holding.keys():

diff --git a/daklib/queue.py b/daklib/queue.py
index 77f402502ca0b6952ea400ec7d1a9903bc4c1992..5d791c5526fb70437e52b5f7f89d013ebaa62ec0 100755 (executable)
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -36,6 +36,8 @@ import time
 import apt_inst
 import apt_pkg
 import utils
+import commands
+import shutil
 from types import *
 
 from dak_exceptions import *
@@ -47,6 +49,7 @@ from dbconn import *
 from summarystats import SummaryStats
 from utils import parse_changes
 from textutils import fix_maintainer
+from binary import Binary
 
 ###############################################################################
 
@@ -211,7 +214,7 @@ def lookup_uid_from_fingerprint(fpr, session):
     # This is a stupid default, but see the comments below
     is_dm = False
 
-    user = get_uid_from_fingerprint(changes["fingerprint"], session)
+    user = get_uid_from_fingerprint(fpr, session)
 
     if user is not None:
         uid = user.uid
@@ -221,8 +224,8 @@ def lookup_uid_from_fingerprint(fpr, session):
             uid_name = user.name
 
         # Check the relevant fingerprint (which we have to have)
-        for f in uid.fingerprint:
-            if f.fingerprint == changes['fingerprint']:
+        for f in user.fingerprint:
+            if f.fingerprint == fpr:
                 is_dm = f.keyring.debian_maintainer
                 break
 
@@ -255,6 +258,7 @@ class Upload(object):
 
     """
     def __init__(self):
+        self.logger = None
         self.pkg = Changes()
         self.reset()
 
@@ -350,6 +354,7 @@ class Upload(object):
                  This is simply to prevent us even trying things later which will
                  fail because we couldn't properly parse the file.
         """
+        Cnf = Config()
         self.pkg.changes_file = filename
 
         # Parse the .changes field into a dictionary
@@ -367,7 +372,7 @@ class Upload(object):
 
         # Parse the Files field from the .changes into another dictionary
         try:
-            self.pkg.files.update(build_file_list(self.pkg.changes))
+            self.pkg.files.update(utils.build_file_list(self.pkg.changes))
         except ParseChangesError, line:
             self.rejects.append("%s: parse error, can't grok: %s." % (filename, line))
             return False
@@ -604,9 +609,9 @@ class Upload(object):
         entry["maintainer"] = control.Find("Maintainer", "")
 
         if f.endswith(".udeb"):
-            files[f]["dbtype"] = "udeb"
+            self.pkg.files[f]["dbtype"] = "udeb"
         elif f.endswith(".deb"):
-            files[f]["dbtype"] = "deb"
+            self.pkg.files[f]["dbtype"] = "deb"
         else:
             self.rejects.append("%s is neither a .deb or a .udeb." % (f))
 
@@ -622,7 +627,7 @@ class Upload(object):
             source_version = m.group(2)
 
         if not source_version:
-            source_version = files[f]["version"]
+            source_version = self.pkg.files[f]["version"]
 
         entry["source package"] = source
         entry["source version"] = source_version
@@ -680,7 +685,8 @@ class Upload(object):
         # Check the version and for file overwrites
         self.check_binary_against_db(f, session)
 
-        b = Binary(f).scan_package()
+        b = Binary(f)
+        b.scan_package()
         if len(b.rejects) > 0:
             for j in b.rejects:
                 self.rejects.append(j)
@@ -726,6 +732,7 @@ class Upload(object):
     def per_suite_file_checks(self, f, suite, session):
         cnf = Config()
         entry = self.pkg.files[f]
+        archive = utils.where_am_i()
 
         # Skip byhand
         if entry.has_key("byhand"):
@@ -760,7 +767,7 @@ class Upload(object):
 
         # Determine the location
         location = cnf["Dir::Pool"]
-        l = get_location(location, component, archive, session)
+        l = get_location(location, entry["component"], archive, session)
         if l is None:
             self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive))
             entry["location id"] = -1
@@ -785,7 +792,7 @@ class Upload(object):
 
         # Check for packages that have moved from one component to another
         entry['suite'] = suite
-        res = get_binary_components(files[f]['package'], suite, entry["architecture"], session)
+        res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session)
         if res.rowcount > 0:
             entry["othercomponents"] = res.fetchone()[0]
 
@@ -834,7 +841,7 @@ class Upload(object):
         has_binaries = False
         has_source = False
 
-        s = DBConn().session()
+        session = DBConn().session()
 
         for f, entry in self.pkg.files.items():
             # Ensure the file does not already exist in one of the accepted directories
@@ -934,7 +941,7 @@ class Upload(object):
 
         # Build up the file list of files mentioned by the .dsc
         try:
-            self.pkg.dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1))
+            self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1))
         except NoFilesFieldError:
             self.rejects.append("%s: no Files: field." % (dsc_filename))
             return False
@@ -954,7 +961,7 @@ class Upload(object):
         # Validate the source and version fields
         if not re_valid_pkg_name.match(self.pkg.dsc["source"]):
             self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"]))
-        if not re_valid_version.match(dsc["version"]):
+        if not re_valid_version.match(self.pkg.dsc["version"]):
             self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"]))
 
         # Bumping the version number of the .dsc breaks extraction by stable's
@@ -994,7 +1001,7 @@ class Upload(object):
 
         # Ensure there is a .tar.gz in the .dsc file
         has_tar = False
-        for f in dsc_files.keys():
+        for f in self.pkg.dsc_files.keys():
             m = re_issource.match(f)
             if not m:
                 self.rejects.append("%s: %s in Files field not recognised as source." % (dsc_filename, f))
@@ -1007,7 +1014,7 @@ class Upload(object):
             self.rejects.append("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename))
 
         # Ensure source is newer than existing source in target suites
-        self.check_source_against_db(dsc_filename, session)
+        self.check_source_against_db(dsc_filename)
 
         self.check_dsc_against_db(dsc_filename)
 
@@ -1023,8 +1030,8 @@ class Upload(object):
 
         # Find the .dsc (again)
         dsc_filename = None
-        for f in self.files.keys():
-            if files[f]["type"] == "dsc":
+        for f in self.pkg.files.keys():
+            if self.pkg.files[f]["type"] == "dsc":
                 dsc_filename = f
 
         # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
@@ -1032,7 +1039,7 @@ class Upload(object):
             return
 
         # Create a symlink mirror of the source files in our temporary directory
-        for f in self.files.keys():
+        for f in self.pkg.files.keys():
             m = re_issource.match(f)
             if m:
                 src = os.path.join(source_dir, f)
@@ -1063,7 +1070,7 @@ class Upload(object):
             return
 
         # Get the upstream version
-        upstr_version = re_no_epoch.sub('', dsc["version"])
+        upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"])
         if re_strip_revision.search(upstr_version):
             upstr_version = re_strip_revision.sub('', upstr_version)
 
@@ -1114,6 +1121,7 @@ class Upload(object):
             shutil.rmtree(tmpdir)
         except OSError, e:
             if e.errno != errno.EACCES:
+                print "foobar"
                 utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
 
             self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"]))
@@ -1124,7 +1132,8 @@ class Upload(object):
             if result != 0:
                 utils.fubar("'%s' failed with result %s." % (cmd, result))
             shutil.rmtree(tmpdir)
-        except:
+        except Exception, e:
+            print "foobar2 (%s)" % e
             utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"]))
 
     ###########################################################################
@@ -1138,7 +1147,7 @@ class Upload(object):
 
         # We need to deal with the original changes blob, as the fields we need
         # might not be in the changes dict serialised into the .dak anymore.
-        orig_changes = parse_deb822(self.pkg.changes['filecontents'])
+        orig_changes = utils.parse_deb822(self.pkg.changes['filecontents'])
 
         # Copy the checksums over to the current changes dict.  This will keep
         # the existing modifications to it intact.
@@ -1164,7 +1173,7 @@ class Upload(object):
                 for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc):
                     self.rejects.append(j)
 
-    def check_hashes():
+    def check_hashes(self):
         for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum):
             self.rejects.append(m)
 
@@ -1177,8 +1186,7 @@ class Upload(object):
         for m in utils.check_size(".dsc", self.pkg.dsc_files):
             self.rejects.append(m)
 
-        for m in utils.ensure_hashes(self.pkg.changes, dsc, files, dsc_files):
-            self.rejects.append(m)
+        self.ensure_hashes()
 
     ###########################################################################
     def check_urgency(self):
@@ -1199,11 +1207,13 @@ class Upload(object):
     #  travel can cause errors on extraction]
 
     def check_timestamps(self):
+        Cnf = Config()
+
         future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"])
         past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y"))
         tar = TarTime(future_cutoff, past_cutoff)
 
-        for filename, entry in self.pkg.files.keys():
+        for filename, entry in self.pkg.files.items():
             if entry["type"] == "deb":
                 tar.reset()
                 try:
@@ -1388,8 +1398,8 @@ distribution."""
                 del self.Subst["__BUG_NUMBER__"]
                 del self.Subst["__STABLE_WARNING__"]
 
-        if action:
-            self.Logger.log(["closing bugs"] + bugs)
+        if action and self.logger:
+            self.logger.log(["closing bugs"] + bugs)
 
         summary += "\n"
 
@@ -1426,7 +1436,7 @@ distribution."""
         self.Subst["__SHORT_SUMMARY__"] = short_summary
 
         for dist in self.pkg.changes["distribution"].keys():
-            announce_list = Cnf.Find("Suite::%s::Announce" % (dist))
+            announce_list = cnf.Find("Suite::%s::Announce" % (dist))
             if announce_list == "" or lists_done.has_key(announce_list):
                 continue
 
@@ -1481,9 +1491,10 @@ distribution."""
             targetdir = cnf["Dir::Queue::Accepted"]
 
         print "Accepting."
-        self.Logger.log(["Accepting changes", self.pkg.changes_file])
+       if self.logger:
+            self.logger.log(["Accepting changes", self.pkg.changes_file])
 
-        self.write_dot_dak(targetdir)
+        self.pkg.write_dot_dak(targetdir)
 
         # Move all the files into the accepted directory
         utils.move(self.pkg.changes_file, targetdir)
@@ -1754,7 +1765,8 @@ distribution."""
         if not cnf["Dinstall::Options::No-Mail"]:
             utils.send_mail(reject_mail_message)
 
-        self.Logger.log(["rejected", pkg.changes_file])
+       if self.logger:
+            self.logger.log(["rejected", pkg.changes_file])
 
         return 0
 
@@ -1921,12 +1933,12 @@ distribution."""
         q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all']))
 
         self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ],
-                                       file, files[file]["version"], sourceful=False)
+                                       file, self.pkg.files[file]["version"], sourceful=False)
 
         # Check for any existing copies of the file
-        q = session.query(DBBinary).filter_by(files[file]["package"])
-        q = q.filter_by(version=files[file]["version"])
-        q = q.join(Architecture).filter_by(arch_string=files[file]["architecture"])
+        q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"])
+        q = q.filter_by(version=self.pkg.files[file]["version"])
+        q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"])
 
         if q.count() > 0:
             self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file))
@@ -2007,7 +2019,7 @@ distribution."""
                                 # TODO: Don't delete the entry, just mark it as not needed
                                 # This would fix the stupidity of changing something we often iterate over
                                 # whilst we're doing it
-                                del files[dsc_name]
+                                del self.pkg.files[dsc_name]
                                 self.pkg.orig_tar_gz = os.path.join(i.location.path, i.filename)
                                 match = 1