From 4db4a1577f7971458d41966083b82e46d50b8a04 Mon Sep 17 00:00:00 2001 From: Mark Hymers Date: Sat, 19 Sep 2009 13:40:29 +0000 Subject: [PATCH] miscellaneous fixups from testing Signed-off-by: Mark Hymers --- dak/process_unchecked.py | 15 +++++-- daklib/changes.py | 3 +- daklib/dbconn.py | 42 ++++++++++++++++---- daklib/holding.py | 3 +- daklib/queue.py | 84 +++++++++++++++++++++++----------------- 5 files changed, 97 insertions(+), 50 deletions(-) diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index 7ff9a0f3..fab21cbf 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -52,7 +52,7 @@ from debian_bundle import deb822 from daklib.dbconn import * from daklib.binary import Binary from daklib import daklog -from daklib import queue +from daklib.queue import * from daklib import utils from daklib.textutils import fix_maintainer from daklib.dak_exceptions import * @@ -433,10 +433,16 @@ def acknowledge_new(u, summary, short_summary): # the .changes file. def process_it(changes_file): + global Logger + cnf = Config() + holding = Holding() + u = Upload() u.pkg.changes_file = changes_file + u.pkg.directory = os.getcwd() + u.logger = Logger # Some defaults in case we can't fully process the .changes file u.pkg.changes["maintainer2047"] = cnf["Dinstall::MyEmailAddress"] @@ -466,18 +472,19 @@ def process_it(changes_file): # Absolutize the filename to avoid the requirement of being in the # same directory as the .changes file. - copy_to_holding(os.path.abspath(changes_file)) + holding.copy_to_holding(os.path.abspath(changes_file)) # Relativize the filename so we use the copy in holding # rather than the original... changespath = os.path.basename(u.pkg.changes_file) - changes["fingerprint"] = utils.check_signature(changespath, reject) + (u.pkg.changes["fingerprint"], rejects) = utils.check_signature(changespath) - if changes["fingerprint"]: + if u.pkg.changes["fingerprint"]: valid_changes_p = u.load_changes(changespath) else: valid_changes_p = False + u.rejects.extend(rejects) if valid_changes_p: while u.reprocess: diff --git a/daklib/changes.py b/daklib/changes.py index 5cd26e2a..1bb90753 100755 --- a/daklib/changes.py +++ b/daklib/changes.py @@ -150,7 +150,6 @@ class Changes(object): or the text of a warning if there are """ - conf = Config() summary = "" # Abandon the check if it's a non-sourceful upload @@ -166,7 +165,7 @@ class Changes(object): entry["override section"]) if entry["priority"] != "-": - if entry["priority"] != entry["override_priority"]: + if entry["priority"] != entry["override priority"]: summary += "%s: package says priority is %s, override says %s.\n" % (name, entry["priority"], entry["override priority"]) diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 22fa262c..f32a94bb 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -417,6 +417,9 @@ def get_or_set_contents_file_id(filename, session=None): session.add(cf) if privatetrans: session.commit() + session.close() + else: + session.flush() return cf.cafilename_id else: return q.one().cafilename_id @@ -491,7 +494,7 @@ class ContentFilepath(object): __all__.append('ContentFilepath') -def get_or_set_contents_path_id(filepath, session): +def get_or_set_contents_path_id(filepath, session=None): """ Returns database id for given path. @@ -520,6 +523,9 @@ def get_or_set_contents_path_id(filepath, session): session.add(cf) if privatetrans: session.commit() + session.close() + else: + session.flush() return cf.cafilepath_id else: return q.one().cafilepath_id @@ -566,27 +572,40 @@ def insert_content_paths(binary_id, fullpaths, session=None): privatetrans = True try: + # Insert paths + pathcache = {} for fullpath in fullpaths: + # Get the necessary IDs ... (path, file) = os.path.split(fullpath) - # Get the necessary IDs ... + filepath_id = get_or_set_contents_path_id(path, session) + filename_id = get_or_set_contents_file_id(file, session) + + pathcache[fullpath] = (filepath_id, filename_id) + + for fullpath, dat in pathcache.items(): ca = ContentAssociation() ca.binary_id = binary_id - ca.filename_id = get_or_set_contents_file_id(file) - ca.filepath_id = get_or_set_contents_path_id(path) + ca.filepath_id = dat[0] + ca.filename_id = dat[1] session.add(ca) # Only commit if we set up the session ourself if privatetrans: session.commit() + session.close() + else: + session.flush() return True + except: traceback.print_exc() # Only rollback if we set up the session ourself if privatetrans: session.rollback() + session.close() return False @@ -1156,23 +1175,32 @@ def insert_pending_content_paths(package, fullpaths, session=None): q.delete() # Insert paths + pathcache = {} for fullpath in fullpaths: (path, file) = os.path.split(fullpath) if path.startswith( "./" ): path = path[2:] + filepath_id = get_or_set_contents_path_id(path, session) + filename_id = get_or_set_contents_file_id(file, session) + + pathcache[fullpath] = (filepath_id, filename_id) + + for fullpath, dat in pathcache.items(): pca = PendingContentAssociation() pca.package = package['Package'] pca.version = package['Version'] - pca.filename_id = get_or_set_contents_file_id(file, session) - pca.filepath_id = get_or_set_contents_path_id(path, session) + pca.filepath_id = dat[0] + pca.filename_id = dat[1] pca.architecture = arch_id session.add(pca) # Only commit if we set up the session ourself if privatetrans: session.commit() + else: + session.flush() return True except: @@ -1319,7 +1347,7 @@ class Queue(object): dest = os.path.join(dest_dir, file_entry) # TODO: Move into database as above - if Cnf.FindB("Dinstall::SecurityQueueBuild"): + if conf.FindB("Dinstall::SecurityQueueBuild"): # Copy it since the original won't be readable by www-data utils.copy(src, dest) else: diff --git a/daklib/holding.py b/daklib/holding.py index d77eecde..0c472d12 100755 --- a/daklib/holding.py +++ b/daklib/holding.py @@ -28,6 +28,7 @@ Simple singleton class for storing info about Holding directory import os from errno import ENOENT, EEXIST, EACCES +import shutil from singleton import Singleton from config import Config @@ -74,7 +75,7 @@ class Holding(Singleton): return None - def clean(self, filename): + def clean(self): cwd = os.getcwd() os.chdir(self.holding_dir) for f in self.in_holding.keys(): diff --git a/daklib/queue.py b/daklib/queue.py index 77f40250..5d791c55 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -36,6 +36,8 @@ import time import apt_inst import apt_pkg import utils +import commands +import shutil from types import * from dak_exceptions import * @@ -47,6 +49,7 @@ from dbconn import * from summarystats import SummaryStats from utils import parse_changes from textutils import fix_maintainer +from binary import Binary ############################################################################### @@ -211,7 +214,7 @@ def lookup_uid_from_fingerprint(fpr, session): # This is a stupid default, but see the comments below is_dm = False - user = get_uid_from_fingerprint(changes["fingerprint"], session) + user = get_uid_from_fingerprint(fpr, session) if user is not None: uid = user.uid @@ -221,8 +224,8 @@ def lookup_uid_from_fingerprint(fpr, session): uid_name = user.name # Check the relevant fingerprint (which we have to have) - for f in uid.fingerprint: - if f.fingerprint == changes['fingerprint']: + for f in user.fingerprint: + if f.fingerprint == fpr: is_dm = f.keyring.debian_maintainer break @@ -255,6 +258,7 @@ class Upload(object): """ def __init__(self): + self.logger = None self.pkg = Changes() self.reset() @@ -350,6 +354,7 @@ class Upload(object): This is simply to prevent us even trying things later which will fail because we couldn't properly parse the file. """ + Cnf = Config() self.pkg.changes_file = filename # Parse the .changes field into a dictionary @@ -367,7 +372,7 @@ class Upload(object): # Parse the Files field from the .changes into another dictionary try: - self.pkg.files.update(build_file_list(self.pkg.changes)) + self.pkg.files.update(utils.build_file_list(self.pkg.changes)) except ParseChangesError, line: self.rejects.append("%s: parse error, can't grok: %s." % (filename, line)) return False @@ -604,9 +609,9 @@ class Upload(object): entry["maintainer"] = control.Find("Maintainer", "") if f.endswith(".udeb"): - files[f]["dbtype"] = "udeb" + self.pkg.files[f]["dbtype"] = "udeb" elif f.endswith(".deb"): - files[f]["dbtype"] = "deb" + self.pkg.files[f]["dbtype"] = "deb" else: self.rejects.append("%s is neither a .deb or a .udeb." % (f)) @@ -622,7 +627,7 @@ class Upload(object): source_version = m.group(2) if not source_version: - source_version = files[f]["version"] + source_version = self.pkg.files[f]["version"] entry["source package"] = source entry["source version"] = source_version @@ -680,7 +685,8 @@ class Upload(object): # Check the version and for file overwrites self.check_binary_against_db(f, session) - b = Binary(f).scan_package() + b = Binary(f) + b.scan_package() if len(b.rejects) > 0: for j in b.rejects: self.rejects.append(j) @@ -726,6 +732,7 @@ class Upload(object): def per_suite_file_checks(self, f, suite, session): cnf = Config() entry = self.pkg.files[f] + archive = utils.where_am_i() # Skip byhand if entry.has_key("byhand"): @@ -760,7 +767,7 @@ class Upload(object): # Determine the location location = cnf["Dir::Pool"] - l = get_location(location, component, archive, session) + l = get_location(location, entry["component"], archive, session) if l is None: self.rejects.append("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive)) entry["location id"] = -1 @@ -785,7 +792,7 @@ class Upload(object): # Check for packages that have moved from one component to another entry['suite'] = suite - res = get_binary_components(files[f]['package'], suite, entry["architecture"], session) + res = get_binary_components(self.pkg.files[f]['package'], suite, entry["architecture"], session) if res.rowcount > 0: entry["othercomponents"] = res.fetchone()[0] @@ -834,7 +841,7 @@ class Upload(object): has_binaries = False has_source = False - s = DBConn().session() + session = DBConn().session() for f, entry in self.pkg.files.items(): # Ensure the file does not already exist in one of the accepted directories @@ -934,7 +941,7 @@ class Upload(object): # Build up the file list of files mentioned by the .dsc try: - self.pkg.dsc_files.update(utils.build_file_list(dsc, is_a_dsc=1)) + self.pkg.dsc_files.update(utils.build_file_list(self.pkg.dsc, is_a_dsc=1)) except NoFilesFieldError: self.rejects.append("%s: no Files: field." % (dsc_filename)) return False @@ -954,7 +961,7 @@ class Upload(object): # Validate the source and version fields if not re_valid_pkg_name.match(self.pkg.dsc["source"]): self.rejects.append("%s: invalid source name '%s'." % (dsc_filename, self.pkg.dsc["source"])) - if not re_valid_version.match(dsc["version"]): + if not re_valid_version.match(self.pkg.dsc["version"]): self.rejects.append("%s: invalid version number '%s'." % (dsc_filename, self.pkg.dsc["version"])) # Bumping the version number of the .dsc breaks extraction by stable's @@ -994,7 +1001,7 @@ class Upload(object): # Ensure there is a .tar.gz in the .dsc file has_tar = False - for f in dsc_files.keys(): + for f in self.pkg.dsc_files.keys(): m = re_issource.match(f) if not m: self.rejects.append("%s: %s in Files field not recognised as source." % (dsc_filename, f)) @@ -1007,7 +1014,7 @@ class Upload(object): self.rejects.append("%s: no .tar.gz or .orig.tar.gz in 'Files' field." % (dsc_filename)) # Ensure source is newer than existing source in target suites - self.check_source_against_db(dsc_filename, session) + self.check_source_against_db(dsc_filename) self.check_dsc_against_db(dsc_filename) @@ -1023,8 +1030,8 @@ class Upload(object): # Find the .dsc (again) dsc_filename = None - for f in self.files.keys(): - if files[f]["type"] == "dsc": + for f in self.pkg.files.keys(): + if self.pkg.files[f]["type"] == "dsc": dsc_filename = f # If there isn't one, we have nothing to do. (We have reject()ed the upload already) @@ -1032,7 +1039,7 @@ class Upload(object): return # Create a symlink mirror of the source files in our temporary directory - for f in self.files.keys(): + for f in self.pkg.files.keys(): m = re_issource.match(f) if m: src = os.path.join(source_dir, f) @@ -1063,7 +1070,7 @@ class Upload(object): return # Get the upstream version - upstr_version = re_no_epoch.sub('', dsc["version"]) + upstr_version = re_no_epoch.sub('', self.pkg.dsc["version"]) if re_strip_revision.search(upstr_version): upstr_version = re_strip_revision.sub('', upstr_version) @@ -1114,6 +1121,7 @@ class Upload(object): shutil.rmtree(tmpdir) except OSError, e: if e.errno != errno.EACCES: + print "foobar" utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"])) self.rejects.append("%s: source tree could not be cleanly removed." % (self.pkg.dsc["source"])) @@ -1124,7 +1132,8 @@ class Upload(object): if result != 0: utils.fubar("'%s' failed with result %s." % (cmd, result)) shutil.rmtree(tmpdir) - except: + except Exception, e: + print "foobar2 (%s)" % e utils.fubar("%s: couldn't remove tmp dir for source tree." % (self.pkg.dsc["source"])) ########################################################################### @@ -1138,7 +1147,7 @@ class Upload(object): # We need to deal with the original changes blob, as the fields we need # might not be in the changes dict serialised into the .dak anymore. - orig_changes = parse_deb822(self.pkg.changes['filecontents']) + orig_changes = utils.parse_deb822(self.pkg.changes['filecontents']) # Copy the checksums over to the current changes dict. This will keep # the existing modifications to it intact. @@ -1164,7 +1173,7 @@ class Upload(object): for j in utils._ensure_dsc_hash(self.pkg.dsc, self.pkg.dsc_files, hashname, hashfunc): self.rejects.append(j) - def check_hashes(): + def check_hashes(self): for m in utils.check_hash(".changes", self.pkg.files, "md5", apt_pkg.md5sum): self.rejects.append(m) @@ -1177,8 +1186,7 @@ class Upload(object): for m in utils.check_size(".dsc", self.pkg.dsc_files): self.rejects.append(m) - for m in utils.ensure_hashes(self.pkg.changes, dsc, files, dsc_files): - self.rejects.append(m) + self.ensure_hashes() ########################################################################### def check_urgency(self): @@ -1199,11 +1207,13 @@ class Upload(object): # travel can cause errors on extraction] def check_timestamps(self): + Cnf = Config() + future_cutoff = time.time() + int(Cnf["Dinstall::FutureTimeTravelGrace"]) past_cutoff = time.mktime(time.strptime(Cnf["Dinstall::PastCutoffYear"],"%Y")) tar = TarTime(future_cutoff, past_cutoff) - for filename, entry in self.pkg.files.keys(): + for filename, entry in self.pkg.files.items(): if entry["type"] == "deb": tar.reset() try: @@ -1388,8 +1398,8 @@ distribution.""" del self.Subst["__BUG_NUMBER__"] del self.Subst["__STABLE_WARNING__"] - if action: - self.Logger.log(["closing bugs"] + bugs) + if action and self.logger: + self.logger.log(["closing bugs"] + bugs) summary += "\n" @@ -1426,7 +1436,7 @@ distribution.""" self.Subst["__SHORT_SUMMARY__"] = short_summary for dist in self.pkg.changes["distribution"].keys(): - announce_list = Cnf.Find("Suite::%s::Announce" % (dist)) + announce_list = cnf.Find("Suite::%s::Announce" % (dist)) if announce_list == "" or lists_done.has_key(announce_list): continue @@ -1481,9 +1491,10 @@ distribution.""" targetdir = cnf["Dir::Queue::Accepted"] print "Accepting." - self.Logger.log(["Accepting changes", self.pkg.changes_file]) + if self.logger: + self.logger.log(["Accepting changes", self.pkg.changes_file]) - self.write_dot_dak(targetdir) + self.pkg.write_dot_dak(targetdir) # Move all the files into the accepted directory utils.move(self.pkg.changes_file, targetdir) @@ -1754,7 +1765,8 @@ distribution.""" if not cnf["Dinstall::Options::No-Mail"]: utils.send_mail(reject_mail_message) - self.Logger.log(["rejected", pkg.changes_file]) + if self.logger: + self.logger.log(["rejected", pkg.changes_file]) return 0 @@ -1921,12 +1933,12 @@ distribution.""" q = q.join(Architecture).filter(Architecture.arch_string.in_([self.pkg.files[file]["architecture"], 'all'])) self.cross_suite_version_check([ (x.suite.suite_name, x.binary.version) for x in q.all() ], - file, files[file]["version"], sourceful=False) + file, self.pkg.files[file]["version"], sourceful=False) # Check for any existing copies of the file - q = session.query(DBBinary).filter_by(files[file]["package"]) - q = q.filter_by(version=files[file]["version"]) - q = q.join(Architecture).filter_by(arch_string=files[file]["architecture"]) + q = session.query(DBBinary).filter_by(package=self.pkg.files[file]["package"]) + q = q.filter_by(version=self.pkg.files[file]["version"]) + q = q.join(Architecture).filter_by(arch_string=self.pkg.files[file]["architecture"]) if q.count() > 0: self.rejects.append("%s: can not overwrite existing copy already in the archive." % (file)) @@ -2007,7 +2019,7 @@ distribution.""" # TODO: Don't delete the entry, just mark it as not needed # This would fix the stupidity of changing something we often iterate over # whilst we're doing it - del files[dsc_name] + del self.pkg.files[dsc_name] self.pkg.orig_tar_gz = os.path.join(i.location.path, i.filename) match = 1 -- 2.39.5