From cd4e8043f530e38f975d877438bdd2707477487e Mon Sep 17 00:00:00 2001 From: James Troup Date: Wed, 8 May 2002 11:52:31 +0000 Subject: [PATCH] dir rationalization and changes for s.d.o --- jennifer | 144 ++++++++++++++++++++++------------------------- katie | 47 ++++++++-------- katie.py | 168 ++++++++++++++++++++++++++++++++++--------------------- 3 files changed, 196 insertions(+), 163 deletions(-) diff --git a/jennifer b/jennifer index 83a55885..1f1016bc 100755 --- a/jennifer +++ b/jennifer @@ -1,8 +1,8 @@ #!/usr/bin/env python # Checks Debian packages from Incoming -# Copyright (C) 2000, 2001 James Troup -# $Id: jennifer,v 1.15 2002-04-22 11:06:57 troup Exp $ +# Copyright (C) 2000, 2001, 2002 James Troup +# $Id: jennifer,v 1.16 2002-05-08 11:52:31 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,7 +34,6 @@ import apt_inst, apt_pkg; import db_access, katie, logging, utils; from types import *; -from string import lower; ################################################################################ @@ -44,7 +43,7 @@ re_is_changes = re.compile (r"(.+?)_(.+?)_(.+?)\.changes$"); ################################################################################ # Globals -jennifer_version = "$Revision: 1.15 $"; +jennifer_version = "$Revision: 1.16 $"; Cnf = None; Options = None; @@ -295,7 +294,7 @@ def copy_to_holding(filename): base_filename = os.path.basename(filename); - dest = Cnf["Dir::QueueHoldingDir"] + '/' + base_filename; + dest = Cnf["Dir::Queue::Holding"] + '/' + base_filename; try: fd = os.open(dest, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0640); os.close(fd); @@ -332,7 +331,7 @@ def clean_holding(): global in_holding; cwd = os.getcwd(); - os.chdir(Cnf["Dir::QueueHoldingDir"]); + os.chdir(Cnf["Dir::Queue::Holding"]); for file in in_holding.keys(): if os.path.exists(file): if string.find(file, '/') != -1: @@ -398,51 +397,6 @@ def check_changes(): if katie.re_isanum.match (i) == None: reject("`%s' from Closes field isn't a number." % (i)); - # Ensure there is a target distribution - if changes["distribution"].keys() == []: - reject("huh? Distribution field is empty in changes file."); - - # Map frozen to unstable if frozen doesn't exist - if changes["distribution"].has_key("frozen") and not Cnf.has_key("Suite::Frozen"): - del changes["distribution"]["frozen"] - changes["distribution"]["unstable"] = 1; - reject("Mapping frozen to unstable.",""); - - # Map testing to unstable - if changes["distribution"].has_key("testing"): - if len(changes["distribution"].keys()) > 1: - del changes["distribution"]["testing"]; - reject("Ignoring testing as a target suite.", "Warning: "); - else: - reject("invalid distribution 'testing'."); - - # Ensure target distributions exist - for i in changes["distribution"].keys(): - if not Cnf.has_key("Suite::%s" % (i)): - reject("Unknown distribution `%s'." % (i)); - - # Map unreleased arches from stable to unstable - if changes["distribution"].has_key("stable"): - for i in changes["architecture"].keys(): - if not Cnf.has_key("Suite::Stable::Architectures::%s" % (i)): - reject("Mapping stable to unstable for unreleased arch %s." % (i),""); - del changes["distribution"]["stable"]; - changes["distribution"]["unstable"] = 1; - break; - - # Map arches not being released from frozen to unstable - if changes["distribution"].has_key("frozen"): - for i in changes["architecture"].keys(): - if not Cnf.has_key("Suite::Frozen::Architectures::%s" % (i)): - reject("Mapping frozen to unstable for non-releasing arch `%s'." % (i),""); - del changes["distribution"]["frozen"] - changes["distribution"]["unstable"] = 1; - - # Map stable uploads to proposed-updates - if changes["distribution"].has_key("stable"): - reject("Mapping stable to updates.",""); - del changes["distribution"]["stable"]; - changes["distribution"]["proposed-updates"] = 1; # chopversion = no epoch; chopversion2 = no epoch and no revision (e.g. for .orig.tar.gz comparison) changes["chopversion"] = utils.re_no_epoch.sub('', changes["version"]) @@ -452,13 +406,53 @@ def check_changes(): # of the queue directories. base_filename = os.path.basename(filename); for dir in [ "Accepted", "Byhand", "Done", "New" ]: - if os.path.exists(Cnf["Dir::Queue%sDir" % (dir) ]+'/'+base_filename): + if os.path.exists(Cnf["Dir::Queue::%s" % (dir) ]+'/'+base_filename): reject("a changes file with the same name already exists in the %s directory." % (dir)); return 1; ################################################################################ +def check_distributions(): + "Check and map the Distribution field of a .changes file." + + # Handle suite mappings + if Cnf.has_key("SuiteMappings"): + for map in Cnf.ValueList("SuiteMappings"): + args = string.split(map); + type = args[0]; + if type == "map": + (source, dest) = args[1:3]; + if changes["distribution"].has_key(source): + del changes["distribution"][source] + changes["distribution"][dest] = 1; + reject("Mapping %s to %s." % (source, dest),""); + elif type == "map-unreleased": + (source, dest) = args[1:3]; + if changes["distribution"].has_key(source): + for arch in changes["architecture"].keys(): + if not Cnf.has_key("Suite::%s::Architectures::%s" % (source, arch)): + reject("Mapping %s to %s for unreleased architecture %s." % (source, dest, arch),""); + del changes["distribution"][source]; + changes["distribution"][dest] = 1; + break; + elif type == "ignore": + suite = args[1]; + if changes["distribution"].has_key(suite): + del changes["distribution"][suite]; + reject("Ignoring %s as a target suite." % (suite), "Warning: "); + + # Ensure there is (still) a target distribution + if changes["distribution"].keys() == []: + reject("no valid distribution."); + + # Ensure target distributions exist + for suite in changes["distribution"].keys(): + if not Cnf.has_key("Suite::%s" % (suite)): + reject("Unknown distribution `%s'." % (suite)); + +################################################################################ + def check_files(): global reprocess @@ -480,7 +474,7 @@ def check_files(): for file in file_keys: # Ensure the file does not already exist in one of the accepted directories for dir in [ "Accepted", "Byhand", "New" ]: - if os.path.exists(Cnf["Dir::Queue%sDir" % (dir) ]+'/'+file): + if os.path.exists(Cnf["Dir::Queue::%s" % (dir) ]+'/'+file): reject("%s file already exists in the %s directory." % (file, dir)); if not utils.re_taint_free.match(file): reject("!!WARNING!! tainted filename: '%s'." % (file)); @@ -524,7 +518,8 @@ def check_files(): reject("%s: control file lists name as `%s', which isn't in changes file." % (file, control.Find("Package", ""))); # Ensure the architecture of the .deb is one we know about. - if not Cnf.has_key("Suite::Unstable::Architectures::%s" % (control.Find("Architecture", ""))): + default_suite = Cnf.get("Dinstall::DefaultSuite", "Unstable") + if not Cnf.has_key("Suite::%s::Architectures::%s" % (default_suite, control.Find("Architecture", ""))): reject("Unknown architecture '%s'." % (control.Find("Architecture", ""))); # Ensure the architecture of the .deb is one of the ones @@ -591,11 +586,11 @@ def check_files(): # Check in one of the other directories source_epochless_version = utils.re_no_epoch.sub('', source_version); dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version); - if os.path.exists(Cnf["Dir::QueueByhandDir"] + '/' + dsc_filename): + if os.path.exists(Cnf["Dir::Queue::Byhand"] + '/' + dsc_filename): files[file]["byhand"] = 1; - elif os.path.exists(Cnf["Dir::QueueNewDir"] + '/' + dsc_filename): + elif os.path.exists(Cnf["Dir::Queue::New"] + '/' + dsc_filename): files[file]["new"] = 1; - elif not os.path.exists(Cnf["Dir::QueueAcceptedDir"] + '/' + dsc_filename): + elif not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename): reject("no source found for %s %s (%s)." % (source_package, source_version, file)); # Checks for a source package... @@ -664,7 +659,7 @@ def check_files(): reject("file '%s' has invalid priority '%s' [contains '/']." % (file, files[file]["priority"])); # Determine the location - location = Cnf["Dir::PoolDir"]; + location = Cnf["Dir::Pool"]; location_id = db_access.get_location_id (location, component, archive); if location_id == -1: reject("[INTERNAL ERROR] couldn't determine location (Component: %s, Archive: %s)" % (component, archive)); @@ -797,7 +792,7 @@ def check_urgency (): if not Cnf.has_key("Urgency::Valid::%s" % changes["urgency"]): reject("%s is not a valid urgency; it will be treated as %s by testing." % (changes["urgency"], Cnf["Urgency::Default"]), "Warning: "); changes["urgency"] = Cnf["Urgency::Default"]; - changes["urgency"] = lower(changes["urgency"]); + changes["urgency"] = string.lower(changes["urgency"]); ################################################################################ @@ -963,10 +958,7 @@ def action (): def accept (summary, short_summary): Katie.accept(summary, short_summary); - - # Check for override disparities - if not Cnf["Dinstall::Options::No-Mail"]: - Katie.check_override(); + Katie.check_override(); # Finally, remove the originals from the unchecked directory os.chdir (pkg.directory); @@ -980,19 +972,18 @@ def do_byhand (summary): print "Moving to BYHAND holding area." Logger.log(["Moving to byhand", pkg.changes_file]); - Katie.dump_vars(Cnf["Dir::QueueByhandDir"]); + Katie.dump_vars(Cnf["Dir::Queue::Byhand"]); file_keys = files.keys(); # Move all the files into the byhand directory - utils.move (pkg.changes_file, Cnf["Dir::QueueByhandDir"]); + utils.move (pkg.changes_file, Cnf["Dir::Queue::Byhand"]); for file in file_keys: - utils.move (file, Cnf["Dir::QueueByhandDir"], perms=0660); + utils.move (file, Cnf["Dir::Queue::Byhand"], perms=0660); # Check for override disparities - if not Cnf["Dinstall::Options::No-Mail"]: - Katie.Subst["__SUMMARY__"] = summary; - Katie.check_override(); + Katie.Subst["__SUMMARY__"] = summary; + Katie.check_override(); # Finally remove the originals. os.chdir (pkg.directory); @@ -1008,19 +999,19 @@ def acknowledge_new (summary): print "Moving to NEW holding area." Logger.log(["Moving to new", pkg.changes_file]); - Katie.dump_vars(Cnf["Dir::QueueNewDir"]); + Katie.dump_vars(Cnf["Dir::Queue::New"]); file_keys = files.keys(); - # Move all the files into the accepted directory - utils.move (pkg.changes_file, Cnf["Dir::QueueNewDir"]); + # Move all the files into the 'new' directory + utils.move (pkg.changes_file, Cnf["Dir::Queue::New"]); for file in file_keys: - utils.move (file, Cnf["Dir::QueueNewDir"], perms=0660); + utils.move (file, Cnf["Dir::Queue::New"], perms=0660); if not Options["No-Mail"]: print "Sending new ack."; Subst["__SUMMARY__"] = summary; - new_ack_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.new","r").read()); + new_ack_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.new"); utils.send_mail(new_ack_message,""); # Finally remove the originals. @@ -1060,7 +1051,7 @@ def process_it (changes_file): # If this is the Real Thing(tm), copy things into a private # holding directory first to avoid replacable file races. if not Options["No-Action"]: - os.chdir(Cnf["Dir::QueueHoldingDir"]); + os.chdir(Cnf["Dir::Queue::Holding"]); copy_to_holding(pkg.changes_file); # Relativize the filename so we use the copy in holding # rather than the original... @@ -1069,6 +1060,7 @@ def process_it (changes_file): changes_valid = check_changes(); if changes_valid: while reprocess: + check_distributions(); check_files(); check_md5sums(); check_dsc(); @@ -1116,7 +1108,7 @@ def main(): # Check that we aren't going to clash with the daily cron job - if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]: + if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later."); # Obtain lock if not in no-action mode and initialize the log diff --git a/katie b/katie index 08e6d4fa..27d909b7 100755 --- a/katie +++ b/katie @@ -1,8 +1,8 @@ #!/usr/bin/env python # Installs Debian packages -# Copyright (C) 2000, 2001 James Troup -# $Id: katie,v 1.79 2002-04-24 01:56:24 troup Exp $ +# Copyright (C) 2000, 2001, 2002 James Troup +# $Id: katie,v 1.80 2002-05-08 11:52:31 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,7 +39,7 @@ import db_access, katie, logging, utils; ############################################################################### # Globals -katie_version = "$Revision: 1.79 $"; +katie_version = "$Revision: 1.80 $"; Cnf = None; Options = None; @@ -73,7 +73,7 @@ class Urgency_Log: self.Cnf = Cnf; self.timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())); # Create the log directory if it doesn't exist - self.log_dir = Cnf["Dir::UrgencyLogDir"]; + self.log_dir = Cnf["Dir::UrgencyLog"]; if not os.path.exists(self.log_dir): umask = os.umask(00000); os.makedirs(self.log_dir, 02775); @@ -226,11 +226,11 @@ def do_reject (): Subst["__REJECTOR_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]; Subst["__REJECT_MESSAGE__"] = reject_message; Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; - reject_mail_message = utils.TemplateSubst(Subst,utils.open_file(Cnf["Dir::TemplatesDir"]+"/katie.unaccept").read()); + reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.unaccept"); # Write the rejection email out as the .reason file reason_filename = os.path.basename(pkg.changes_file[:-8]) + ".reason"; - reject_filename = Cnf["Dir::QueueRejectDir"] + '/' + reason_filename; + reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename; # If we fail here someone is probably trying to exploit the race # so let's just raise an exception ... if os.path.exists(reject_filename): @@ -306,7 +306,7 @@ def install (): source_version = files[file]["source version"]; filename = files[file]["pool name"] + file; if not files[file].has_key("location id") or not files[file]["location id"]: - files[file]["location id"] = db_access.get_location_id(Cnf["Dir::PoolDir"],files[file]["component"],utils.where_am_i()); + files[file]["location id"] = db_access.get_location_id(Cnf["Dir::Pool"],files[file]["component"],utils.where_am_i()); if not files[file].has_key("files id") or not files[file]["files id"]: files[file]["files id"] = db_access.set_files_id (filename, files[file]["size"], files[file]["md5sum"], files[file]["location id"]) source_id = db_access.get_source_id (source, source_version); @@ -337,7 +337,7 @@ def install (): legacy_filename = qid["path"]+qid["filename"]; pool_location = utils.poolify (changes["source"], files[file]["component"]); pool_filename = pool_location + os.path.basename(qid["filename"]); - destination = Cnf["Dir::PoolDir"] + pool_location + destination = Cnf["Dir::Pool"] + pool_location utils.move(legacy_filename, destination); # Then Update the DB's files table q = projectB.query("UPDATE files SET filename = '%s', location = '%s' WHERE id = '%s'" % (pool_filename, dsc_location_id, qid["files_id"])); @@ -356,13 +356,13 @@ def install (): new_filename = utils.poolify(changes["source"], dsc_component) + os.path.basename(old_filename); new_files_id = db_access.get_files_id(new_filename, file_size, file_md5sum, dsc_location_id); if new_files_id == None: - utils.copy(old_filename, Cnf["Dir::PoolDir"] + new_filename); + utils.copy(old_filename, Cnf["Dir::Pool"] + new_filename); new_files_id = db_access.set_files_id(new_filename, file_size, file_md5sum, dsc_location_id); projectB.query("UPDATE dsc_files SET file = %s WHERE source = %s AND file = %s" % (new_files_id, source_id, orig_tar_id)); # Install the files into the pool for file in files.keys(): - destination = Cnf["Dir::PoolDir"] + files[file]["pool name"] + file; + destination = Cnf["Dir::Pool"] + files[file]["pool name"] + file; utils.move(file, destination); Logger.log(["installed", file, files[file]["type"], files[file]["size"], files[file]["architecture"]]); install_bytes = install_bytes + float(files[file]["size"]); @@ -370,7 +370,7 @@ def install (): # Copy the .changes file across for suite which need it. for suite in changes["distribution"].keys(): if Cnf.has_key("Suite::%s::CopyChanges" % (suite)): - utils.copy(pkg.changes_file, Cnf["Dir::RootDir"] + Cnf["Suite::%s::CopyChanges" % (suite)]); + utils.copy(pkg.changes_file, Cnf["Dir::Root"] + Cnf["Suite::%s::CopyChanges" % (suite)]); # and the .katie file... if Cnf.has_key("Suite::%s::CopyKatie" % (suite)): utils.copy(Katie.pkg.changes_file[:-8]+".katie", Cnf["Suite::%s::CopyKatie" % (suite)]); @@ -379,13 +379,13 @@ def install (): # Move the .changes into the 'done' directory try: - utils.move (pkg.changes_file, os.path.join(Cnf["Dir::QueueDoneDir"], os.path.basename(pkg.changes_file))); + utils.move (pkg.changes_file, os.path.join(Cnf["Dir::Queue::Done"], os.path.basename(pkg.changes_file))); except: utils.warn("couldn't move changes file '%s' to DONE directory. [Got %s]" % (os.path.basename(pkg.changes_file), sys.exc_type)); os.unlink(Katie.pkg.changes_file[:-8]+".katie"); - if changes["architecture"].has_key("source"): + if changes["architecture"].has_key("source") and Urgency_Logger: Urgency_Logger.log(dsc["source"], dsc["version"], changes["urgency"]); # Undo the work done in katie.py(accept) to help auto-building @@ -400,7 +400,7 @@ def install (): projectB.query("UPDATE unstable_accepted SET in_accepted = 'f', last_used = '%s' WHERE filename = '%s'" % (now_date, dest)); # Update the symlink to point to the new location in the pool pool_location = utils.poolify (changes["source"], files[file]["component"]); - src = os.path.join(Cnf["Dir::PoolDir"], pool_location, os.path.basename(file)); + src = os.path.join(Cnf["Dir::Pool"], pool_location, os.path.basename(file)); os.unlink(dest); os.symlink(src, dest); # Update last_used on any non-upload .orig.tar.gz symlink @@ -461,8 +461,8 @@ def stable_install (summary, short_summary): utils.move (pkg.changes_file, Cnf["Dir::Morgue"] + '/katie/' + os.path.basename(pkg.changes_file)); ## Update the Stable ChangeLog file - new_changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog"; - changelog_filename = Cnf["Dir::RootDir"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog"; + new_changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + ".ChangeLog"; + changelog_filename = Cnf["Dir::Root"] + Cnf["Suite::Stable::ChangeLogBase"] + "ChangeLog"; if os.path.exists(new_changelog_filename): os.unlink (new_changelog_filename); @@ -489,7 +489,7 @@ def stable_install (summary, short_summary): if not Options["No-Mail"] and changes["architecture"].has_key("source"): Subst["__SUITE__"] = " into stable"; Subst["__SUMMARY__"] = summary; - mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/katie.installed","r").read()); + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.installed"); utils.send_mail(mail_message, ""); Katie.announce(short_summary, 1) @@ -550,7 +550,7 @@ def main(): # Check that we aren't going to clash with the daily cron job - if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::RootDir"])) and not Options["No-Lock"]: + if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later."); # If running from within proposed-updates; assume an install to stable @@ -562,7 +562,7 @@ def main(): lock_fd = os.open(Cnf["Dinstall::LockFile"], os.O_RDWR | os.O_CREAT); fcntl.lockf(lock_fd, FCNTL.F_TLOCK); Logger = Katie.Logger = logging.Logger(Cnf, "katie"); - if not installing_to_stable: + if not installing_to_stable and Cnf.get("Dir::UrgencyLog"): Urgency_Logger = Urgency_Log(Cnf); # Initialize the substitution template mapping global @@ -571,7 +571,8 @@ def main(): Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]); else: Subst["__BCC__"] = bcc; - Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"]; + if Cnf.has_key("Dinstall::StableRejector"): + Subst["__STABLE_REJECTOR__"] = Cnf["Dinstall::StableRejector"]; # Sort the .changes files so that we process sourceful ones first changes_files.sort(utils.changes_compare); @@ -590,8 +591,10 @@ def main(): if not Options["No-Action"]: Logger.close(); - if not installing_to_stable: + if Urgency_Logger: Urgency_Logger.close(); +############################################################################### + if __name__ == '__main__': - main() + main(); diff --git a/katie.py b/katie.py index b2ac53a9..4595cb2d 100644 --- a/katie.py +++ b/katie.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # Utility functions for katie -# Copyright (C) 2001 James Troup -# $Id: katie.py,v 1.17 2002-04-29 22:00:44 troup Exp $ +# Copyright (C) 2001, 2002 James Troup +# $Id: katie.py,v 1.18 2002-05-08 11:52:31 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -54,7 +54,7 @@ class nmu_p: self.group_maint = {}; self.Cnf = Cnf; if Cnf.get("Dinstall::GroupOverrideFilename"): - filename = Cnf["Dir::OverrideDir"] + Cnf["Dinstall::GroupOverrideFilename"]; + filename = Cnf["Dir::Override"] + Cnf["Dinstall::GroupOverrideFilename"]; file = utils.open_file(filename); for line in file.readlines(): line = lower(string.strip(utils.re_comments.sub('', line))); @@ -152,7 +152,8 @@ class Katie: for i in [ "package", "version", "architecture", "type", "size", "md5sum", "component", "location id", "source package", "source version", "maintainer", "dbtype", "files id", - "new", "section", "priority", "oldfiles", "othercomponents" ]: + "new", "section", "priority", "oldfiles", "othercomponents", + "pool name" ]: if files[file].has_key(i): d_files[file][i] = files[file][i]; ## changes @@ -216,6 +217,11 @@ class Katie: if self.Cnf.has_key("Dinstall::TrackingServer") and changes.has_key("source"): Subst["__MAINTAINER_TO__"] = Subst["__MAINTAINER_TO__"] + "\nBcc: %s@%s" % (changes["source"], self.Cnf["Dinstall::TrackingServer"]) + # Apply any global override of the Maintainer field + if self.Cnf.get("Dinstall::OverrideMaintainer"): + Subst["__MAINTAINER_TO__"] = self.Cnf["Dinstall::OverrideMaintainer"]; + Subst["__MAINTAINER_FROM__"] = self.Cnf["Dinstall::OverrideMaintainer"]; + Subst["__REJECT_MESSAGE__"] = reject_message; Subst["__SOURCE__"] = changes.get("source", "Unknown"); Subst["__VERSION__"] = changes.get("version", "Unknown"); @@ -265,70 +271,85 @@ class Katie: ########################################################################### - def announce (self, short_summary, action): + def close_bugs (self, summary, action): + changes = self.pkg.changes; Subst = self.Subst; Cnf = self.Cnf; - changes = self.pkg.changes; - dsc = self.pkg.dsc; - # Only do announcements for source uploads with a recent dpkg-dev installed - if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"): - return "" - - lists_done = {} - summary = "" - Subst["__SHORT_SUMMARY__"] = short_summary; + bugs = changes["closes"].keys(); - for dist in changes["distribution"].keys(): - list = Cnf.Find("Suite::%s::Announce" % (dist)) - if list == "" or lists_done.has_key(list): - continue - lists_done[list] = 1 - summary = summary + "Announcing to %s\n" % (list) + if not bugs: + return summary; - if action: - Subst["__ANNOUNCE_LIST_ADDRESS__"] = list; - if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"): - Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]) - mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.announce","r").read()); - utils.send_mail (mail_message, "") - - bugs = changes["closes"].keys() - bugs.sort() + bugs.sort(); if not self.nmu.is_an_nmu(self.pkg): - summary = summary + "Closing bugs: " + summary = summary + "Closing bugs: "; for bug in bugs: - summary = summary + "%s " % (bug) + summary = summary + "%s " % (bug); if action: Subst["__BUG_NUMBER__"] = bug; if changes["distribution"].has_key("stable"): Subst["__STABLE_WARNING__"] = """ - Note that this package is not part of the released stable Debian - distribution. It may have dependencies on other unreleased software, - or other instabilities. Please take care if you wish to install it. - The update will eventually make its way into the next released Debian - distribution.""" +Note that this package is not part of the released stable Debian +distribution. It may have dependencies on other unreleased software, +or other instabilities. Please take care if you wish to install it. +The update will eventually make its way into the next released Debian +distribution."""; else: Subst["__STABLE_WARNING__"] = ""; - mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.bug-close","r").read()); - utils.send_mail (mail_message, "") + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-close"); + utils.send_mail (mail_message, ""); if action: self.Logger.log(["closing bugs"]+bugs); else: # NMU - summary = summary + "Setting bugs to severity fixed: " - control_message = "" + summary = summary + "Setting bugs to severity fixed: "; + control_message = ""; for bug in bugs: - summary = summary + "%s " % (bug) - control_message = control_message + "tag %s + fixed\n" % (bug) + summary = summary + "%s " % (bug); + control_message = control_message + "tag %s + fixed\n" % (bug); if action and control_message != "": Subst["__CONTROL_MESSAGE__"] = control_message; - mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.bug-nmu-fixed","r").read()); - utils.send_mail (mail_message, "") + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.bug-nmu-fixed"); + utils.send_mail (mail_message, ""); if action: self.Logger.log(["setting bugs to fixed"]+bugs); - summary = summary + "\n" + summary = summary + "\n"; + return summary; + + ########################################################################### + + def announce (self, short_summary, action): + Subst = self.Subst; + Cnf = self.Cnf; + changes = self.pkg.changes; + dsc = self.pkg.dsc; + + # Only do announcements for source uploads with a recent dpkg-dev installed + if float(changes.get("format", 0)) < 1.6 or not changes["architecture"].has_key("source"): + return ""; - return summary + lists_done = {}; + summary = ""; + Subst["__SHORT_SUMMARY__"] = short_summary; + + for dist in changes["distribution"].keys(): + list = Cnf.Find("Suite::%s::Announce" % (dist)); + if list == "" or lists_done.has_key(list): + continue; + lists_done[list] = 1; + summary = summary + "Announcing to %s\n" % (list); + + if action: + Subst["__ANNOUNCE_LIST_ADDRESS__"] = list; + if Cnf.get("Dinstall::TrackingServer") and changes["architecture"].has_key("source"): + Subst["__ANNOUNCE_LIST_ADDRESS__"] = Subst["__ANNOUNCE_LIST_ADDRESS__"] + "\nBcc: %s@%s" % (changes["source"], Cnf["Dinstall::TrackingServer"]); + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.announce"); + utils.send_mail (mail_message, ""); + + if Cnf.get("Dinstall::CloseBugs"): + summary = self.close_bugs(summary, action); + + return summary; ########################################################################### @@ -340,13 +361,13 @@ class Katie: print "Accepting." self.Logger.log(["Accepting changes",self.pkg.changes_file]); - self.dump_vars(Cnf["Dir::QueueAcceptedDir"]); + self.dump_vars(Cnf["Dir::Queue::Accepted"]); # Move all the files into the accepted directory - utils.move(self.pkg.changes_file, Cnf["Dir::QueueAcceptedDir"]); + utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]); file_keys = files.keys(); for file in file_keys: - utils.move(file, Cnf["Dir::QueueAcceptedDir"]); + utils.move(file, Cnf["Dir::Queue::Accepted"]); self.accept_bytes = self.accept_bytes + float(files[file]["size"]) self.accept_count = self.accept_count + 1; @@ -355,16 +376,16 @@ class Katie: if not Cnf["Dinstall::Options::No-Mail"]: Subst["__SUITE__"] = ""; Subst["__SUMMARY__"] = summary; - mail_message = utils.TemplateSubst(Subst,open(Cnf["Dir::TemplatesDir"]+"/jennifer.accepted","r").read()); + mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/jennifer.accepted"); utils.send_mail(mail_message, "") self.announce(short_summary, 1) # Special support to enable clean auto-building of accepted packages - if Cnf.get("Dinstall::SpecialAcceptedAutoBuild") and \ + if Cnf.FindB("Dinstall::SpecialAcceptedAutoBuild") and \ self.pkg.changes["distribution"].has_key("unstable"): self.projectB.query("BEGIN WORK"); for file in file_keys: - src = os.path.join(Cnf["Dir::QueueAcceptedDir"], file); + src = os.path.join(Cnf["Dir::Queue::Accepted"], file); dest = os.path.join(Cnf["Dir::AcceptedAutoBuild"], file); # Create a symlink to it os.symlink(src, dest); @@ -398,9 +419,15 @@ class Katie: Subst = self.Subst; changes = self.pkg.changes; files = self.pkg.files; + Cnf = self.Cnf; - # Only check section & priority on sourceful uploads - if not changes["architecture"].has_key("source"): + # Abandon the check if: + # a) it's a non-sourceful upload + # b) override disparity checks have been disabled + # c) we're not sending mail + if not changes["architecture"].has_key("source") or \ + not Cnf.FindB("Dinstall::OverrideDisparityCheck") or \ + Cnf["Dinstall::Options::No-Mail"]: return; summary = ""; @@ -422,7 +449,7 @@ class Katie: return; Subst["__SUMMARY__"] = summary; - mail_message = utils.TemplateSubst(Subst,utils.open_file(self.Cnf["Dir::TemplatesDir"]+"/jennifer.override-disparity").read()); + mail_message = utils.TemplateSubst(Subst,self.Cnf["Dir::Templates"]+"/jennifer.override-disparity"); utils.send_mail (mail_message, ""); ########################################################################### @@ -438,13 +465,13 @@ class Katie: # Skip any files which don't exist or which we don't have permission to copy. if os.access(file,os.R_OK) == 0: continue; - dest_file = os.path.join(Cnf["Dir::QueueRejectDir"], file); + dest_file = os.path.join(Cnf["Dir::Queue::Reject"], file); try: os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644); except OSError, e: # File exists? Let's try and move it to the morgue if errno.errorcode[e.errno] == 'EEXIST': - morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueRejectDir"],file); + morgue_file = os.path.join(Cnf["Dir::Morgue"],Cnf["Dir::MorgueReject"],file); try: morgue_file = utils.find_next_free(morgue_file); except utils.tried_too_hard_exc: @@ -504,7 +531,7 @@ class Katie: pkg = self.pkg; reason_filename = pkg.changes_file[:-8] + ".reason"; - reject_filename = Cnf["Dir::QueueRejectDir"] + '/' + reason_filename; + reject_filename = Cnf["Dir::Queue::Reject"] + '/' + reason_filename; # Move all the files into the reject directory reject_files = pkg.files.keys() + [pkg.changes_file]; @@ -522,7 +549,7 @@ class Katie: Subst["__CC__"] = "X-Katie-Rejection: automatic (moo)"; os.write(fd, reject_message); os.close(fd); - reject_mail_message = utils.TemplateSubst(Subst,utils.open_file(Cnf["Dir::TemplatesDir"]+"/katie.rejected").read()); + reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected"); else: # Build up the rejection email user_email_address = utils.whoami() + " <%s>" % (Cnf["Dinstall::MyAdminAddress"]); @@ -530,7 +557,7 @@ class Katie: Subst["__REJECTOR_ADDRESS__"] = user_email_address; Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message; Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"]; - reject_mail_message = utils.TemplateSubst(Subst,utils.open_file(Cnf["Dir::TemplatesDir"]+"/katie.rejected").read()); + reject_mail_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/katie.rejected"); # Write the rejection email out as the .reason file os.write(fd, reject_mail_message); @@ -684,7 +711,7 @@ class Katie: actual_size = int(files[dsc_file]["size"]); found = "%s in incoming" % (dsc_file) # Check the file does not already exist in the archive - q = self.projectB.query("SELECT f.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); + q = self.projectB.query("SELECT f.size, f.md5sum FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); # "It has not broken them. It has fixed a # brokenness. Your crappy hack exploited a bug in @@ -695,8 +722,19 @@ class Katie: # the same name and version.)" # -- ajk@ on d-devel@l.d.o - if q.getresult() != []: - self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file)); + ql = q.getresult(); + if ql: + match = 0; + if dsc_file[-12:] == ".orig.tar.gz": + for i in ql: + if int(files[dsc_file]["size"]) == int(i[0]) and \ + files[dsc_file]["md5sum"] == i[1]: + self.reject("ignoring %s, since it's already in the archive." % (dsc_file), "Warning: "); + del files[dsc_file]; + match = 1; + + if not match: + self.reject("can not overwrite existing copy of '%s' already in the archive." % (dsc_file)); elif dsc_file[-12:] == ".orig.tar.gz": # Check in the pool q = self.projectB.query("SELECT l.path, f.filename, l.type, f.id, l.id FROM files f, location l WHERE (f.filename ~ '/%s$' OR f.filename = '%s') AND l.id = f.location" % (utils.regex_safe(dsc_file), dsc_file)); @@ -735,13 +773,13 @@ class Katie: else: # Not there? Check the queue directories... - in_unchecked = os.path.join(self.Cnf["Dir::QueueUncheckedDir"],dsc_file); + in_unchecked = os.path.join(self.Cnf["Dir::Queue::Unchecked"],dsc_file); # See process_it() in jennifer for explanation of this if os.path.exists(in_unchecked): return (self.reject_message, in_unchecked); else: for dir in [ "Accepted", "New", "Byhand" ]: - in_otherdir = os.path.join(self.Cnf["Dir::Queue%sDir" % (dir)],dsc_file); + in_otherdir = os.path.join(self.Cnf["Dir::Queue::%s" % (dir)],dsc_file); if os.path.exists(in_otherdir): actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir)); actual_size = os.stat(in_otherdir)[stat.ST_SIZE]; -- 2.39.5