From: Frank Lichtenheld Date: Mon, 26 Oct 2009 19:19:27 +0000 (+0000) Subject: Convert process_new to new APIs X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eb7609bc253b856d1463610e4daa4f4c7aec7e90;p=dak Convert process_new to new APIs All code only used in security and/or stable updates is commented out for now and not converted, yet --- diff --git a/dak/process_new.py b/dak/process_new.py index 3a8dbc7b..d73ca356 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -6,6 +6,7 @@ @contact: Debian FTP Master @copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup @copyright: 2009 Joerg Jaspert +@copyright: 2009 Frank Lichtenheld @license: GNU General Public License version 2 or later """ # This program is free software; you can redistribute it and/or modify @@ -55,40 +56,30 @@ import pwd import apt_pkg, apt_inst import examine_package -from daklib import database +from daklib.dbconn import * +from daklib.queue import * from daklib import daklog -from daklib import queue from daklib import utils from daklib.regexes import re_no_epoch, re_default_answer, re_isanum from daklib.dak_exceptions import CantOpenError, AlreadyLockedError, CantGetLockError from daklib.summarystats import SummaryStats +from daklib.config import Config # Globals -Cnf = None #: Configuration, apt_pkg.Configuration Options = None -Upload = None -projectB = None #: database connection, pgobject Logger = None Priorities = None Sections = None -reject_message = "" - ################################################################################ ################################################################################ ################################################################################ -def reject (str, prefix="Rejected: "): - global reject_message - if str: - reject_message += prefix + str + "\n" - -def recheck(): - global reject_message - files = Upload.pkg.files - reject_message = "" +def recheck(upload, session): + files = upload.pkg.files + cnf = Config() for f in files.keys(): # The .orig.tar.gz can disappear out from under us is it's a # duplicate of one in the archive. @@ -98,32 +89,31 @@ def recheck(): if files[f]["type"] == "deb": source_version = files[f]["source version"] source_package = files[f]["source package"] - if not Upload.pkg.changes["architecture"].has_key("source") \ - and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()): + if not upload.pkg.changes["architecture"].has_key("source") \ + and not upload.source_exists(source_package, source_version, upload.pkg.changes["distribution"].keys()): source_epochless_version = re_no_epoch.sub('', source_version) dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version) found = 0 for q in ["Accepted", "Embargoed", "Unembargoed", "Newstage"]: - if Cnf.has_key("Dir::Queue::%s" % (q)): - if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename): + if cnf.has_key("Dir::Queue::%s" % (q)): + if os.path.exists(cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename): found = 1 if not found: - reject("no source found for %s %s (%s)." % (source_package, source_version, f)) + upload.rejects.append("no source found for %s %s (%s)." % (source_package, source_version, f)) # Version and file overwrite checks if files[f]["type"] == "deb": - reject(Upload.check_binary_against_db(f), "") + upload.check_binary_against_db(f, session) elif files[f]["type"] == "dsc": - reject(Upload.check_source_against_db(f), "") - (reject_msg, is_in_incoming) = Upload.check_dsc_against_db(f) - reject(reject_msg, "") + upload.check_source_against_db(f, session) + upload.check_dsc_against_db(f, session) - if reject_message.find("Rejected") != -1: + if len(upload.rejects) > 0: answer = "XXX" if Options["No-Action"] or Options["Automatic"] or Options["Trainee"]: answer = 'S' - print "REJECT\n" + reject_message, + print "REJECT\n" + upload.rejects.join("\n"), prompt = "[R]eject, Skip, Quit ?" while prompt.find(answer) == -1: @@ -134,8 +124,8 @@ def recheck(): answer = answer[:1].upper() if answer == 'R': - Upload.do_reject(0, reject_message) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + upload.do_reject(manual=0, reject_message=upload.rejects.join("\n")) + os.unlink(upload.pkg.changes_file[:-8]+".dak") return 0 elif answer == 'S': return 0 @@ -186,7 +176,7 @@ def sg_compare (a, b): # Sort by time of oldest upload return cmp(a["oldest"], b["oldest"]) -def sort_changes(changes_files): +def sort_changes(changes_files, session): """Sort into source groups, then sort each source group by version, have source, filename. Finally, sort the source groups by have note, time of oldest upload of each source upload.""" @@ -197,11 +187,11 @@ def sort_changes(changes_files): cache = {} # Read in all the .changes files for filename in changes_files: + u = Upload() try: - Upload.pkg.changes_file = filename - Upload.init_vars() - Upload.update_vars() - cache[filename] = copy.copy(Upload.pkg.changes) + u.pkg.load_dot_dak(filename) + u.update_subst() + cache[filename] = copy.copy(u.pkg.changes) cache[filename]["filename"] = filename except: sorted_list.append(filename) @@ -216,9 +206,8 @@ def sort_changes(changes_files): per_source[source]["list"].append(cache[filename]) # Determine oldest time and have note status for each source group for source in per_source.keys(): - q = projectB.query("SELECT 1 FROM source WHERE source = '%s'" % source) - ql = q.getresult() - per_source[source]["source_in_database"] = len(ql)>0 + q = session.query(DBSource).filter_by(source = source).all() + per_source[source]["source_in_database"] = len(q)>0 source_list = per_source[source]["list"] first = source_list[0] oldest = os.stat(first["filename"])[stat.ST_MTIME] @@ -227,7 +216,7 @@ def sort_changes(changes_files): mtime = os.stat(d["filename"])[stat.ST_MTIME] if mtime < oldest: oldest = mtime - have_note += (database.has_new_comment(d["source"], d["version"], True)) + have_note += has_new_comment(d["source"], d["version"], session) per_source[source]["oldest"] = oldest if not have_note: per_source[source]["note_state"] = 0; # none @@ -246,12 +235,11 @@ def sort_changes(changes_files): ################################################################################ class Section_Completer: - def __init__ (self): + def __init__ (self, session): self.sections = [] self.matches = [] - q = projectB.query("SELECT section FROM section") - for i in q.getresult(): - self.sections.append(i[0]) + for s, in session.query(Section.section): + self.sections.append(s) def complete(self, text, state): if state == 0: @@ -268,12 +256,11 @@ class Section_Completer: ############################################################ class Priority_Completer: - def __init__ (self): + def __init__ (self, session): self.priorities = [] self.matches = [] - q = projectB.query("SELECT priority FROM priority") - for i in q.getresult(): - self.priorities.append(i[0]) + for p, in session.query(Priority.priority): + self.priorities.append(p) def complete(self, text, state): if state == 0: @@ -289,9 +276,9 @@ class Priority_Completer: ################################################################################ -def print_new (new, indexed, file=sys.stdout): - queue.check_valid(new) - broken = 0 +def print_new (new, upload, indexed, file=sys.stdout): + check_valid(new) + broken = False index = 0 for pkg in new.keys(): index += 1 @@ -299,21 +286,22 @@ def print_new (new, indexed, file=sys.stdout): priority = new[pkg]["priority"] if new[pkg]["section id"] == -1: section += "[!]" - broken = 1 + broken = True if new[pkg]["priority id"] == -1: priority += "[!]" - broken = 1 + broken = True if indexed: line = "(%s): %-20s %-20s %-20s" % (index, pkg, priority, section) else: line = "%-20s %-20s %-20s" % (pkg, priority, section) line = line.strip()+'\n' file.write(line) - note = database.get_new_comments(Upload.pkg.changes.get("source")) - if len(note) > 0: - for line in note: - print line - return broken, note + notes = get_new_comments(upload.pkg.changes.get("source")) + for note in notes: + print "\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \ + % (note.author, note.version, note.notedate, note.comment) + print "-" * 72 + return broken, len(notes) > 0 ################################################################################ @@ -326,11 +314,11 @@ def index_range (index): ################################################################################ ################################################################################ -def edit_new (new): +def edit_new (new, upload): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() temp_file = os.fdopen(fd, 'w') - print_new (new, 0, temp_file) + print_new (new, upload, indexed=0, file=temp_file) temp_file.close() # Spawn an editor on that file editor = os.environ.get("EDITOR","vi") @@ -360,14 +348,14 @@ def edit_new (new): if priority.endswith("[!]"): priority = priority[:-3] for f in new[pkg]["files"]: - Upload.pkg.files[f]["section"] = section - Upload.pkg.files[f]["priority"] = priority + upload.pkg.files[f]["section"] = section + upload.pkg.files[f]["priority"] = priority new[pkg]["section"] = section new[pkg]["priority"] = priority ################################################################################ -def edit_index (new, index): +def edit_index (new, upload, index): priority = new[index]["priority"] section = new[index]["section"] ftype = new[index]["type"] @@ -426,19 +414,19 @@ def edit_index (new, index): readline.set_completer(None) for f in new[index]["files"]: - Upload.pkg.files[f]["section"] = section - Upload.pkg.files[f]["priority"] = priority + upload.pkg.files[f]["section"] = section + upload.pkg.files[f]["priority"] = priority new[index]["priority"] = priority new[index]["section"] = section return new ################################################################################ -def edit_overrides (new): +def edit_overrides (new, upload, session): print done = 0 while not done: - print_new (new, 1) + print_new (new, upload, indexed=1) new_index = {} index = 0 for i in new.keys(): @@ -462,17 +450,17 @@ def edit_overrides (new): got_answer = 1 if answer == 'E': - edit_new(new) + edit_new(new, upload) elif answer == 'D': done = 1 else: - edit_index (new, new_index[answer]) + edit_index (new, upload, new_index[answer]) return new ################################################################################ -def edit_note(note): +def edit_note(note, upload, session): # Write the current data to a temporary file (fd, temp_filename) = utils.temp_filename() editor = os.environ.get("EDITOR","vi") @@ -499,19 +487,26 @@ def edit_note(note): end() sys.exit(0) - database.add_new_comment(Upload.pkg.changes["source"], Upload.pkg.changes["version"], newnote, utils.whoami(), bool(Options["Trainee"])) + comment = NewComment() + comment.package = upload.pkg.changes["source"] + comment.version = upload.pkg.changes["version"] + comment.comment = newnote + comment.author = utils.whoami() + comment.trainee = bool(Options["Trainee"]) + session.add(comment) + session.commit() ################################################################################ -def check_pkg (): +def check_pkg (upload): try: less_fd = os.popen("less -R -", 'w', 0) stdout_fd = sys.stdout try: sys.stdout = less_fd - changes = utils.parse_changes (Upload.pkg.changes_file) - examine_package.display_changes(changes['distribution'], Upload.pkg.changes_file) - files = Upload.pkg.files + changes = utils.parse_changes (upload.pkg.changes_file) + examine_package.display_changes(changes['distribution'], upload.pkg.changes_file) + files = upload.pkg.files for f in files.keys(): if files[f].has_key("new"): ftype = files[f]["type"] @@ -536,49 +531,50 @@ def check_pkg (): ## FIXME: horribly Debian specific -def do_bxa_notification(): - files = Upload.pkg.files - summary = "" - for f in files.keys(): - if files[f]["type"] == "deb": - control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f))) - summary += "\n" - summary += "Package: %s\n" % (control.Find("Package")) - summary += "Description: %s\n" % (control.Find("Description")) - Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary - bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification") - utils.send_mail(bxa_mail) +# def do_bxa_notification(): +# files = Upload.pkg.files +# summary = "" +# for f in files.keys(): +# if files[f]["type"] == "deb": +# control = apt_pkg.ParseSection(apt_inst.debExtractControl(utils.open_file(f))) +# summary += "\n" +# summary += "Package: %s\n" % (control.Find("Package")) +# summary += "Description: %s\n" % (control.Find("Description")) +# Upload.Subst["__BINARY_DESCRIPTIONS__"] = summary +# bxa_mail = utils.TemplateSubst(Upload.Subst,Cnf["Dir::Templates"]+"/process-new.bxa_notification") +# utils.send_mail(bxa_mail) ################################################################################ -def add_overrides (new): - changes = Upload.pkg.changes - files = Upload.pkg.files +def add_overrides (new, upload, session): + changes = upload.pkg.changes + files = upload.pkg.files srcpkg = changes.get("source") - projectB.query("BEGIN WORK") for suite in changes["suite"].keys(): - suite_id = database.get_suite_id(suite) + suite_id = get_suite(suite).suite_id for pkg in new.keys(): - component_id = database.get_component_id(new[pkg]["component"]) - type_id = database.get_override_type_id(new[pkg]["type"]) + component_id = get_component(new[pkg]["component"]).component_id + type_id = get_override_type(new[pkg]["type"]).overridetype_id priority_id = new[pkg]["priority id"] section_id = new[pkg]["section id"] Logger.log(["%s overrides" % (srcpkg), suite, new[pkg]["component"], new[pkg]["type"], new[pkg]["priority"], new[pkg]["section"]]) - projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)) + session.execute("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (:sid, :cid, :tid, :pkg, :pid, :sectid, '')", + { 'sid': suite.suiteid, 'cid': component_id, 'tid':type_id, 'pkg': pkg, 'pid': priority_id, 'sectid': section_id}) for f in new[pkg]["files"]: if files[f].has_key("new"): del files[f]["new"] del new[pkg] - projectB.query("COMMIT WORK") + session.commit() - if Cnf.FindB("Dinstall::BXANotify"): - do_bxa_notification() +# if Cnf.FindB("Dinstall::BXANotify"): +# do_bxa_notification() ################################################################################ -def prod_maintainer (note): +def prod_maintainer (note, upload): + cnf = Config() # Here we prepare an editor and get them ready to prod... (fd, temp_filename) = utils.temp_filename() temp_file = os.fdopen(fd, 'w') @@ -611,39 +607,40 @@ def prod_maintainer (note): sys.exit(0) # Otherwise, do the proding... user_email_address = utils.whoami() + " <%s>" % ( - Cnf["Dinstall::MyAdminAddress"]) + cnf["Dinstall::MyAdminAddress"]) - Subst = Upload.Subst + Subst = upload.Subst Subst["__FROM_ADDRESS__"] = user_email_address Subst["__PROD_MESSAGE__"] = prod_message - Subst["__CC__"] = "Cc: " + Cnf["Dinstall::MyEmailAddress"] + Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"] prod_mail_message = utils.TemplateSubst( - Subst,Cnf["Dir::Templates"]+"/process-new.prod") + Subst,cnf["Dir::Templates"]+"/process-new.prod") # Send the prod mail if appropriate - if not Cnf["Dinstall::Options::No-Mail"]: + if not cnf["Dinstall::Options::No-Mail"]: utils.send_mail(prod_mail_message) print "Sent proding message" ################################################################################ -def do_new(): +def do_new(upload, session): print "NEW\n" - files = Upload.pkg.files - changes = Upload.pkg.changes + files = upload.pkg.files + changes = upload.pkg.changes + cnf = Config() # Make a copy of distribution we can happily trample on changes["suite"] = copy.copy(changes["distribution"]) # Fix up the list of target suites for suite in changes["suite"].keys(): - override = Cnf.Find("Suite::%s::OverrideSuite" % (suite)) + override = cnf.Find("Suite::%s::OverrideSuite" % (suite)) if override: - (olderr, newerr) = (database.get_suite_id(suite) == -1, - database.get_suite_id(override) == -1) + (olderr, newerr) = (get_suite(suite, session) == None, + get_suite(override, session) == None) if olderr or newerr: (oinv, newinv) = ("", "") if olderr: oinv = "invalid " @@ -654,15 +651,14 @@ def do_new(): changes["suite"][override] = 1 # Validate suites for suite in changes["suite"].keys(): - suite_id = database.get_suite_id(suite) - if suite_id == -1: + if get_suite(suite, session) is None: utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite)) # The main NEW processing loop done = 0 while not done: # Find out what's new - new = queue.determine_new(changes, files, projectB) + new = determine_new(changes, files) if not new: break @@ -671,7 +667,7 @@ def do_new(): if Options["No-Action"] or Options["Automatic"]: answer = 'S' - (broken, note) = print_new(new, 0) + (broken, note) = print_new(new, upload, indexed=0) prompt = "" if not broken and not note: @@ -698,36 +694,43 @@ def do_new(): if answer == 'A' and not Options["Trainee"]: try: check_daily_lock() - done = add_overrides (new) - Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (Upload.pkg.changes_file)]) + done = add_overrides (new, upload, session) + Logger.log([utils.getusername(), "NEW ACCEPT: %s" % (upload.pkg.changes_file)]) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'C': - check_pkg() + check_pkg(upload) elif answer == 'E' and not Options["Trainee"]: - new = edit_overrides (new) + new = edit_overrides (new, upload, session) elif answer == 'M' and not Options["Trainee"]: - aborted = Upload.do_reject(manual=1, + aborted = upload.do_reject(manual=1, reject_message=Options["Manual-Reject"], - note=database.get_new_comments(changes.get("source", ""))) + note=get_new_comments(changes.get("source", ""), session=session)) if not aborted: - Logger.log([utils.getusername(), "NEW REJECT: %s" % (Upload.pkg.changes_file)]) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + Logger.log([utils.getusername(), "NEW REJECT: %s" % (upload.pkg.changes_file)]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'N': - edit_note(database.get_new_comments(changes.get("source", ""))) + edit_note(get_new_comments(changes.get("source", ""), session=session), + upload, session) elif answer == 'P' and not Options["Trainee"]: - prod_maintainer(database.get_new_comments(changes.get("source", ""))) - Logger.log([utils.getusername(), "NEW PROD: %s" % (Upload.pkg.changes_file)]) + prod_maintainer(get_new_comments(changes.get("source", ""), session=session), + upload) + Logger.log([utils.getusername(), "NEW PROD: %s" % (upload.pkg.changes_file)]) elif answer == 'R' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear note (y/N)? ").lower() if confirm == "y": - database.delete_new_comments(changes.get("source"), changes.get("version")) + for c in get_new_comments(changes.get("source", ""), changes.get("version", ""), session=session): + session.delete(c) + session.commit() elif answer == 'O' and not Options["Trainee"]: confirm = utils.our_raw_input("Really clear all notes (y/N)? ").lower() if confirm == "y": - database.delete_all_new_comments(changes.get("source")) + for c in get_new_comments(changes.get("source", ""), session=session): + session.delete(c) + session.commit() + elif answer == 'S': done = 1 elif answer == 'Q': @@ -751,53 +754,10 @@ def usage (exit_code=0): ################################################################################ -def init(): - global Cnf, Options, Logger, Upload, projectB, Sections, Priorities - - Cnf = utils.get_conf() - - Arguments = [('a',"automatic","Process-New::Options::Automatic"), - ('h',"help","Process-New::Options::Help"), - ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"), - ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), - ('t',"trainee","Process-New::Options::Trainee"), - ('n',"no-action","Process-New::Options::No-Action")] - - for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]: - if not Cnf.has_key("Process-New::Options::%s" % (i)): - Cnf["Process-New::Options::%s" % (i)] = "" - - changes_files = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) - if len(changes_files) == 0 and not Cnf.get("Process-New::Options::Comments-Dir",""): - changes_files = utils.get_changes_files(Cnf["Dir::Queue::New"]) - - Options = Cnf.SubTree("Process-New::Options") - - if Options["Help"]: - usage() - - Upload = queue.Upload(Cnf) - - if not Options["No-Action"]: - try: - Logger = Upload.Logger = daklog.Logger(Cnf, "process-new") - except CantOpenError, e: - Options["Trainee"] = "True" - - projectB = Upload.projectB - - Sections = Section_Completer() - Priorities = Priority_Completer() - readline.parse_and_bind("tab: complete") - - return changes_files - -################################################################################ - -def do_byhand(): +def do_byhand(upload, session): done = 0 while not done: - files = Upload.pkg.files + files = upload.pkg.files will_install = 1 byhand = [] @@ -832,14 +792,14 @@ def do_byhand(): done = 1 for f in byhand: del files[f] - Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (Upload.pkg.changes_file)]) + Logger.log([utils.getusername(), "BYHAND ACCEPT: %s" % (upload.pkg.changes_file)]) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" elif answer == 'M': - Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (Upload.pkg.changes_file)]) - Upload.do_reject(1, Options["Manual-Reject"]) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + Logger.log([utils.getusername(), "BYHAND REJECT: %s" % (upload.pkg.changes_file)]) + upload.do_reject(manual=1, reject_message=Options["Manual-Reject"]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") done = 1 elif answer == 'S': done = 1 @@ -854,13 +814,15 @@ def check_daily_lock(): Raises CantGetLockError if the dinstall daily.lock exists. """ + cnf = Config() try: - os.open(Cnf["Process-New::DinstallLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL) + os.open(cnf["Process-New::DinstallLockFile"], + os.O_RDONLY | os.O_CREAT | os.O_EXCL) except OSError, e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: raise CantGetLockError - os.unlink(Cnf["Process-New::DinstallLockFile"]) + os.unlink(cnf["Process-New::DinstallLockFile"]) @contextlib.contextmanager @@ -872,7 +834,7 @@ def lock_package(package): @param package: source package name to lock """ - path = os.path.join(Cnf["Process-New::LockDir"], package) + path = os.path.join(Config()["Process-New::LockDir"], package) try: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) except OSError, e: @@ -885,115 +847,118 @@ def lock_package(package): finally: os.unlink(path) -def move_to_dir (dest, perms=0660, changesperms=0664): - utils.move (Upload.pkg.changes_file, dest, perms=changesperms) - file_keys = Upload.pkg.files.keys() - for f in file_keys: - utils.move (f, dest, perms=perms) - -def is_source_in_queue_dir(qdir): - entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"]) - and x.endswith(".changes") ] - for entry in entries: - # read the .dak - u = queue.Upload(Cnf) - u.pkg.changes_file = os.path.join(qdir, entry) - u.update_vars() - if not u.pkg.changes["architecture"].has_key("source"): - # another binary upload, ignore - continue - if Upload.pkg.changes["version"] != u.pkg.changes["version"]: - # another version, ignore - continue - # found it! - return True - return False - -def move_to_holding(suite, queue_dir): - print "Moving to %s holding area." % (suite.upper(),) - if Options["No-Action"]: - return - Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file]) - Upload.dump_vars(queue_dir) - move_to_dir(queue_dir, perms=0664) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") - -def _accept(): +# def move_to_dir (upload, dest, perms=0660, changesperms=0664): +# utils.move (upload.pkg.changes_file, dest, perms=changesperms) +# file_keys = upload.pkg.files.keys() +# for f in file_keys: +# utils.move (f, dest, perms=perms) + +# def is_source_in_queue_dir(qdir): +# entries = [ x for x in os.listdir(qdir) if x.startswith(Upload.pkg.changes["source"]) +# and x.endswith(".changes") ] +# for entry in entries: +# # read the .dak +# u = queue.Upload(Cnf) +# u.pkg.changes_file = os.path.join(qdir, entry) +# u.update_vars() +# if not u.pkg.changes["architecture"].has_key("source"): +# # another binary upload, ignore +# continue +# if Upload.pkg.changes["version"] != u.pkg.changes["version"]: +# # another version, ignore +# continue +# # found it! +# return True +# return False + +# def move_to_holding(suite, queue_dir): +# print "Moving to %s holding area." % (suite.upper(),) +# if Options["No-Action"]: +# return +# Logger.log(["Moving to %s" % (suite,), Upload.pkg.changes_file]) +# Upload.dump_vars(queue_dir) +# move_to_dir(queue_dir, perms=0664) +# os.unlink(Upload.pkg.changes_file[:-8]+".dak") + +def _accept(upload): if Options["No-Action"]: return - (summary, short_summary) = Upload.build_summaries() - Upload.accept(summary, short_summary, targetdir=Cnf["Dir::Queue::Newstage"]) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") - -def do_accept_stableupdate(suite, q): - queue_dir = Cnf["Dir::Queue::%s" % (q,)] - if not Upload.pkg.changes["architecture"].has_key("source"): - # It is not a sourceful upload. So its source may be either in p-u - # holding, in new, in accepted or already installed. - if is_source_in_queue_dir(queue_dir): - # It's in p-u holding, so move it there. - print "Binary-only upload, source in %s." % (q,) - move_to_holding(suite, queue_dir) - elif Upload.source_exists(Upload.pkg.changes["source"], - Upload.pkg.changes["version"]): - # dak tells us that there is source available. At time of - # writing this means that it is installed, so put it into - # accepted. - print "Binary-only upload, source installed." - Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) - _accept() - elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]): - # The source is in accepted, the binary cleared NEW: accept it. - print "Binary-only upload, source in accepted." - Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) - _accept() - elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]): - # It's in NEW. We expect the source to land in p-u holding - # pretty soon. - print "Binary-only upload, source in new." - move_to_holding(suite, queue_dir) - elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]): - # It's in newstage. Accept into the holding area - print "Binary-only upload, source in newstage." - Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) - _accept() - else: - # No case applicable. Bail out. Return will cause the upload - # to be skipped. - print "ERROR" - print "Stable update failed. Source not found." - return - else: - # We are handling a sourceful upload. Move to accepted if currently - # in p-u holding and to p-u holding otherwise. - if is_source_in_queue_dir(queue_dir): - print "Sourceful upload in %s, accepting." % (q,) - _accept() - else: - move_to_holding(suite, queue_dir) - -def do_accept(): + (summary, short_summary) = upload.build_summaries() + upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"]) + os.unlink(upload.pkg.changes_file[:-8]+".dak") + +# def do_accept_stableupdate(upload,suite, q): +# cnf = Config() +# queue_dir = cnf["Dir::Queue::%s" % (q,)] +# if not upload.pkg.changes["architecture"].has_key("source"): +# # It is not a sourceful upload. So its source may be either in p-u +# # holding, in new, in accepted or already installed. +# if is_source_in_queue_dir(queue_dir): +# # It's in p-u holding, so move it there. +# print "Binary-only upload, source in %s." % (q,) +# move_to_holding(suite, queue_dir) +# elif Upload.source_exists(Upload.pkg.changes["source"], +# Upload.pkg.changes["version"]): +# # dak tells us that there is source available. At time of +# # writing this means that it is installed, so put it into +# # accepted. +# print "Binary-only upload, source installed." +# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) +# _accept() +# elif is_source_in_queue_dir(Cnf["Dir::Queue::Accepted"]): +# # The source is in accepted, the binary cleared NEW: accept it. +# print "Binary-only upload, source in accepted." +# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) +# _accept() +# elif is_source_in_queue_dir(Cnf["Dir::Queue::New"]): +# # It's in NEW. We expect the source to land in p-u holding +# # pretty soon. +# print "Binary-only upload, source in new." +# move_to_holding(suite, queue_dir) +# elif is_source_in_queue_dir(Cnf["Dir::Queue::Newstage"]): +# # It's in newstage. Accept into the holding area +# print "Binary-only upload, source in newstage." +# Logger.log([utils.getusername(), "PUNEW ACCEPT: %s" % (Upload.pkg.changes_file)]) +# _accept() +# else: +# # No case applicable. Bail out. Return will cause the upload +# # to be skipped. +# print "ERROR" +# print "Stable update failed. Source not found." +# return +# else: +# # We are handling a sourceful upload. Move to accepted if currently +# # in p-u holding and to p-u holding otherwise. +# if is_source_in_queue_dir(queue_dir): +# print "Sourceful upload in %s, accepting." % (q,) +# _accept() +# else: +# move_to_holding(suite, queue_dir) + +def do_accept(upload): print "ACCEPT" + cnf = Config() if not Options["No-Action"]: - (summary, short_summary) = Upload.build_summaries() - if Cnf.FindB("Dinstall::SecurityQueueHandling"): - Upload.dump_vars(Cnf["Dir::Queue::Embargoed"]) - move_to_dir(Cnf["Dir::Queue::Embargoed"]) - Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"]) - # Check for override disparities - Upload.Subst["__SUMMARY__"] = summary - else: + (summary, short_summary) = upload.build_summaries() +# if cnf.FindB("Dinstall::SecurityQueueHandling"): +# upload.dump_vars(cnf["Dir::Queue::Embargoed"]) +# move_to_dir(cnf["Dir::Queue::Embargoed"]) +# upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) +# # Check for override disparities +# upload.Subst["__SUMMARY__"] = summary +# else: # Stable updates need to be copied to proposed-updates holding # area instead of accepted. Sourceful uploads need to go # to it directly, binaries only if the source has not yet been # accepted into p-u. for suite, q in [("proposed-updates", "ProposedUpdates"), ("oldstable-proposed-updates", "OldProposedUpdates")]: - if not Upload.pkg.changes["distribution"].has_key(suite): + if not upload.pkg.changes["distribution"].has_key(suite): continue - return do_accept_stableupdate(suite, q) + utils.fubar("stable accept not supported yet") +# return do_accept_stableupdate(suite, q) # Just a normal upload, accept it... - _accept() + _accept(upload) def check_status(files): new = byhand = 0 @@ -1004,30 +969,29 @@ def check_status(files): new = 1 return (new, byhand) -def do_pkg(changes_file): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - Upload.update_subst() - files = Upload.pkg.files +def do_pkg(changes_file, session): + u = Upload() + u.pkg.load_dot_dak(changes_file) + u.update_subst() + files = u.pkg.files try: - with lock_package(Upload.pkg.changes["source"]): - if not recheck(): + with lock_package(u.pkg.changes["source"]): + if not recheck(u, session): return (new, byhand) = check_status(files) if new or byhand: if new: - do_new() + do_new(u, session) if byhand: - do_byhand() + do_byhand(u, session) (new, byhand) = check_status(files) if not new and not byhand: try: check_daily_lock() - do_accept() + do_accept(u) except CantGetLockError: print "Hello? Operator! Give me the number for 911!" print "Dinstall in the locked area, cant process packages, come back later" @@ -1052,87 +1016,122 @@ def end(): ################################################################################ -def do_comments(dir, opref, npref, line, fn): - for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: - lines = open("%s/%s" % (dir, comm)).readlines() - if len(lines) == 0 or lines[0] != line + "\n": continue - changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_") - and x.endswith(".changes") ] - changes_files = sort_changes(changes_files) - for f in changes_files: - f = utils.validate_changes_file_arg(f, 0) - if not f: continue - print "\n" + f - fn(f, "".join(lines[1:])) - - if opref != npref and not Options["No-Action"]: - newcomm = npref + comm[len(opref):] - os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm)) +# def do_comments(dir, opref, npref, line, fn): +# for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]: +# lines = open("%s/%s" % (dir, comm)).readlines() +# if len(lines) == 0 or lines[0] != line + "\n": continue +# changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_") +# and x.endswith(".changes") ] +# changes_files = sort_changes(changes_files) +# for f in changes_files: +# f = utils.validate_changes_file_arg(f, 0) +# if not f: continue +# print "\n" + f +# fn(f, "".join(lines[1:])) + +# if opref != npref and not Options["No-Action"]: +# newcomm = npref + comm[len(opref):] +# os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm)) + +# ################################################################################ + +# def comment_accept(changes_file, comments): +# Upload.pkg.changes_file = changes_file +# Upload.init_vars() +# Upload.update_vars() +# Upload.update_subst() +# files = Upload.pkg.files + +# if not recheck(): +# return # dak wants to REJECT, crap + +# (new, byhand) = check_status(files) +# if not new and not byhand: +# do_accept() + +# ################################################################################ + +# def comment_reject(changes_file, comments): +# Upload.pkg.changes_file = changes_file +# Upload.init_vars() +# Upload.update_vars() +# Upload.update_subst() + +# if not recheck(): +# pass # dak has its own reasons to reject as well, which is fine + +# reject(comments) +# print "REJECT\n" + reject_message, +# if not Options["No-Action"]: +# Upload.do_reject(0, reject_message) +# os.unlink(Upload.pkg.changes_file[:-8]+".dak") ################################################################################ -def comment_accept(changes_file, comments): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - Upload.update_subst() - files = Upload.pkg.files +def main(): + global Options, Logger, Sections, Priorities - if not recheck(): - return # dak wants to REJECT, crap + cnf = Config() + session = DBConn().session() + + Arguments = [('a',"automatic","Process-New::Options::Automatic"), + ('h',"help","Process-New::Options::Help"), + ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"), + ('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"), + ('t',"trainee","Process-New::Options::Trainee"), + ('n',"no-action","Process-New::Options::No-Action")] - (new, byhand) = check_status(files) - if not new and not byhand: - do_accept() + for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir", "trainee"]: + if not cnf.has_key("Process-New::Options::%s" % (i)): + cnf["Process-New::Options::%s" % (i)] = "" -################################################################################ + changes_files = apt_pkg.ParseCommandLine(cnf.Cnf,Arguments,sys.argv) + if len(changes_files) == 0 and not cnf.get("Process-New::Options::Comments-Dir",""): + changes_files = utils.get_changes_files(cnf["Dir::Queue::New"]) -def comment_reject(changes_file, comments): - Upload.pkg.changes_file = changes_file - Upload.init_vars() - Upload.update_vars() - Upload.update_subst() + Options = cnf.SubTree("Process-New::Options") - if not recheck(): - pass # dak has its own reasons to reject as well, which is fine + if Options["Help"]: + usage() - reject(comments) - print "REJECT\n" + reject_message, if not Options["No-Action"]: - Upload.do_reject(0, reject_message) - os.unlink(Upload.pkg.changes_file[:-8]+".dak") + try: + Logger = daklog.Logger(cnf, "process-new") + except CantOpenError, e: + Options["Trainee"] = "True" -################################################################################ + Sections = Section_Completer(session) + Priorities = Priority_Completer(session) + readline.parse_and_bind("tab: complete") -def main(): - changes_files = init() - if len(changes_files) > 50: + if len(changes_files) > 1: sys.stderr.write("Sorting changes...\n") - changes_files = sort_changes(changes_files) + changes_files = sort_changes(changes_files, session) # Kill me now? **FIXME** - Cnf["Dinstall::Options::No-Mail"] = "" - bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $" - if Cnf.has_key("Dinstall::Bcc"): - Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (Cnf["Dinstall::Bcc"]) - else: - Upload.Subst["__BCC__"] = bcc - - commentsdir = Cnf.get("Process-New::Options::Comments-Dir","") - if commentsdir: - if changes_files != []: - sys.stderr.write("Can't specify any changes files if working with comments-dir") - sys.exit(1) - do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept) - do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject) - else: + cnf["Dinstall::Options::No-Mail"] = "" +# bcc = "X-DAK: dak process-new\nX-Katie: lisa $Revision: 1.31 $" +# if cnf.has_key("Dinstall::Bcc"): +# Upload.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) +# else: +# Upload.Subst["__BCC__"] = bcc + +# commentsdir = cnf.get("Process-New::Options::Comments-Dir","") +# if commentsdir: +# if changes_files != []: +# sys.stderr.write("Can't specify any changes files if working with comments-dir") +# sys.exit(1) +# do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept) +# do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject) +# else: + if True: for changes_file in changes_files: changes_file = utils.validate_changes_file_arg(changes_file, 0) if not changes_file: continue print "\n" + changes_file - do_pkg (changes_file) + do_pkg (changes_file, session) end()