From: Mark Hymers Date: Sat, 31 Oct 2009 12:41:34 +0000 (+0000) Subject: initial policy queue implementation in process_upload X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=61a3eb37937eb827a8e69827a9d0fe06b0c8ed03;p=dak initial policy queue implementation in process_upload Signed-off-by: Mark Hymers --- diff --git a/dak/process_upload.py b/dak/process_upload.py index cf1594a3..97117b8a 100755 --- a/dak/process_upload.py +++ b/dak/process_upload.py @@ -125,21 +125,53 @@ Checks Debian packages from Incoming ## pu: create files for BTS ## pu: create entry in queue_build ## pu: check overrides -import errno + +# Integrity checks +## GPG +## Parsing changes (check for duplicates) +## Parse dsc +## file list checks + +# New check layout (TODO: Implement) +## Permission checks +### suite mappings +### ACLs +### version checks (suite) +### override checks + +## Source checks +### copy orig +### unpack +### BTS changelog +### src contents +### lintian +### urgency log + +## Binary checks +### timestamps +### control checks +### src relation check +### contents + +## Database insertion (? copy from stuff) +### BYHAND / NEW / Policy queues +### Pool + +## Queue builds + +from errno import EACCES, EAGAIN import fcntl import os import sys -#from datetime import datetime import traceback import apt_pkg +from sqlalchemy.orm.exc import NoResultFound from daklib import daklog from daklib.queue import * from daklib.queue_install import * from daklib import utils from daklib.dbconn import * -#from daklib.dak_exceptions import * -#from daklib.regexes import re_default_answer, re_issource, re_fdnic from daklib.urgencylog import UrgencyLog from daklib.summarystats import SummaryStats from daklib.holding import Holding @@ -164,13 +196,14 @@ def usage (exit_code=0): ############################################################################### -def action(u): +def action(u, dbc): cnf = Config() holding = Holding() + session = DBConn().session() # changes["distribution"] may not exist in corner cases # (e.g. unreadable changes files) - if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], DictType): + if not u.pkg.changes.has_key("distribution") or not isinstance(u.pkg.changes["distribution"], dict): u.pkg.changes["distribution"] = {} (summary, short_summary) = u.build_summaries() @@ -193,6 +226,8 @@ def action(u): if Options["Automatic"]: answer = 'R' else: + # Are we headed for NEW / BYHAND / AUTOBYHAND? + # Note that policy queues are no longer handled here qu = determine_target(u) if qu: print "%s for %s\n%s%s" % ( qu.upper(), ", ".join(u.pkg.changes["distribution"].keys()), pi, summary) @@ -205,10 +240,38 @@ def action(u): if Options["Automatic"]: answer = queuekey else: - print "ACCEPT\n" + pi + summary, - prompt = "[A]ccept, Skip, Quit ?" - if Options["Automatic"]: - answer = 'A' + # TODO: FIX THIS BY HAVING ADDED TO changes TABLE earlier + try: + dbc = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one() + except NoResultFound, e: + dbc = None + + # Does suite have a policy_queue configured + divert = False + for s in u.pkg.changes["distribution"].keys(): + suite = get_suite(s, session) + if suite.policy_queue: + if not dbc or dbc.approved_for_id != su.policy_queue.policy_queue_id: + # This routine will check whether the upload is a binary + # upload when the source is already in the target suite. If + # so, we skip the policy queue, otherwise we go there. + divert = package_to_suite(u, suite.suite_name, session=session) + if divert: + print "%s for %s\n%s%s" % ( su.policy_queue.queue_name.upper(), + ", ".join(u.pkg.changes["distribution"].keys()), + pi, summary) + queuekey = "P" + prompt = "[P]olicy, Skip, Quit ?" + policyqueue = su.policy_queue + if Options["Automatic"]: + answer = 'P' + break + + if not divert: + print "ACCEPT\n" + pi + summary, + prompt = "[A]ccept, Skip, Quit ?" + if Options["Automatic"]: + answer = 'A' while prompt.find(answer) == -1: answer = utils.our_raw_input(prompt) @@ -217,8 +280,6 @@ def action(u): answer = m.group(1) answer = answer[:1].upper() - session = DBConn().session() - if answer == 'R': os.chdir(u.pkg.directory) u.do_reject(0, pi) @@ -227,6 +288,10 @@ def action(u): u.accept(summary, short_summary, session) u.check_override() u.remove() + elif answer == 'P': + u.pkg.add_known_changes(holding.holding_dir, session) + package_to_queue(u, summary, short_summary, policyqueue, perms=0664, announce=None) + u.remove() elif answer == queuekey: u.pkg.add_known_changes(holding.holding_dir, session) QueueInfo[qu]["process"](u, summary, short_summary, session) @@ -252,6 +317,9 @@ def process_it(changes_file): holding = Holding() + # TODO: Actually implement using pending* tables so that we don't lose track + # of what is where + u = Upload() u.pkg.changes_file = changes_file u.pkg.directory = os.getcwd() diff --git a/daklib/queue_install.py b/daklib/queue_install.py index c8fa39e0..d6b651b2 100644 --- a/daklib/queue_install.py +++ b/daklib/queue_install.py @@ -31,60 +31,37 @@ from daklib import utils from daklib.dbconn import * from daklib.config import Config -############################################################################### - -def determine_target(u): - cnf = Config() - - queues = [ "New", "Autobyhand", "Byhand" ] - if cnf.FindB("Dinstall::SecurityQueueHandling"): - queues += [ "Unembargo", "Embargo" ] - else: - queues += [ "OldStableUpdate", "StableUpdate" ] - - target = None - for q in queues: - if QueueInfo[q]["is"](u): - target = q - break - - return target - ################################################################################ -def package_to_suite(u, suite): +def package_to_suite(u, suite_name, session): if not u.pkg.changes["distribution"].has_key(suite): return False ret = True if not u.pkg.changes["architecture"].has_key("source"): - s = DBConn().session() - q = s.query(SrcAssociation.sa_id) + q = session.query(SrcAssociation.sa_id) q = q.join(Suite).filter_by(suite_name=suite) q = q.join(DBSource).filter_by(source=u.pkg.changes['source']) q = q.filter_by(version=u.pkg.changes['version']).limit(1) # NB: Careful, this logic isn't what you would think it is - # Source is already in {old-,}proposed-updates so no need to hold - # Instead, we don't move to the holding area, we just do an ACCEPT + # Source is already in the target suite so no need to go to policy + # Instead, we don't move to the policy area, we just do an ACCEPT if q.count() > 0: ret = False - s.close() - return ret -def package_to_queue(u, summary, short_summary, queue, perms=0660, build=True, announce=None): +def package_to_queue(u, summary, short_summary, queue, perms=0660, announce=None): cnf = Config() - dir = cnf["Dir::Queue::%s" % queue] + dir = queue.path - print "Moving to %s holding area" % queue.upper() - u.logger.log(["Moving to %s" % queue, u.pkg.changes_file]) + print "Moving to %s policy queue" % queue.queue_name.upper() + u.logger.log(["Moving to %s" % queue.queue_name, u.pkg.changes_file]) u.move_to_dir(dir, perms=perms) - if build: - get_or_set_queue(queue.lower()).autobuild_upload(u.pkg, dir) + # TODO: Put building logic in here? We used to take a build=bool argument # Check for override disparities u.check_override() @@ -100,64 +77,49 @@ def package_to_queue(u, summary, short_summary, queue, perms=0660, build=True, a ################################################################################ -def is_unembargo(u): - session = DBConn().session() - cnf = Config() - - q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", u.pkg.changes) - if q.rowcount > 0: - session.close() - return True - - oldcwd = os.getcwd() - os.chdir(cnf["Dir::Queue::Disembargo"]) - disdir = os.getcwd() - os.chdir(oldcwd) - - ret = False - - if u.pkg.directory == disdir: - if u.pkg.changes["architecture"].has_key("source"): - session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", u.pkg.changes) - session.commit() - - ret = True - - session.close() - - return ret - -def queue_unembargo(u, summary, short_summary, session=None): - return package_to_queue(u, summary, short_summary, "Unembargoed", - perms=0660, build=True, announce='process-unchecked.accepted') - -################################################################################ - -def is_embargo(u): - # if embargoed queues are enabled always embargo - return True - -def queue_embargo(u, summary, short_summary, session=None): - return package_to_queue(u, summary, short_summary, "Unembargoed", - perms=0660, build=True, announce='process-unchecked.accepted') - -################################################################################ - -def is_stableupdate(u): - return package_to_suite(u, 'proposed-updates') - -def do_stableupdate(u, summary, short_summary, session=None): - return package_to_queue(u, summary, short_summary, "ProposedUpdates", - perms=0664, build=False, announce=None) - -################################################################################ - -def is_oldstableupdate(u): - return package_to_suite(u, 'oldstable-proposed-updates') - -def do_oldstableupdate(u, summary, short_summary, session=None): - return package_to_queue(u, summary, short_summary, "OldProposedUpdates", - perms=0664, build=False, announce=None) +# TODO: This logic needs to be replaced with policy queues before we upgrade +# security master + +#def is_unembargo(u): +# session = DBConn().session() +# cnf = Config() +# +# q = session.execute("SELECT package FROM disembargo WHERE package = :source AND version = :version", u.pkg.changes) +# if q.rowcount > 0: +# session.close() +# return True +# +# oldcwd = os.getcwd() +# os.chdir(cnf["Dir::Queue::Disembargo"]) +# disdir = os.getcwd() +# os.chdir(oldcwd) +# +# ret = False +# +# if u.pkg.directory == disdir: +# if u.pkg.changes["architecture"].has_key("source"): +# session.execute("INSERT INTO disembargo (package, version) VALUES (:package, :version)", u.pkg.changes) +# session.commit() +# +# ret = True +# +# session.close() +# +# return ret +# +#def queue_unembargo(u, summary, short_summary, session=None): +# return package_to_queue(u, summary, short_summary, "Unembargoed", +# perms=0660, build=True, announce='process-unchecked.accepted') +# +################################################################################# +# +#def is_embargo(u): +# # if embargoed queues are enabled always embargo +# return True +# +#def queue_embargo(u, summary, short_summary, session=None): +# return package_to_queue(u, summary, short_summary, "Unembargoed", +# perms=0660, build=True, announce='process-unchecked.accepted') ################################################################################ @@ -275,12 +237,22 @@ def acknowledge_new(u, summary, short_summary, session=None): # q-unapproved hax0ring QueueInfo = { - "New": { "is": is_new, "process": acknowledge_new }, - "Autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand }, - "Byhand" : { "is": is_byhand, "process": do_byhand }, - "OldStableUpdate" : { "is": is_oldstableupdate, - "process": do_oldstableupdate }, - "StableUpdate" : { "is": is_stableupdate, "process": do_stableupdate }, - "Unembargo" : { "is": is_unembargo, "process": queue_unembargo }, - "Embargo" : { "is": is_embargo, "process": queue_embargo }, + "new": { "is": is_new, "process": acknowledge_new }, + "autobyhand" : { "is" : is_autobyhand, "process": do_autobyhand }, + "byhand" : { "is": is_byhand, "process": do_byhand }, } + +def determine_target(u): + cnf = Config() + + # Statically handled queues + target = None + + for q in QueueInfo.keys(): + if QueueInfo[q]["is"](u): + target = q + + return target + +############################################################################### +