From bf38dcbe75f32f221887eeda8fce0e81e64db115 Mon Sep 17 00:00:00 2001 From: Mark Hymers Date: Sat, 31 Oct 2009 21:09:04 +0000 Subject: [PATCH] Fix up hand off to policy and buildd queues Signed-off-by: Mark Hymers --- dak/process_new.py | 4 ++-- dak/process_upload.py | 40 +++++++++++++++++++++++----------------- daklib/changes.py | 5 +++++ daklib/dbconn.py | 19 +++++++++++++------ daklib/queue.py | 32 ++++++++++++++++++++------------ daklib/queue_install.py | 31 ++++++++++++++++++++----------- 6 files changed, 83 insertions(+), 48 deletions(-) diff --git a/dak/process_new.py b/dak/process_new.py index bec55df5..f89e40c8 100755 --- a/dak/process_new.py +++ b/dak/process_new.py @@ -821,7 +821,7 @@ def _accept(upload): if Options["No-Action"]: return (summary, short_summary) = upload.build_summaries() - upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"]) + upload.accept(summary, short_summary, targetqueue) os.unlink(upload.pkg.changes_file[:-8]+".dak") def do_accept(upload): @@ -832,7 +832,7 @@ def do_accept(upload): if cnf.FindB("Dinstall::SecurityQueueHandling"): upload.dump_vars(cnf["Dir::Queue::Embargoed"]) - upload.move_to_dir(cnf["Dir::Queue::Embargoed"]) + upload.move_to_queue(get_queue('embargoed')) upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"]) # Check for override disparities upload.Subst["__SUMMARY__"] = summary diff --git a/dak/process_upload.py b/dak/process_upload.py index 2f904c35..623fe031 100755 --- a/dak/process_upload.py +++ b/dak/process_upload.py @@ -196,10 +196,9 @@ def usage (exit_code=0): ############################################################################### -def action(u): +def action(u, session): cnf = Config() holding = Holding() - session = DBConn().session() # changes["distribution"] may not exist in corner cases # (e.g. unreadable changes files) @@ -216,6 +215,11 @@ def action(u): pi = u.package_info() + try: + chg = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one() + except NoResultFound, e: + chg = None + if len(u.rejects) > 0: if u.upload_too_new(): print "SKIP (too new)\n" + pi, @@ -240,18 +244,12 @@ def action(u): if Options["Automatic"]: answer = queuekey else: - # TODO: FIX THIS BY HAVING ADDED TO changes TABLE earlier - try: - dbc = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one() - except NoResultFound, e: - dbc = None - # Does suite have a policy_queue configured divert = False for s in u.pkg.changes["distribution"].keys(): suite = get_suite(s, session) if suite.policy_queue: - if not dbc or dbc.approved_for_id != su.policy_queue.policy_queue_id: + if not chg or chg.approved_for_id != su.policy_queue.policy_queue_id: # This routine will check whether the upload is a binary # upload when the source is already in the target suite. If # so, we skip the policy queue, otherwise we go there. @@ -284,17 +282,25 @@ def action(u): os.chdir(u.pkg.directory) u.do_reject(0, pi) elif answer == 'A': - u.pkg.add_known_changes(holding.holding_dir, session) + if not chg: + chg = u.pkg.add_known_changes(holding.holding_dir, session) u.accept(summary, short_summary, session) u.check_override() + session.commit() u.remove() elif answer == 'P': - u.pkg.add_known_changes(holding.holding_dir, session) - package_to_queue(u, summary, short_summary, policyqueue, perms=0664, announce=None) + if not chg: + chg = u.pkg.add_known_changes(holding.holding_dir, session) + move_to_queue(u, policyqueue, perms=0664, announce=None) + chg.in_queue = policyqueue.queue_id + session.add(chg) + session.commit() u.remove() elif answer == queuekey: - u.pkg.add_known_changes(holding.holding_dir, session) - QueueInfo[qu]["process"](u, summary, short_summary, session) + if not chg: + chg = u.pkg.add_known_changes(holding.holding_dir, session) + QueueInfo[qu]["process"](u, summary, short_summary, chg, session) + session.commit() u.remove() elif answer == 'Q': sys.exit(0) @@ -308,7 +314,7 @@ def cleanup(): if not Options["No-Action"]: h.clean() -def process_it(changes_file): +def process_it(changes_file, session): global Logger Logger.log(["Processing changes file", changes_file]) @@ -377,7 +383,7 @@ def process_it(changes_file): u.check_timestamps() u.check_signed_by_key() - action(u) + action(u, session) except (SystemExit, KeyboardInterrupt): cleanup() @@ -466,7 +472,7 @@ def main(): for changes_file in changes_files: print "\n" + changes_file session = DBConn().session() - process_it(changes_file) + process_it(changes_file, session) session.close() if summarystats.accept_count: diff --git a/daklib/changes.py b/daklib/changes.py index 3f305c9c..c1f8f5ba 100755 --- a/daklib/changes.py +++ b/daklib/changes.py @@ -204,6 +204,7 @@ class Changes(object): else: multivalues[key] = self.changes[key].keys() + # TODO: Use ORM session.execute( """INSERT INTO changes (changesname, seen, source, binaries, architecture, version, @@ -223,6 +224,10 @@ class Changes(object): 'changedby': self.changes["changed-by"], 'date': self.changes["date"]} ) + session.commit() + + return session.query(DBChange).filter_by(changesname = self.changes_file).one() + def unknown_files_fields(self, name): return sorted(list( set(self.files[name].keys()) - set(CHANGESFIELDS_FILES))) diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 779c0e29..8d8e6af8 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -458,7 +458,7 @@ class BuildQueue(object): qf = QueueFile() qf.queue_id = self.queue_id qf.lastused = datetime.now() - qf.filename = dest + qf.filename = poolfile_basename targetpath = qf.fullpath queuepath = os.path.join(self.path, poolfile_basename) @@ -1965,6 +1965,7 @@ __all__.append('get_source_in_suite') def add_dsc_to_db(u, filename, session=None): entry = u.pkg.files[filename] source = DBSource() + pfs = [] source.source = u.pkg.dsc["source"] source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch @@ -1983,6 +1984,7 @@ def add_dsc_to_db(u, filename, session=None): filename = entry["pool name"] + filename poolfile = add_poolfile(filename, entry, dsc_location_id, session) session.flush() + pfs.append(poolfile) entry["files id"] = poolfile.file_id source.poolfile_id = entry["files id"] @@ -2026,6 +2028,7 @@ def add_dsc_to_db(u, filename, session=None): # FIXME: needs to check for -1/-2 and or handle exception if found and obj is not None: files_id = obj.file_id + pfs.append(obj) # If still not found, add it if files_id is None: @@ -2033,6 +2036,7 @@ def add_dsc_to_db(u, filename, session=None): dentry["sha1sum"] = dfentry["sha1sum"] dentry["sha256sum"] = dfentry["sha256sum"] poolfile = add_poolfile(filename, dentry, dsc_location_id, session) + pfs.append(poolfile) files_id = poolfile.file_id df.poolfile_id = files_id @@ -2062,7 +2066,7 @@ def add_dsc_to_db(u, filename, session=None): session.flush() - return dsc_component, dsc_location_id + return dsc_component, dsc_location_id, pfs __all__.append('add_dsc_to_db') @@ -2090,11 +2094,12 @@ def add_deb_to_db(u, filename, session=None): if not entry.get("location id", None): entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id - if not entry.get("files id", None): + if entry.get("files id", None): + poolfile = get_poolfile_by_id(bin.poolfile_id) + bin.poolfile_id = entry["files id"] + else: poolfile = add_poolfile(filename, entry, entry["location id"], session) - entry["files id"] = poolfile.file_id - - bin.poolfile_id = entry["files id"] + bin.poolfile_id = entry["files id"] = poolfile.file_id # Find source id bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session) @@ -2125,6 +2130,8 @@ def add_deb_to_db(u, filename, session=None): # session.rollback() # raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename) + return poolfile + __all__.append('add_deb_to_db') ################################################################################ diff --git a/daklib/queue.py b/daklib/queue.py index 5f9fccb4..66424d86 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -1842,15 +1842,19 @@ distribution.""" print "Installing." self.logger.log(["installing changes", self.pkg.changes_file]) + poolfiles = [] + # Add the .dsc file to the DB first for newfile, entry in self.pkg.files.items(): if entry["type"] == "dsc": - dsc_component, dsc_location_id = add_dsc_to_db(self, newfile, session) + dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session) + for j in pfs: + poolfiles.append(j) # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb) for newfile, entry in self.pkg.files.items(): if entry["type"] == "deb": - add_deb_to_db(self, newfile, session) + poolfiles.append(add_deb_to_db(self, newfile, session)) # If this is a sourceful diff only upload that is moving # cross-component we need to copy the .orig files into the new @@ -1885,6 +1889,8 @@ distribution.""" session.add(dscf) session.flush() + poolfiles.append(newf) + # Install the files into the pool for newfile, entry in self.pkg.files.items(): destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile) @@ -1958,12 +1964,14 @@ distribution.""" os.rename(temp_filename, filename) os.chmod(filename, 0644) - # This routine returns None on success or an error on failure - # TODO: Replace queue copying using the new queue.add_file_from_pool routine - # and by looking up which queues in suite.copy_queues - #res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"]) - #if res: - # utils.fubar(res) + session.commit() + + # Set up our copy queues (e.g. buildd queues) + for suite_name in self.pkg.changes["distribution"].keys(): + suite = get_suite(suite_name, session) + for q in suite.copyqueues: + for f in poolfiles: + q.add_file_from_pool(f) session.commit() @@ -2025,15 +2033,15 @@ distribution.""" ########################################################################### - def move_to_dir (self, dest, perms=0660, changesperms=0664): + def move_to_queue (self, queue): """ - Move files to dest with certain perms/changesperms + Move files to a destination queue using the permissions in the table """ h = Holding() utils.move(os.path.join(h.holding_dir, self.pkg.changes_file), - dest, perms=changesperms) + dest, perms=int(queue.changesperms, 8)) for f in self.pkg.files.keys(): - utils.move(os.path.join(h.holding_dir, f), dest, perms=perms) + utils.move(os.path.join(h.holding_dir, f), dest, perms=int(queue.perms, 8)) ########################################################################### diff --git a/daklib/queue_install.py b/daklib/queue_install.py index e6aee579..b4a38b96 100644 --- a/daklib/queue_install.py +++ b/daklib/queue_install.py @@ -53,15 +53,17 @@ def package_to_suite(u, suite_name, session): return ret -def package_to_queue(u, summary, short_summary, queue, perms=0660, announce=None): +def package_to_queue(u, summary, short_summary, queue, chg, session, announce=None): cnf = Config() dir = queue.path print "Moving to %s policy queue" % queue.queue_name.upper() u.logger.log(["Moving to %s" % queue.queue_name, u.pkg.changes_file]) - u.move_to_dir(dir, perms=perms) - # TODO: Put building logic in here? We used to take a build=bool argument + u.move_to_queue(queue) + chg.in_queue = queue.queue_id + session.add(chg) + session.commit() # Check for override disparities u.check_override() @@ -162,7 +164,7 @@ def is_autobyhand(u): return any_auto and all_auto -def do_autobyhand(u, summary, short_summary, session=None): +def do_autobyhand(u, summary, short_summary, chg, session): print "Attempting AUTOBYHAND." byhandleft = True for f, entry in u.pkg.files.items(): @@ -192,7 +194,7 @@ def do_autobyhand(u, summary, short_summary, session=None): byhandleft = True if byhandleft: - do_byhand(u, summary, short_summary, session) + do_byhand(u, summary, short_summary, chg, session) else: u.accept(summary, short_summary, session) u.check_override() @@ -205,9 +207,10 @@ def is_byhand(u): return True return False -def do_byhand(u, summary, short_summary, session=None): - return package_to_queue(u, summary, short_summary, "Byhand", - perms=0660, build=False, announce=None) +def do_byhand(u, summary, short_summary, chg, session): + return package_to_queue(u, summary, short_summary, + get_queue('byhand'), chg, session, + announce=None) ################################################################################ @@ -217,13 +220,18 @@ def is_new(u): return True return False -def acknowledge_new(u, summary, short_summary, session=None): +def acknowledge_new(u, summary, short_summary, chg, session): cnf = Config() print "Moving to NEW queue." u.logger.log(["Moving to new", u.pkg.changes_file]) - u.move_to_dir(cnf["Dir::Queue::New"], perms=0640, changesperms=0644) + q = get_queue('new', session) + + u.move_to_queue(q) + chg.in_queue = q.queue_id + session.add(chg) + session.commit() if not cnf["Dinstall::Options::No-Mail"]: print "Sending new ack." @@ -248,9 +256,10 @@ def determine_target(u): # Statically handled queues target = None - for q in QueueInfo.keys(): + for q in ["new", "autobyhand", "byhand"]: if QueueInfo[q]["is"](u): target = q + break return target -- 2.39.5