]> err.no Git - dak/commitdiff
Fix up hand off to policy and buildd queues
authorMark Hymers <mhy@debian.org>
Sat, 31 Oct 2009 21:09:04 +0000 (21:09 +0000)
committerMark Hymers <mhy@debian.org>
Sat, 31 Oct 2009 21:09:04 +0000 (21:09 +0000)
Signed-off-by: Mark Hymers <mhy@debian.org>
dak/process_new.py
dak/process_upload.py
daklib/changes.py
daklib/dbconn.py
daklib/queue.py
daklib/queue_install.py

index bec55df55c64b19a05b68bc6817cb2016866e229..f89e40c8ad667ef7fa00a23292c35234340eb823 100755 (executable)
@@ -821,7 +821,7 @@ def _accept(upload):
     if Options["No-Action"]:
         return
     (summary, short_summary) = upload.build_summaries()
-    upload.accept(summary, short_summary, targetdir=Config()["Dir::Queue::Newstage"])
+    upload.accept(summary, short_summary, targetqueue)
     os.unlink(upload.pkg.changes_file[:-8]+".dak")
 
 def do_accept(upload):
@@ -832,7 +832,7 @@ def do_accept(upload):
 
         if cnf.FindB("Dinstall::SecurityQueueHandling"):
             upload.dump_vars(cnf["Dir::Queue::Embargoed"])
-            upload.move_to_dir(cnf["Dir::Queue::Embargoed"])
+            upload.move_to_queue(get_queue('embargoed'))
             upload.queue_build("embargoed", cnf["Dir::Queue::Embargoed"])
             # Check for override disparities
             upload.Subst["__SUMMARY__"] = summary
index 2f904c3596a283355cd4c8d527771c20ac21d523..623fe031d64454eac134d201c5a76ded08494102 100755 (executable)
@@ -196,10 +196,9 @@ def usage (exit_code=0):
 
 ###############################################################################
 
-def action(u):
+def action(u, session):
     cnf = Config()
     holding = Holding()
-    session = DBConn().session()
 
     # changes["distribution"] may not exist in corner cases
     # (e.g. unreadable changes files)
@@ -216,6 +215,11 @@ def action(u):
 
     pi = u.package_info()
 
+    try:
+        chg = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one()
+    except NoResultFound, e:
+        chg = None
+
     if len(u.rejects) > 0:
         if u.upload_too_new():
             print "SKIP (too new)\n" + pi,
@@ -240,18 +244,12 @@ def action(u):
             if Options["Automatic"]:
                 answer = queuekey
         else:
-            # TODO: FIX THIS BY HAVING ADDED TO changes TABLE earlier
-            try:
-                dbc = session.query(DBChange).filter_by(changesname=os.path.basename(u.pkg.changes_file)).one()
-            except NoResultFound, e:
-                dbc = None
-
             # Does suite have a policy_queue configured
             divert = False
             for s in u.pkg.changes["distribution"].keys():
                 suite = get_suite(s, session)
                 if suite.policy_queue:
-                    if not dbc or dbc.approved_for_id != su.policy_queue.policy_queue_id:
+                    if not chg or chg.approved_for_id != su.policy_queue.policy_queue_id:
                         # This routine will check whether the upload is a binary
                         # upload when the source is already in the target suite.  If
                         # so, we skip the policy queue, otherwise we go there.
@@ -284,17 +282,25 @@ def action(u):
         os.chdir(u.pkg.directory)
         u.do_reject(0, pi)
     elif answer == 'A':
-        u.pkg.add_known_changes(holding.holding_dir, session)
+        if not chg:
+            chg = u.pkg.add_known_changes(holding.holding_dir, session)
         u.accept(summary, short_summary, session)
         u.check_override()
+        session.commit()
         u.remove()
     elif answer == 'P':
-        u.pkg.add_known_changes(holding.holding_dir, session)
-        package_to_queue(u, summary, short_summary, policyqueue, perms=0664, announce=None)
+        if not chg:
+            chg = u.pkg.add_known_changes(holding.holding_dir, session)
+        move_to_queue(u, policyqueue, perms=0664, announce=None)
+        chg.in_queue = policyqueue.queue_id
+        session.add(chg)
+        session.commit()
         u.remove()
     elif answer == queuekey:
-        u.pkg.add_known_changes(holding.holding_dir, session)
-        QueueInfo[qu]["process"](u, summary, short_summary, session)
+        if not chg:
+            chg = u.pkg.add_known_changes(holding.holding_dir, session)
+        QueueInfo[qu]["process"](u, summary, short_summary, chg, session)
+        session.commit()
         u.remove()
     elif answer == 'Q':
         sys.exit(0)
@@ -308,7 +314,7 @@ def cleanup():
     if not Options["No-Action"]:
         h.clean()
 
-def process_it(changes_file):
+def process_it(changes_file, session):
     global Logger
 
     Logger.log(["Processing changes file", changes_file])
@@ -377,7 +383,7 @@ def process_it(changes_file):
             u.check_timestamps()
             u.check_signed_by_key()
 
-        action(u)
+        action(u, session)
 
     except (SystemExit, KeyboardInterrupt):
         cleanup()
@@ -466,7 +472,7 @@ def main():
     for changes_file in changes_files:
         print "\n" + changes_file
         session = DBConn().session()
-        process_it(changes_file)
+        process_it(changes_file, session)
         session.close()
 
     if summarystats.accept_count:
index 3f305c9c0fe4cbc7f3082bd739e38400e0aebf6a..c1f8f5ba89f5168f0d322c7d82fe04b1164be17c 100755 (executable)
@@ -204,6 +204,7 @@ class Changes(object):
             else:
                 multivalues[key] = self.changes[key].keys()
 
+        # TODO: Use ORM
         session.execute(
             """INSERT INTO changes
               (changesname, seen, source, binaries, architecture, version,
@@ -223,6 +224,10 @@ class Changes(object):
                 'changedby':    self.changes["changed-by"],
                 'date':         self.changes["date"]} )
 
+        session.commit()
+
+        return session.query(DBChange).filter_by(changesname = self.changes_file).one()
+
     def unknown_files_fields(self, name):
         return sorted(list( set(self.files[name].keys()) -
                             set(CHANGESFIELDS_FILES)))
index 779c0e2936a3fa270fe68b290c22b9d614d3f2d6..8d8e6af8ea4e43dc24845983bb5099eb8ef0f194 100755 (executable)
@@ -458,7 +458,7 @@ class BuildQueue(object):
         qf = QueueFile()
         qf.queue_id = self.queue_id
         qf.lastused = datetime.now()
-        qf.filename = dest
+        qf.filename = poolfile_basename
 
         targetpath = qf.fullpath
         queuepath = os.path.join(self.path, poolfile_basename)
@@ -1965,6 +1965,7 @@ __all__.append('get_source_in_suite')
 def add_dsc_to_db(u, filename, session=None):
     entry = u.pkg.files[filename]
     source = DBSource()
+    pfs = []
 
     source.source = u.pkg.dsc["source"]
     source.version = u.pkg.dsc["version"] # NB: not files[file]["version"], that has no epoch
@@ -1983,6 +1984,7 @@ def add_dsc_to_db(u, filename, session=None):
         filename = entry["pool name"] + filename
         poolfile = add_poolfile(filename, entry, dsc_location_id, session)
         session.flush()
+        pfs.append(poolfile)
         entry["files id"] = poolfile.file_id
 
     source.poolfile_id = entry["files id"]
@@ -2026,6 +2028,7 @@ def add_dsc_to_db(u, filename, session=None):
             # FIXME: needs to check for -1/-2 and or handle exception
             if found and obj is not None:
                 files_id = obj.file_id
+                pfs.append(obj)
 
             # If still not found, add it
             if files_id is None:
@@ -2033,6 +2036,7 @@ def add_dsc_to_db(u, filename, session=None):
                 dentry["sha1sum"] = dfentry["sha1sum"]
                 dentry["sha256sum"] = dfentry["sha256sum"]
                 poolfile = add_poolfile(filename, dentry, dsc_location_id, session)
+                pfs.append(poolfile)
                 files_id = poolfile.file_id
 
         df.poolfile_id = files_id
@@ -2062,7 +2066,7 @@ def add_dsc_to_db(u, filename, session=None):
 
     session.flush()
 
-    return dsc_component, dsc_location_id
+    return dsc_component, dsc_location_id, pfs
 
 __all__.append('add_dsc_to_db')
 
@@ -2090,11 +2094,12 @@ def add_deb_to_db(u, filename, session=None):
     if not entry.get("location id", None):
         entry["location id"] = get_location(cnf["Dir::Pool"], entry["component"], session=session).location_id
 
-    if not entry.get("files id", None):
+    if entry.get("files id", None):
+        poolfile = get_poolfile_by_id(bin.poolfile_id)
+        bin.poolfile_id = entry["files id"]
+    else:
         poolfile = add_poolfile(filename, entry, entry["location id"], session)
-        entry["files id"] = poolfile.file_id
-
-    bin.poolfile_id = entry["files id"]
+        bin.poolfile_id = entry["files id"] = poolfile.file_id
 
     # Find source id
     bin_sources = get_sources_from_name(entry["source package"], entry["source version"], session=session)
@@ -2125,6 +2130,8 @@ def add_deb_to_db(u, filename, session=None):
     #    session.rollback()
     #    raise MissingContents, "No contents stored for package %s, and couldn't determine contents of %s" % (bin.package, filename)
 
+    return poolfile
+
 __all__.append('add_deb_to_db')
 
 ################################################################################
index 5f9fccb4073dfc8e5854c552ddd262ca7c791ec6..66424d86eb6a5e30424026bb3f851ae49ef5819c 100755 (executable)
@@ -1842,15 +1842,19 @@ distribution."""
         print "Installing."
         self.logger.log(["installing changes", self.pkg.changes_file])
 
+        poolfiles = []
+
         # Add the .dsc file to the DB first
         for newfile, entry in self.pkg.files.items():
             if entry["type"] == "dsc":
-                dsc_component, dsc_location_id = add_dsc_to_db(self, newfile, session)
+                dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
+                for j in pfs:
+                    poolfiles.append(j)
 
         # Add .deb / .udeb files to the DB (type is always deb, dbtype is udeb/deb)
         for newfile, entry in self.pkg.files.items():
             if entry["type"] == "deb":
-                add_deb_to_db(self, newfile, session)
+                poolfiles.append(add_deb_to_db(self, newfile, session))
 
         # If this is a sourceful diff only upload that is moving
         # cross-component we need to copy the .orig files into the new
@@ -1885,6 +1889,8 @@ distribution."""
                     session.add(dscf)
                     session.flush()
 
+                    poolfiles.append(newf)
+
         # Install the files into the pool
         for newfile, entry in self.pkg.files.items():
             destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
@@ -1958,12 +1964,14 @@ distribution."""
             os.rename(temp_filename, filename)
             os.chmod(filename, 0644)
 
-        # This routine returns None on success or an error on failure
-        # TODO: Replace queue copying using the new queue.add_file_from_pool routine
-        #       and by looking up which queues in suite.copy_queues
-        #res = get_queue('accepted').autobuild_upload(self.pkg, cnf["Dir::Queue::Accepted"])
-        #if res:
-        #    utils.fubar(res)
+        session.commit()
+
+        # Set up our copy queues (e.g. buildd queues)
+        for suite_name in self.pkg.changes["distribution"].keys():
+            suite = get_suite(suite_name, session)
+            for q in suite.copyqueues:
+                for f in poolfiles:
+                    q.add_file_from_pool(f)
 
         session.commit()
 
@@ -2025,15 +2033,15 @@ distribution."""
 
     ###########################################################################
 
-    def move_to_dir (self, dest, perms=0660, changesperms=0664):
+    def move_to_queue (self, queue):
         """
-        Move files to dest with certain perms/changesperms
+        Move files to a destination queue using the permissions in the table
         """
         h = Holding()
         utils.move(os.path.join(h.holding_dir, self.pkg.changes_file),
-                   dest, perms=changesperms)
+                   dest, perms=int(queue.changesperms, 8))
         for f in self.pkg.files.keys():
-            utils.move(os.path.join(h.holding_dir, f), dest, perms=perms)
+            utils.move(os.path.join(h.holding_dir, f), dest, perms=int(queue.perms, 8))
 
     ###########################################################################
 
index e6aee579faaaba1a61f9efd8c2f3ca4a75df9a93..b4a38b96e4488ff697745bcf86382b9ee9ce27c0 100644 (file)
@@ -53,15 +53,17 @@ def package_to_suite(u, suite_name, session):
 
     return ret
 
-def package_to_queue(u, summary, short_summary, queue, perms=0660, announce=None):
+def package_to_queue(u, summary, short_summary, queue, chg, session, announce=None):
     cnf = Config()
     dir = queue.path
 
     print "Moving to %s policy queue" % queue.queue_name.upper()
     u.logger.log(["Moving to %s" % queue.queue_name, u.pkg.changes_file])
 
-    u.move_to_dir(dir, perms=perms)
-    # TODO: Put building logic in here?  We used to take a build=bool argument
+    u.move_to_queue(queue)
+    chg.in_queue = queue.queue_id
+    session.add(chg)
+    session.commit()
 
     # Check for override disparities
     u.check_override()
@@ -162,7 +164,7 @@ def is_autobyhand(u):
 
     return any_auto and all_auto
 
-def do_autobyhand(u, summary, short_summary, session=None):
+def do_autobyhand(u, summary, short_summary, chg, session):
     print "Attempting AUTOBYHAND."
     byhandleft = True
     for f, entry in u.pkg.files.items():
@@ -192,7 +194,7 @@ def do_autobyhand(u, summary, short_summary, session=None):
             byhandleft = True
 
     if byhandleft:
-        do_byhand(u, summary, short_summary, session)
+        do_byhand(u, summary, short_summary, chg, session)
     else:
         u.accept(summary, short_summary, session)
         u.check_override()
@@ -205,9 +207,10 @@ def is_byhand(u):
             return True
     return False
 
-def do_byhand(u, summary, short_summary, session=None):
-    return package_to_queue(u, summary, short_summary, "Byhand",
-                            perms=0660, build=False, announce=None)
+def do_byhand(u, summary, short_summary, chg, session):
+    return package_to_queue(u, summary, short_summary,
+                            get_queue('byhand'), chg, session,
+                            announce=None)
 
 ################################################################################
 
@@ -217,13 +220,18 @@ def is_new(u):
             return True
     return False
 
-def acknowledge_new(u, summary, short_summary, session=None):
+def acknowledge_new(u, summary, short_summary, chg, session):
     cnf = Config()
 
     print "Moving to NEW queue."
     u.logger.log(["Moving to new", u.pkg.changes_file])
 
-    u.move_to_dir(cnf["Dir::Queue::New"], perms=0640, changesperms=0644)
+    q = get_queue('new', session)
+
+    u.move_to_queue(q)
+    chg.in_queue = q.queue_id
+    session.add(chg)
+    session.commit()
 
     if not cnf["Dinstall::Options::No-Mail"]:
         print "Sending new ack."
@@ -248,9 +256,10 @@ def determine_target(u):
     # Statically handled queues
     target = None
 
-    for q in QueueInfo.keys():
+    for q in ["new", "autobyhand", "byhand"]:
         if QueueInfo[q]["is"](u):
             target = q
+            break
 
     return target