From: Joerg Jaspert Date: Thu, 29 Oct 2009 16:41:31 +0000 (+0100) Subject: Start the known_changes table X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=556d1eea7b0849a71d53a5e1ab0e9bff8a625e93;p=dak Start the known_changes table Signed-off-by: Joerg Jaspert --- diff --git a/dak/dakdb/update20.py b/dak/dakdb/update20.py new file mode 100755 index 00000000..a666326c --- /dev/null +++ b/dak/dakdb/update20.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# coding=utf8 + +""" +Adding table to get rid of queue/done checks + +@contact: Debian FTP Master +@copyright: 2009 Joerg Jaspert +@license: GNU General Public License version 2 or later +""" + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + + +################################################################################ + +import psycopg2 +import time +import os +import datetime +from daklib.dak_exceptions import DBUpdateError, InvalidDscError, ChangesUnicodeError +from daklib.config import Config +from daklib.utils import parse_changes, warn, gpgv_get_status_output, process_gpgv_output + +################################################################################ + +def check_signature (sig_filename, data_filename=""): + keyrings = [ + "/home/joerg/keyring/keyrings/debian-keyring.gpg", + "/home/joerg/keyring/keyrings/debian-keyring.pgp", + "/home/joerg/keyring/keyrings/debian-maintainers.gpg", + "/home/joerg/keyring/keyrings/debian-role-keys.gpg", + "/home/joerg/keyring/keyrings/emeritus-keyring.pgp", + "/home/joerg/keyring/keyrings/emeritus-keyring.gpg", + "/home/joerg/keyring/keyrings/removed-keys.gpg", + "/home/joerg/keyring/keyrings/removed-keys.pgp" + ] + + keyringargs = " ".join(["--keyring %s" % x for x in keyrings ]) + + # Build the command line + status_read, status_write = os.pipe() + cmd = "gpgv --status-fd %s %s %s" % (status_write, keyringargs, sig_filename) + + # Invoke gpgv on the file + (output, status, exit_status) = gpgv_get_status_output(cmd, status_read, status_write) + + # Process the status-fd output + (keywords, internal_error) = process_gpgv_output(status) + + # If we failed to parse the status-fd output, let's just whine and bail now + if internal_error: + warn("Couldn't parse signature") + return (None, rejects) + + # usually one would check for bad things here. We, however, do not care. + + # Next check gpgv exited with a zero return code + if exit_status: + warn("Couldn't parse signature") + return (None, rejects) + + # Sanity check the good stuff we expect + if not keywords.has_key("VALIDSIG"): + warn("Couldn't parse signature") + else: + args = keywords["VALIDSIG"] + if len(args) < 1: + warn("Couldn't parse signature") + else: + fingerprint = args[0] + + return (fingerprint, []) + +################################################################################ + +def do_update(self): + print "Adding known_changes table" + + try: + c = self.db.cursor() + c.execute(""" + CREATE TABLE known_changes ( + id SERIAL PRIMARY KEY, + changesname TEXT NOT NULL, + seen TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), + source TEXT NOT NULL, + binaries TEXT NOT NULL, + architecture TEXT NOT NULL, + version TEXT NOT NULL, + distribution TEXT NOT NULL, + urgency TEXT NOT NULL, + maintainer TEXT NOT NULL, + changedby TEXT NOT NULL, + date TEXT NOT NULL, + UNIQUE (changesname) + ) + """) + c.execute("CREATE INDEX changesname_ind ON known_changes(changesname)") + c.execute("CREATE INDEX changestimestamp_ind ON known_changes(seen)") + c.execute("CREATE INDEX changessource_ind ON known_changes(source)") + c.execute("CREATE INDEX changesdistribution_ind ON known_changes(distribution)") + c.execute("CREATE INDEX changesurgency_ind ON known_changes(urgency)") + + print "Done. Now looking for old changes files" + count = 0 + failure = 0 + cnf = Config() + for directory in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]: + checkdir = cnf["Dir::Queue::%s" % (directory) ] + if os.path.exists(checkdir): + print "Looking into %s" % (checkdir) + for dirpath, dirnames, filenames in os.walk(checkdir, topdown=False): + if not filenames: + # Empty directory (or only subdirectories), next + continue + for changesfile in filenames: + if not changesfile.endswith(".changes"): + # Only interested in changes files. + continue + try: + count += 1 + print "Directory %s, file %7d, failures %3d. (%s)" % (dirpath[-10:], count, failure, changesfile) + changes = parse_changes(os.path.join(dirpath, changesfile), signing_rules=-1) + (changes["fingerprint"], _) = check_signature(os.path.join(dirpath, changesfile)) + except InvalidDscError, line: + warn("syntax error in .dsc file '%s', line %s." % (f, line)) + failure += 1 + except ChangesUnicodeError: + warn("found invalid changes file, not properly utf-8 encoded") + failure += 1 + + filetime = datetime.datetime.fromtimestamp(os.path.getctime(os.path.join(dirpath, changesfile))) + c.execute("INSERT INTO known_changes(changesname, seen, source, binaries, architecture, version, distribution, urgency, maintainer, changedby, date) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" , [changesfile, filetime, changes["source"], changes["binary"], changes["architecture"], changes["version"], changes["distribution"], changes["urgency"], changes["maintainer"], changes["changed-by"], changes["date"]]) + + c.execute("UPDATE config SET value = '20' WHERE name = 'db_revision'") + self.db.commit() + + except psycopg2.ProgrammingError, msg: + self.db.rollback() + raise DBUpdateError, "Unable to apply source format update 15, rollback issued. Error message : %s" % (str(msg)) diff --git a/daklib/dbconn.py b/daklib/dbconn.py index 6d5497fc..2b0a8492 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -901,6 +901,41 @@ __all__.append('get_or_set_keyring') ################################################################################ +class KnownChange(object): + def __init__(self, *args, **kwargs): + pass + + def __repr__(self): + return '' % self.changesname + +__all__.append('KnownChange') + +@session_wrapper +def get_knownchange(filename, session=None): + """ + returns knownchange object for given C{filename}. + + @type archive: string + @param archive: the name of the arhive + + @type session: Session + @param session: Optional SQLA session object (a temporary one will be + generated if not supplied) + + @rtype: Archive + @return: Archive object for the given name (None if not present) + + """ + q = session.query(KnownChange).filter_by(changesname=filename) + + try: + return q.one() + except NoResultFound: + return None + +__all__.append('get_knownchange') + +################################################################################ class Location(object): def __init__(self, *args, **kwargs): pass @@ -2116,6 +2151,7 @@ class DBConn(Singleton): self.tbl_files = Table('files', self.db_meta, autoload=True) self.tbl_fingerprint = Table('fingerprint', self.db_meta, autoload=True) self.tbl_keyrings = Table('keyrings', self.db_meta, autoload=True) + self.tbl_known_changes = Table('known_changes', self.db_meta, autoload=True) self.tbl_location = Table('location', self.db_meta, autoload=True) self.tbl_maintainer = Table('maintainer', self.db_meta, autoload=True) self.tbl_new_comments = Table('new_comments', self.db_meta, autoload=True) @@ -2218,6 +2254,9 @@ class DBConn(Singleton): properties = dict(keyring_name = self.tbl_keyrings.c.name, keyring_id = self.tbl_keyrings.c.id)) + mapper(KnownChange, self.tbl_known_changes, + properties = dict(known_change_id = self.tbl_known_changes.c.id)) + mapper(Location, self.tbl_location, properties = dict(location_id = self.tbl_location.c.id, component_id = self.tbl_location.c.component, diff --git a/daklib/queue.py b/daklib/queue.py index d73b4797..7e7f468f 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -452,9 +452,8 @@ class Upload(object): # Check there isn't already a changes file of the same name in one # of the queue directories. base_filename = os.path.basename(filename) - for d in [ "Accepted", "Byhand", "Done", "New", "ProposedUpdates", "OldProposedUpdates" ]: - if os.path.exists(os.path.join(Cnf["Dir::Queue::%s" % (d) ], base_filename)): - self.rejects.append("%s: a file with this name already exists in the %s directory." % (base_filename, d)) + if get_knownchange(base_filename): + self.rejects.append("%s: a file with this name already exists." % (base_filename)) # Check the .changes is non-empty if not self.pkg.files: