From 076d0a9a0f88c8e86fc206b491f73ea6e8c71fa0 Mon Sep 17 00:00:00 2001 From: Luca Falavigna Date: Wed, 21 Apr 2010 21:26:27 +0000 Subject: [PATCH] Refactorize make-changelog handling, only for source uploads ATM Signed-off-by: Luca Falavigna --- dak/dakdb/update33.py | 11 ++++-- dak/make_changelog.py | 83 ++++++++++++++++++------------------------- dak/process_policy.py | 18 ---------- daklib/dbconn.py | 24 ------------- daklib/queue.py | 18 ++++++++++ 5 files changed, 61 insertions(+), 93 deletions(-) diff --git a/dak/dakdb/update33.py b/dak/dakdb/update33.py index 28e5a28a..5c0b3f41 100644 --- a/dak/dakdb/update33.py +++ b/dak/dakdb/update33.py @@ -2,7 +2,7 @@ # coding=utf8 """ -Implement changelogs table +Implement changelogs related tables @contact: Debian FTP Master @copyright: 2010 Luca Falavigna @@ -39,7 +39,12 @@ def do_update(self): print __doc__ try: c = self.db.cursor() - c.execute('CREATE TABLE changelogs (source text, version debversion, suite text, changelog text)') + c.execute('ALTER TABLE changes ADD COLUMN changelog_id integer') + c.execute('CREATE TABLE changelogs_text (id serial PRIMARY KEY NOT NULL, changelog text)') + c.execute("GRANT SELECT ON changelogs_text TO public") + c.execute("GRANT ALL ON changelogs_text TO ftpmaster") + c.execute('CREATE VIEW changelogs AS SELECT cl.id, source, version, architecture, changelog \ + FROM changes c JOIN changelogs_text cl ON cl.id = c.changelog_id') c.execute("GRANT SELECT ON changelogs TO public") c.execute("GRANT ALL ON changelogs TO ftpmaster") c.execute("UPDATE config SET value = '33' WHERE name = 'db_revision'") @@ -47,4 +52,4 @@ def do_update(self): except psycopg2.ProgrammingError, msg: self.db.rollback() - raise DBUpdateError, 'Unable to apply build_queue update 32, rollback issued. Error message : %s' % (str(msg)) + raise DBUpdateError, 'Unable to apply build_queue update 33, rollback issued. Error message : %s' % (str(msg)) diff --git a/dak/make_changelog.py b/dak/make_changelog.py index e7505a1c..852fabd4 100644 --- a/dak/make_changelog.py +++ b/dak/make_changelog.py @@ -53,13 +53,9 @@ import sys import apt_pkg from daklib.dbconn import * from daklib import utils -from daklib.queue import Upload ################################################################################ -suites = {'proposed-updates': 'proposedupdates', - 'oldstable-proposed-updates': 'oldproposedupdates'} - def usage (exit_code=0): print """Usage: make-changelog -s -b [OPTION]... Generate changelog between two suites @@ -72,50 +68,38 @@ Options: sys.exit(exit_code) -def get_new_packages(suite, base_suite): - """ - Returns a dict of sources and versions where version is newer in base. - """ - - suite_sources = dict() - base_suite_sources = dict() - new_in_suite = dict() - session = DBConn().session() - - # Get source details from given suites - for i in get_all_sources_in_suite(suite, session): - suite_sources[i[0]] = i[1] - for i in get_all_sources_in_suite(base_suite, session): - base_suite_sources[i[0]] = i[1] - - # Compare if version in suite is greater than the base_suite one - for i in suite_sources.keys(): - if i not in suite_sources.keys(): - new_in_suite[i] = (suite_sources[i], 0) - elif apt_pkg.VersionCompare(suite_sources[i], base_suite_sources[i]) > 0: - new_in_suite[i] = (suite_sources[i], base_suite_sources[i]) - - return new_in_suite - -def generate_changelog(suite, source, versions): +def get_source_uploads(suite, base_suite, session): """ - Generates changelog data returned from changelogs table + Returns changelogs for source uploads where version is newer than base. """ - query = """ - SELECT changelog FROM changelogs - WHERE suite = :suite - AND source = :source - AND version > :base - AND version <= :current - ORDER BY source, version DESC""" - session = DBConn().session() - result = session.execute(query, {'suite': suites[suite], 'source': source, \ - 'base': versions[1], 'current': versions[0]}) - session.commit() - for r in result.fetchall(): - for i in range(0, len(r)): - print r[i] + query = """WITH base AS ( + SELECT source, max(version) AS version + FROM source_suite + WHERE suite_name = :base_suite + GROUP BY source + UNION SELECT source, CAST(0 AS debversion) AS version + FROM source_suite + WHERE suite_name = :suite + EXCEPT SELECT source, CAST(0 AS debversion) AS version + FROM source_suite + WHERE suite_name = :base_suite + ORDER BY source), + cur_suite AS ( + SELECT source, max(version) AS version + FROM source_suite + WHERE suite_name = :suite + GROUP BY source) + SELECT DISTINCT c.source, c.version, c.changelog + FROM changelogs c + JOIN base b on b.source = c.source + JOIN cur_suite cs ON cs.source = c.source + WHERE c.version > b.version + AND c.version <= cs.version + AND c.architecture LIKE '%source%' + ORDER BY c.source, c.version DESC""" + + return session.execute(query, {'suite': suite, 'base_suite': base_suite}) def main(): Cnf = utils.get_conf() @@ -139,9 +123,12 @@ def main(): if not get_suite(s): utils.fubar('Invalid suite "%s"' % s) - new_packages = get_new_packages(suite, base_suite) - for package in sorted(new_packages.keys()): - generate_changelog(suite, package, new_packages[package]) + session = DBConn().session() + uploads = get_source_uploads(suite, base_suite, session) + session.commit() + + for u in uploads: + print u[2] if __name__ == '__main__': main() diff --git a/dak/process_policy.py b/dak/process_policy.py index 83ee19b5..d1377b97 100755 --- a/dak/process_policy.py +++ b/dak/process_policy.py @@ -65,9 +65,6 @@ def do_comments(dir, srcqueue, opref, npref, line, fn, session): continue fn(f, srcqueue, "".join(lines[1:]), session) - if len(changes_files) and not Options["No-Action"]: - store_changelog(changes_files[0], srcqueue) - if opref != npref and not Options["No-Action"]: newcomm = npref + comm[len(opref):] os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm)) @@ -116,21 +113,6 @@ def comment_reject(changes_file, srcqueue, comments, session): ################################################################################ -def store_changelog(changes_file, srcqueue): - Cnf = Config() - u = Upload() - u.pkg.changes_file = os.path.join(Cnf['Dir::Queue::Newstage'], changes_file) - u.load_changes(u.pkg.changes_file) - u.update_subst() - query = """INSERT INTO changelogs (source, version, suite, changelog) - VALUES (:source, :version, :suite, :changelog)""" - session = DBConn().session() - session.execute(query, {'source': u.pkg.changes['source'], 'version': u.pkg.changes['version'], \ - 'suite': srcqueue.queue_name, 'changelog': u.pkg.changes['changes']}) - session.commit() - -################################################################################ - def main(): global Options, Logger diff --git a/daklib/dbconn.py b/daklib/dbconn.py index fbebec50..ea11a145 100755 --- a/daklib/dbconn.py +++ b/daklib/dbconn.py @@ -2212,30 +2212,6 @@ def get_source_in_suite(source, suite, session=None): __all__.append('get_source_in_suite') -@session_wrapper -def get_all_sources_in_suite(suitename, session=None): - """ - Returns list of sources and versions found in a given suite - - - B{suite} - a suite name, eg. I{unstable} - - @type suite: string - @param suite: the suite name - - @rtype: dictionary - @return: the version for I{source} in I{suite} - - """ - query = """SELECT source, version FROM source_suite - WHERE suite_name=:suitename - ORDER BY source""" - - vals = {'suitename': suitename} - - return session.execute(query, vals) - -__all__.append('get_all_sources_in_suite') - ################################################################################ @session_wrapper diff --git a/daklib/queue.py b/daklib/queue.py index c177602c..519899fb 100755 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -1899,6 +1899,9 @@ distribution.""" # Make sure that our source object is up-to-date session.expire(source) + # Add changelog information to the database + self.store_changelog() + # Install the files into the pool for newfile, entry in self.pkg.files.items(): destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile) @@ -2672,3 +2675,18 @@ distribution.""" os.chdir(cwd) return too_new + + def store_changelog(self): + session = DBConn().session() + + # Add current changelog text into changelogs_text table, return created ID + query = "INSERT INTO changelogs_text (changelog) VALUES (:changelog) RETURNING id" + ID = session.execute(query, {'changelog': self.pkg.changes['changes']}).fetchone()[0] + + # Link ID to the upload available in changes table + query = """UPDATE changes SET changelog_id = :id WHERE source = :source + AND version = :version AND architecture LIKE '%source%'""" + session.execute(query, {'id': ID, 'source': self.pkg.changes['source'], \ + 'version': self.pkg.changes['version']}) + + session.commit() -- 2.39.5