]> err.no Git - dak/commitdiff
Remove old contents code and tables.
authorTorsten Werner <twerner@debian.org>
Mon, 21 Mar 2011 08:24:12 +0000 (09:24 +0100)
committerTorsten Werner <twerner@debian.org>
Mon, 21 Mar 2011 08:34:59 +0000 (09:34 +0100)
Signed-off-by: Torsten Werner <twerner@debian.org>
dak/dakdb/update43.py [new file with mode: 0755]
dak/update_db.py
daklib/binary.py [deleted file]
daklib/dbconn.py
daklib/queue.py

diff --git a/dak/dakdb/update43.py b/dak/dakdb/update43.py
new file mode 100755 (executable)
index 0000000..b9c7f58
--- /dev/null
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# coding=utf8
+
+"""
+Remove old contents tables that are no longer needed by the current
+implementation.
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2011 Torsten Werner <twerner@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+################################################################################
+
+import psycopg2
+from daklib.dak_exceptions import DBUpdateError
+from socket import gethostname;
+
+################################################################################
+def do_update(self):
+    """
+    Remove old contents tables that are no longer needed by the current
+    implementation.
+    """
+    print __doc__
+    try:
+        c = self.db.cursor()
+
+        # remove useless type casts
+        for table in ('pending_bin_contents', 'deb_contents', 'udeb_contents'):
+            c.execute("DROP TABLE %s" % table)
+
+        c.execute("UPDATE config SET value = '43' WHERE name = 'db_revision'")
+        self.db.commit()
+
+    except psycopg2.ProgrammingError, msg:
+        self.db.rollback()
+        raise DBUpdateError, 'Unable to apply sick update 43, rollback issued. Error message : %s' % (str(msg))
index 928396c531d702e252f2da63f59df5795d24fab4..71bac95514980ab67ca814e30f543052e9a9b66a 100755 (executable)
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
 ################################################################################
 
 Cnf = None
-required_database_schema = 42
+required_database_schema = 43
 
 ################################################################################
 
diff --git a/daklib/binary.py b/daklib/binary.py
deleted file mode 100644 (file)
index d1f78f3..0000000
+++ /dev/null
@@ -1,367 +0,0 @@
-#!/usr/bin/python
-
-"""
-Functions related debian binary packages
-
-@contact: Debian FTPMaster <ftpmaster@debian.org>
-@copyright: 2009  Mike O'Connor <stew@debian.org>
-@license: GNU General Public License version 2 or later
-"""
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-################################################################################
-
-# <Ganneff> are we going the xorg way?
-# <Ganneff> a dak without a dak.conf?
-# <stew> automatically detect the wrong settings at runtime?
-# <Ganneff> yes!
-# <mhy> well, we'll probably always need dak.conf (how do you get the database setting
-# <mhy> but removing most of the config into the database seems sane
-# <Ganneff> mhy: dont spoil the fun
-# <Ganneff> mhy: and i know how. we nmap localhost and check all open ports
-# <Ganneff> maybe one answers to sql
-# <stew> we will discover projectb via avahi
-# <mhy> you're both sick
-# <mhy> really fucking sick
-
-################################################################################
-
-import os
-import sys
-import shutil
-import tarfile
-import commands
-import traceback
-import atexit
-
-try:
-    # starting with squeeze
-    from debian import deb822
-except:
-    # up to lenny
-    from debian_bundle import deb822
-
-from dbconn import *
-from config import Config
-import utils
-
-################################################################################
-
-__all__ = []
-
-################################################################################
-
-class Binary(object):
-    def __init__(self, filename, reject=None):
-        """
-        @type filename: string
-        @param filename: path of a .deb
-
-        @type reject: function
-        @param reject: a function to log reject messages to
-        """
-        self.filename = filename
-        self.tmpdir = None
-        self.chunks = None
-        self.wrapped_reject = reject
-        # Store rejects for later use
-        self.rejects = []
-
-    def reject(self, message):
-        """
-        if we were given a reject function, send the reject message,
-        otherwise send it to stderr.
-        """
-        print >> sys.stderr, message
-        self.rejects.append(message)
-        if self.wrapped_reject:
-            self.wrapped_reject(message)
-
-    def __del__(self):
-        """
-        make sure we cleanup when we are garbage collected.
-        """
-        self._cleanup()
-
-    def _cleanup(self):
-        """
-        we need to remove the temporary directory, if we created one
-        """
-        if self.tmpdir and os.path.exists(self.tmpdir):
-            shutil.rmtree(self.tmpdir)
-            self.tmpdir = None
-
-    def __scan_ar(self):
-        # get a list of the ar contents
-        if not self.chunks:
-
-            cmd = "ar t %s" % (self.filename)
-            (result, output) = commands.getstatusoutput(cmd)
-            if result != 0:
-                rejected = True
-                print("%s: 'ar t' invocation failed." % (self.filename))
-                self.reject("%s: 'ar t' invocation failed." % (self.filename))
-                self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
-            self.chunks = output.split('\n')
-
-
-
-    def __unpack(self):
-        # Internal function which extracts the contents of the .ar to
-        # a temporary directory
-
-        if not self.tmpdir:
-            tmpdir = utils.temp_dirname()
-            cwd = os.getcwd()
-            try:
-                os.chdir( tmpdir )
-                cmd = "ar x %s %s %s" % (os.path.join(cwd,self.filename), self.chunks[1], self.chunks[2])
-                (result, output) = commands.getstatusoutput(cmd)
-                if result != 0:
-                    print("%s: '%s' invocation failed." % (self.filename, cmd))
-                    self.reject("%s: '%s' invocation failed." % (self.filename, cmd))
-                    self.reject(utils.prefix_multi_line_string(output, " [ar output:] "))
-                else:
-                    self.tmpdir = tmpdir
-                    atexit.register( self._cleanup )
-
-            finally:
-                os.chdir( cwd )
-
-    def valid_deb(self, relaxed=False):
-        """
-        Check deb contents making sure the .deb contains:
-          1. debian-binary
-          2. control.tar.gz
-          3. data.tar.gz or data.tar.bz2
-        in that order, and nothing else.
-        """
-        self.__scan_ar()
-        rejected = not self.chunks
-        if relaxed:
-            if len(self.chunks) < 3:
-                rejected = True
-                self.reject("%s: found %d chunks, expected at least 3." % (self.filename, len(self.chunks)))
-        else:
-            if len(self.chunks) != 3:
-                rejected = True
-                self.reject("%s: found %d chunks, expected 3." % (self.filename, len(self.chunks)))
-        if self.chunks[0] != "debian-binary":
-            rejected = True
-            self.reject("%s: first chunk is '%s', expected 'debian-binary'." % (self.filename, self.chunks[0]))
-        if not rejected and self.chunks[1] != "control.tar.gz":
-            rejected = True
-            self.reject("%s: second chunk is '%s', expected 'control.tar.gz'." % (self.filename, self.chunks[1]))
-        if not rejected and self.chunks[2] not in [ "data.tar.bz2", "data.tar.gz" ]:
-            rejected = True
-            self.reject("%s: third chunk is '%s', expected 'data.tar.gz' or 'data.tar.bz2'." % (self.filename, self.chunks[2]))
-
-        return not rejected
-
-    def scan_package(self, bootstrap_id=0, relaxed=False, session=None):
-        """
-        Unpack the .deb, do sanity checking, and gather info from it.
-
-        Currently information gathering consists of getting the contents list. In
-        the hopefully near future, it should also include gathering info from the
-        control file.
-
-        @type bootstrap_id: int
-        @param bootstrap_id: the id of the binary these packages
-          should be associated or zero meaning we are not bootstrapping
-          so insert into a temporary table
-
-        @return: True if the deb is valid and contents were imported
-        """
-        result = False
-        rejected = not self.valid_deb(relaxed)
-        if not rejected:
-            self.__unpack()
-
-
-            cwd = os.getcwd()
-            if not rejected and self.tmpdir:
-                try:
-                    os.chdir(self.tmpdir)
-                    if self.chunks[1] == "control.tar.gz":
-                        control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
-                        control.extract('./control', self.tmpdir )
-                    if self.chunks[2] == "data.tar.gz":
-                        data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
-                    elif self.chunks[2] == "data.tar.bz2":
-                        data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
-
-                    if bootstrap_id:
-                        result = insert_content_paths(bootstrap_id, [tarinfo.name for tarinfo in data if not tarinfo.isdir()], session)
-                    else:
-                        pkgs = deb822.Packages.iter_paragraphs(file(os.path.join(self.tmpdir,'control')))
-                        pkg = pkgs.next()
-                        result = insert_pending_content_paths(pkg,
-                                                              self.filename.endswith('.udeb'),
-                                                              [tarinfo.name for tarinfo in data if not tarinfo.isdir()],
-                                                              session)
-
-                except:
-                    traceback.print_exc()
-
-            os.chdir(cwd)
-        self._cleanup()
-        return result
-
-    def check_utf8_package(self, package):
-        """
-        Unpack the .deb, do sanity checking, and gather info from it.
-
-        Currently information gathering consists of getting the contents list. In
-        the hopefully near future, it should also include gathering info from the
-        control file.
-
-        @type package: string
-        @param package: the name of the package to be checked
-
-        @rtype: boolean
-        @return: True if the deb is valid and contents were imported
-        """
-        rejected = not self.valid_deb(True)
-        self.__unpack()
-
-        if not rejected and self.tmpdir:
-            cwd = os.getcwd()
-            try:
-                os.chdir(self.tmpdir)
-                if self.chunks[1] == "control.tar.gz":
-                    control = tarfile.open(os.path.join(self.tmpdir, "control.tar.gz" ), "r:gz")
-                    control.extract('control', self.tmpdir )
-                if self.chunks[2] == "data.tar.gz":
-                    data = tarfile.open(os.path.join(self.tmpdir, "data.tar.gz"), "r:gz")
-                elif self.chunks[2] == "data.tar.bz2":
-                    data = tarfile.open(os.path.join(self.tmpdir, "data.tar.bz2" ), "r:bz2")
-
-                for tarinfo in data:
-                    try:
-                        unicode( tarinfo.name )
-                    except:
-                        print >> sys.stderr, "E: %s has non-unicode filename: %s" % (package,tarinfo.name)
-
-                result = True
-
-            except:
-                traceback.print_exc()
-                result = False
-
-            os.chdir(cwd)
-
-        return result
-
-__all__.append('Binary')
-
-
-def copy_temporary_contents(binary, bin_association, reject, session=None):
-    """
-    copy the previously stored contents from the temp table to the permanant one
-
-    during process-unchecked, the deb should have been scanned and the
-    contents stored in pending_content_associations
-    """
-
-    cnf = Config()
-
-    privatetrans = False
-    if session is None:
-        session = DBConn().session()
-        privatetrans = True
-
-    arch = get_architecture(archname, session=session)
-
-    pending = session.query(PendingBinContents).filter_by(package=binary.package,
-                                                          version=binary.version,
-                                                          arch=binary.arch).first()
-
-    if pending:
-        # This should NOT happen.  We should have added contents
-        # during process-unchecked.  if it did, log an error, and send
-        # an email.
-        subst = {
-            "__PACKAGE__": package,
-            "__VERSION__": version,
-            "__ARCH__": arch,
-            "__TO_ADDRESS__": cnf["Dinstall::MyAdminAddress"],
-            "__DAK_ADDRESS__": cnf["Dinstall::MyEmailAddress"] }
-
-        message = utils.TemplateSubst(subst, cnf["Dir::Templates"]+"/missing-contents")
-        utils.send_mail(message)
-
-        # rescan it now
-        exists = Binary(deb, reject).scan_package()
-
-        if not exists:
-            # LOG?
-            return False
-
-    component = binary.poolfile.location.component
-    override = session.query(Override).filter_by(package=binary.package,
-                                                 suite=bin_association.suite,
-                                                 component=component.id).first()
-    if not override:
-        # LOG?
-        return False
-
-
-    if not override.overridetype.type.endswith('deb'):
-        return True
-
-    if override.overridetype.type == "udeb":
-        table = "udeb_contents"
-    elif override.overridetype.type == "deb":
-        table = "deb_contents"
-    else:
-        return False
-
-
-    if component.name == "main":
-        component_str = ""
-    else:
-        component_str = component.name + "/"
-
-    vals = { 'package':binary.package,
-             'version':binary.version,
-             'arch':binary.architecture,
-             'binary_id': binary.id,
-             'component':component_str,
-             'section':override.section.section
-             }
-
-    session.execute( """INSERT INTO %s
-    (binary_id,package,version.component,arch,section,filename)
-    SELECT :binary_id, :package, :version, :component, :arch, :section
-    FROM pending_bin_contents pbc
-    WHERE pbc.package=:package
-    AND pbc.version=:version
-    AND pbc.arch=:arch""" % table, vals )
-
-    session.execute( """DELETE from pending_bin_contents package=:package
-    AND version=:version
-    AND arch=:arch""", vals )
-
-    if privatetrans:
-        session.commit()
-        session.close()
-
-    return exists
-
-__all__.append('copy_temporary_contents')
-
-
index 9b757d13487887382595daa0cb23b55742301411..4f6b02d8fdfec3fd640551e63070fc893c3e8436 100755 (executable)
@@ -77,9 +77,6 @@ import warnings
 warnings.filterwarnings('ignore', \
     "The SQLAlchemy PostgreSQL dialect has been renamed from 'postgres' to 'postgresql'.*", \
     SADeprecationWarning)
-# TODO: sqlalchemy needs some extra configuration to correctly reflect
-# the ind_deb_contents_* indexes - we ignore the warnings at the moment
-warnings.filterwarnings("ignore", 'Predicate of partial index', SAWarning)
 
 
 ################################################################################
@@ -1948,111 +1945,6 @@ __all__.append('get_override_type')
 
 ################################################################################
 
-class DebContents(object):
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def __repr__(self):
-        return '<DebConetnts %s: %s>' % (self.package.package,self.file)
-
-__all__.append('DebContents')
-
-
-class UdebContents(object):
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def __repr__(self):
-        return '<UdebConetnts %s: %s>' % (self.package.package,self.file)
-
-__all__.append('UdebContents')
-
-class PendingBinContents(object):
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def __repr__(self):
-        return '<PendingBinContents %s>' % self.contents_id
-
-__all__.append('PendingBinContents')
-
-def insert_pending_content_paths(package,
-                                 is_udeb,
-                                 fullpaths,
-                                 session=None):
-    """
-    Make sure given paths are temporarily associated with given
-    package
-
-    @type package: dict
-    @param package: the package to associate with should have been read in from the binary control file
-    @type fullpaths: list
-    @param fullpaths: the list of paths of the file being associated with the binary
-    @type session: SQLAlchemy session
-    @param session: Optional SQLAlchemy session.  If this is passed, the caller
-    is responsible for ensuring a transaction has begun and committing the
-    results or rolling back based on the result code.  If not passed, a commit
-    will be performed at the end of the function
-
-    @return: True upon success, False if there is a problem
-    """
-
-    privatetrans = False
-
-    if session is None:
-        session = DBConn().session()
-        privatetrans = True
-
-    try:
-        arch = get_architecture(package['Architecture'], session)
-        arch_id = arch.arch_id
-
-        # Remove any already existing recorded files for this package
-        q = session.query(PendingBinContents)
-        q = q.filter_by(package=package['Package'])
-        q = q.filter_by(version=package['Version'])
-        q = q.filter_by(architecture=arch_id)
-        q.delete()
-
-        for fullpath in fullpaths:
-
-            if fullpath.startswith( "./" ):
-                fullpath = fullpath[2:]
-
-            pca = PendingBinContents()
-            pca.package = package['Package']
-            pca.version = package['Version']
-            pca.file = fullpath
-            pca.architecture = arch_id
-
-            if isudeb:
-                pca.type = 8 # gross
-            else:
-                pca.type = 7 # also gross
-            session.add(pca)
-
-        # Only commit if we set up the session ourself
-        if privatetrans:
-            session.commit()
-            session.close()
-        else:
-            session.flush()
-
-        return True
-    except Exception, e:
-        traceback.print_exc()
-
-        # Only rollback if we set up the session ourself
-        if privatetrans:
-            session.rollback()
-            session.close()
-
-        return False
-
-__all__.append('insert_pending_content_paths')
-
-################################################################################
-
 class PolicyQueue(object):
     def __init__(self, *args, **kwargs):
         pass
@@ -2940,7 +2832,6 @@ class DBConn(object):
             'maintainer',
             'new_comments',
             'override_type',
-            'pending_bin_contents',
             'policy_queue',
             'priority',
             'section',
@@ -2958,13 +2849,11 @@ class DBConn(object):
             'changes_pending_files_map',
             'changes_pending_source_files',
             'changes_pool_files',
-            'deb_contents',
             # TODO: the maintainer column in table override should be removed.
             'override',
             'suite_architectures',
             'suite_src_formats',
             'suite_build_queue_copy',
-            'udeb_contents',
         )
 
         views = (
@@ -3028,30 +2917,6 @@ class DBConn(object):
                properties = dict(archive_id = self.tbl_archive.c.id,
                                  archive_name = self.tbl_archive.c.name))
 
-        mapper(PendingBinContents, self.tbl_pending_bin_contents,
-               properties = dict(contents_id =self.tbl_pending_bin_contents.c.id,
-                                 filename = self.tbl_pending_bin_contents.c.filename,
-                                 package = self.tbl_pending_bin_contents.c.package,
-                                 version = self.tbl_pending_bin_contents.c.version,
-                                 arch = self.tbl_pending_bin_contents.c.arch,
-                                 otype = self.tbl_pending_bin_contents.c.type))
-
-        mapper(DebContents, self.tbl_deb_contents,
-               properties = dict(binary_id=self.tbl_deb_contents.c.binary_id,
-                                 package=self.tbl_deb_contents.c.package,
-                                 suite=self.tbl_deb_contents.c.suite,
-                                 arch=self.tbl_deb_contents.c.arch,
-                                 section=self.tbl_deb_contents.c.section,
-                                 filename=self.tbl_deb_contents.c.filename))
-
-        mapper(UdebContents, self.tbl_udeb_contents,
-               properties = dict(binary_id=self.tbl_udeb_contents.c.binary_id,
-                                 package=self.tbl_udeb_contents.c.package,
-                                 suite=self.tbl_udeb_contents.c.suite,
-                                 arch=self.tbl_udeb_contents.c.arch,
-                                 section=self.tbl_udeb_contents.c.section,
-                                 filename=self.tbl_udeb_contents.c.filename))
-
         mapper(BuildQueue, self.tbl_build_queue,
                properties = dict(queue_id = self.tbl_build_queue.c.id))
 
index a7e2d5a9b8a3823f2424a69763718a6660b4c6a2..b3e26d7bed50945dc1a47b4cf525c0fb7f309a64 100755 (executable)
@@ -53,7 +53,6 @@ from dbconn import *
 from summarystats import SummaryStats
 from utils import parse_changes, check_dsc_files
 from textutils import fix_maintainer
-from binary import Binary
 from lintian import parse_lintian_output, generate_reject_messages
 
 # suppress some deprecation warnings in squeeze related to apt_pkg
@@ -866,13 +865,6 @@ class Upload(object):
         # Check the version and for file overwrites
         self.check_binary_against_db(f, session)
 
-        # Temporarily disable contents generation until we change the table storage layout
-        #b = Binary(f)
-        #b.scan_package()
-        #if len(b.rejects) > 0:
-        #    for j in b.rejects:
-        #        self.rejects.append(j)
-
     def source_file_checks(self, f, session):
         entry = self.pkg.files[f]