From f0f02944f8a5744e3f10989c3afb53af2753bc15 Mon Sep 17 00:00:00 2001 From: Ansgar Burchardt Date: Wed, 10 Apr 2013 23:04:00 +0200 Subject: [PATCH] Remove dak/metadata.py and daklib/metadata.py. dak imports package metadata at installation time for some time already. There is no longer need for an extra subcommand to do so. --- dak/metadata.py | 117 --------------------------------------------- daklib/metadata.py | 113 ------------------------------------------- 2 files changed, 230 deletions(-) delete mode 100755 dak/metadata.py delete mode 100644 daklib/metadata.py diff --git a/dak/metadata.py b/dak/metadata.py deleted file mode 100755 index b4c1d585..00000000 --- a/dak/metadata.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -""" -Import data for Package/Sources files from .deb and .dsc files -@copyright: 2011 Torsten Werner -@copyright: 2011 Mark Hymers -@license: GNU General Public License version 2 or later -""" - -################################################################################ - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -################################################################################ - -# < mvo> that screams for consolidation in libapt at least (that then in turn can -# use libdpkg ... ) - I guess the "d" means delayed ;) - -# (whilst discussing adding xz support to dak, and therefore python-apt, and -# therefore libapt-pkg) - -################################################################################ - -import sys -import apt_pkg - -from daklib.config import Config -from daklib.dbconn import * -from daklib.metadata import MetadataScanner -from daklib import daklog -from daklib import utils - -################################################################################ - -def usage (exit_code=0): - print """Usage: dak metadata [options] subcommand - -SUBCOMMANDS - scan-source - scan the dsc files in the existing pool and load metadata into the database - - scan-binary - scan the deb files in the existing pool and load metadata into the database - -OPTIONS - -h, --help - show this help and exit - -OPTIONS for scan - -l, --limit=NUMBER - maximum number of items to scan -""" - sys.exit(exit_code) - -################################################################################ - -def scan_all(cnf, mode, limit): - Logger = daklog.Logger('metadata scan (%s)' % mode) - result = MetadataScanner.scan_all(mode, limit) - processed = '%(processed)d %(type)s processed' % result - remaining = '%(remaining)d %(type)s remaining' % result - Logger.log([processed, remaining]) - Logger.close() - -################################################################################ - -def main(): - cnf = Config() - cnf['Metadata::Options::Help'] = '' - cnf['Metadata::Options::Suite'] = '' - cnf['Metadata::Options::Limit'] = '' - cnf['Metadata::Options::Force'] = '' - arguments = [('h', "help", 'Metadata::Options::Help'), - ('s', "suite", 'Metadata::Options::Suite', "HasArg"), - ('l', "limit", 'Metadata::Options::Limit', "HasArg"), - ('f', "force", 'Metadata::Options::Force'), - ] - args = apt_pkg.parse_commandline(cnf.Cnf, arguments, sys.argv) - options = cnf.subtree('Metadata::Options') - - if (len(args) != 1) or options['Help']: - usage() - - limit = None - if len(options['Limit']) > 0: - limit = int(options['Limit']) - - if args[0] == 'scan-source': - scan_all(cnf, 'source', limit) - return - elif args[0] == 'scan-binary': - scan_all(cnf, 'binary', limit) - return - - suite_names = utils.split_args(options['Suite']) - - force = bool(options['Force']) - - if args[0] == 'generate': - raise NotImplementError - - usage() - - -if __name__ == '__main__': - main() diff --git a/daklib/metadata.py b/daklib/metadata.py deleted file mode 100644 index 793b073c..00000000 --- a/daklib/metadata.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -""" -Helper code for packages and sources generation. - -@contact: Debian FTPMaster -@copyright: 2011 Torsten Werner -@copyright: 2011 Mark Hymers -@license: GNU General Public License version 2 or later -""" - -################################################################################ - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -################################################################################ - -from daklib.dbconn import * -from daklib.config import Config - -from multiprocessing import Pool -from subprocess import Popen, PIPE - -import os.path - -class MetadataScanner(object): - ''' - MetadataScanner provides a threadsafe method scan() to scan the metadata of - a DBSource or DBBinary object depending on what is passed as dbclass''' - - def __init__(self, dbclass, pkid, verbose=True): - ''' - The argument binary_id is the id of the DBBinary object that - - should be scanned. - ''' - self.verbose = True - self.dbclass = dbclass - self.pkid = pkid - - def scan(self, dummy_arg = None): - ''' - This method does the actual scan and fills in the associated metadata - property. It commits any changes to the database. The argument dummy_arg - is ignored but needed by our threadpool implementation. - ''' - obj = None - fullpath = 'UNKNOWN PATH' - - session = DBConn().session() - try: - obj = session.query(self.dbclass).get(self.pkid) - fullpath = obj.poolfile.fullpath - import_metadata_into_db(obj, session=session) - if self.verbose: - print "Imported %s (%s)" % (self.pkid, fullpath) - session.commit() - except Exception as e: - print "Failed to import %s [id=%s; fullpath=%s]" % (self.dbclass.__name__, self.pkid, fullpath) - print "Exception: ", e - session.rollback() - - session.close() - - @classmethod - def scan_all(class_, scantype='source', limit = None): - ''' - The class method scan_all() scans all sources using multiple threads. - The number of sources to be scanned can be limited with the limit - argument. Returns the number of processed and remaining files as a - dict. - ''' - session = DBConn().session() - if scantype == 'source': - dbclass = DBSource - query = session.query(DBSource).filter(~DBSource.source_id.in_(session.query(SourceMetadata.source_id.distinct()))) - t = 'sources' - else: - # Otherwise binary - dbclass = DBBinary - query = session.query(DBBinary).filter(~DBBinary.binary_id.in_(session.query(BinaryMetadata.binary_id.distinct()))) - t = 'binaries' - - remaining = query.count - if limit is not None: - query = query.limit(limit) - processed = query.count() - pool = Pool(processes=10) - for obj in query.yield_per(100): - pool.apply_async(scan_helper, (dbclass, obj.pkid, )) - pool.close() - pool.join() - remaining = remaining() - session.close() - return { 'processed': processed, 'remaining': remaining , 'type': t} - -def scan_helper(dbclass, source_id): - ''' - This function runs in a subprocess. - ''' - scanner = MetadataScanner(dbclass, source_id) - scanner.scan() -- 2.39.5