From b36379705f3f2a52788ad19cf1da339c4edf339d Mon Sep 17 00:00:00 2001 From: Torsten Werner Date: Sat, 26 Mar 2011 13:04:15 +0100 Subject: [PATCH] Split binary contents into components. Signed-off-by: Torsten Werner --- dak/contents.py | 22 ++++++---- dak/generate_index_diffs.py | 14 +++--- daklib/contents.py | 85 +++++++++++-------------------------- daklib/filewriter.py | 6 +-- 4 files changed, 47 insertions(+), 80 deletions(-) diff --git a/dak/contents.py b/dak/contents.py index ee904b2a..c2d04eb6 100755 --- a/dak/contents.py +++ b/dak/contents.py @@ -70,6 +70,9 @@ OPTIONS for generate -s, --suite={stable,testing,unstable,...} only operate on specified suite names + -c, --component={main,contrib,non-free} + only operate on specified components + -f, --force write Contents files for suites marked as untouchable, too @@ -81,9 +84,9 @@ OPTIONS for scan-source and scan-binary ################################################################################ -def write_all(cnf, suite_names = [], force = None): +def write_all(cnf, suite_names = [], component_names = [], force = None): Logger = daklog.Logger(cnf.Cnf, 'contents generate') - ContentsWriter.write_all(Logger, suite_names, force) + ContentsWriter.write_all(Logger, suite_names, component_names, force) Logger.close() ################################################################################ @@ -112,12 +115,14 @@ def main(): cnf = Config() cnf['Contents::Options::Help'] = '' cnf['Contents::Options::Suite'] = '' + cnf['Contents::Options::Component'] = '' cnf['Contents::Options::Limit'] = '' cnf['Contents::Options::Force'] = '' - arguments = [('h', "help", 'Contents::Options::Help'), - ('s', "suite", 'Contents::Options::Suite', "HasArg"), - ('l', "limit", 'Contents::Options::Limit', "HasArg"), - ('f', "force", 'Contents::Options::Force'), + arguments = [('h', "help", 'Contents::Options::Help'), + ('s', "suite", 'Contents::Options::Suite', "HasArg"), + ('c', "component", 'Contents::Options::Component', "HasArg"), + ('l', "limit", 'Contents::Options::Limit', "HasArg"), + ('f', "force", 'Contents::Options::Force'), ] args = apt_pkg.ParseCommandLine(cnf.Cnf, arguments, sys.argv) options = cnf.SubTree('Contents::Options') @@ -137,12 +142,13 @@ def main(): binary_scan_all(cnf, limit) return - suite_names = utils.split_args(options['Suite']) + suite_names = utils.split_args(options['Suite']) + component_names = utils.split_args(options['Component']) force = bool(options['Force']) if args[0] == 'generate': - write_all(cnf, suite_names, force) + write_all(cnf, suite_names, component_names, force) return usage() diff --git a/dak/generate_index_diffs.py b/dak/generate_index_diffs.py index 7e4b0058..3a69f3a5 100755 --- a/dak/generate_index_diffs.py +++ b/dak/generate_index_diffs.py @@ -352,14 +352,6 @@ def main(): for archobj in architectures: architecture = archobj.arch_string - if architecture != "source": - # Process Contents - file = "%s/Contents-%s" % (Cnf["Dir::Root"] + tree, - architecture) - storename = "%s/%s_contents_%s" % (Options["TempDir"], suite, architecture) - genchanges(Options, file + ".diff", storename, file, \ - Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents)) - # use sections instead of components since dak.conf # treats "foo/bar main" as suite "foo", suitesuffix "bar" and # component "bar/main". suck. @@ -373,6 +365,12 @@ def main(): longarch = "binary-%s"% (architecture) packages = "Packages" maxsuite = maxpackages + # Process Contents + file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component, + longarch) + storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture) + genchanges(Options, file + ".diff", storename, file, \ + Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents)) file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree, component, longarch, packages) diff --git a/daklib/contents.py b/daklib/contents.py index a906b54d..f5f27488 100755 --- a/daklib/contents.py +++ b/daklib/contents.py @@ -41,7 +41,7 @@ class BinaryContentsWriter(object): ''' BinaryContentsWriter writes the Contents-$arch.gz files. ''' - def __init__(self, suite, architecture, overridetype, component = None): + def __init__(self, suite, architecture, overridetype, component): self.suite = suite self.architecture = architecture self.overridetype = overridetype @@ -58,15 +58,14 @@ class BinaryContentsWriter(object): params = { 'suite': self.suite.suite_id, 'overridesuite': overridesuite.suite_id, + 'component': self.component.component_id, 'arch_all': get_architecture('all', self.session).arch_id, 'arch': self.architecture.arch_id, 'type_id': self.overridetype.overridetype_id, 'type': self.overridetype.overridetype, } - if self.component is not None: - params['component'] = self.component.component_id - sql = ''' + sql = ''' create temp table newest_binaries ( id integer primary key, package text); @@ -88,34 +87,6 @@ unique_override as where o.suite = :overridesuite and o.type = :type_id and o.section = s.id and o.component = :component) -select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package) as pkglist - from newest_binaries b, bin_contents bc, unique_override o - where b.id = bc.binary_id and o.package = b.package - group by bc.file''' - - else: - sql = ''' -create temp table newest_binaries ( - id integer primary key, - package text); - -create index newest_binaries_by_package on newest_binaries (package); - -insert into newest_binaries (id, package) - select distinct on (package) id, package from binaries - where type = :type and - (architecture = :arch_all or architecture = :arch) and - id in (select bin from bin_associations where suite = :suite) - order by package, version desc; - -with - -unique_override as - (select distinct on (o.package, s.section) o.package, s.section - from override o, section s - where o.suite = :overridesuite and o.type = :type_id and o.section = s.id - order by o.package, s.section, o.modified desc) - select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package) as pkglist from newest_binaries b, bin_contents bc, unique_override o where b.id = bc.binary_id and o.package = b.package @@ -151,10 +122,9 @@ select bc.file, string_agg(o.section || '/' || b.package, ',' order by b.package ''' values = { 'suite': self.suite.suite_name, + 'debtype': self.overridetype.overridetype, 'architecture': self.architecture.arch_string, } - if self.component is not None: - values['component'] = self.component.component_name return BinaryContentsFileWriter(**values) def get_header(self): @@ -264,7 +234,7 @@ select sc.file, string_agg(s.source, ',' order by s.source) as pkglist writer.close() -def binary_helper(suite_id, arch_id, overridetype_id, component_id = None): +def binary_helper(suite_id, arch_id, overridetype_id, component_id): ''' This function is called in a new subprocess and multiprocessing wants a top level function. @@ -273,12 +243,9 @@ def binary_helper(suite_id, arch_id, overridetype_id, component_id = None): suite = Suite.get(suite_id, session) architecture = Architecture.get(arch_id, session) overridetype = OverrideType.get(overridetype_id, session) - log_message = [suite.suite_name, architecture.arch_string, overridetype.overridetype] - if component_id is None: - component = None - else: - component = Component.get(component_id, session) - log_message.append(component.component_name) + component = Component.get(component_id, session) + log_message = [suite.suite_name, architecture.arch_string, \ + overridetype.overridetype, component.component_name] contents_writer = BinaryContentsWriter(suite, architecture, overridetype, component) contents_writer.write_file() return log_message @@ -309,7 +276,7 @@ class ContentsWriter(object): class_.logger.log(result) @classmethod - def write_all(class_, logger, suite_names = [], force = False): + def write_all(class_, logger, suite_names = [], component_names = [], force = False): ''' Writes all Contents files for suites in list suite_names which defaults to all 'touchable' suites if not specified explicitely. Untouchable @@ -320,33 +287,29 @@ class ContentsWriter(object): suite_query = session.query(Suite) if len(suite_names) > 0: suite_query = suite_query.filter(Suite.suite_name.in_(suite_names)) + component_query = session.query(Component) + if len(component_names) > 0: + component_query = component_query.filter(Component.component_name.in_(component_names)) if not force: suite_query = suite_query.filter_by(untouchable = False) deb_id = get_override_type('deb', session).overridetype_id udeb_id = get_override_type('udeb', session).overridetype_id - main_id = get_component('main', session).component_id - contrib_id = get_component('contrib', session).component_id - non_free_id = get_component('non-free', session).component_id pool = Pool() for suite in suite_query: suite_id = suite.suite_id - # handle source packages - pool.apply_async(source_helper, (suite_id, main_id), - callback = class_.log_result) - pool.apply_async(source_helper, (suite_id, contrib_id), - callback = class_.log_result) - pool.apply_async(source_helper, (suite_id, non_free_id), - callback = class_.log_result) - for architecture in suite.get_architectures(skipsrc = True, skipall = True): - arch_id = architecture.arch_id - # handle 'deb' packages - pool.apply_async(binary_helper, (suite_id, arch_id, deb_id), \ - callback = class_.log_result) - # handle 'udeb' packages for 'main' and 'non-free' - pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, main_id), \ - callback = class_.log_result) - pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, non_free_id), \ + for component in component_query: + component_id = component.component_id + # handle source packages + pool.apply_async(source_helper, (suite_id, component_id), callback = class_.log_result) + for architecture in suite.get_architectures(skipsrc = True, skipall = True): + arch_id = architecture.arch_id + # handle 'deb' packages + pool.apply_async(binary_helper, (suite_id, arch_id, deb_id, component_id), \ + callback = class_.log_result) + # handle 'udeb' packages + pool.apply_async(binary_helper, (suite_id, arch_id, udeb_id, component_id), \ + callback = class_.log_result) pool.close() pool.join() session.close() diff --git a/daklib/filewriter.py b/daklib/filewriter.py index 714531af..a3c16ea8 100755 --- a/daklib/filewriter.py +++ b/daklib/filewriter.py @@ -93,10 +93,10 @@ class BinaryContentsFileWriter(BaseFileWriter): 'bzip2': False } flags.update(keywords) - if 'component' in flags: + if flags['debtype'] == 'deb': template = "dists/%(suite)s/%(component)s/Contents-%(architecture)s" - else: - template = "dists/%(suite)s/Contents-%(architecture)s" + else: # udeb + template = "dists/%(suite)s/%(component)s/Contents-udeb-%(architecture)s" BaseFileWriter.__init__(self, template, **flags) class SourceContentsFileWriter(BaseFileWriter): -- 2.39.5