From 78a2020dbc565ee964741f9d1536d0c7ff3367e3 Mon Sep 17 00:00:00 2001 From: Mark Hymers Date: Sat, 22 May 2010 13:16:52 +0000 Subject: [PATCH] kill make_suite_file_list Signed-off-by: Mark Hymers --- dak/dak.py | 2 - dak/make_suite_file_list.py | 397 ------------------------------- docs/README.first | 5 +- docs/README.stable-point-release | 2 +- 4 files changed, 3 insertions(+), 403 deletions(-) delete mode 100755 dak/make_suite_file_list.py diff --git a/dak/dak.py b/dak/dak.py index 9be84300..8d7776a5 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -75,8 +75,6 @@ def init(): ("dominate", "Remove obsolete source and binary associations from suites"), - ("make-suite-file-list", - "OBSOLETE: replaced by dominate and generate-filelist"), ("make-pkg-file-mapping", "Generate package <-> file mapping"), ("generate-filelist", diff --git a/dak/make_suite_file_list.py b/dak/make_suite_file_list.py deleted file mode 100755 index 8c131009..00000000 --- a/dak/make_suite_file_list.py +++ /dev/null @@ -1,397 +0,0 @@ -#!/usr/bin/env python - -""" -Generate file lists used by apt-ftparchive to generate Packages and Sources files -@contact: Debian FTP Master -@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup -@license: GNU General Public License version 2 or later -""" - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -################################################################################ - -# I'm doing it in python btw.. nothing against your monster -# SQL, but the python wins in terms of speed and readiblity -# bah -# you suck!!!!! -# sorry :( -# you are not!!! -# you mock my SQL!!!! -# you want have contest of skillz?????? -# all your skillz are belong to my sql!!!! -# yo momma are belong to my python!!!! -# yo momma was SQLin' like a pig last night! - -################################################################################ - -import copy -import os -import sys -import apt_pkg - -from daklib.dbconn import * -from daklib.config import Config -from daklib import daklog -from daklib import utils - -################################################################################ - -Logger = None #: Logger object -Options = None #: Parsed CommandLine arguments - -################################################################################ - -def usage (exit_code=0): - print """Usage: dak make-suite-file-list [OPTION] -Write out file lists suitable for use with apt-ftparchive. - - -a, --architecture=ARCH only write file lists for this architecture - -c, --component=COMPONENT only write file lists for this component - -f, --force ignore Untouchable suite directives in dak.conf - -h, --help show this help and exit - -n, --no-delete don't delete older versions - -s, --suite=SUITE only write file lists for this suite - -ARCH, COMPONENT and SUITE can be space separated lists, e.g. - --architecture=\"m68k i386\"""" - sys.exit(exit_code) - -################################################################################ - -def version_cmp(a, b): - return -apt_pkg.VersionCompare(a[0], b[0]) - -##################################################### - -def delete_packages(delete_versions, pkg, dominant_arch, suite, - dominant_version, delete_table, delete_col, packages, session): - suite_o = get_suite(suite.lower(), session) - suite_id = suite_o.suite_id - for version in delete_versions: - delete_unique_id = version[1] - if not packages.has_key(delete_unique_id): - continue - delete_version = version[0] - delete_id = packages[delete_unique_id]["sourceid"] - delete_arch = packages[delete_unique_id]["arch"] - if Options["Force"] or not suite_o.untouchable: - if Options["No-Delete"]: - print "Would delete %s_%s_%s in %s in favour of %s_%s" % (pkg, delete_arch, delete_version, suite, dominant_version, dominant_arch) - else: - Logger.log(["dominated", pkg, delete_arch, delete_version, dominant_version, dominant_arch]) - # TODO: Fix properly - session.execute("DELETE FROM %s WHERE suite = :suiteid AND %s = :delid" % (delete_table, delete_col), {'suiteid': suite_id, 'delid': delete_id}) - del packages[delete_unique_id] - else: - if Options["No-Delete"]: - print "Would delete %s_%s_%s in favour of %s_%s, but %s is untouchable" % (pkg, delete_arch, delete_version, dominant_version, dominant_arch, suite) - else: - Logger.log(["dominated but untouchable", pkg, delete_arch, delete_version, dominant_version, dominant_arch]) - -##################################################### - -def resolve_arch_all_vs_any(versions, packages, session): - """ Per-suite&pkg: resolve arch-all, vs. arch-any, assumes only one arch-all """ - arch_all_version = None - arch_any_versions = copy.copy(versions) - for i in arch_any_versions: - unique_id = i[1] - arch = packages[unique_id]["arch"] - if arch == "all": - arch_all_versions = [i] - arch_all_version = i[0] - arch_any_versions.remove(i) - # Sort arch: any versions into descending order - arch_any_versions.sort(version_cmp) - highest_arch_any_version = arch_any_versions[0][0] - - pkg = packages[unique_id]["pkg"] - suite = packages[unique_id]["suite"] - delete_table = "bin_associations" - delete_col = "bin" - - if apt_pkg.VersionCompare(highest_arch_any_version, arch_all_version) < 1: - # arch: all dominates - delete_packages(arch_any_versions, pkg, "all", suite, - arch_all_version, delete_table, delete_col, packages, session) - else: - # arch: any dominates - delete_packages(arch_all_versions, pkg, "any", suite, - highest_arch_any_version, delete_table, delete_col, - packages, session) - -##################################################### - -def remove_duplicate_versions(versions, packages, session): - """ Per-suite&pkg&arch: resolve duplicate versions """ - # Sort versions into descending order - versions.sort(version_cmp) - dominant_versions = versions[0] - dominated_versions = versions[1:] - (dominant_version, dominant_unqiue_id) = dominant_versions - pkg = packages[dominant_unqiue_id]["pkg"] - arch = packages[dominant_unqiue_id]["arch"] - suite = packages[dominant_unqiue_id]["suite"] - if arch == "source": - delete_table = "src_associations" - delete_col = "source" - else: # !source - delete_table = "bin_associations" - delete_col = "bin" - # Remove all but the highest - delete_packages(dominated_versions, pkg, arch, suite, - dominant_version, delete_table, delete_col, packages, session) - return [dominant_versions] - -################################################################################ - -def cleanup(packages, session): - # Build up the index used by the clean up functions - d = {} - for unique_id in packages.keys(): - suite = packages[unique_id]["suite"] - pkg = packages[unique_id]["pkg"] - arch = packages[unique_id]["arch"] - version = packages[unique_id]["version"] - d.setdefault(suite, {}) - d[suite].setdefault(pkg, {}) - d[suite][pkg].setdefault(arch, []) - d[suite][pkg][arch].append([version, unique_id]) - # Clean up old versions - for suite in d.keys(): - for pkg in d[suite].keys(): - for arch in d[suite][pkg].keys(): - versions = d[suite][pkg][arch] - if len(versions) > 1: - d[suite][pkg][arch] = remove_duplicate_versions(versions, packages, session) - - # Arch: all -> any and vice versa - for suite in d.keys(): - for pkg in d[suite].keys(): - arches = d[suite][pkg] - # If we don't have any arch: all; we've nothing to do - if not arches.has_key("all"): - continue - # Check to see if we have arch: all and arch: !all (ignoring source) - num_arches = len(arches.keys()) - if arches.has_key("source"): - num_arches -= 1 - # If we do, remove the duplicates - if num_arches > 1: - versions = [] - for arch in arches.keys(): - if arch != "source": - versions.extend(d[suite][pkg][arch]) - resolve_arch_all_vs_any(versions, packages, session) - -################################################################################ - -def write_filelist(suite, component, arch, type, list, packages, dislocated_files): - cnf = Config() - - # Work out the filename - if arch != "source": - if type == "udeb": - arch = "debian-installer_binary-%s" % (arch) - elif type == "deb": - arch = "binary-%s" % (arch) - filename = os.path.join(cnf["Dir::Lists"], "%s_%s_%s.list" % (suite, component, arch)) - output = utils.open_file(filename, "w") - # Generate the final list of files - files = {} - for fileid in list: - path = packages[fileid]["path"] - filename = packages[fileid]["filename"] - file_id = packages[fileid]["file_id"] - pkg = packages[fileid]["pkg"] - if suite == "stable" and dislocated_files.has_key(file_id): - filename = dislocated_files[file_id] - else: - filename = path + filename - if files.has_key(pkg): - utils.warn("%s (in %s/%s, %s) is duplicated." % (pkg, suite, component, filename)) - else: - files[pkg] = filename - # Sort the files since apt-ftparchive doesn't - pkgs = files.keys() - pkgs.sort() - # Write the list of files out - for pkg in pkgs: - output.write(files[pkg]+'\n') - output.close() - -################################################################################ - -def write_filelists(packages, dislocated_files, session): - # Build up the index to iterate over - cnf = Config() - - d = {} - for unique_id in packages.keys(): - suite = packages[unique_id]["suite"] - component = packages[unique_id]["component"] - arch = packages[unique_id]["arch"] - packagetype = packages[unique_id]["filetype"] - d.setdefault(suite, {}) - d[suite].setdefault(component, {}) - d[suite][component].setdefault(arch, {}) - d[suite][component][arch].setdefault(packagetype, []) - d[suite][component][arch][packagetype].append(unique_id) - # Flesh out the index - if not Options["Suite"]: - suites = cnf.SubTree("Suite").List() - else: - suites = utils.split_args(Options["Suite"]) - for suite in [ i.lower() for i in suites ]: - d.setdefault(suite, {}) - if not Options["Component"]: - components = cnf.ValueList("Suite::%s::Components" % (suite)) - else: - components = utils.split_args(Options["Component"]) - udeb_components = cnf.ValueList("Suite::%s::UdebComponents" % (suite)) - for component in components: - d[suite].setdefault(component, {}) - if component in udeb_components: - binary_types = [ "deb", "udeb" ] - else: - binary_types = [ "deb" ] - if not Options["Architecture"]: - architectures = [ a.arch_string for a in get_suite_architectures(suite, session=session) ] - else: - architectures = utils.split_args(Options["Architecture"]) - for arch in [ i.lower() for i in architectures ]: - d[suite][component].setdefault(arch, {}) - if arch == "source": - types = [ "dsc" ] - else: - types = binary_types - for packagetype in types: - d[suite][component][arch].setdefault(packagetype, []) - # Then walk it - for suite in d.keys(): - if cnf.has_key("Suite::%s::Components" % (suite)): - for component in d[suite].keys(): - for arch in d[suite][component].keys(): - if arch == "all": - continue - for packagetype in d[suite][component][arch].keys(): - filelist = d[suite][component][arch][packagetype] - # If it's a binary, we need to add in the arch: all debs too - if arch != "source": - archall_suite = cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite)) - if archall_suite: - filelist.extend(d[archall_suite][component]["all"][packagetype]) - elif d[suite][component].has_key("all") and \ - d[suite][component]["all"].has_key(packagetype): - filelist.extend(d[suite][component]["all"][packagetype]) - write_filelist(suite, component, arch, packagetype, filelist, - packages, dislocated_files) - else: # something broken - utils.warn("Suite %s has no components." % (suite)) - -################################################################################ - -def do_da_do_da(): - cnf = Config() - - # If we're only doing a subset of suites, ensure we do enough to - # be able to do arch: all mapping. - if Options["Suite"]: - suites = utils.split_args(Options["Suite"]) - for suite in suites: - archall_suite = cnf.get("Make-Suite-File-List::ArchAllMap::%s" % (suite)) - if archall_suite and archall_suite not in suites: - utils.warn("Adding %s as %s maps Arch: all from it." % (archall_suite, suite)) - suites.append(archall_suite) - Options["Suite"] = ",".join(suites) - - (con_suites, con_architectures, con_components, check_source) = \ - utils.parse_args(Options) - - session = DBConn().session() - - dislocated_files = {} - - # TODO: Fix this properly - query_args = {'con_suites': con_suites, 'con_architectures': con_architectures, 'con_components': con_components} - - query = """ -SELECT b.id, b.package, a.arch_string, b.version, l.path, f.filename, c.name, - f.id, su.suite_name, b.type - FROM binaries b, bin_associations ba, architecture a, files f, location l, - component c, suite su - WHERE b.id = ba.bin AND b.file = f.id AND b.architecture = a.id - AND f.location = l.id AND l.component = c.id AND ba.suite = su.id - %(con_suites)s %(con_architectures)s %(con_components)s""" % query_args - if check_source: - query += """ -UNION -SELECT s.id, s.source, 'source', s.version, l.path, f.filename, c.name, f.id, - su.suite_name, 'dsc' - FROM source s, src_associations sa, files f, location l, component c, suite su - WHERE s.id = sa.source AND s.file = f.id AND f.location = l.id - AND l.component = c.id AND sa.suite = su.id %(con_suites)s %(con_components)s""" % query_args - - # Build up the main index of packages - packages = {} - unique_id = 0 - - q = session.execute(query) - for i in q.fetchall(): - (sourceid, pkg, arch, version, path, filename, component, file_id, suite, filetype) = i - - # 'id' comes from either 'binaries' or 'source', so it's not unique - unique_id += 1 - packages[unique_id] = dict(sourceid=sourceid, pkg=pkg, arch=arch, version=version, - path=path, filename=filename, - component=component, file_id=file_id, - suite=suite, filetype = filetype) - cleanup(packages, session) - session.commit() - - # has been replaced by 'dak generate-filelist': - #write_filelists(packages, dislocated_files, session) - -################################################################################ - -def main(): - global Options, Logger - - cnf = Config() - Arguments = [('a', "architecture", "Make-Suite-File-List::Options::Architecture", "HasArg"), - ('c', "component", "Make-Suite-File-List::Options::Component", "HasArg"), - ('h', "help", "Make-Suite-File-List::Options::Help"), - ('n', "no-delete", "Make-Suite-File-List::Options::No-Delete"), - ('f', "force", "Make-Suite-File-List::Options::Force"), - ('s', "suite", "Make-Suite-File-List::Options::Suite", "HasArg")] - for i in ["architecture", "component", "help", "no-delete", "suite", "force" ]: - if not cnf.has_key("Make-Suite-File-List::Options::%s" % (i)): - cnf["Make-Suite-File-List::Options::%s" % (i)] = "" - apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) - Options = cnf.SubTree("Make-Suite-File-List::Options") - if Options["Help"]: - usage() - - DBConn() - - Logger = daklog.Logger(cnf.Cnf, "make-suite-file-list") - do_da_do_da() - Logger.close() - -######################################################################################### - -if __name__ == '__main__': - main() diff --git a/docs/README.first b/docs/README.first index c1eb3d36..7d0d4074 100644 --- a/docs/README.first +++ b/docs/README.first @@ -25,9 +25,8 @@ o To process queue/: o To generate indices files: - * dak make-suite-file-list - generates file lists for apt-ftparchive - and removes obsolete packages from - suites + * dak dominate - removes obsolete packages from suites + * dak generate-filelist - generates file lists for apt-ftparchive * dak generate-releases - generates Release o To clean things up: diff --git a/docs/README.stable-point-release b/docs/README.stable-point-release index b62ab1c0..b1e0d778 100644 --- a/docs/README.stable-point-release +++ b/docs/README.stable-point-release @@ -15,7 +15,7 @@ o Update the 'Debian.r' symlink in dists/ o Clean up dists/stable/ChangeLog (add header, basically) o Update fields in suite table in postgresql (see below) -o Run 'dak make-suite-file-list --force -s stable' +o Run 'dak dominate --force -s stable' o Run 'dak generate-filelist -s stable' o Run apt-ftparchive generate apt.conf.stable o Run 'dak generate-releases --force-touch --apt-conf apt.conf.stable stable' -- 2.39.5