From 3ca8a114bb074ff029e99cf28d308550f74f4c4f Mon Sep 17 00:00:00 2001 From: James Troup Date: Mon, 20 Jan 2003 19:13:21 +0000 Subject: [PATCH] Loads of changes to improve output --- rene | 213 +++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 179 insertions(+), 34 deletions(-) diff --git a/rene b/rene index e9a9c04b..d221ad03 100755 --- a/rene +++ b/rene @@ -2,7 +2,7 @@ # Check for obsolete binary packages # Copyright (C) 2000, 2001, 2002 James Troup -# $Id: rene,v 1.16 2003-01-02 18:10:02 troup Exp $ +# $Id: rene,v 1.17 2003-01-20 19:13:21 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,9 +24,11 @@ # you might as well write some letters to God about how unfair entropy # is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au +## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add support for non-US ?, add overrides, avoid ANAIS for duplicated packages + ################################################################################ -import commands, pg, os, sys, tempfile; +import commands, pg, os, string, sys, tempfile, time; import utils, db_access; import apt_pkg; @@ -34,6 +36,7 @@ import apt_pkg; Cnf = None; projectB = None; +suite_id = None; ################################################################################ @@ -46,8 +49,17 @@ Check for obsolete or duplicated packages. ################################################################################ +def add_nbs(nbs_d, source, version, package): + if not nbs_d.has_key(source): + nbs_d[source] = {}; + if not nbs_d[source].has_key(version): + nbs_d[source][version] = {}; + nbs_d[source][version][package] = ""; + +################################################################################ + def main (): - global Cnf, projectB; + global Cnf, projectB, suite_id; Cnf = utils.get_conf(); @@ -66,13 +78,29 @@ def main (): db_access.init(Cnf, projectB); bin_pkgs = {}; - miss_src = {}; src_pkgs = {}; source_binaries = {}; + bins_in_suite = {}; + nbs = {}; + source_versions = {}; + + anais_output = ""; + duplicate_bins = {}; suite = "unstable"; suite_id = db_access.get_suite_id(suite); + bin_not_built = {}; + + # Initalize a large hash table of all binary packages + before = time.time(); + sys.stderr.write("[Getting a list of binary packages in %s..." % (suite)); + q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = 5 AND ba.bin = b.id"); + ql = q.getresult(); + sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))); + for i in ql: + bins_in_suite[i[0]] = ""; + components = Cnf.ValueList("Suite::%s::Components" % (suite)); for component in components: filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component); @@ -88,23 +116,28 @@ def main (): Sources = apt_pkg.ParseTagFile(sources); while Sources.Step(): source = Sources.Section.Find('Package'); + source_version = Sources.Section.Find('Version'); architecture = Sources.Section.Find('Architecture'); binaries = Sources.Section.Find('Binary'); + binaries_list = map(string.strip, binaries.split(',')); + + # Check for binaries not built on any architecture. + for binary in binaries_list: + if not bins_in_suite.has_key(binary): + if not bin_not_built.has_key(source): + bin_not_built[source] = {}; + bin_not_built[source][binary] = ""; # Check for packages built on architectures they shouldn't be. if architecture != "any" and architecture != "all": architectures = {}; for arch in architecture.split(): architectures[arch.strip()] = ""; - for binary in binaries.split(','): - binary = binary.strip(); + for binary in binaries_list: q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary)); ql = q.getresult(); - if not ql: - utils.warn("%s lists %s as a binary, but it doesn't seem to exist in %s?" % (source, binary, suite)); - # Loop around twice; first to get the latest 'valid' version versions = []; - for i in q.getresult(): + for i in ql: arch = i[0]; version = i[1]; if architectures.has_key(arch): @@ -114,32 +147,39 @@ def main (): latest_version = versions.pop() else: latest_version = None; - # ... then to check for 'invalid' architectures - for i in q.getresult(): + # Check for 'invalid' architectures + versions_d = {} + for i in ql: arch = i[0]; version = i[1]; if not architectures.has_key(arch): - print "[%s]: %s appears for %s (vs. '%s')" % (source, binary, arch, architecture), - if not latest_version: - print "** mwaap, mwapp! Ignore me **"; - continue; - if apt_pkg.VersionCompare(latest_version, version) != -1: - print "- out of date.", - else: - print "- current.", - print "[%s vs %s (%s)]" % (latest_version, version, arch); + if not versions_d.has_key(version): + versions_d[version] = []; + versions_d[version].append(arch) + + if versions_d != {}: + anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture); + versions = versions_d.keys(); + versions.sort(apt_pkg.VersionCompare); + for version in versions: + arches = versions_d[version]; + arches.sort(); + anais_output += " o %s: %s\n" % (version, ", ".join(arches)); # Check for duplicated packages and build indices for checking "no source" later source_index = component + '/' + source; if src_pkgs.has_key(source): print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]); src_pkgs[source] = source_index; - for binary in binaries.split(','): - binary = binary.strip(); + for binary in binaries_list: if bin_pkgs.has_key(binary): - print " binary %s is duplicated in source packages %s and %s" % (binary, source, bin_pkgs[binary]); + key = "%s~%s" % (source, bin_pkgs[binary]); + if not duplicate_bins.has_key(key): + duplicate_bins[key] = []; + duplicate_bins[key].append(binary); bin_pkgs[binary] = source; source_binaries[source] = binaries; + source_versions[source] = source_version; sources.close(); os.unlink(temp_filename); @@ -153,21 +193,35 @@ def main (): while Packages.Step(): package = Packages.Section.Find('Package'); source = Packages.Section.Find('Source', ""); + version = Packages.Section.Find('Version'); if source == "": source = package; if source.find("(") != -1: - m = utils.re_extract_src_version.match(source) - source = m.group(1) - if not bin_pkgs.has_key(package) and not miss_src.has_key(package): - miss_src[package] = 1; - print " %s has no source [%s: %s]" % (package, source, source_binaries.get(source, "(source does not exist)")); + m = utils.re_extract_src_version.match(source); + source = m.group(1); + version = m.group(2); + if not bin_pkgs.has_key(package): + if not nbs.has_key(source): + nbs[source] = {}; + if not nbs[source].has_key(package): + nbs[source][package] = {}; + nbs[source][package][version] = ""; packages.close(); - # Check for packages in experimental obsoleted by versions in unstable - # - # [If melanie was callable from python, we could auto-remove these - # packages...] + dubious_nbs = {}; + real_nbs = {}; + for source in nbs.keys(): + for package in nbs[source].keys(): + versions = nbs[source][package].keys(); + versions.sort(apt_pkg.VersionCompare); + latest_version = versions.pop(); + source_version = source_versions.get(source,"0"); + if apt_pkg.VersionCompare(latest_version, source_version) == 0: + add_nbs(dubious_nbs, source, latest_version, package); + else: + add_nbs(real_nbs, source, latest_version, package); + # Check for packages in experimental obsoleted by versions in unstable suite_id = db_access.get_suite_id("unstable"); q = projectB.query(""" SELECT s.source, s.version AS experimental, s2.version AS unstable @@ -177,8 +231,99 @@ SELECT s.source, s.version AS experimental, s2.version AS unstable AND versioncmp(s.version, s2.version) < 0""" % (suite_id)); ql = q.getresult(); if ql: + nviu_to_remove = []; + print "Newer version in unstable"; + print "-------------------------"; + print ; + for i in ql: + (source, experimental_version, unstable_version) = i; + print " o %s (%s, %s)" % (source, experimental_version, unstable_version); + nviu_to_remove.extend(source); + print + print "Suggested command:" + print " melanie -m \"[rene] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove)); print - print q + + print "Not Built from Source"; + print "---------------------"; + print ; + + nbs_to_remove = []; + nbs_keys = real_nbs.keys(); + nbs_keys.sort(); + for source in nbs_keys: + binaries = source_binaries.get(source, "(source does not exist)") + print " * %s_%s builds: %s" % (source, + source_versions.get(source, "??"), + source_binaries.get(source, "(source does not exist)")); + print " but no longer builds:" + versions = real_nbs[source].keys(); + versions.sort(apt_pkg.VersionCompare); + for version in versions: + packages = real_nbs[source][version].keys(); + packages.sort(); + for pkg in packages: + # *cough* FIXME + if pkg.find("pcmcia") == -1: + nbs_to_remove.append(pkg); + print " o %s: %s" % (version, ", ".join(packages)); + + print ; + + print "Suggested command:" + print " melanie -m \"[rene] NBS\" -b %s" % (" ".join(nbs_to_remove)); + print + + print "="*75 + print + + print "Unbuilt binary packages"; + print "-----------------------"; + print + keys = bin_not_built.keys(); + keys.sort(); + for source in keys: + binaries = bin_not_built[source].keys(); + binaries.sort(); + print " o %s: %s" % (source, ", ".join(binaries)); + print ; + + print "Built from multiple source packages"; + print "-----------------------------------"; + print ; + keys = duplicate_bins.keys(); + keys.sort(); + for key in keys: + (source_a, source_b) = key.split("~"); + print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key])); + print ; + + print "Architecture Not Allowed In Source"; + print "----------------------------------"; + print anais_output; + print ; + + print "Dubious NBS"; + print "-----------"; + print ; + + dubious_nbs_keys = dubious_nbs.keys(); + dubious_nbs_keys.sort(); + for source in dubious_nbs_keys: + binaries = source_binaries.get(source, "(source does not exist)") + print " * %s_%s builds: %s" % (source, + source_versions.get(source, "??"), + source_binaries.get(source, "(source does not exist)")); + print " won't admit to building:" + versions = dubious_nbs[source].keys(); + versions.sort(apt_pkg.VersionCompare); + for version in versions: + packages = dubious_nbs[source][version].keys(); + packages.sort(); + print " o %s: %s" % (version, ", ".join(packages)); + + print ; + ################################################################################ -- 2.39.5