From dd0ee5b67c650470139ffa8a50ae6b83a92ca76a Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Tue, 15 Nov 2005 09:50:32 +0000 Subject: [PATCH] merge months of changes on ftp-master, see ChangeLog --- ChangeLog | 61 ++++++++ Makefile | 2 +- TODO | 2 +- apt.conf | 36 +++-- apt.conf.stable | 20 ++- cindy | 304 +++++++++++++++++++++++++++++------- cron.daily | 30 +++- cron.hourly | 1 + denise | 54 +------ heidi | 6 +- helena | 290 +++++++++++++++++++++++++++++++--- jennifer | 14 +- katie.conf | 106 +++++++++++-- katie.py | 16 +- kelly | 20 ++- lisa | 34 ++-- logging.py | 16 +- madison | 8 +- melanie | 6 +- natalie | 7 +- nina | 8 +- pseudo-packages.description | 1 + pseudo-packages.maintainers | 13 +- saffron | 12 +- vars | 3 +- ziyi | 17 +- 26 files changed, 859 insertions(+), 228 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6c5f73f0..87c10777 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,64 @@ +2005-11-15 Anthony Towns + + * Merge of changes from spohr, by various people. + + * tiffani: new script to do patches to Packages, Sources and Contents + files for quicker downloads. + * ziyi: update to authenticate tiffani generated files + + * dak: new script to provide a single binary with less arbitrary names + for access to dak functionality. + + * cindy: script implemented + + * saffron: cope with suites that don't have a Priority specified + * heidi: use get_suite_id() + * denise: don't hardcode stable and unstable, or limit udebs to unstable + * denise: remove override munging for testing (now done by cindy) + * helena: expanded help, added new, sort and age options, and fancy headers + * jennifer: require description, add a reject for missing dsc file + * jennifer: change lock file + * kelly: propogation support + * lisa: honour accepted lock, use mtime not ctime, add override type_id + * madison: don't say "dep-retry" + * melanie: bug fix in output (missing %) + * natalie: cope with maintainer_override == None; add type_id for overrides + * nina: use mtime, not ctime + + * katie.py: propogation bug fixes + * logging.py: add debugging support, use | as the logfile separator + + * katie.conf: updated signing key (4F368D5D) + * katie.conf: changed lockfile to dinstall.lock + * katie.conf: added Lisa::AcceptedLockFile, Dir::Lock + * katie.conf: added tiffani, cindy support + * katie.conf: updated to match 3.0r6 release + * katie.conf: updated to match sarge's release + + * apt.conf: update for sarge's release + * apt.conf.stable: update for sarge's release + * apt.conf: bump daily max Contents change to 25MB from 12MB + + * cron.daily: add accepted lock and invoke cindy + * cron.daily: add daily.lock + * cron.daily: invoke tiffani + * cron.daily: rebuild accepted buildd stuff + * cron.daily: save rene-daily output on the web site + * cron.daily: disable billie + * cron.daily: add stats pr0n + + * cron.hourly: invoke helena + + * pseudo-packages.maintainers,.descriptions: miscellaneous updates + * vars: add lockdir, add etch to copyoverrides + * Makefile: add -Ipostgresql/server to CXXFLAGS + + * docs/: added README.quotes + * docs/: added manpages for alicia, catherine, charisma, cindy, heidi, + julia, katie, kelly, lisa, madison, melanie, natalie, rhona. + + * TODO: correct spelling of "conflicts" + 2005-05-28 James Troup * helena (process_changes_files): use MTIME rather than CTIME (the diff --git a/Makefile b/Makefile index 32e5417e..68ead583 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f -CXXFLAGS = -I/usr/include/postgresql/ -fPIC -Wall +CXXFLAGS = -I/usr/include/postgresql/ -I/usr/include/postgresql/server/ -fPIC -Wall CFLAGS = -fPIC -Wall LDFLAGS = -fPIC LIBS = -lapt-pkg diff --git a/TODO b/TODO index b7c24611..e8a3d6a9 100644 --- a/TODO +++ b/TODO @@ -227,7 +227,7 @@ Others * arch != {any,all} * build-depends wrong (via andrea) * suid - * conficlits + * conflicts * notification/stats to admin daily o trap fernanda exiting o distinguish binary only versus others (neuro) diff --git a/apt.conf b/apt.conf index 0f8d5c32..591a30e7 100644 --- a/apt.conf +++ b/apt.conf @@ -11,7 +11,7 @@ Default Sources::Compress "gzip"; Contents::Compress "gzip"; DeLinkLimit 0; - MaxContentsChange 12000; + MaxContentsChange 25000; FileMode 0664; } @@ -26,9 +26,9 @@ tree "dists/proposed-updates" SourceFileList "/org/ftp.debian.org/database/dists/proposed-updates_$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc source"; - BinOverride "override.woody.$(SECTION)"; - ExtraOverride "override.woody.extra.$(SECTION)"; - SrcOverride "override.woody.$(SECTION).src"; + BinOverride "override.sarge.$(SECTION)"; + ExtraOverride "override.sarge.extra.$(SECTION)"; + SrcOverride "override.sarge.$(SECTION).src"; Contents " "; }; @@ -39,9 +39,11 @@ tree "dists/testing" SourceFileList "/org/ftp.debian.org/database/dists/testing_$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc source"; - BinOverride "override.sarge.$(SECTION)"; - ExtraOverride "override.sarge.extra.$(SECTION)"; - SrcOverride "override.sarge.$(SECTION).src"; + BinOverride "override.etch.$(SECTION)"; + ExtraOverride "override.etch.extra.$(SECTION)"; + SrcOverride "override.etch.$(SECTION).src"; + Packages::Compress ". gzip bzip2"; + Sources::Compress "gzip bzip2"; }; tree "dists/testing-proposed-updates" @@ -50,9 +52,9 @@ tree "dists/testing-proposed-updates" SourceFileList "/org/ftp.debian.org/database/dists/testing-proposed-updates_$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc source"; - BinOverride "override.sarge.$(SECTION)"; - ExtraOverride "override.sarge.extra.$(SECTION)"; - SrcOverride "override.sarge.$(SECTION).src"; + BinOverride "override.etch.$(SECTION)"; + ExtraOverride "override.etch.extra.$(SECTION)"; + SrcOverride "override.etch.$(SECTION).src"; Contents " "; }; @@ -65,7 +67,7 @@ tree "dists/unstable" BinOverride "override.sid.$(SECTION)"; ExtraOverride "override.sid.extra.$(SECTION)"; SrcOverride "override.sid.$(SECTION).src"; - Packages::Compress ". gzip bzip2"; + Packages::Compress "gzip bzip2"; Sources::Compress "gzip bzip2"; }; @@ -76,11 +78,11 @@ tree "dists/testing/main" FileList "/org/ftp.debian.org/database/dists/testing_main_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; Architectures "alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc"; - BinOverride "override.sarge.main.$(SECTION)"; - SrcOverride "override.sarge.main.src"; + BinOverride "override.etch.main.$(SECTION)"; + SrcOverride "override.etch.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents " "; + Contents "$(DIST)/../Contents-udeb"; }; tree "dists/testing-proposed-updates/main" @@ -88,8 +90,8 @@ tree "dists/testing-proposed-updates/main" FileList "/org/ftp.debian.org/database/dists/testing-proposed-updates_main_$(SECTION)_binary-$(ARCH).list"; Sections "debian-installer"; Architectures "alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc"; - BinOverride "override.sarge.main.$(SECTION)"; - SrcOverride "override.sarge.main.src"; + BinOverride "override.etch.main.$(SECTION)"; + SrcOverride "override.etch.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; Contents " "; @@ -104,7 +106,7 @@ tree "dists/unstable/main" SrcOverride "override.sid.main.src"; BinCacheDB "packages-debian-installer-$(ARCH).db"; Packages::Extensions ".udeb"; - Contents " "; + Contents "Contents $(DIST)/../Contents-udeb"; }; // Experimental diff --git a/apt.conf.stable b/apt.conf.stable index 12a6e1ba..05cd2123 100644 --- a/apt.conf.stable +++ b/apt.conf.stable @@ -25,7 +25,21 @@ tree "dists/stable" SourceFileList "/org/ftp.debian.org/database/dists/stable_$(SECTION)_source.list"; Sections "main contrib non-free"; Architectures "alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc source"; - BinOverride "override.woody.$(SECTION)"; - ExtraOverride "override.woody.extra.$(SECTION)"; - SrcOverride "override.woody.$(SECTION).src"; + BinOverride "override.sarge.$(SECTION)"; + ExtraOverride "override.sarge.extra.$(SECTION)"; + SrcOverride "override.sarge.$(SECTION).src"; +}; + +// debian-installer + +tree "dists/stable/main" +{ + FileList "/org/ftp.debian.org/database/dists/stable_main_$(SECTION)_binary-$(ARCH).list"; + Sections "debian-installer"; + Architectures "alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc"; + BinOverride "override.sarge.main.$(SECTION)"; + SrcOverride "override.sarge.main.src"; + BinCacheDB "packages-debian-installer-$(ARCH).db"; + Packages::Extensions ".udeb"; + Contents " "; }; diff --git a/cindy b/cindy index 50f0e676..b95997f7 100755 --- a/cindy +++ b/cindy @@ -1,8 +1,9 @@ #!/usr/bin/env python -# Cruft checker for overrides +# Cruft checker and hole filler for overrides # Copyright (C) 2000, 2001, 2002, 2004 James Troup -# $Id: cindy,v 1.13 2004-11-27 19:23:40 troup Exp $ +# Copyright (C) 2005 Jeroen van Wolffelaar +# $Id: cindy,v 1.14 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,24 +29,36 @@ # plan for accepted to be in the DB. # ###################################################################### -# NB[2]: cindy entirely breaks for suites that share overrides, -# e.g. experimental in Debian. - -# NB[3]: cindy ENTIRELY breaks for 'source-only'-upload based distros -# like Ubuntu. Go Cindy. +# cindy should now work fine being done during cron.daily, for example just +# before denise (after kelly and jenna). At that point, queue/accepted should +# be empty and installed, so... Cindy does now take into account suites +# sharing overrides + +# TODO: +# * Only update out-of-sync overrides when corresponding versions are equal to +# some degree +# * consistency checks like: +# - section=debian-installer only for udeb and # dsc +# - priority=source iff dsc +# - (suite, package, 'dsc') is unique, +# - just as (suite, package, (u)deb) (yes, across components!) +# - sections match their component (each component has an own set of sections, +# could probably be reduced...) ################################################################################ -import pg, sys; -import utils, db_access; +import pg, sys, os; +import utils, db_access, logging; import apt_pkg; ################################################################################ -Cnf = None; Options = None; projectB = None; -override = {} +Logger = None +sections = {} +priorities = {} +blacklist = {} ################################################################################ @@ -53,18 +66,29 @@ def usage (exit_code=0): print """Usage: cindy Check for cruft in overrides. + -n, --no-action don't do anything -h, --help show this help and exit""" sys.exit(exit_code) ################################################################################ -def process(suite, component, type): - global override; +def gen_blacklist(dir): + for entry in os.listdir(dir): + entry = entry.split('_')[0] + blacklist[entry] = 1 + +def process(osuite, affected_suites, originosuite, component, type): + global Logger, Options, projectB, sections, priorities; - suite_id = db_access.get_suite_id(suite); - if suite_id == -1: - utils.fubar("Suite '%s' not recognised." % (suite)); + osuite_id = db_access.get_suite_id(osuite); + if osuite_id == -1: + utils.fubar("Suite '%s' not recognised." % (osuite)); + originosuite_id = None + if originosuite: + originosuite_id = db_access.get_suite_id(originosuite); + if originosuite_id == -1: + utils.fubar("Suite '%s' not recognised." % (originosuite)); component_id = db_access.get_component_id(component); if component_id == -1: @@ -74,83 +98,251 @@ def process(suite, component, type): if type_id == -1: utils.fubar("Type '%s' not recognised. (Valid types are deb, udeb and dsc)" % (type)); dsc_type_id = db_access.get_override_type_id("dsc"); + deb_type_id = db_access.get_override_type_id("deb") + + source_priority_id = db_access.get_priority_id("source") if type == "deb" or type == "udeb": packages = {}; q = projectB.query(""" SELECT b.package FROM binaries b, bin_associations ba, files f, location l, component c - WHERE b.id = ba.bin AND f.id = b.file AND l.id = f.location - AND c.id = l.component AND ba.suite = %s AND c.id = %s -""" % (suite_id, component_id)); + WHERE b.type = '%s' AND b.id = ba.bin AND f.id = b.file AND l.id = f.location + AND c.id = l.component AND ba.suite IN (%s) AND c.id = %s +""" % (type, ",".join(map(str,affected_suites)), component_id)); for i in q.getresult(): - packages[i[0]] = ""; + packages[i[0]] = 0; src_packages = {}; q = projectB.query(""" SELECT s.source FROM source s, src_associations sa, files f, location l, component c WHERE s.id = sa.source AND f.id = s.file AND l.id = f.location - AND c.id = l.component AND sa.suite = %s AND c.id = %s -""" % (suite_id, component_id)); + AND c.id = l.component AND sa.suite IN (%s) AND c.id = %s +""" % (",".join(map(str,affected_suites)), component_id)); for i in q.getresult(): - src_packages[i[0]] = ""; + src_packages[i[0]] = 0; - q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (suite_id, component_id, type_id)); + # ----------- + # Drop unused overrides + + q = projectB.query("SELECT package, priority, section, maintainer FROM override WHERE suite = %s AND component = %s AND type = %s" % (osuite_id, component_id, type_id)); projectB.query("BEGIN WORK"); - for i in q.getresult(): - package = i[0]; - if type == "deb" or type == "udeb": - if not packages.has_key(package): - if not src_packages.has_key(package): - print "DELETING: %s" % (package); - if not Options["No-Action"]: - projectB.query("DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (package, suite_id, component_id, type_id)); - else: - print "MAKING SOURCE: %s" % (package); - if not Options["No-Action"]: - projectB.query("DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (package, suite_id, component_id, type_id)); - # Then if source doesn't already have a copy, insert it into source - q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (package, suite_id, component_id, dsc_type_id)); - if not q.getresult() and not Options["No-Action"]: - projectB.query("INSERT INTO override (package, suite, component, priority, section, type, maintainer) VALUES ('%s', %s, %s, %s, %s, %s, '%s')" % (package, suite_id, component_id, i[1], i[2], dsc_type_id, i[3])); - else: # dsc - if not src_packages.has_key(package): - print "DELETING: %s" % (package); + if type == "dsc": + for i in q.getresult(): + package = i[0]; + if src_packages.has_key(package): + src_packages[package] = 1 + else: + if blacklist.has_key(package): + utils.warn("%s in incoming, not touching" % package) + continue + Logger.log(["removing unused override", osuite, component, + type, package, priorities[i[1]], sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""DELETE FROM override WHERE package = + '%s' AND suite = %s AND component = %s AND type = + %s""" % (package, osuite_id, component_id, type_id)); + # create source overrides based on binary overrides, as source + # overrides not always get created + q = projectB.query(""" SELECT package, priority, section, + maintainer FROM override WHERE suite = %s AND component = %s + """ % (osuite_id, component_id)); + for i in q.getresult(): + package = i[0] + if not src_packages.has_key(package) or src_packages[package]: + continue + src_packages[package] = 1 + + Logger.log(["add missing override", osuite, component, + type, package, "source", sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""INSERT INTO override (package, suite, + component, priority, section, type, maintainer) VALUES + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, + osuite_id, component_id, source_priority_id, i[2], + dsc_type_id, i[3])); + # Check whether originosuite has an override for us we can + # copy + if originosuite: + q = projectB.query("""SELECT origin.package, origin.priority, + origin.section, origin.maintainer, target.priority, + target.section, target.maintainer FROM override origin LEFT + JOIN override target ON (origin.package = target.package AND + target.suite=%s AND origin.component = target.component AND origin.type = + target.type) WHERE origin.suite = %s AND origin.component = %s + AND origin.type = %s""" % + (osuite_id, originosuite_id, component_id, type_id)); + for i in q.getresult(): + package = i[0] + if not src_packages.has_key(package) or src_packages[package]: + if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): + Logger.log(["syncing override", osuite, component, + type, package, "source", sections[i[5]], i[6], "source", sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""UPDATE override SET section=%s, + maintainer='%s' WHERE package='%s' AND + suite=%s AND component=%s AND type=%s""" % + (i[2], i[3], package, osuite_id, component_id, + dsc_type_id)); + continue + # we can copy + src_packages[package] = 1 + Logger.log(["copying missing override", osuite, component, + type, package, "source", sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""INSERT INTO override (package, suite, + component, priority, section, type, maintainer) VALUES + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, + osuite_id, component_id, source_priority_id, i[2], + dsc_type_id, i[3])); + + for package, hasoverride in src_packages.items(): + if not hasoverride: + utils.warn("%s has no override!" % package) + + else: # binary override + for i in q.getresult(): + package = i[0]; + if packages.has_key(package): + packages[package] = 1 + else: + if blacklist.has_key(package): + utils.warn("%s in incoming, not touching" % package) + continue + Logger.log(["removing unused override", osuite, component, + type, package, priorities[i[1]], sections[i[2]], i[3]]) if not Options["No-Action"]: - projectB.query("DELETE FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (package, suite_id, component_id, type_id)); + projectB.query("""DELETE FROM override WHERE package = + '%s' AND suite = %s AND component = %s AND type = + %s""" % (package, osuite_id, component_id, type_id)); + + # Check whether originosuite has an override for us we can + # copy + if originosuite: + q = projectB.query("""SELECT origin.package, origin.priority, + origin.section, origin.maintainer, target.priority, + target.section, target.maintainer FROM override origin LEFT + JOIN override target ON (origin.package = target.package AND + target.suite=%s AND origin.component = target.component AND + origin.type = target.type) WHERE origin.suite = %s AND + origin.component = %s AND origin.type = %s""" % (osuite_id, + originosuite_id, component_id, type_id)); + for i in q.getresult(): + package = i[0] + if not packages.has_key(package) or packages[package]: + if i[4] and (i[1] != i[4] or i[2] != i[5] or i[3] != i[6]): + Logger.log(["syncing override", osuite, component, + type, package, priorities[i[4]], sections[i[5]], + i[6], priorities[i[1]], sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""UPDATE override SET priority=%s, section=%s, + maintainer='%s' WHERE package='%s' AND + suite=%s AND component=%s AND type=%s""" % + (i[1], i[2], i[3], package, osuite_id, + component_id, type_id)); + continue + # we can copy + packages[package] = 1 + Logger.log(["copying missing override", osuite, component, + type, package, priorities[i[1]], sections[i[2]], i[3]]) + if not Options["No-Action"]: + projectB.query("""INSERT INTO override (package, suite, + component, priority, section, type, maintainer) VALUES + ('%s', %s, %s, %s, %s, %s, '%s')""" % (package, osuite_id, component_id, i[1], i[2], type_id, i[3])); + + for package, hasoverride in packages.items(): + if not hasoverride: + utils.warn("%s has no override!" % package) + projectB.query("COMMIT WORK"); + sys.stdout.flush() ################################################################################ def main (): - global Cnf, Options, projectB, override; + global Logger, Options, projectB, sections, priorities; Cnf = utils.get_conf() Arguments = [('h',"help","Cindy::Options::Help"), ('n',"no-action", "Cindy::Options::No-Action")]; for i in [ "help", "no-action" ]: - if not Cnf.has_key("Cindy::Options::%s" % (i)): - Cnf["Cindy::Options::%s" % (i)] = ""; + if not Cnf.has_key("Cindy::Options::%s" % (i)): + Cnf["Cindy::Options::%s" % (i)] = ""; apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); Options = Cnf.SubTree("Cindy::Options") if Options["Help"]: - usage(); + usage(); projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); db_access.init(Cnf, projectB); - suite = "unstable" - print "Processing %s..." % (suite); - for component in Cnf.SubTree("Component").List(): - if component == "mixed": - continue; # Ick - for otype in Cnf.ValueList("OverrideType"): - print "Processing %s [%s - %s]..." % (suite, component, otype); - process(suite, component, otype); + # init sections, priorities: + q = projectB.query("SELECT id, section FROM section") + for i in q.getresult(): + sections[i[0]] = i[1] + q = projectB.query("SELECT id, priority FROM priority") + for i in q.getresult(): + priorities[i[0]] = i[1] + + if not Options["No-Action"]: + Logger = logging.Logger(Cnf, "cindy") + else: + Logger = logging.Logger(Cnf, "cindy", 1) + + gen_blacklist(Cnf["Dir::Queue::Accepted"]) + + for osuite in Cnf.SubTree("Cindy::OverrideSuites").List(): + if "1" != Cnf["Cindy::OverrideSuites::%s::Process" % osuite]: + continue + + osuite = osuite.lower() + + originosuite = None + originremark = "" + try: + originosuite = Cnf["Cindy::OverrideSuites::%s::OriginSuite" % osuite]; + originosuite = originosuite.lower() + originremark = " taking missing from %s" % originosuite + except KeyError: + pass + + print "Processing %s%s..." % (osuite, originremark); + # Get a list of all suites that use the override file of 'osuite' + ocodename = Cnf["Suite::%s::codename" % osuite] + suites = [] + for suite in Cnf.SubTree("Suite").List(): + if ocodename == Cnf["Suite::%s::OverrideCodeName" % suite]: + suites.append(suite) + + q = projectB.query("SELECT id FROM suite WHERE suite_name in (%s)" \ + % ", ".join(map(repr, suites)).lower()) + + suiteids = [] + for i in q.getresult(): + suiteids.append(i[0]) + + if len(suiteids) != len(suites) or len(suiteids) < 1: + utils.fubar("Couldn't find id's of all suites: %s" % suites) + + for component in Cnf.SubTree("Component").List(): + if component == "mixed": + continue; # Ick + # It is crucial for the dsc override creation based on binary + # overrides that 'dsc' goes first + otypes = Cnf.ValueList("OverrideType") + otypes.remove("dsc") + otypes = ["dsc"] + otypes + for otype in otypes: + print "Processing %s [%s - %s] using %s..." \ + % (osuite, component, otype, suites); + sys.stdout.flush() + process(osuite, suiteids, originosuite, component, otype); + + Logger.close() ################################################################################ diff --git a/cron.daily b/cron.daily index 5cb1cebd..f257df22 100755 --- a/cron.daily +++ b/cron.daily @@ -1,6 +1,6 @@ #! /bin/sh # -# Executed daily via cron, out of troup's crontab. +# Executed daily via cron, out of katie's crontab. set -e export SCRIPTVARS=/org/ftp.debian.org/katie/vars @@ -11,13 +11,17 @@ export SCRIPTVARS=/org/ftp.debian.org/katie/vars echo Archive maintenance started at $(date +%X) NOTICE="$ftpdir/Archive_Maintenance_In_Progress" +LOCKCU="$lockdir/daily.lock" +LOCKAC="$lockdir/unchecked.lock" cleanup() { rm -f "$NOTICE" + rm -f "$LOCKCU" } trap cleanup 0 rm -f "$NOTICE" +lockfile -l 3600 $LOCKCU cat > "$NOTICE" <> # Generate Packages and Sources files cd $masterdir apt-ftparchive generate apt.conf +# Generate *.diff/ incremental updates +tiffani # Generate Release files ziyi @@ -76,12 +86,19 @@ ziyi rhona shania +# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to +# update this before wanna-build is updated. +psql projectb -A -t -q -c "SELECT filename FROM accepted_autobuild WHERE suite = 5 AND in_accepted = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list +apt-ftparchive generate apt.conf.buildd + mkmaintainers copyoverrides mklslar mkchecksums - rm -f $NOTICE +sudo -u archvsync /home/archvsync/pushmerkel + +rm -f $LOCKCU echo Archive maintenance finished at $(date +%X) ################################################################################ @@ -101,13 +118,13 @@ echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skippin # Send a report on NEW/BYHAND packages helena | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org # and one on crufty packages -rene | mail -e -s "rene run for $(date +%D)" ftpmaster@ftp-master.debian.org +rene | tee $webdir/rene-daily.txt | mail -e -s "rene run for $(date +%D)" ftpmaster@ftp-master.debian.org ################################################################################ # Run billie -time billie +#time billie ################################################################################ @@ -116,3 +133,8 @@ ulimit -m 90000 -d 90000 -s 10000 -v 90000 run-parts --report /org/ftp.debian.org/scripts/distmnt echo Daily cron scripts successful. +# Stats pr0n + +cd $masterdir +update-ftpstats $base/log/* > $base/misc/ftpstats.data +R --slave --vanilla < $base/misc/ftpstats.R diff --git a/cron.hourly b/cron.hourly index 89d125e3..85af9120 100644 --- a/cron.hourly +++ b/cron.hourly @@ -8,3 +8,4 @@ export SCRIPTVARS=/org/ftp.debian.org/katie/vars cd $masterdir julia +helena -n > $webdir/new.html diff --git a/denise b/denise index 96774609..cae1bb6d 100755 --- a/denise +++ b/denise @@ -2,7 +2,7 @@ # Output override files for apt-ftparchive and indices/ # Copyright (C) 2000, 2001, 2002, 2004 James Troup -# $Id: denise,v 1.17 2004-11-27 17:56:42 troup Exp $ +# $Id: denise,v 1.18 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -96,7 +96,11 @@ def main (): projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); db_access.init(Cnf, projectB); - for suite in [ "stable", "unstable" ]: + for suite in Cnf.SubTree("Cindy::OverrideSuites").List(): + if Cnf.has_key("Suite::%s::Untouchable" % suite) and Cnf["Suite::%s::Untouchable" % suite] != 0: + continue + suite = suite.lower() + sys.stderr.write("Processing %s...\n" % (suite)); override_suite = Cnf["Suite::%s::OverrideCodeName" % (suite)]; for component in Cnf.SubTree("Component").List(): @@ -106,7 +110,7 @@ def main (): if otype == "deb": suffix = ""; elif otype == "udeb": - if suite != "unstable" or component != "main": + if component != "main": continue; # Ick2 suffix = ".debian-installer"; elif otype == "dsc": @@ -116,50 +120,6 @@ def main (): do_list(output_file, suite, component, otype); output_file.close(); - # Munge the override file for testing by using unstable's where - # possible and falling back on stable's where it's not. - - sys.stderr.write("Processing testing...\n"); - suite = "testing"; - suite_id = db_access.get_suite_id(suite); - override_suite = Cnf["Suite::%s::OverrideCodeName" % (suite)]; - for component in Cnf.SubTree("Component").List(): - if component == "mixed": - continue; - component_id = db_access.get_component_id(component); - for otype in Cnf.ValueList("OverrideType"): - if otype == "deb" or otype == "udeb": - if otype == "deb": - suffix = ""; - elif otype == "udeb": - if component != "main": - continue; - suffix = ".debian-installer"; - q = projectB.query("SELECT DISTINCT b.package FROM bin_associations ba, binaries b, files f, location l WHERE ba.suite = %s AND l.component = %s AND b.type = '%s' AND ba.bin = b.id AND b.file = f.id AND f.location = l.id" % (suite_id, component_id, otype)); - elif otype == "dsc": - q = projectB.query("SELECT DISTINCT s.source FROM src_associations sa, source s, files f, location l WHERE sa.suite = %s AND l.component = %s AND sa.source = s.id AND s.file = f.id AND f.location = l.id" % (suite_id, component_id)); - suffix = ".src"; - filename = "%s/override.%s.%s%s" % (Cnf["Dir::Override"], override_suite, component.replace("non-US/", ""), suffix); - output_file = utils.open_file(filename, 'w'); - for i in q.getresult(): - package = i[0]; - if otype == "deb" or otype == "dsc": - if override["unstable"][component][otype].has_key(package): - output_file.write(utils.result_join(override["unstable"][component][otype][package])+'\n'); - elif override["stable"][component][otype].has_key(package): - output_file.write(utils.result_join(override["stable"][component][otype][package])+'\n'); - else: - if otype == "dsc" and (override["unstable"][component]["deb"].has_key(package) or override["stable"][component]["deb"].has_key(package)): - continue; # source falls back on binary; so accept silently - utils.warn("Can't find override entry for testing package '%s' (component %s, type %s)." % (package, component, otype)); - elif otype == "udeb": - if component == "main" and override["unstable"][component][otype].has_key(package): - output_file.write(utils.result_join(override["unstable"][component][otype][package])+'\n'); - else: - utils.warn("Can't find override entry for testing package '%s' (component %s, type %s)." % (package, component, otype)); - - output_file.close(); - ################################################################################ if __name__ == '__main__': diff --git a/heidi b/heidi index 2a603024..2454b234 100755 --- a/heidi +++ b/heidi @@ -1,8 +1,8 @@ #!/usr/bin/env python # Manipulate suite tags -# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: heidi,v 1.18 2004-03-11 00:20:51 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: heidi,v 1.19 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -259,7 +259,7 @@ def main (): for i in ("add", "list", "remove", "set"): if Cnf["Heidi::Options::%s" % (i)] != "": suite = Cnf["Heidi::Options::%s" % (i)]; - if not Cnf.has_key("Suite::%s" % (suite)): + if db_access.get_suite_id(suite) == -1: utils.fubar("Unknown suite '%s'." %(suite)); else: if action: diff --git a/helena b/helena index 73374610..d33b19e3 100755 --- a/helena +++ b/helena @@ -1,8 +1,8 @@ #!/usr/bin/env python # Produces a report on NEW and BYHAND packages -# Copyright (C) 2001, 2002, 2003 James Troup -# $Id: helena,v 1.5 2003-07-15 17:29:26 troup Exp $ +# Copyright (C) 2001, 2002, 2003, 2005 James Troup +# $Id: helena,v 1.6 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -38,9 +38,12 @@ import copy, glob, os, stat, sys, time; import apt_pkg; import katie, utils; +import encodings.utf_8, encodings.latin_1, string; Cnf = None; Katie = None; +direction = []; +row_number = 0; ################################################################################ @@ -48,7 +51,19 @@ def usage(exit_code=0): print """Usage: helena Prints a report of packages in queue directories (usually new and byhand). - -h, --help show this help and exit.""" + -h, --help show this help and exit. + -n, --new produce html-output + -s, --sort=key sort output according to key, see below. + -a, --age=key if using sort by age, how should time be treated? + If not given a default of hours will be used. + + Sorting Keys: ao=age, oldest first. an=age, newest first. + na=name, ascending nd=name, descending + nf=notes, first nl=notes, last + + Age Keys: m=minutes, h=hours, d=days, w=weeks, o=months, y=years + +""" sys.exit(exit_code) ################################################################################ @@ -104,6 +119,149 @@ def sg_compare (a, b): ############################################################ +def sortfunc(a,b): + for sorting in direction: + (sortkey, way, time) = sorting; + ret = 0 + if time == "m": + x=int(a[sortkey]/60) + y=int(b[sortkey]/60) + elif time == "h": + x=int(a[sortkey]/3600) + y=int(b[sortkey]/3600) + elif time == "d": + x=int(a[sortkey]/86400) + y=int(b[sortkey]/86400) + elif time == "w": + x=int(a[sortkey]/604800) + y=int(b[sortkey]/604800) + elif time == "o": + x=int(a[sortkey]/2419200) + y=int(b[sortkey]/2419200) + elif time == "y": + x=int(a[sortkey]/29030400) + y=int(b[sortkey]/29030400) + else: + x=a[sortkey] + y=b[sortkey] + if x < y: + ret = -1 + elif x > y: + ret = 1 + if ret != 0: + if way < 0: + ret = ret*-1 + return ret + return 0 + +############################################################ + +def header(): + print """ + + Debian NEW and BYHAND Packages + + + + +
+ + + + Debian Project +
+
+ + + + + + + + + + +
+ Debian NEW and BYHAND Packages +
+ +
+ """ + +def footer(): + print "

Timestamp: %s (UTC)

" % (time.strftime("%d.%m.%Y / %H:%M:%S", time.gmtime())) + print "

Hint: Age is the youngest upload of the package, if there is more than one version.

" + print "

You may want to look at the REJECT-FAQ for possible reasons why one of the above packages may get rejected.

" + print """ + Valid HTML 4.01! + + Valid CSS! + """ + print "" + +def table_header(type): + print "

Summary for: %s

" % (type) + print """
+ + + + + + + + + + """ + +def table_footer(type, source_count, total_count): + print "
PackageVersionArchDistributionAgeMaintainerCloses

\n" + print "

Package count in %s: %s\n" % (type, source_count) + print "
Total Package count: %s

\n" % (total_count) + +def force_to_latin(s): + """Forces a string to Latin-1.""" + latin1_s = unicode(s,'utf-8'); + return latin1_s.encode('iso8859-1', 'replace'); + + +def table_row(source, version, arch, last_mod, maint, distribution, closes): + + global row_number; + + if row_number % 2 != 0: + print "" + else: + print "" + + tdclass = "sid" + for dist in distribution: + if dist == "experimental": + tdclass = "exp"; + print "%s" % (tdclass, source); + print "" % (tdclass) + for vers in version.split(): + print "%s
" % (vers); + print "%s" % (tdclass, arch, tdclass); + for dist in distribution: + print "%s
" % (dist); + print "%s" % (tdclass, last_mod); + (name, mail) = maint.split(":"); + name = force_to_latin(name); + + print "%s" % (tdclass, mail, name); + print "" % (tdclass) + for close in closes: + print "#%s
" % (close, close); + print ""; + row_number+=1; + +############################################################ + def process_changes_files(changes_files, type): msg = ""; cache = {}; @@ -129,12 +287,16 @@ def process_changes_files(changes_files, type): for source in per_source.keys(): source_list = per_source[source]["list"]; first = source_list[0]; - oldest = os.stat(first["filename"])[stat.ST_CTIME]; + oldest = os.stat(first["filename"])[stat.ST_MTIME]; have_note = 0; for d in per_source[source]["list"]: - ctime = os.stat(d["filename"])[stat.ST_CTIME]; - if ctime < oldest: - oldest = ctime; + mtime = os.stat(d["filename"])[stat.ST_MTIME]; + if Cnf.has_key("Helena::Options::New"): + if mtime > oldest: + oldest = mtime; + else: + if mtime < oldest: + oldest = mtime; have_note += (d.has_key("lisa note")); per_source[source]["oldest"] = oldest; if not have_note: @@ -150,6 +312,11 @@ def process_changes_files(changes_files, type): max_source_len = 0; max_version_len = 0; max_arch_len = 0; + maintainer = {}; + maint=""; + distribution=""; + closes=""; + source_exists=""; for i in per_source_items: last_modified = time.time()-i[1]["oldest"]; source = i[1]["list"][0]["source"]; @@ -158,6 +325,18 @@ def process_changes_files(changes_files, type): arches = {}; versions = {}; for j in i[1]["list"]: + if Cnf.has_key("Helena::Options::New"): + try: + (maintainer["maintainer822"], maintainer["maintainer2047"], + maintainer["maintainername"], maintainer["maintaineremail"]) = \ + utils.fix_maintainer (j["maintainer"]); + except utils.ParseMaintError, msg: + print "Problems while parsing maintainer address\n"; + maintainer["maintainername"] = "Unknown"; + maintainer["maintaineremail"] = "Unknown"; + maint="%s:%s" % (maintainer["maintainername"], maintainer["maintaineremail"]); + distribution=j["distribution"].keys(); + closes=j["closes"].keys(); for arch in j["architecture"].keys(): arches[arch] = ""; version = j["version"]; @@ -174,23 +353,77 @@ def process_changes_files(changes_files, type): note = " | [N]"; else: note = ""; - entries.append([source, version_list, arch_list, note, time_pp(last_modified)]); + entries.append([source, version_list, arch_list, note, last_modified, maint, distribution, closes]); + + # direction entry consists of "Which field, which direction, time-consider" where + # time-consider says how we should treat last_modified. Thats all. + + # Look for the options for sort and then do the sort. + age = "h" + if Cnf.has_key("Helena::Options::Age"): + age = Cnf["Helena::Options::Age"] + if Cnf.has_key("Helena::Options::New"): + # If we produce html we always have oldest first. + direction.append([4,-1,"ao"]); + else: + if Cnf.has_key("Helena::Options::Sort"): + for i in Cnf["Helena::Options::Sort"].split(","): + if i == "ao": + # Age, oldest first. + direction.append([4,-1,age]); + elif i == "an": + # Age, newest first. + direction.append([4,1,age]); + elif i == "na": + # Name, Ascending. + direction.append([0,1,0]); + elif i == "nd": + # Name, Descending. + direction.append([0,-1,0]); + elif i == "nl": + # Notes last. + direction.append([3,1,0]); + elif i == "nf": + # Notes first. + direction.append([3,-1,0]); + entries.sort(lambda x, y: sortfunc(x, y)) + # Yes, in theory you can add several sort options at the commandline with. But my mind is to small + # at the moment to come up with a real good sorting function that considers all the sidesteps you + # have with it. (If you combine options it will simply take the last one at the moment). + # Will be enhanced in the future. + + if Cnf.has_key("Helena::Options::New"): + direction.append([4,1,"ao"]); + entries.sort(lambda x, y: sortfunc(x, y)) + # Output for a html file. First table header. then table_footer. + # Any line between them is then a printed from subroutine table_row. + if len(entries) > 0: + table_header(type.upper()); + for entry in entries: + (source, version_list, arch_list, note, last_modified, maint, distribution, closes) = entry; + table_row(source, version_list, arch_list, time_pp(last_modified), maint, distribution, closes); + total_count = len(changes_files); + source_count = len(per_source_items); + table_footer(type.upper(), source_count, total_count); + else: + # The "normal" output without any formatting. + format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len) + + msg = ""; + for entry in entries: + (source, version_list, arch_list, note, last_modified, undef, undef, undef) = entry; + msg += format % (source, version_list, arch_list, note, time_pp(last_modified)); + + if msg: + total_count = len(changes_files); + source_count = len(per_source_items); + print type.upper(); + print "-"*len(type); + print + print msg; + print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count)); + print - format="%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (max_source_len, max_version_len, max_arch_len) - msg = ""; - for entry in entries: - (source, version_list, arch_list, note, last_modified) = entry; - msg += format % (source, version_list, arch_list, note, last_modified); - - if msg: - total_count = len(changes_files); - source_count = len(per_source_items); - print type.upper(); - print "-"*len(type); - print - print msg; - print "%s %s source package%s / %s %s package%s in total." % (source_count, type, plural(source_count), total_count, type, plural(total_count)); - print ################################################################################ @@ -198,7 +431,10 @@ def main(): global Cnf, Katie; Cnf = utils.get_conf(); - Arguments = [('h',"help","Helena::Options::Help")]; + Arguments = [('h',"help","Helena::Options::Help"), + ('n',"new","Helena::Options::New"), + ('s',"sort","Helena::Options::Sort", "HasArg"), + ('a',"age","Helena::Options::Age", "HasArg")]; for i in [ "help" ]: if not Cnf.has_key("Helena::Options::%s" % (i)): Cnf["Helena::Options::%s" % (i)] = ""; @@ -211,6 +447,9 @@ def main(): Katie = katie.Katie(Cnf); + if Cnf.has_key("Helena::Options::New"): + header(); + directories = Cnf.ValueList("Helena::Directories"); if not directories: directories = [ "byhand", "new" ]; @@ -219,6 +458,9 @@ def main(): changes_files = glob.glob("%s/*.changes" % (Cnf["Dir::Queue::%s" % (directory)])); process_changes_files(changes_files, directory); + if Cnf.has_key("Helena::Options::New"): + footer(); + ################################################################################ if __name__ == '__main__': diff --git a/jennifer b/jennifer index 0abbfa7e..92055bf2 100755 --- a/jennifer +++ b/jennifer @@ -2,7 +2,7 @@ # Checks Debian packages from Incoming # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 James Troup -# $Id: jennifer,v 1.56 2005-01-18 22:18:31 troup Exp $ +# $Id: jennifer,v 1.57 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -45,7 +45,7 @@ re_strip_revision = re.compile(r"-([^-]+)$"); ################################################################################ # Globals -jennifer_version = "$Revision: 1.56 $"; +jennifer_version = "$Revision: 1.57 $"; Cnf = None; Options = None; @@ -201,7 +201,7 @@ def check_changes(): # Check for mandatory fields for i in ("source", "binary", "architecture", "version", "distribution", - "maintainer", "files", "changes"): + "maintainer", "files", "changes", "description"): if not changes.has_key(i): reject("%s: Missing mandatory field `%s'." % (filename, i)); return 0 # Avoid errors during later tests @@ -299,7 +299,7 @@ def check_distributions(): # give these as "uploaded-to(non-mapped) suites-to-add-when-upload-obsoletes" # # changes["distribution-version"] looks like: {'testing': 'testing-proposed-updates'} - if args[1] in changes["distribution"]: + if changes["distribution"].has_key(args[1]): changes.setdefault("distribution-version", {}) for suite in args[2:]: changes["distribution-version"][suite]=suite @@ -535,7 +535,8 @@ def check_files(): # Check the version and for file overwrites reject(Katie.check_binary_against_db(file),""); - check_deb_ar(file, control) + # [JT - 2005/05/31; disabled for now, will go back on post-sarge] + #check_deb_ar(file, control) # Checks for a source package... else: @@ -667,6 +668,7 @@ def check_dsc(): # If there isn't one, we have nothing to do. (We have reject()ed the upload already) if not dsc_filename: + reject("source uploads must contain a dsc file"); return 0; # Parse the .dsc file @@ -1241,7 +1243,7 @@ def main(): # Check that we aren't going to clash with the daily cron job - if not Options["No-Action"] and os.path.exists("%s/Archive_Maintenance_In_Progress" % (Cnf["Dir::Root"])) and not Options["No-Lock"]: + if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (Cnf["Dir::Lock"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later."); # Obtain lock if not in no-action mode and initialize the log diff --git a/katie.conf b/katie.conf index d7b4c5f1..40e0597f 100644 --- a/katie.conf +++ b/katie.conf @@ -4,7 +4,7 @@ Dinstall GPGKeyring "/org/keyring.debian.org/keyrings/debian-keyring.gpg"; SigningKeyring "/org/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg"; SigningPubKeyring "/org/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg"; - SigningKeyIds "1DB114E0"; + SigningKeyIds "4F368D5D"; SendmailCommand "/usr/sbin/sendmail -odq -oi -t"; MyEmailAddress "Debian Installer "; MyAdminAddress "ftpmaster@debian.org"; @@ -13,7 +13,7 @@ Dinstall BugServer "bugs.debian.org"; PackagesServer "packages.debian.org"; TrackingServer "packages.qa.debian.org"; - LockFile "/org/ftp.debian.org/katie/lock"; + LockFile "/org/ftp.debian.org/lock/dinstall.lock"; Bcc "archive@ftp-master.debian.org"; GroupOverrideFilename "override.group-maint"; FutureTimeTravelGrace 28800; // 8 hours @@ -34,6 +34,15 @@ Dinstall }; }; +Tiffani +{ + Options + { + TempDir "/org/ftp.debian.org/tiffani"; + MaxDiffs { Default 90; }; + }; +}; + Alicia { MyEmailAddress "Debian FTP Masters "; @@ -101,7 +110,7 @@ Neve Lauren { StableRejector "Martin (Joey) Schulze "; - MoreInfoURL "http://people.debian.org/~joey/3.0r4/"; + MoreInfoURL "http://people.debian.org/~joey/3.0r6/"; }; Emilie @@ -125,10 +134,36 @@ Rhona MorgueSubDir "rhona"; }; -Suite +Lisa { + AcceptedLockFile "/org/ftp.debian.org/lock/unchecked.lock"; +}; - Stable +Cindy +{ + OverrideSuites + { + Stable + { + Process "0"; + }; + + Testing + { + Process "1"; + OriginSuite "Unstable"; + }; + + Unstable + { + Process "1"; + }; + }; +}; + +Suite +{ + Oldstable { Components { @@ -153,14 +188,52 @@ Suite sparc; }; Announce "debian-changes@lists.debian.org"; - Version "3.0r4"; + Version "3.0r6"; Origin "Debian"; - Description "Debian 3.0r4 Released 31st December 2004"; + Description "Debian 3.0r6 Released 31 May 2005"; CodeName "woody"; OverrideCodeName "woody"; + Priority "1"; + Untouchable "1"; + }; + + Stable + { + Components + { + main; + contrib; + non-free; + }; + Architectures + { + source; + all; + alpha; + arm; + hppa; + i386; + ia64; + m68k; + mips; + mipsel; + powerpc; + s390; + sparc; + }; + Announce "debian-changes@lists.debian.org"; + Version "3.1r0"; + Origin "Debian"; + Description "Debian 3.1r0 Released 06 June 2005"; + CodeName "sarge"; + OverrideCodeName "sarge"; Priority "3"; Untouchable "1"; ChangeLogBase "dists/stable/"; + UdebComponents + { + main; + }; }; Proposed-Updates @@ -190,11 +263,11 @@ Suite Announce "debian-changes@lists.debian.org"; CopyChanges "dists/proposed-updates/"; CopyKatie "/org/ftp.debian.org/queue/proposed-updates/"; - Version "3.0-updates"; + Version "3.1-updates"; Origin "Debian"; - Description "Debian 3.0 Proposed Updates - Not Released"; + Description "Debian 3.1 Proposed Updates - Not Released"; CodeName "proposed-updates"; - OverrideCodeName "woody"; + OverrideCodeName "sarge"; OverrideSuite "stable"; Priority "4"; VersionChecks @@ -214,6 +287,10 @@ Suite Stable; }; }; + UdebComponents + { + main; + }; }; Testing @@ -243,8 +320,8 @@ Suite Announce "debian-testing-changes@lists.debian.org"; Origin "Debian"; Description "Debian Testing distribution - Not Released"; - CodeName "sarge"; - OverrideCodeName "sarge"; + CodeName "etch"; + OverrideCodeName "etch"; Priority "5"; UdebComponents { @@ -280,8 +357,8 @@ Suite Origin "Debian"; Description "Debian Testing distribution updates - Not Released"; CodeName "testing-proposed-updates"; - OverrideCodeName "sarge"; - OverrideSuite "unstable"; + OverrideCodeName "etch"; + OverrideSuite "etch"; Priority "6"; VersionChecks { @@ -428,6 +505,7 @@ Dir PoolRoot "pool/"; Lists "/org/ftp.debian.org/database/dists/"; Log "/org/ftp.debian.org/log/"; + Lock "/org/ftp.debian.org/lock"; Morgue "/org/ftp.debian.org/morgue/"; MorgueReject "reject"; Override "/org/ftp.debian.org/scripts/override/"; diff --git a/katie.py b/katie.py index f952b34f..569abf87 100644 --- a/katie.py +++ b/katie.py @@ -2,7 +2,7 @@ # Utility functions for katie # Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup -# $Id: katie.py,v 1.53 2005-01-18 22:18:55 troup Exp $ +# $Id: katie.py,v 1.54 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -813,7 +813,8 @@ distribution."""; # # And - we really should complain to the dorks who configured dak self.reject("%s is mapped to, but not enhanced by %s - adding anyways" % (suite, addsuite), "Warning: ") - self.pkg.changes["distribution"][addsuite] = 1 + self.pkg.changes.setdefault("propdistribution", {}) + self.pkg.changes["propdistribution"][addsuite] = 1 cansave = 1 elif not target_version: # not targets_version is true when the package is NEW @@ -822,17 +823,18 @@ distribution."""; self.reject("Won't propogate NEW packages.") elif apt_pkg.VersionCompare(new_version, add_version) < 0: # propogation would be redundant. no need to reject though. - #self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: "); - self.reject("foo", "Warning: ") + self.reject("ignoring versionconflict: %s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite), "Warning: ") cansave = 1 elif apt_pkg.VersionCompare(new_version, add_version) > 0 and \ - apt_pkg.VersionCompare(add_version, target_version) == 0: + apt_pkg.VersionCompare(add_version, target_version) >= 0: # propogate!! - self.pkg.changes["distribution"][addsuite] = 1 + self.reject("Propogating upload to %s" % (addsuite), "Warning: ") + self.pkg.changes.setdefault("propdistribution", {}) + self.pkg.changes["propdistribution"][addsuite] = 1 cansave = 1 if not cansave: - self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)); + self.reject("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (file, existent_version, suite, new_version, target_suite)) ################################################################################ diff --git a/kelly b/kelly index 68d59a0b..12bcb5a3 100755 --- a/kelly +++ b/kelly @@ -2,7 +2,7 @@ # Installs Debian packages from queue/accepted into the pool # Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: kelly,v 1.15 2005-01-14 14:07:17 ajt Exp $ +# $Id: kelly,v 1.16 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,7 +37,7 @@ import db_access, katie, logging, utils; ############################################################################### # Globals -kelly_version = "$Revision: 1.15 $"; +kelly_version = "$Revision: 1.16 $"; Cnf = None; Options = None; @@ -107,6 +107,8 @@ def reject (str, prefix="Rejected: "): # frozen between accept and our run time. def check(): + propogate={} + nopropogate={} for file in files.keys(): # The .orig.tar.gz can disappear out from under us is it's a # duplicate of one in the archive. @@ -129,6 +131,20 @@ def check(): (reject_msg, is_in_incoming) = Katie.check_dsc_against_db(file); reject(reject_msg, ""); + # propogate in the case it is in the override tables: + if changes.has_key("propdistribution"): + for suite in changes["propdistribution"].keys(): + if Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): + propogate[suite] = 1 + else: + nopropogate[suite] = 1 + + for suite in propogate.keys(): + if suite in nopropogate: + continue + changes["distribution"][suite] = 1 + + for file in files.keys(): # Check the package is still in the override tables for suite in changes["distribution"].keys(): if not Katie.in_override_p(files[file]["package"], files[file]["component"], suite, files[file].get("dbtype",""), file): diff --git a/lisa b/lisa index 99319f55..d98c0df6 100755 --- a/lisa +++ b/lisa @@ -1,8 +1,8 @@ #!/usr/bin/env python # Handles NEW and BYHAND packages -# Copyright (C) 2001, 2002, 2003, 2004 James Troup -# $Id: lisa,v 1.30 2004-04-01 17:13:11 troup Exp $ +# Copyright (C) 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: lisa,v 1.31 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,12 +37,12 @@ ################################################################################ -import copy, errno, os, readline, stat, sys; +import copy, errno, os, readline, stat, sys, time; import apt_pkg, apt_inst; import db_access, fernanda, katie, logging, utils; # Globals -lisa_version = "$Revision: 1.30 $"; +lisa_version = "$Revision: 1.31 $"; Cnf = None; Options = None; @@ -253,12 +253,12 @@ def sort_changes(changes_files): for source in per_source.keys(): source_list = per_source[source]["list"]; first = source_list[0]; - oldest = os.stat(first["filename"])[stat.ST_CTIME]; + oldest = os.stat(first["filename"])[stat.ST_MTIME]; have_note = 0; for d in per_source[source]["list"]: - ctime = os.stat(d["filename"])[stat.ST_CTIME]; - if ctime < oldest: - oldest = ctime; + mtime = os.stat(d["filename"])[stat.ST_MTIME]; + if mtime < oldest: + oldest = mtime; have_note += (d.has_key("lisa note")); per_source[source]["oldest"] = oldest; if not have_note: @@ -629,7 +629,7 @@ def add_overrides (new): type_id = db_access.get_override_type_id(new[pkg]["type"]); priority_id = new[pkg]["priority id"]; section_id = new[pkg]["section id"]; - projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" % (suite_id, component_id, type_id, pkg, priority_id, section_id)); + projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, pkg, priority_id, section_id)); for file in new[pkg]["files"]: if files[file].has_key("new"): del files[file]["new"]; @@ -870,9 +870,25 @@ def do_byhand(): def do_accept(): print "ACCEPT"; if not Options["No-Action"]: + retry = 0; + while retry < 10: + try: + lock_fd = os.open(Cnf["Lisa::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL); + retry = 10; + except OSError, e: + if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST': + retry += 1; + if (retry >= 10): + utils.fubar("Couldn't obtain lock; assuming jennifer is already running."); + else: + print("Unable to get accepted lock (try %d of 10)" % retry); + time.sleep(60); + else: + raise; (summary, short_summary) = Katie.build_summaries(); Katie.accept(summary, short_summary); os.unlink(Katie.pkg.changes_file[:-8]+".katie"); + os.unlink(Cnf["Lisa::AcceptedLockFile"]); def check_status(files): new = byhand = 0; diff --git a/logging.py b/logging.py index ae8ee6f2..5fe0cf63 100644 --- a/logging.py +++ b/logging.py @@ -2,7 +2,7 @@ # Logging functions # Copyright (C) 2001, 2002 James Troup -# $Id: logging.py,v 1.3 2002-10-16 02:47:32 troup Exp $ +# $Id: logging.py,v 1.4 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,7 +20,7 @@ ################################################################################ -import os, pwd, time; +import os, pwd, time, sys; import utils; ################################################################################ @@ -31,7 +31,7 @@ class Logger: logfile = None; program = None; - def __init__ (self, Cnf, program): + def __init__ (self, Cnf, program, debug=0): "Initialize a new Logger object" self.Cnf = Cnf; self.program = program; @@ -42,9 +42,11 @@ class Logger: os.makedirs(logdir, 02775); # Open the logfile logfilename = "%s/%s" % (logdir, time.strftime("%Y-%m")); - logfile = utils.open_file(logfilename, 'a'); - # Seek to the end of the logfile - logfile.seek(0,2); + logfile = None + if debug: + logfile = sys.stderr + else: + logfile = utils.open_file(logfilename, 'a'); self.logfile = logfile; # Log the start of the program user = pwd.getpwuid(os.getuid())[0]; @@ -59,7 +61,7 @@ class Logger: # Force the contents of the list to be string.join-able details = map(str, details); # Write out the log in TSV - self.logfile.write("~".join(details)+'\n'); + self.logfile.write("|".join(details)+'\n'); # Flush the output to enable tail-ing self.logfile.flush(); diff --git a/madison b/madison index c9f65777..5a17892b 100755 --- a/madison +++ b/madison @@ -1,8 +1,8 @@ #!/usr/bin/env python # Display information about package(s) (suite, version, etc.) -# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: madison,v 1.32 2005-02-08 22:43:45 troup Exp $ +# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 James Troup +# $Id: madison,v 1.33 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -187,9 +187,9 @@ SELECT s.source, s.version, 'source', su.suite_name, c.name, m.name for arch in arches: sys.stdout.write("%s %s %s\n" % (pkg, version, arch)); if Options["GreaterOrEqual"]: - print "\ndep-retry %s (>= %s)" % (pkg, highver[pkg]) + print "\n%s (>= %s)" % (pkg, highver[pkg]) if Options["GreaterThan"]: - print "\ndep-retry %s (>> %s)" % (pkg, highver[pkg]) + print "\n%s (>> %s)" % (pkg, highver[pkg]) if not results: sys.exit(1); diff --git a/melanie b/melanie index 0ca9abda..3b2596cc 100755 --- a/melanie +++ b/melanie @@ -2,7 +2,7 @@ # General purpose package removal tool for ftpmaster # Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup -# $Id: melanie,v 1.43 2004-11-27 18:12:57 troup Exp $ +# $Id: melanie,v 1.44 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -162,7 +162,7 @@ def reverse_depends_check(removals, suites): if unsat == len(dep): component = p2c[package]; if component != "main": - what = "%s/%s" (package, component); + what = "%s/%s" % (package, component); else: what = "** %s" % (package); print "%s has an unsatisfied dependency on %s: %s" % (what, architecture, utils.pp_deps(dep)); @@ -509,7 +509,7 @@ def main (): Subst["__BCC__"] = "Bcc: " + ", ".join(bcc); else: Subst["__BCC__"] = "X-Filler: 42"; - Subst["__CC__"] = "X-Katie: melanie $Revision: 1.43 $"; + Subst["__CC__"] = "X-Katie: melanie $Revision: 1.44 $"; if carbon_copy: Subst["__CC__"] += "\nCc: " + ", ".join(carbon_copy); Subst["__SUITE_LIST__"] = suites_list; diff --git a/natalie b/natalie index 2c43a5d2..4a5d62bc 100755 --- a/natalie +++ b/natalie @@ -2,7 +2,7 @@ # Manipulate override files # Copyright (C) 2000, 2001, 2002, 2003 James Troup -# $Id: natalie,v 1.6 2003-04-15 16:03:31 troup Exp $ +# $Id: natalie,v 1.7 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -164,7 +164,8 @@ def process_file (file, suite, component, type, action): (old_priority_id, old_section_id, old_maintainer_override, old_priority, old_section) = original[package]; if action == "add" or old_priority_id == priority_id and \ old_section_id == section_id and \ - old_maintainer_override == maintainer_override: + ((old_maintainer_override == maintainer_override) or \ + (old_maintainer_override == "" and maintainer_override == None)): # If it's unchanged or we're in 'add only' mode, ignore it c_skipped += 1; continue; @@ -190,7 +191,7 @@ def process_file (file, suite, component, type, action): projectB.query("INSERT INTO override (suite, component, type, package, priority, section, maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '%s')" % (suite_id, component_id, type_id, package, priority_id, section_id, maintainer_override)); else: - projectB.query("INSERT INTO override (suite, component, type, package, priority, section) VALUES (%s, %s, %s, '%s', %s, %s)" + projectB.query("INSERT INTO override (suite, component, type, package, priority, section,maintainer) VALUES (%s, %s, %s, '%s', %s, %s, '')" % (suite_id, component_id, type_id, package, priority_id, section_id)); if not update_p: diff --git a/nina b/nina index 71922122..a628b2c4 100755 --- a/nina +++ b/nina @@ -1,7 +1,7 @@ #!/usr/bin/env python -# Copyright (C) 2004 James Troup -# $Id: nina,v 1.1 2004-11-27 19:25:59 troup Exp $ +# Copyright (C) 2004, 2005 James Troup +# $Id: nina,v 1.2 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,8 +31,8 @@ def main(): files = glob.glob("%s/*" % (Cnf["Dir::Queue::Done"])); for filename in files: if os.path.isfile(filename): - ctime = time.gmtime(os.stat(filename)[stat.ST_CTIME]); - dirname = time.strftime("%Y/%m/%d", ctime); + mtime = time.gmtime(os.stat(filename)[stat.ST_MTIME]); + dirname = time.strftime("%Y/%m/%d", mtime); if not os.path.exists(dirname): print "Creating: %s" % (dirname); os.makedirs(dirname); diff --git a/pseudo-packages.description b/pseudo-packages.description index 3910e3b0..42f1a634 100644 --- a/pseudo-packages.description +++ b/pseudo-packages.description @@ -24,3 +24,4 @@ mirrors Problems with the official mirrors security.debian.org The Debian Security Team installation-reports Reports of installation problems with stable & testing upgrade-reports Reports of upgrade problems for stable & testing +release-notes Problems with the Release Notes diff --git a/pseudo-packages.maintainers b/pseudo-packages.maintainers index 87b35778..eb3d305e 100644 --- a/pseudo-packages.maintainers +++ b/pseudo-packages.maintainers @@ -4,17 +4,17 @@ installation Debian Install Team cdrom Debian CD-ROM Team boot-floppy Debian Install Team press press@debian.org -bugs.debian.org Adam Heath and others +bugs.debian.org Debian Bug Tracking Team ftp.debian.org James Troup and others qa.debian.org debian-qa@lists.debian.org -nonus.debian.org James Troup and others -www.debian.org James Treacy and others -mirrors Josip Rodin and others -listarchives Josip Rodin and others +nonus.debian.org Michael Beattie and others +www.debian.org Debian WWW Team +mirrors Debian Mirrors Team +listarchives Debian List Archive Team project debian-project@lists.debian.org general debian-devel@lists.debian.org kernel Debian Kernel Team -lists.debian.org Martin Schulze and others +lists.debian.org Debian Listmaster Team spam spam@debian.org slink-cd Steve McIntyre potato-cd Steve McIntyre @@ -24,3 +24,4 @@ tech-ctte Technical Committee security.debian.org Debian Security Team installation-reports Debian Install Team upgrade-reports Debian Testing Group +release-notes Debian Documentation Team diff --git a/saffron b/saffron index 5d497c19..a11b902e 100755 --- a/saffron +++ b/saffron @@ -2,7 +2,7 @@ # Various statistical pr0nography fun and games # Copyright (C) 2000, 2001, 2002, 2003 James Troup -# $Id: saffron,v 1.2 2003-02-07 14:53:47 troup Exp $ +# $Id: saffron,v 1.3 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -109,8 +109,14 @@ def longest(list): return longest; def suite_sort(a, b): - a_priority = int(Cnf["Suite::%s::Priority" % (a)]); - b_priority = int(Cnf["Suite::%s::Priority" % (b)]); + if Cnf.has_key("Suite::%s::Priority" % (a)): + a_priority = int(Cnf["Suite::%s::Priority" % (a)]); + else: + a_priority = 0; + if Cnf.has_key("Suite::%s::Priority" % (b)): + b_priority = int(Cnf["Suite::%s::Priority" % (b)]); + else: + b_priority = 0; return cmp(a_priority, b_priority); def output_format(suite): diff --git a/vars b/vars index d921984f..a4e3b5d0 100644 --- a/vars +++ b/vars @@ -9,6 +9,7 @@ archs="alpha arm hppa hurd-i386 i386 ia64 m68k mips mipsel powerpc s390 sh sparc scriptdir=$base/scripts masterdir=$base/katie/ dbdir=$base/database/ +lockdir=$base/lock/ overridedir=$scriptdir/override extoverridedir=$scriptdir/external-overrides @@ -19,7 +20,7 @@ incoming=$base/incoming ftpgroup=debadmin -copyoverrides="potato.contrib potato.contrib.src potato.main potato.main.src potato.non-free potato.non-free.src woody.contrib woody.contrib.src woody.main woody.main.src woody.non-free woody.non-free.src sarge.contrib sarge.contrib.src sarge.main sarge.main.src sarge.non-free sarge.non-free.src sid.contrib sid.contrib.src sid.main sid.main.debian-installer sid.main.src sid.non-free sid.non-free.src sid.extra.contrib sid.extra.main sid.extra.non-free woody.extra.contrib woody.extra.main woody.extra.non-free sarge.extra.contrib sarge.extra.main sarge.extra.non-free" +copyoverrides="etch.contrib etch.contrib.src etch.main etch.main.src etch.non-free etch.non-free.src etch.extra.main etch.extra.non-free etch.extra.contrib etch.main.debian-installer woody.contrib woody.contrib.src woody.main woody.main.src woody.non-free woody.non-free.src sarge.contrib sarge.contrib.src sarge.main sarge.main.src sarge.non-free sarge.non-free.src sid.contrib sid.contrib.src sid.main sid.main.debian-installer sid.main.src sid.non-free sid.non-free.src sid.extra.contrib sid.extra.main sid.extra.non-free woody.extra.contrib woody.extra.main woody.extra.non-free sarge.extra.contrib sarge.extra.main sarge.extra.non-free" PATH=$masterdir:$PATH umask 022 diff --git a/ziyi b/ziyi index 10210190..49a19fb7 100755 --- a/ziyi +++ b/ziyi @@ -3,7 +3,7 @@ # Create all the Release files # Copyright (C) 2001, 2002 Anthony Towns -# $Id: ziyi,v 1.26 2003-07-15 09:43:18 ajt Exp $ +# $Id: ziyi,v 1.27 2005-11-15 09:50:32 ajt Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,6 +48,13 @@ If no SUITE is given Release files are generated for all suites.""" ################################################################################ +def add_tiffani (files, path, indexstem): + index = "%s.diff/Index" % (indexstem) + filepath = "%s/%s" % (path, index) + if os.path.exists(filepath): + #print "ALERT: there was a tiffani file %s" % (filepath) + files.append(index) + def compressnames (tree,type,file): compress = AptCnf.get("%s::%s::Compress" % (tree,type), AptCnf.get("Default::%s::Compress" % (type), ". gzip")) result = [] @@ -204,8 +211,10 @@ def main (): for sec in AptCnf["tree::%s::Sections" % (tree)].split(): for arch in AptCnf["tree::%s::Architectures" % (tree)].split(): if arch == "source": - for file in compressnames("tree::%s" % (tree), "Sources", "%s/%s/Sources" % (sec, arch)): + filepath = "%s/%s/Sources" % (sec, arch) + for file in compressnames("tree::%s" % (tree), "Sources", filepath): files.append(file) + add_tiffani(files, Cnf["Dir::Root"] + tree, filepath) else: disks = "%s/disks-%s" % (sec, arch) diskspath = Cnf["Dir::Root"]+tree+"/"+disks @@ -214,8 +223,10 @@ def main (): if os.path.exists("%s/%s/md5sum.txt" % (diskspath, dir)): files.append("%s/%s/md5sum.txt" % (disks, dir)) - for file in compressnames("tree::%s" % (tree), "Packages", "%s/binary-%s/Packages" % (sec, arch)): + filepath = "%s/binary-%s/Packages" % (sec, arch) + for file in compressnames("tree::%s" % (tree), "Packages", filepath): files.append(file) + add_tiffani(files, Cnf["Dir::Root"] + tree, filepath) if arch == "source": rel = "%s/%s/Release" % (sec, arch) -- 2.39.5