From 5144a464766f5c2ed592b5fa81e2943992f0c1fe Mon Sep 17 00:00:00 2001 From: James Troup Date: Wed, 20 Dec 2000 08:15:35 +0000 Subject: [PATCH] Check size in .dsc and fix up rhona to work on auric. --- THANKS | 1 + TODO | 13 ++++--- apt.conf-non-US | 2 +- cron.daily | 3 +- cron.daily-non-US | 5 ++- katie | 10 ++++-- katie.conf | 4 +-- katie.conf-non-US | 4 +-- madison | 87 +++++++++++++++++++++++++++++++++++++++++++++++ mkchecksums | 6 ++-- rhona | 45 +++++++++++++++++------- vars | 1 + vars-non-US | 1 + 13 files changed, 149 insertions(+), 33 deletions(-) create mode 100755 madison diff --git a/THANKS b/THANKS index 8fbe7241..e32fb87d 100644 --- a/THANKS +++ b/THANKS @@ -6,6 +6,7 @@ this would be possible. Adam Heath Anthony Towns Antti-Juhani Kaijanaho +Ben Collins Brendan O'Dea Chris Leishman Daniel Jacobwitz diff --git a/TODO b/TODO index e8cbcd3d..6cfeddad 100644 --- a/TODO +++ b/TODO @@ -6,18 +6,17 @@ Urgent Less Urgent ----------- - o CD building scripts need fixing + o Need to merge non-non-US and non-US DBs. + o need a poolifier that will poolify X mb a day.. (catherine) + o Need to vacuum daily + + == o Log files for jenna, rhona and katie. o Optimize all the queries by using EXAMINE and building some INDEXs. o enclose all the setting SQL stuff in transactions (mostly done). - o clear out maintainers table o jenna needs to use order by to keep aj from going nutzo - - == - - o need a poolifier that will poolify X mb a day.. (catherine) - o Need to merge non-non-US and non-US DBs. + o remove empty directories in the pool == diff --git a/apt.conf-non-US b/apt.conf-non-US index 4f08607d..da7f5d44 100644 --- a/apt.conf-non-US +++ b/apt.conf-non-US @@ -31,7 +31,7 @@ tree "dists/stable/non-US" External-Links false; }; -tree "dists/testing" +tree "dists/testing/non-US" { FileList "/org/non-us.debian.org/database/dists/testing_non-us/$(SECTION)_binary-$(ARCH).list"; SourceFileList "/org/non-us.debian.org/database/dists/testing_non-us/$(SECTION)_source.list"; diff --git a/cron.daily b/cron.daily index d1fa06b8..445ce11a 100755 --- a/cron.daily +++ b/cron.daily @@ -62,7 +62,8 @@ for i in proposed-updates_-_binary-*; do cat $i >> proposed-updates_-_binary.lis cd $masterdir apt-ftparchive generate apt.conf -#rhona +# Clean out old packages +rhona mkmaintainers copyoverrides diff --git a/cron.daily-non-US b/cron.daily-non-US index 3e2767b1..bfcb880f 100755 --- a/cron.daily-non-US +++ b/cron.daily-non-US @@ -55,7 +55,10 @@ cd /org/non-us.debian.org/database/dists/ for i in proposed-updates_-_binary-*; do cat $i >> proposed-updates_-_binary.list; done cd $masterdir apt-ftparchive generate apt.conf-non-US -#rhona + +# Clean out old packages +rhona + cd $indices charisma > .new-maintainers mv -f .new-maintainers Maintainers diff --git a/katie b/katie index 010d5611..ed35d8dc 100755 --- a/katie +++ b/katie @@ -2,7 +2,7 @@ # Installs Debian packaes # Copyright (C) 2000 James Troup -# $Id: katie,v 1.14 2000-12-19 21:06:20 troup Exp $ +# $Id: katie,v 1.15 2000-12-20 08:15:35 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -460,7 +460,8 @@ def check_dsc (): # locations of an .orig.tar.gz. for dsc_file in dsc_files.keys(): if files.has_key(dsc_file): - actual_md5 = files[dsc_file]["md5sum"] + actual_md5 = files[dsc_file]["md5sum"]; + actual_size = int(files[dsc_file]["size"]); found = "%s in incoming" % (dsc_file) # Check the file does not already exist in the archive if not changes.has_key("stable upload"): @@ -474,6 +475,7 @@ def check_dsc (): if len(ql) > 0: old_file = ql[0][0] + ql[0][1]; actual_md5 = apt_pkg.md5sum(utils.open_file(old_file,"r")); + actual_size = os.stat(old_file)[stat.ST_SIZE]; found = old_file; suite_type = ql[0][2]; dsc_files[dsc_file]["files id"] = ql[0][3]; # need this for updating dsc_files in install() @@ -500,7 +502,9 @@ def check_dsc (): reject_message = reject_message + "Rejected: %s refers to %s, but I can't find it in Incoming." % (file, dsc_file); continue; if actual_md5 != dsc_files[dsc_file]["md5sum"]: - reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file) + reject_message = reject_message + "Rejected: md5sum for %s doesn't match %s.\n" % (found, file); + if actual_size != int(dsc_files[dsc_file]["size"]): + reject_message = reject_message + "Rejected: size for %s doesn't match %s.\n" % (found, file); if string.find(reject_message, "Rejected:") != -1: return 0 diff --git a/katie.conf b/katie.conf index 556ff745..bcb175e5 100644 --- a/katie.conf +++ b/katie.conf @@ -85,11 +85,11 @@ Neve Rhona { - Options // Currently don't do anything + Options { Debug ""; Help ""; - Interactive ""; + No-Action ""; Version ""; }; diff --git a/katie.conf-non-US b/katie.conf-non-US index 1584b96b..a1b7f362 100644 --- a/katie.conf-non-US +++ b/katie.conf-non-US @@ -85,11 +85,11 @@ Neve Rhona { - Options // Currently don't do anything + Options { Debug ""; Help ""; - Interactive ""; + No-Action ""; Version ""; }; diff --git a/madison b/madison new file mode 100755 index 00000000..7d0222d9 --- /dev/null +++ b/madison @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +# 'Fix' stable to make debian-cd and dpkg -BORGiE users happy +# Copyright (C) 2000 James Troup +# $Id: madison,v 1.1 2000-12-20 08:15:35 troup Exp $ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# And, lo, a great and menacing voice rose from the depths, and with +# great wrath and vehemence it's voice boomed across the +# land... ``hehehehehehe... that *tickles*'' +# -- aj on IRC + +################################################################################ + +import pg, sys, os, string +import utils, db_access +import apt_pkg; + +################################################################################ + +Cnf = None; +projectB = None; + +################################################################################ + +def main (): + global Cnf, projectB; + + apt_pkg.init(); + + Cnf = apt_pkg.newConfiguration(); + apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + + Arguments = [('d',"debug","Claire::Options::Debug", "IntVal"), + ('h',"help","Claire::Options::Help"), + ('v',"version","Claire::Options::Version")]; + + packages = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + + projectB = pg.connect('projectb', 'localhost'); + + db_access.init(Cnf, projectB); + + for package in packages: + q = projectB.query("SELECT b.package, b.version, a.arch_string, su.suite_name FROM binaries b, architecture a, suite su, bin_associations ba WHERE b.package = '%s' AND a.id = b.architecture AND su.id = ba.suite AND b.id = ba.bin" % (package)); + d = {}; + for i in q.getresult(): + package = i[0]; + version = i[1]; + architecture = i[2]; + suite = i[3]; + key = (version, suite); + if not d.has_key(key): + d[key] = []; + d[key].append(architecture); + + keys = d.keys(); + keys.sort(); + for i in keys: + (version, suite) = i; + sys.stdout.write("%10s | %10s | %13s | " % (package, version, suite)); + count = 0; + for arch in d[i]: + if count > 0: + sys.stdout.write(', '); + sys.stdout.write(arch); + count = count + 1; + sys.stdout.write('\n'); + +####################################################################################### + +if __name__ == '__main__': + main() + diff --git a/mkchecksums b/mkchecksums index d614fc1f..b99fd866 100755 --- a/mkchecksums +++ b/mkchecksums @@ -1,16 +1,16 @@ #!/bin/sh # Update the md5sums file -# $Id: mkchecksums,v 1.1 2000-11-24 00:20:11 troup Exp $ +# $Id: mkchecksums,v 1.2 2000-12-20 08:15:35 troup Exp $ set -e . $SCRIPTVARS -dsynclist=$indices/dsync.list +dsynclist=$dbdir/dsync.list md5list=$indices/md5sums echo -n "Creating md5 / dsync index file ... " cd "$ftpdir" dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 -dsync-flist -q md5sums $dsynclist | tee $md5list | gzip -9n > ${md5list}.gz +dsync-flist -q md5sums $dsynclist | gzip -9n > ${md5list}.gz dsync-flist -q link-dups $dsynclist || true diff --git a/rhona b/rhona index 8446c019..3f3be031 100755 --- a/rhona +++ b/rhona @@ -2,7 +2,7 @@ # rhona, cleans up unassociated binary and source packages # Copyright (C) 2000 James Troup -# $Id: rhona,v 1.5 2000-12-19 21:06:20 troup Exp $ +# $Id: rhona,v 1.6 2000-12-20 08:15:35 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -150,8 +150,9 @@ def check_sources(): if Cnf.Find("Suite::%s::Untouchable" % (i[0])): untouchable = 1; else: - projectB.query("DELETE FROM src_associations WHERE source = %s" % (id)); - + if not Cnf["Rhona::Options::No-Action"]: + projectB.query("DELETE FROM src_associations WHERE source = %s" % (id)); + # We can't delete binary-less source-only packages if # they're in an untouchable suite (i.e. stable)... if untouchable: @@ -171,7 +172,8 @@ def check_sources(): # this source package's reference to it from dsc_files. # So just clear out all references to the source file in # dsc_files now. - projectB.query("DELETE FROM dsc_files WHERE source = %s" % (id)); + if not Cnf["Rhona::Options::No-Action"]: + projectB.query("DELETE FROM dsc_files WHERE source = %s" % (id)); projectB.query("COMMIT WORK"); @@ -247,7 +249,11 @@ def clean_binaries(): # source also removed (if possible). print "Cleaning binaries from the DB..." - projectB.query("DELETE FROM binaries WHERE file IN (SELECT id FROM files WHERE last_used < '%s')" % (delete_date)); + if not Cnf["Rhona::Options::No-Action"]: + before = time.time(); + sys.stdout.write("[Deleting from binaries table... "); + projectB.query("DELETE FROM binaries WHERE EXISTS (SELECT id FROM files WHERE binaries.file = files.id AND files.last_used <= '%s')" % (delete_date)); + sys.stdout.write("done. (%d)]\n" % (int(time.time()-before))); def clean(): global delete_date; @@ -262,7 +268,12 @@ def clean(): os.mkdir(dest); # Delete from source (dsc_file should already be done!) - projectB.query("DELETE FROM source WHERE file IN (SELECT id FROM files WHERE last_used <= '%s')" % (delete_date)); + if not Cnf["Rhona::Options::No-Action"]: + before = time.time(); + sys.stdout.write("[Deleting from source table... "); + projectB.query("DELETE FROM source WHERE EXISTS (SELECT id FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date)); + sys.stdout.write("done. (%d)]\n" % (int(time.time()-before))); + # Delete files from the pool q = projectB.query("SELECT l.path, f.filename FROM location l, files f WHERE f.last_used <= '%s' AND l.id = f.location" % (delete_date)); for i in q.getresult(): @@ -273,18 +284,26 @@ def clean(): if os.path.isfile(filename): if os.path.islink(filename): count = count + 1; - #print "Removing symlink %s..." % (filename); - os.unlink(filename); + if Cnf["Rhona::Options::No-Action"]: + print "Removing symlink %s..." % (filename); + else: + os.unlink(filename); else: size = size + os.stat(filename)[stat.ST_SIZE]; count = count + 1; - #print "Cleaning %s to %s..." % (filename, dest); - utils.move(filename, dest); + if Cnf["Rhona::Options::No-Action"]: + print "Cleaning %s to %s..." % (filename, dest); + else: + utils.move(filename, dest); else: sys.stderr.write("%s is neither symlink nor file?!\n" % (filename)); sys.exit(1); # delete from files - projectB.query("DELETE FROM files WHERE last_used <= '%s'" % (delete_date)); + if not Cnf["Rhona::Options::No-Action"]: + before = time.time(); + sys.stdout.write("[Deleting from files table... "); + projectB.query("DELETE FROM files WHERE last_used <= '%s'" % (delete_date)); + sys.stdout.write("done. (%d)]\n" % (int(time.time()-before))); if count > 0: sys.stderr.write("Cleaned %d files, %s.\n" % (count, utils.size_type(size))); @@ -308,7 +327,8 @@ def clean_maintainers(): projectB.query("BEGIN WORK"); for id in all.keys(): if not used.has_key(id): - projectB.query("DELETE FROM maintainer WHERE id = %s" % (id)); + if not Cnf["Rhona::Options::No-Action"]: + projectB.query("DELETE FROM maintainer WHERE id = %s" % (id)); count = count + 1; projectB.query("COMMIT WORK"); @@ -328,7 +348,6 @@ def main(): Arguments = [('D',"debug","Rhona::Options::Debug", "IntVal"), ('h',"help","Rhona::Options::Help"), ('n',"no-action","Rhona::Options::No-Action"), - ('v',"verbose","Rhona::Options::Verbose"), ('V',"version","Rhona::Options::Version")]; apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); diff --git a/vars b/vars index cdbf61dd..84699f2d 100644 --- a/vars +++ b/vars @@ -6,6 +6,7 @@ archs="alpha arm hppa hurd-i386 i386 m68k powerpc sparc mips mipsel sh" scriptdir=/org/ftp.debian.org/scripts masterdir=/org/ftp.debian.org/katie/ +dbdir=/org/ftp.debian.org/database/ overridedir=$scriptdir/override incoming=/org/ftp.debian.org/incoming diff --git a/vars-non-US b/vars-non-US index 5e006379..96bf71c9 100644 --- a/vars-non-US +++ b/vars-non-US @@ -7,6 +7,7 @@ archs="alpha arm hppa hurd-i386 i386 m68k powerpc sparc mips mipsel sh" masterdir=$nonushome/katie overridedir=$nonushome/scripts/override +dbdir=$nonushome/database/ incoming=$nonushome/incoming packagesfiles=packagesfiles-non-US -- 2.39.5