From: Joerg Jaspert Date: Fri, 2 May 2008 20:52:12 +0000 (+0200) Subject: Add logging to cronjobs X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f8d3d222b8464d1f8e96d0b4f2d8538ca239f1a7;p=dak Add logging to cronjobs --- diff --git a/ChangeLog b/ChangeLog index 77026003..d65764e0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,25 @@ +2008-05-02 Joerg Jaspert + + * config/debian/cron.buildd: Cleanup unreachable code. (If someone + still wants it - its in revision control...) + + * config/debian/cron.daily: Modify call to update-ftpstats to only + use files log/2*, instead of log/*, so it ignores the new cron/ + subdir. Makes sure it can't get confused, and is also safe for + nearly thousand years. If this code is really in use in year 3000, + im sure people can adjust it! :) + + * config/debian/vars: Add logdir as a place for cronjob log output + + * config/debian/cron.daily: Use a logfile and be more verbose of + whats going on. + Also moved the commented VACUUM ; VACUUM ANALYZE calls over to + cron.weekly, ... + * config/debian/cron.weekly: likewise, + ... and activate those calls again. Once a week, as an + additional safety bet to be sure the stuff does get run is ok, + even if we have autovacuum by default. + 2008-05-02 Thomas Viehmann * dak/process_unchecked.py (check_hashes): fix typo in diff --git a/config/debian/cron.buildd b/config/debian/cron.buildd index 5e81e765..1bc4d680 100755 --- a/config/debian/cron.buildd +++ b/config/debian/cron.buildd @@ -4,32 +4,3 @@ # ssh buildd@buildd /org/wanna-build/trigger.often exit 0 - -cleanup() { - rm -f "$LOCKFILE" - kill -TERM $SSH_PID -} - -ARCHS="alpha arm hppa i386 ia64 m68k mips mipsel powerpc sparc s390" - -set -e -export SCRIPTVARS=/org/ftp.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -LOCKFILE="/org/wanna-build/tmp/DB_Maintenance_In_Progress" - -if [ ! -e "$ftpdir/Archive_Maintenance_In_Progress" ]; then - if lockfile -r3 $LOCKFILE; then - trap cleanup 0 - cd /org/incoming.debian.org/buildd - cp /org/wanna-build/tmp/Sources.unstable-old Sources - gzip -cd Sources.gz >> Sources - for a in $ARCHS; do - cp /org/wanna-build/tmp/Packages.unstable.$a-old Packages - gzip -cd /org/incoming.debian.org/buildd/Packages.gz >> Packages - quinn-diff -i -a /org/buildd.debian.org/web/quinn-diff/Packages-arch-specific -A $a 2>/dev/null | perl -pi -e 's#^(non-free)/.*$##msg' | wanna-build -b $a/build-db --merge-partial-quinn 2> /dev/null - wanna-build -A $a -b $a/build-db --merge-packages Packages 2>/dev/null - done - rm -f Sources Packages - fi -fi diff --git a/config/debian/cron.daily b/config/debian/cron.daily index b3d43502..09d6a6b3 100755 --- a/config/debian/cron.daily +++ b/config/debian/cron.daily @@ -8,6 +8,11 @@ export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars ################################################################################ +# Start logging +NOW=`date "+%Y.%m.%d-%H:%M:%S"` +LOGFILE="$logdir/daily_${NOW}.log" +exec > "$LOGFILE" 2>&1 + echo Archive maintenance started at $(date +%X) TS=0 @@ -22,6 +27,8 @@ cleanup() { if [ "$lockac" -eq "1" ]; then rm -f "$LOCKAC" fi + echo "Cleanup" + rm -f "$LOGFILE" } lockfile -l 3600 $LOCKCU trap cleanup 0 @@ -36,6 +43,7 @@ You should not mirror the archive during this period. EOF # Push merkels qa user, so the qa pages can show "dinstall is running" information +echo "Telling merkels QA user that we start dinstall" ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 qa@merkel.debian.org sleep 1 ################################################################################ @@ -46,6 +54,7 @@ pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Updating Bugs docu, Mirror list and mailing-lists.txt" cd $configdir $scriptsdir/update-bugdoctxt $scriptsdir/update-mirrorlists @@ -54,12 +63,14 @@ $scriptsdir/update-mailingliststxt ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Doing automated p-u-new processing" cd $queuedir/p-u-new date -u -R >> REPORT dak process-new -a -C COMMENTS >> REPORT echo >> REPORT TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Doing automated o-p-u-new processing" cd $queuedir/o-p-u-new date -u -R >> REPORT dak process-new -a -C COMMENTS >> REPORT @@ -70,6 +81,7 @@ echo >> REPORT TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) lockfile $LOCKAC lockac=1 +echo "Processing queue/accepted" cd $accepted rm -f REPORT dak process-accepted -pa *.changes | tee REPORT | \ @@ -78,31 +90,30 @@ chgrp debadmin REPORT chmod 664 REPORT TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) - +echo "Checking for cruft in overrides" dak check-overrides rm -f $LOCKAC lockac=0 +echo "Fixing symlinks in $ftpdir" symlinks -d -r $ftpdir TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Generating suite file lists for apt-ftparchive" dak make-suite-file-list TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Updating fingerprints" # Update fingerprints dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg # Generate override files +echo "Writing overrides into text files" cd $overridedir dak make-overrides -# Update task overrides for testing and unstable -# [JT 2004-02-04 disabled; copying in by hand for now] -#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sarge.extra.main -#cat $extoverridedir/task | perl -ne 'print if /^\S+\sTask\s\S+(,\s*\S+)*$/;' > override.sid.extra.main - # FIXME rm -f override.sid.all3 for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done @@ -110,16 +121,20 @@ for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Generate Packages and Sources files +echo "Generating Packages and Sources files" cd $configdir apt-ftparchive generate apt.conf # Generate *.diff/ incremental updates +echo "Generating pdiff files" dak generate-index-diffs # Generate Release files +echo "Generating Release files" dak generate-releases TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Clean out old packages +echo "Cleanup old packages/files" dak clean-suites dak clean-queues @@ -127,12 +142,14 @@ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Needs to be rebuilt, as files have moved. Due to unaccepts, we need to # update this before wanna-build is updated. +echo "Regenerating wanna-build/buildd information" psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list symlinks -d /srv/incoming.debian.org/buildd > /dev/null apt-ftparchive generate apt.conf.buildd TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) +echo "Running various scripts from $scriptsdir" cd $scriptsdir ./mkmaintainers ./copyoverrides @@ -141,6 +158,7 @@ cd $scriptsdir ./mkchecksums # rm -f $NOTICE +echo "Trigger daily wanna-build run" ssh buildd@buildd /org/wanna-build/trigger.daily rm -f $LOCKCU @@ -157,13 +175,6 @@ pg_dump projectb > $POSTDUMP TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) -# Vacuum the database -# (JJ, 20-04-2008) disabled, as we have autovacuum set to on in postgres. -# refer to http://www.postgresql.org/docs/current/static/routine-vacuuming.html#AUTOVACUUM -# which says "Beginning in PostgreSQL 8.1, there is an optional feature called autovacuum, -# whose purpose is to automate the execution of VACUUM and ANALYZE commands." -# echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$" - echo "Expiring old database dumps..." (cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*") @@ -172,38 +183,38 @@ echo "Expiring old database dumps..." TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Send a report on NEW/BYHAND packages +echo "Nagging ftpteam about NEW/BYHAND packages" dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-master.debian.org # and one on crufty packages +echo "Sending information about crufty packages" dak cruft-report > $webdir/cruft-report-daily.txt dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org -$scriptsdir/dm-monitor >$webdir/dm-uploaders.html - +echo "Updating DM html page" $scriptsdir/dm-monitor >$webdir/dm-uploaders.html ################################################################################ # Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached +echo "Trigger merkels projectb sync" ssh -2 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 -# Run mirror-split - -#time dak mirror-split - ################################################################################ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) ulimit -m 90000 -d 90000 -s 10000 -v 200000 +echo "Using run-parts to run scripts in $base/scripts/distmnt" run-parts --report $base/scripts/distmnt -echo Daily cron scripts successful. +echo "Daily cron scripts successful." TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Stats pr0n +echo "Updating stats data" cd $configdir $scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data R --slave --vanilla < $base/misc/ftpstats.R @@ -211,14 +222,14 @@ R --slave --vanilla < $base/misc/ftpstats.R TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Clean up apt-ftparchive's databases - +echo "Clean up apt-ftparchive's databases" cd $configdir apt-ftparchive -q clean apt.conf TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) # Compress psql backups older than a week, but no more than 20 of them - +echo "Compress old psql backups" (cd $base/backup/ find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mtime +7 | sort | head -n20 | while read dumpname; do @@ -227,4 +238,10 @@ TS=$(($TS+1)); echo Archive maintenance timestamp $TS: $(date +%X) done ) +echo "Finally, all is done, sending mail and compressing logfile" +exec > /dev/null 2>&1 + +cat "$LOGFILE" | mail -s "Log for cron.daily run of $(date +%Y.%m.%d)" cron@ftp-master.debian.org +bzip2 -9 "$LOGFILE" + ################################################################################ diff --git a/config/debian/cron.weekly b/config/debian/cron.weekly index f7fa9c04..99d16359 100755 --- a/config/debian/cron.weekly +++ b/config/debian/cron.weekly @@ -7,22 +7,43 @@ set -u export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars . $SCRIPTVARS +# Start logging +NOW=`date "+%Y.%m.%d-%H:%M:%S"` +LOGFILE="$logdir/weekly_${NOW}.log" +exec > "$LOGFILE" 2>&1 + +cleanup() { + echo "Cleanup" + rm -f "$LOGFILE" +} +trap cleanup 0 + ################################################################################ # Purge empty directories +echo "Purging empty directories in $ftpdir/pool/" if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then find $ftpdir/pool/ -type d -empty | xargs rmdir; fi -# Clean up apt-ftparchive's databases - # Split queue/done +echo "Splitting queue/done" dak split-done > /dev/null +# Vacuum the database +echo "VACUUM; VACUUM ANALYZE;" | psql --no-psqlrc projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$" + +# Clean up apt-ftparchive's databases cd $configdir +echo "Cleanup apt-ftparchive's database" apt-ftparchive -q clean apt.conf apt-ftparchive -q clean apt.conf.buildd +echo "Finally, all is done, compressing logfile" +exec > /dev/null 2>&1 + +bzip2 -9 "$LOGFILE" + ################################################################################ diff --git a/config/debian/vars b/config/debian/vars index b88a83d2..3f993fad 100644 --- a/config/debian/vars +++ b/config/debian/vars @@ -14,6 +14,7 @@ dbdir=$base/database/ lockdir=$base/lock/ overridedir=$scriptdir/override extoverridedir=$scriptdir/external-overrides +logdir=$base/log/cron/ queuedir=$base/queue/ unchecked=$queuedir/unchecked/