ts() {
TS=$(($TS+1));
- echo Archive maintenance timestamp $TS: $(date +%H:%M:%S)
+ echo "Archive maintenance timestamp $TS ($1): $(date +%H:%M:%S)"
}
-echo Archive maintenance started at $(date +%H:%M:%S)
-TS=0
+TS=-1
+ts "startup"
NOTICE="$ftpdir/Archive_Maintenance_In_Progress"
LOCKCU="$lockdir/daily.lock"
# Push merkels qa user, so the qa pages can show "dinstall is running" information
echo "Telling merkels QA user that we start dinstall"
ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 || true
-ts
+ts "init"
################################################################################
echo "Creating pre-daily-cron-job backup of projectb database..."
pg_dump projectb > $base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
+ts "pg_dump1"
################################################################################
-ts
echo "Updating Bugs docu, Mirror list and mailing-lists.txt"
cd $configdir
$scriptsdir/update-bugdoctxt
$scriptsdir/update-mirrorlists
$scriptsdir/update-mailingliststxt
$scriptsdir/update-pseudopackages.sh
+ts "External Updates"
################################################################################
-ts
echo "Doing automated p-u-new processing"
cd $queuedir/p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT
+ts "p-u-new"
-ts
echo "Doing automated o-p-u-new processing"
cd $queuedir/o-p-u-new
date -u -R >> REPORT
dak process-new -a -C COMMENTS >> REPORT || true
echo >> REPORT
+ts "o-p-u-new"
################################################################################
-ts
echo "Synchronizing i18n package descriptions"
# First sync their newest data
echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR."
echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail debian-l10n-devel@lists.alioth.debian.org
fi
+ts "i18n 1"
################################################################################
-ts
lockfile $LOCKAC
lockac=1
echo "Processing queue/accepted"
mail -s "Install for $(date +%D)" ftpmaster@ftp-master.debian.org
chgrp debadmin REPORT
chmod 664 REPORT
+ts "accepted"
-ts
echo "Checking for cruft in overrides"
dak check-overrides
rm -f $LOCKAC
echo "Fixing symlinks in $ftpdir"
symlinks -d -r $ftpdir
-
-ts
+ts "cruft"
echo "Generating suite file lists for apt-ftparchive"
dak make-suite-file-list
-
-ts
+ts "make-suite-file-list"
echo "Updating fingerprints"
# Update fingerprints
dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg || true
-
-ts
+ts "import-keyring"
# Generate override files
echo "Writing overrides into text files"
cd $overridedir
dak make-overrides
-ts
-
# FIXME
rm -f override.sid.all3
for i in main contrib non-free main.debian-installer; do cat override.sid.$i >> override.sid.all3; done
+ts "overrides"
-ts
# Generate Packages and Sources files
echo "Generating Packages and Sources files"
cd $configdir
apt-ftparchive generate apt.conf
-
-ts
+ts "apt-ftparchive"
# Generate *.diff/ incremental updates
echo "Generating pdiff files"
dak generate-index-diffs
-
-ts
+ts "pdiff"
# Generate Release files
echo "Generating Release files"
dak generate-releases
-
-ts
+ts "release files"
# Clean out old packages
echo "Cleanup old packages/files"
dak clean-suites
dak clean-queues
-
-ts
+ts "cleanup"
# Needs to be rebuilt, as files have moved. Due to unaccepts, we need to
# update this before wanna-build is updated.
psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE suite = 5 AND queue = 0 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list
symlinks -d /srv/incoming.debian.org/buildd > /dev/null
apt-ftparchive generate apt.conf.buildd
-
-ts
+ts "buildd"
echo "Running various scripts from $scriptsdir"
cd $scriptsdir
./mklslar
./mkfilesindices
./mkchecksums
-
-ts
+ts "scripts"
# (Re)generate the hardlinked mirror directory for "public" buildd / mirror access
echo "Regenerating mirror/ hardlink fun"
cd ${mirrordir}
rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
-
-
-ts
+ts "mirror hardlinks"
echo "Trigger daily wanna-build run"
ssh -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
+ts "w-b"
rm -f $NOTICE
rm -f $LOCKCU
-echo Archive maintenance finished at $(date +%H:%M:%S)
+
+ts "locked part finished"
################################################################################
POSTDUMP=$base/backup/dump_$(date +%Y.%m.%d-%H:%M:%S)
pg_dump projectb > $POSTDUMP
(cd $base/backup; ln -sf $POSTDUMP current)
+ts "pg_dump2"
################################################################################
-ts
echo "Expiring old database dumps..."
(cd $base/backup; $scriptsdir/expire_dumps -d . -p -f "dump_*")
+ts "expire_dumps"
################################################################################
-ts
# Send a report on NEW/BYHAND packages
echo "Nagging ftpteam about NEW/BYHAND packages"
dak cruft-report > $webdir/cruft-report-daily.txt
dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org
-
-ts
+ts "reports"
echo "Updating DM html page"
$scriptsdir/dm-monitor >$webdir/dm-uploaders.html
# Push katie@merkel so it syncs the projectb there. Returns immediately, the sync runs detached
echo "Trigger merkels projectb sync"
ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb katie@merkel.debian.org sleep 1 || true
+ts "merkel projectb push"
################################################################################
-ts
ulimit -m 90000 -d 90000 -s 10000 -v 200000
echo "Using run-parts to run scripts in $base/scripts/distmnt"
run-parts --report $base/scripts/distmnt
-
-ts
+ts "run-parts"
echo "Exporting package data foo for i18n project"
STAMP=$(date "+%Y%m%d%H%M")
cd ${scriptdir}
find ./i18n -mtime +2 -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf
-
-ts
+ts "i18n 2"
echo "Daily cron scripts successful."
-ts
-
# Stats pr0n
echo "Updating stats data"
cd $configdir
$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data
R --slave --vanilla < $base/misc/ftpstats.R
-
-ts
+ts "stats"
# Remove the britney lock
rm -f ${BRITNEYLOCK}
echo "Clean up apt-ftparchive's databases"
cd $configdir
apt-ftparchive -q clean apt.conf
-
-ts
+ts "apt-ftparchive cleanup"
# Compress psql backups
echo "Compress old psql backups"
bzip2 -9 "$dumpname"
done
)
-
-ts
+ts "compress"
echo "Removing old dinstall logfiles"
(cd $logdir
rm -f "$dumpname"
done
)
+ts "logremove"
echo "Finally, all is done, sending mail and compressing logfile"
exec > /dev/null 2>&1
projects / dak / commitdiff