From: Joerg Jaspert Date: Sun, 5 Sep 2010 15:08:20 +0000 (+0200) Subject: Update the cron scripts X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=44606e96f011e2d7ab245efb9cc3e1a80043fdc6;p=dak Update the cron scripts Signed-off-by: Joerg Jaspert --- diff --git a/config/backports/cron.buildd b/config/backports/cron.buildd deleted file mode 100755 index 36ea8a3f..00000000 --- a/config/backports/cron.buildd +++ /dev/null @@ -1,11 +0,0 @@ -#! /bin/bash -e - -# Executed hourly via cron, out of katie's crontab. -# stolen from newraff and adjusted by aba on 2005-04-30 -#exit 0 - -export SCRIPTVARS=/srv/backports-master.debian.org/config/backports/vars -. $SCRIPTVARS - -ssh -i $base/s3kr1t/dot-ssh/id_rsa wbadm@buildd.debian.org echo broken -exit 0 diff --git a/config/backports/cron.daily b/config/backports/cron.daily index eaffdf52..3a63a319 100755 --- a/config/backports/cron.daily +++ b/config/backports/cron.daily @@ -1,19 +1,22 @@ -#! /bin/sh +#! /bin/bash # -# Executed daily via cron, out of katie's crontab. set -e +set -u + export SCRIPTVARS=/srv/backports-master.debian.org/config/backports/vars . $SCRIPTVARS ################################################################################ # Clean out old packages -dak clean-suites +dak clean-suites -m 10000 dak clean-queues # Send a report on NEW/BYHAND packages dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@backports.debian.org # and one on crufty packages -dak cruft-report | tee $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@backports.debian.org + +dak cruft-report > $webdir/cruft-report-daily.txt +cat $webdir/cruft-report-daily.txt | mail -e -s "Debian backports archive cruft report for $(date +%D)" ftpmaster@backports.debian.org echo Daily cron scripts successful. diff --git a/config/backports/cron.dinstall b/config/backports/cron.dinstall index 0841f838..15e339f3 100755 --- a/config/backports/cron.dinstall +++ b/config/backports/cron.dinstall @@ -1,7 +1,7 @@ #!/bin/bash # No way I try to deal with a crippled sh just for POSIX foo. -# Copyright (C) 2009 Joerg Jaspert +# Copyright (C) 2009, 2010 Joerg Jaspert # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -35,7 +35,7 @@ set -u set -E # import the general variable set. -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars +export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars . $SCRIPTVARS ######################################################################## @@ -171,14 +171,6 @@ GO=( ) stage $GO -GO=( - FUNC="merkel1" - TIME="init" - ARGS="" - ERR="false" -) -stage $GO & - GO=( FUNC="pgdump_pre" TIME="pg_dump1" @@ -187,41 +179,9 @@ GO=( ) stage $GO -GO=( - FUNC="updates" - TIME="External Updates" - ARGS="" - ERR="false" -) -stage $GO - -GO=( - FUNC="i18n1" - TIME="i18n 1" - ARGS="" - ERR="false" -) -stage $GO - lockfile "$LOCK_ACCEPTED" lockfile "$LOCK_NEW" -GO=( - FUNC="punew" - TIME="p-u-new" - ARGS="proposedupdates" - ERR="false" -) -stage $GO - -#GO=( -# FUNC="opunew" -# TIME="o-p-u-new" -# ARGS="oldproposedupdates" -# ERR="false" -#) -#stage $GO - GO=( FUNC="newstage" TIME="newstage" @@ -256,13 +216,13 @@ GO=( ) stage $GO -GO=( - FUNC="fingerprints" - TIME="import-keyring" - ARGS="" - ERR="false" -) -stage $GO +# GO=( +# FUNC="fingerprints" +# TIME="import-keyring" +# ARGS="" +# ERR="false" +# ) +# stage $GO GO=( FUNC="overrides" @@ -357,14 +317,6 @@ GO=( ) stage $GO -GO=( - FUNC="mkfilesindices" - TIME="mkfilesindices" - ARGS="" - ERR="" -) -stage $GO - GO=( FUNC="mkchecksums" TIME="mkchecksums" @@ -404,14 +356,6 @@ GO=( ) stage $GO & -GO=( - FUNC="transitionsclean" - TIME="transitionsclean" - ARGS="" - ERR="" -) -stage $GO & - GO=( FUNC="reports" TIME="reports" @@ -420,29 +364,13 @@ GO=( ) stage $GO & -GO=( - FUNC="dm" - TIME="" - ARGS="" - ERR="" -) -stage $GO & - -GO=( - FUNC="bts" - TIME="" - ARGS="" - ERR="false" -) -stage $GO & - -GO=( - FUNC="merkel2" - TIME="merkel projectb push" - ARGS="" - ERR="false" -) -stage $GO & +# GO=( +# FUNC="dm" +# TIME="" +# ARGS="" +# ERR="" +# ) +# stage $GO & GO=( FUNC="mirrorpush" @@ -452,14 +380,6 @@ GO=( ) stage $GO & -GO=( - FUNC="i18n2" - TIME="i18n 2" - ARGS="" - ERR="false" -) -stage $GO & - GO=( FUNC="stats" TIME="stats" @@ -468,32 +388,8 @@ GO=( ) stage $GO & -GO=( - FUNC="testingsourcelist" - TIME="" - ARGS="" - ERR="false" -) -stage $GO & - rm -f "${LOCK_BRITNEY}" -GO=( - FUNC="pgdakdev" - TIME="dak-dev db" - ARGS="" - ERR="false" -) -stage $GO & - -GO=( - FUNC="merkel3" - TIME="merkel ddaccessible sync" - ARGS="" - ERR="false" -) -stage $GO & - GO=( FUNC="compress" TIME="compress" diff --git a/config/backports/cron.hourly b/config/backports/cron.hourly deleted file mode 100755 index 3f52806d..00000000 --- a/config/backports/cron.hourly +++ /dev/null @@ -1,119 +0,0 @@ -#! /bin/sh -# -# Executed daily via cron, out of katie's crontab. -set -e -export SCRIPTVARS=/srv/backports-master.debian.org/config/backports/vars -. $SCRIPTVARS - -################################################################################ -cd $accepted - -changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) - -if [ -z "$changes" ]; then - exit 0; -fi - -echo Archive maintenance started at $(date +%X) - -NOTICE="$ftpdir/Archive_Maintenance_In_Progress" -LOCKCU="$lockdir/daily.lock" -LOCKAC="$lockdir/unchecked.lock" - -cleanup() { - rm -f "$NOTICE" - rm -f "$LOCKCU" -} -trap cleanup 0 - -rm -f "$NOTICE" -lockfile -l 3600 $LOCKCU -cat > "$NOTICE" < $base/ftp/indices/package-file.map.bz2 - -# Clean out old packages -# Now in cron.daily. JJ[03.05.2005.] -#rhona -#shania - -cd $scriptsdir -./mkmaintainers -./copyoverrides -./mklslar -./mkchecksums - -rm -f $NOTICE -rm -f $LOCKCU -echo Archive maintenance finished at $(date +%X) - -################################################################################ - -echo "Creating post-hourly-cron-job backup of projectb database..." -POSTDUMP=/srv/backports-master.debian.org/backup/dump_$(date +%Y.%m.%d-%H:%M:%S) -pg_dump projectb > $POSTDUMP -(cd /srv/backports-master.debian.org/backup; ln -sf $POSTDUMP current) - -################################################################################ - -# Vacuum the database -echo "VACUUM; VACUUM ANALYZE;" | psql projectb 2>&1 | grep -v "^NOTICE: Skipping.*only table owner can VACUUM it$" - -################################################################################ - -# Now in cron.daily JJ[03.05.2005] -# Send a report on NEW/BYHAND packages -#helena | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@amd64.debian.net -# and one on crufty package -#rene | mail -e -s "rene run for $(date +%D)" ftpmaster@amd64.debian.net - -################################################################################ - -(cd /srv/backports-master.debian.org/stats; rm -f master.list; ./dmc.pl get >/dev/null 2>&1; \ -./mirror.pl>$ftpdir/README.mirrors.html; cd $ftpdir; /usr/bin/links -dump README.mirrors.html >README.mirrors.txt) - - -################################################################################ - -ulimit -m 90000 -d 90000 -s 10000 -v 90000 - -run-parts --report /srv/backports-master.debian.org/scripts/distmnt - -echo Daily cron scripts successful. diff --git a/config/backports/cron.monthly b/config/backports/cron.monthly index 76b13b14..52f7e80a 100755 --- a/config/backports/cron.monthly +++ b/config/backports/cron.monthly @@ -1,8 +1,9 @@ -#!/bin/sh +#!/bin/bash # -# Run at the beginning of the month via cron, out of katie's crontab. set -e +set -u + export SCRIPTVARS=/srv/backports-master.debian.org/config/backports/vars . $SCRIPTVARS @@ -10,24 +11,22 @@ export SCRIPTVARS=/srv/backports-master.debian.org/config/backports/vars DATE=`date -d yesterday +%y%m` -cd ${basedir}/mail/archive +cd ${base}/mail/archive for m in mail import; do if [ -f $m ]; then mv $m ${m}-$DATE sleep 20 gzip -9 ${m}-$DATE - chgrp debadmin ${m}-$DATE.gz + chgrp backports ${m}-$DATE.gz chmod 660 ${m}-$DATE.gz fi; done DATE=`date +%Y-%m` -cd ${basedir}/log +cd ${base}/log touch $DATE -rm current -ln -s $DATE current +ln -sf $DATE current chmod g+w $DATE -chown dak:debadmin $DATE +chown dak:backports $DATE -dak split-done ################################################################################ diff --git a/config/backports/cron.unchecked b/config/backports/cron.unchecked index 78eccd11..d5ada070 100755 --- a/config/backports/cron.unchecked +++ b/config/backports/cron.unchecked @@ -1,34 +1,104 @@ -#! /bin/sh +#! /bin/bash + +# Copyright (C) 2009 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# exit on errors set -e +# make sure to only use defined variables +set -u +# ERR traps should be inherited from functions too. (And command +# substitutions and subshells and whatnot, but for us the functions is +# the important part here) +set -E + export SCRIPTVARS=/srv/backports-master.debian.org/config/backports/vars . $SCRIPTVARS + +LOCKDAILY="" LOCKFILE="$lockdir/unchecked.lock" +LOCK_NEW="$lockdir/processnew.lock" NOTICE="$lockdir/daily.lock" +LOCK_BUILDD="$lockdir/buildd.lock" + +# our name +PROGRAM="unchecked" + +if [ -e $NOTICE ]; then + exit 0; +fi + +######################################################################## +# Functions # +######################################################################## +# common functions are "outsourced" +. "${configdir}/common" + +STAMP=$(date "+%Y%m%d%H%M") cleanup() { - rm -f "$LOCKFILE" - if [ ! -z $LOCKDAILY ]; then - rm -f "$NOTICE" - fi + rm -f "$LOCKFILE" + if [ ! -z "$LOCKDAILY" ]; then + rm -f "$NOTICE" + fi } -trap cleanup 0 + +function do_buildd () { + if lockfile -r3 $NOTICE; then + LOCKDAILY="YES" + cd $overridedir + dak make-overrides &>/dev/null + rm -f override.sid.all3 override.sid.all3.src + for i in main contrib non-free main.debian-installer; do + cat override.lenny-backports.$i >> override.sid.all3 + if [ "$i" != "main.debian-installer" ]; then + cat override.lenny-backports.$i.src >> override.sid.all3.src + fi + done + make_buildd_dir + wbtrigger + fi +} + +function do_dists () { + cd $configdir + dak generate-filelist + GZIP='--rsyncable' ; export GZIP + dak generate-packages-sources +} + +######################################################################## +# the actual unchecked functions follow # +######################################################################## + +# And use one locale, no matter what the caller has set +export LANG=C +export LC_ALL=C # only run one cron.unchecked -if lockfile -r3 $LOCKFILE; then - cd $unchecked - - changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) - report=$queuedir/REPORT - timestamp=$(date "+%Y-%m-%d %H:%M") - - if [ ! -z "$changes" ]; then - echo "$timestamp": "$changes" >> $report - dak process-unchecked -a $changes >> $report - echo "--" >> $report - else - echo "$timestamp": Nothing to do >> $report - fi -fi +lockfile -r3 $LOCKFILE || exit 0 +trap cleanup 0 -rm -f "$LOCKFILE" +do_newstage +do_unchecked + +if [ ! -z "$changes" ]; then + sync_debbugs + do_buildd +# echo "Starting g-p-s: $(date +%H:%M:%S)" +# do_dists +# echo "Done with g-p-s: $(date +%H:%M:%S)" +fi diff --git a/config/backports/cron.weekly b/config/backports/cron.weekly index 9a563aa5..c4a3a4b3 100755 --- a/config/backports/cron.weekly +++ b/config/backports/cron.weekly @@ -1,22 +1,51 @@ -#!/bin/sh +#!/bin/bash # -# Run once a week via cron, out of katie's crontab. set -e +set -u +# ERR traps should be inherited from functions too. (And command +# substitutions and subshells and whatnot, but for us the functions is +# the important part here) +set -E export SCRIPTVARS=/srv/backports-master.debian.org/config/backports/vars . $SCRIPTVARS +# Start logging +NOW=`date "+%Y.%m.%d-%H:%M:%S"` +LOGFILE="$logdir/weekly_${NOW}.log" +exec > "$LOGFILE" 2>&1 + +cleanup() { + echo "Cleanup" + rm -f "$LOGFILE" +} +trap cleanup 0 + ################################################################################ # Purge empty directories - +echo "Purging empty directories in $ftpdir/pool/" if [ ! -z "$(find $ftpdir/pool/ -type d -empty)" ]; then find $ftpdir/pool/ -type d -empty | xargs rmdir; fi # Clean up apt-ftparchive's databases +echo "Splitting queue/done" +dak split-done + +# Clean up apt-ftparchive's databases cd $configdir +echo "Cleanup apt-ftparchive's database" apt-ftparchive -q clean apt.conf +echo "Fixing symlinks in $ftpdir" +symlinks -d -r $ftpdir + +echo "Finally, all is done, compressing logfile" +exec > /dev/null 2>&1 + +bzip2 -9 "$LOGFILE" + + ################################################################################ diff --git a/config/backports/dak.conf b/config/backports/dak.conf index 9ac88286..f6af1441 100644 --- a/config/backports/dak.conf +++ b/config/backports/dak.conf @@ -96,6 +96,12 @@ Check-Overrides // OriginSuite "Unstable"; }; + squeeze-backports + { + Process "1"; +// OriginSuite "Unstable"; + }; + etch-backports { Process "1"; @@ -189,8 +195,8 @@ Suite non-free; }; - Announce "backports-changes@lists.backports.org"; - Origin "backports.debian.org archive"; + Announce "debian-backports-changes@lists.debian.org"; + Origin "Debian Backports"; Description "Backports for the Lenny Distribution"; CodeName "lenny-backports"; OverrideCodeName "lenny-backports"; @@ -198,6 +204,25 @@ Suite NotAutomatic "yes"; }; + + squeeze-backports + { + Components + { + main; + contrib; + non-free; + }; + + Announce "debian-backports-changes@lists.debian.org"; + Origin "Debian Backports"; + Description "Backports for the Squeeze Distribution"; + CodeName "squeeze-backports"; + OverrideCodeName "squeeze-backports"; + Priority "7"; + NotAutomatic "yes"; + }; + etch-backports { Components @@ -260,10 +285,8 @@ SuiteMappings { "map lenny lenny-backports"; "map lenny-bpo lenny-backports"; - "map etch etch-backports"; -// formi mag des nit -// "map stable etch-backports"; - "map etch-bpo etch-backports"; + "map squeeze squeeze-backports"; + "map squeeze-bpo squeeze-backports"; }; Architectures diff --git a/config/backports/dinstall.functions b/config/backports/dinstall.functions new file mode 100644 index 00000000..0ae81724 --- /dev/null +++ b/config/backports/dinstall.functions @@ -0,0 +1,606 @@ +# -*- mode:sh -*- +# Timestamp. Used for dinstall stat graphs +function ts() { + echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)" +} + +# Cleanup actions +function cleanup() { + rm -f ${LOCK_DAILY} + rm -f ${LOCK_ACCEPTED} +} + +# If we error out this one is called, *FOLLOWED* by cleanup above +function onerror() { + ERRDATE=$(date "+%Y.%m.%d-%H:%M:%S") + + subject="ATTENTION ATTENTION!" + if [ "${error}" = "false" ]; then + subject="${subject} (continued)" + else + subject="${subject} (interrupted)" + fi + subject="${subject} dinstall error at ${ERRDATE} in ${STAGEFILE} - (Be quiet, Brain, or I'll stab you with a Q-tip)" + + cat "${STAGEFILE}.log" | mail -s "${subject}" -a "X-Debian: DAK" cron@backports-master.debian.org +} + +######################################################################## +# the actual dinstall functions follow # +######################################################################## + +# pushing merkels QA user, part one +function merkel1() { + log "Telling merkels QA user that we start dinstall" + ssh -2 -i ~dak/.ssh/push_merkel_qa -o BatchMode=yes -o SetupTimeOut=90 -o ConnectTimeout=90 qa@merkel.debian.org sleep 1 +} + +# Create the postgres dump files +function pgdump_pre() { + log "Creating pre-daily-cron-job backup of $PGDATABASE database..." + pg_dump > $base/backup/dump_pre_$(date +%Y.%m.%d-%H:%M:%S) +} + +function pgdump_post() { + log "Creating post-daily-cron-job backup of $PGDATABASE database..." + cd $base/backup + POSTDUMP=$(date +%Y.%m.%d-%H:%M:%S) + pg_dump > $base/backup/dump_$POSTDUMP + #pg_dumpall --globals-only > $base/backup/dumpall_$POSTDUMP + ln -sf $base/backup/dump_$POSTDUMP current + #ln -sf $base/backup/dumpall_$POSTDUMP currentall +} + +# Load the dak-dev projectb +function pgdakdev() { + # Make sure to unset any possible psql variables so we don't drop the wrong + # f****** database by accident + local PGDATABASE + unset PGDATABASE + local PGHOST + unset PGHOST + local PGPORT + unset PGPORT + local PGUSER + unset PGUSER + cd $base/backup + echo "drop database projectb" | psql -p 5434 template1 + #cat currentall | psql -p 5433 template1 + createdb -p 5434 -T template1 projectb + fgrep -v '\connect' current | psql -p 5434 projectb +} + +# Updating various files +function updates() { + log "Updating Bugs docu, Mirror list and mailing-lists.txt" + cd $configdir + $scriptsdir/update-bugdoctxt + $scriptsdir/update-mirrorlists + $scriptsdir/update-mailingliststxt + $scriptsdir/update-pseudopackages.sh +} + +# Process (oldstable)-proposed-updates "NEW" queue +function punew_do() { + date -u -R >> REPORT + dak process-policy $1 | tee -a REPORT | mail -e -s "NEW changes in $1" debian-release@lists.debian.org + echo >> REPORT +} +function punew() { + log "Doing automated p-u-new processing" + cd "${queuedir}/p-u-new" + punew_do "$1" +} +function opunew() { + log "Doing automated o-p-u-new processing" + cd "${queuedir}/o-p-u-new" + punew_do "$1" +} + +# The first i18n one, syncing new descriptions +function i18n1() { + log "Synchronizing i18n package descriptions" + # First sync their newest data + cd ${scriptdir}/i18nsync + rsync -aq --delete --delete-after ddtp-sync:/does/not/matter . || true + + # Now check if we still know about the packages for which they created the files + # is the timestamp signed by us? + if $(gpgv --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg timestamp.gpg timestamp); then + # now read it. As its signed by us we are sure the content is what we expect, no need + # to do more here. And we only test -d a directory on it anyway. + TSTAMP=$(cat timestamp) + # do we have the dir still? + if [ -d ${scriptdir}/i18n/${TSTAMP} ]; then + # Lets check! + if ${scriptsdir}/ddtp-i18n-check.sh . ${scriptdir}/i18n/${TSTAMP}; then + # Yay, worked, lets copy around + for dir in squeeze sid; do + if [ -d dists/${dir}/ ]; then + cd dists/${dir}/main/i18n + rsync -aq --delete --delete-after . ${ftpdir}/dists/${dir}/main/i18n/. + fi + cd ${scriptdir}/i18nsync + done + else + echo "ARRRR, bad guys, wrong files, ARRR" + echo "Arf, Arf, Arf, bad guys, wrong files, arf, arf, arf" | mail -s "Don't you kids take anything. I'm watching you. I've got eye implants in the back of my head." debian-l10n-devel@lists.alioth.debian.org + fi + else + echo "ARRRR, missing the timestamp ${TSTAMP} directory, not updating i18n, ARRR" + echo "Arf, Arf, Arf, missing the timestamp ${TSTAMP} directory, not updating i18n, arf, arf, arf" | mail -s "Lisa, if you don't like your job you don't strike. You just go in every day and do it really half-assed. That's the American way." debian-l10n-devel@lists.alioth.debian.org + fi + else + echo "ARRRRRRR, could not verify our timestamp signature, ARRR. Don't mess with our files, i18n guys, ARRRRR." + echo "Arf, Arf, Arf, could not verify our timestamp signature, arf. Don't mess with our files, i18n guys, arf, arf, arf" | mail -s "You can't keep blaming yourself. Just blame yourself once, and move on." debian-l10n-devel@lists.alioth.debian.org + fi +} + +function cruft() { + log "Checking for cruft in overrides" + dak check-overrides +} + +function dominate() { + log "Removing obsolete source and binary associations" + dak dominate +} + +function filelist() { + log "Generating file lists for apt-ftparchive" + dak generate-filelist +} + +function fingerprints() { + log "Not updating fingerprints - scripts needs checking" + + log "Updating fingerprints" + dak import-keyring -L /srv/keyring.debian.org/keyrings/debian-keyring.gpg + + OUTFILE=$(mktemp) + dak import-keyring --generate-users "%s" /srv/keyring.debian.org/keyrings/debian-maintainers.gpg >"${OUTFILE}" + + if [ -s "${OUTFILE}" ]; then + /usr/sbin/sendmail -odq -oi -t -f envelope@ftp-master.debian.org < +To: +Subject: Debian Maintainers Keyring changes +Content-Type: text/plain; charset=utf-8 +MIME-Version: 1.0 + +The following changes to the debian-maintainers keyring have just been activated: + +$(cat $OUTFILE) + +Debian distribution maintenance software, +on behalf of the Keyring maintainers + +EOF + fi + rm -f "$OUTFILE" +} + +function overrides() { + log "Writing overrides into text files" + cd $overridedir + dak make-overrides + + # FIXME + rm -f override.sid.all3 + for i in main contrib non-free main.debian-installer; do cat override.lenny-backports.$i >> override.sid.all3; done +} + +function mpfm() { + log "Generating package / file mapping" + dak make-pkg-file-mapping | bzip2 -9 > $base/ftp/indices/package-file.map.bz2 +} + +function packages() { + log "Generating Packages and Sources files" + cd $configdir + GZIP='--rsyncable' ; export GZIP + apt-ftparchive generate apt.conf + #dak generate-packages-sources +} + +function pdiff() { + log "Generating pdiff files" + dak generate-index-diffs +} + +function release() { + log "Generating Release files" + dak generate-releases +} + +function dakcleanup() { + log "Cleanup old packages/files" + dak clean-suites -m 10000 + dak clean-queues +} + +function buildd_dir() { + # Rebuilt the buildd dir to avoid long times of 403 + log "Regenerating the buildd incoming dir" + STAMP=$(date "+%Y%m%d%H%M") + make_buildd_dir +} + +function mklslar() { + cd $ftpdir + + FILENAME=ls-lR + + log "Removing any core files ..." + find -type f -name core -print0 | xargs -0r rm -v + + log "Checking permissions on files in the FTP tree ..." + find -type f \( \! -perm -444 -o -perm +002 \) -ls + find -type d \( \! -perm -555 -o -perm +002 \) -ls + + log "Checking symlinks ..." + symlinks -rd . + + log "Creating recursive directory listing ... " + rm -f .${FILENAME}.new + TZ=UTC ls -lR > .${FILENAME}.new + + if [ -r ${FILENAME}.gz ] ; then + mv -f ${FILENAME}.gz ${FILENAME}.old.gz + mv -f .${FILENAME}.new ${FILENAME} + rm -f ${FILENAME}.patch.gz + zcat ${FILENAME}.old.gz | diff -u - ${FILENAME} | gzip --rsyncable -9cfn - >${FILENAME}.patch.gz + rm -f ${FILENAME}.old.gz + else + mv -f .${FILENAME}.new ${FILENAME} + fi + + gzip --rsyncable -9cfN ${FILENAME} >${FILENAME}.gz + rm -f ${FILENAME} +} + +function mkmaintainers() { + log 'Creating Maintainers index ... ' + + cd $indices + dak make-maintainers ${scriptdir}/masterfiles/pseudo-packages.maintainers | \ + sed -e "s/~[^ ]*\([ ]\)/\1/" | \ + awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-maintainers + + if ! cmp -s .new-maintainers Maintainers || [ ! -f Maintainers ]; then + log "installing Maintainers ... " + mv -f .new-maintainers Maintainers + gzip --rsyncable -9v .new-maintainers.gz + mv -f .new-maintainers.gz Maintainers.gz + else + rm -f .new-maintainers + fi +} + +function mkuploaders() { + log 'Creating Uploaders index ... ' + + cd $indices + dak make-maintainers -u ${scriptdir}/masterfiles/pseudo-packages.maintainers | \ + sed -e "s/~[^ ]*\([ ]\)/\1/" | \ + awk '{printf "%-20s ", $1; for (i=2; i<=NF; i++) printf "%s ", $i; printf "\n";}' > .new-uploaders + + if ! cmp -s .new-uploaders Uploaders || [ ! -f Uploaders ]; then + log "installing Uploaders ... " + mv -f .new-uploaders Uploaders + gzip --rsyncable -9v .new-uploaders.gz + mv -f .new-uploaders.gz Uploaders.gz + else + rm -f .new-uploaders + fi +} + +function copyoverrides() { + log 'Copying override files into public view ...' + + for ofile in $copyoverrides ; do + cd $overridedir + chmod g+w override.$ofile + + cd $indices + + newofile=override.$ofile.gz + rm -f .newover-$ofile.gz + pc="`gzip 2>&1 -9nv <$overridedir/override.$ofile >.newover-$ofile.gz`" + if ! cmp -s .newover-$ofile.gz $newofile || [ ! -f $newofile ]; then + log " installing new $newofile $pc" + mv -f .newover-$ofile.gz $newofile + chmod g+w $newofile + else + rm -f .newover-$ofile.gz + fi + done +} + +function mkfilesindices() { + umask 002 + cd $base/ftp/indices/files/components + + ARCHLIST=$(tempfile) + + log "Querying $PGDATABASE..." + echo 'SELECT l.path, f.filename, a.arch_string FROM location l JOIN files f ON (f.location = l.id) LEFT OUTER JOIN (binaries b JOIN architecture a ON (b.architecture = a.id)) ON (f.id = b.file)' | psql -At | sed 's/|//;s,^/srv/ftp-master.debian.org/ftp,.,' | sort >$ARCHLIST + + includedirs () { + perl -ne 'print; while (m,/[^/]+$,) { $_=$`; print $_ . "\n" unless $d{$_}++; }' + } + poolfirst () { + perl -e '@nonpool=(); while (<>) { if (m,^\./pool/,) { print; } else { push @nonpool, $_; } } print for (@nonpool);' + } + + log "Generating sources list" + ( + sed -n 's/|$//p' $ARCHLIST + cd $base/ftp + find ./dists -maxdepth 1 \! -type d + find ./dists \! -type d | grep "/source/" + ) | sort -u | gzip --rsyncable -9 > source.list.gz + + log "Generating arch lists" + + ARCHES=$( (<$ARCHLIST sed -n 's/^.*|//p'; echo amd64) | grep . | grep -v all | sort -u) + for a in $ARCHES; do + (sed -n "s/|$a$//p" $ARCHLIST + sed -n 's/|all$//p' $ARCHLIST + + cd $base/ftp + find ./dists -maxdepth 1 \! -type d + find ./dists \! -type d | grep -E "(proposed-updates.*_$a.changes$|/main/disks-$a/|/main/installer-$a/|/Contents-$a|/binary-$a/)" + ) | sort -u | gzip --rsyncable -9 > arch-$a.list.gz + done + + log "Generating suite lists" + + suite_list () { + printf 'SELECT DISTINCT l.path, f.filename FROM (SELECT sa.source AS source FROM src_associations sa WHERE sa.suite = %d UNION SELECT b.source AS source FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) WHERE ba.suite = %d) s JOIN dsc_files df ON (s.source = df.source) JOIN files f ON (df.file = f.id) JOIN location l ON (f.location = l.id)\n' $1 $1 | psql -F' ' -A -t + + printf 'SELECT l.path, f.filename FROM bin_associations ba JOIN binaries b ON (ba.bin = b.id) JOIN files f ON (b.file = f.id) JOIN location l ON (f.location = l.id) WHERE ba.suite = %d\n' $1 | psql -F' ' -A -t + } + + printf 'SELECT id, suite_name FROM suite\n' | psql -F' ' -At | + while read id suite; do + [ -e $base/ftp/dists/$suite ] || continue + ( + (cd $base/ftp + distname=$(cd dists; readlink $suite || echo $suite) + find ./dists/$distname \! -type d + for distdir in ./dists/*; do + [ "$(readlink $distdir)" != "$distname" ] || echo $distdir + done + ) + suite_list $id | tr -d ' ' | sed 's,^/srv/ftp-master.debian.org/ftp,.,' + ) | sort -u | gzip --rsyncable -9 > suite-${suite}.list.gz + done + + log "Finding everything on the ftp site to generate sundries" + (cd $base/ftp; find . \! -type d \! -name 'Archive_Maintenance_In_Progress' | sort) >$ARCHLIST + + rm -f sundries.list + zcat *.list.gz | cat - *.list | sort -u | + diff - $ARCHLIST | sed -n 's/^> //p' > sundries.list + + log "Generating files list" + + for a in $ARCHES; do + (echo ./project/trace; zcat arch-$a.list.gz source.list.gz) | + cat - sundries.list dists.list project.list docs.list indices.list | + sort -u | poolfirst > ../arch-$a.files + done + + (cd $base/ftp/ + for dist in sid squeeze; do + find ./dists/$dist/main/i18n/ \! -type d | sort -u | gzip --rsyncable -9 > $base/ftp/indices/files/components/translation-$dist.list.gz + done + ) + + (cat ../arch-i386.files ../arch-amd64.files; zcat suite-proposed-updates.list.gz ; zcat translation-sid.list.gz ; zcat translation-squeeze.list.gz) | + sort -u | poolfirst > ../typical.files + + rm -f $ARCHLIST + log "Done!" +} + +function mkchecksums() { + dsynclist=$dbdir/dsync.list + md5list=$indices/md5sums + + log -n "Creating md5 / dsync index file ... " + + cd "$ftpdir" + ${bindir}/dsync-flist -q generate $dsynclist --exclude $dsynclist --md5 + ${bindir}/dsync-flist -q md5sums $dsynclist | gzip -9n --rsyncable > ${md5list}.gz + ${bindir}/dsync-flist -q link-dups $dsynclist || true +} + +function mirror() { + log "Regenerating \"public\" mirror/ hardlink fun" + DATE_SERIAL=$(date +"%Y%m%d01") + FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} ) + if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then + SERIAL="$DATE_SERIAL" + else + SERIAL="$FILESOAPLUS1" + fi + date -u > ${TRACEFILE} + echo "Using dak v1" >> ${TRACEFILE} + echo "Running on host: $(hostname -f)" >> ${TRACEFILE} + echo "Archive serial: ${SERIAL}" >> ${TRACEFILE} + cd ${mirrordir} + rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. . +} + +function expire() { + log "Expiring old database dumps..." + cd $base/backup + $scriptsdir/expire_dumps -d . -p -f "dump_*" +} + +function transitionsclean() { + log "Removing out of date transitions..." + cd $base + dak transitions -c -a +} + +function reports() { + # Send a report on NEW/BYHAND packages + log "Nagging ftpteam about NEW/BYHAND packages" + dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@backports-master.debian.org + # and one on crufty packages + log "Sending information about crufty packages" + dak cruft-report > $webdir/cruft-report-daily.txt +# dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt + cat $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@backports-master.debian.org +} + +function dm() { + log "Updating DM html page" + $scriptsdir/dm-monitor >$webdir/dm-uploaders.html +} + +function bts() { + log "Categorizing uncategorized bugs filed against ftp.debian.org" + dak bts-categorize +} + +function merkel2() { + # Push dak@merkel so it syncs the projectb there. Returns immediately, the sync runs detached + log "Trigger merkel/flotows $PGDATABASE sync" + ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_projectb dak@merkel.debian.org sleep 1 + # Also trigger flotow, the ftpmaster test box + ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_flotow_projectb dak@flotow.debconf.org sleep 1 +} + +function merkel3() { + # Push dak@merkel to tell it to sync the dd accessible parts. Returns immediately, the sync runs detached + log "Trigger merkels dd accessible parts sync" + ssh -2 -o BatchMode=yes -o SetupTimeOut=30 -o ConnectTimeout=30 -i ~/.ssh/push_merkel_ddaccess dak@merkel.debian.org sleep 1 +} + +function mirrorpush() { + log "Starting the mirrorpush" + date -u > /srv/backports-web.debian.org/underlay/mirrorstart + echo "Using dak v1" >> /srv/backports-web.debian.org/underlay/mirrorstart + echo "Running on host $(hostname -f)" >> /srv/backports-web.debian.org/underlay/mirrorstart + sudo -H -u archvsync /home/archvsync/runmirrors > ~dak/runmirrors.log 2>&1 & +} + +function i18n2() { + log "Exporting package data foo for i18n project" + STAMP=$(date "+%Y%m%d%H%M") + mkdir -p ${scriptdir}/i18n/${STAMP} + cd ${scriptdir}/i18n/${STAMP} + dak control-suite -l stable > lenny + dak control-suite -l testing > squeeze + dak control-suite -l unstable > sid + echo "${STAMP}" > timestamp + gpg --secret-keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/secring.gpg --keyring /srv/ftp-master.debian.org/s3kr1t/dot-gnupg/pubring.gpg --no-options --batch --no-tty --armour --default-key 55BE302B --detach-sign -o timestamp.gpg timestamp + rm -f md5sum + md5sum * > md5sum + cd ${webdir}/ + ln -sfT ${scriptdir}/i18n/${STAMP} i18n + + cd ${scriptdir} + find ./i18n -mindepth 1 -maxdepth 1 -mtime +2 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf +} + +function stats() { + log "Updating stats data" + cd $configdir + #$scriptsdir/update-ftpstats $base/log/* > $base/misc/ftpstats.data + #R --slave --vanilla < $base/misc/ftpstats.R + dak stats arch-space > $webdir/arch-space + dak stats pkg-nums > $webdir/pkg-nums +} + +function aptftpcleanup() { + log "Clean up apt-ftparchive's databases" + cd $configdir + apt-ftparchive -q clean apt.conf +} + +function compress() { + log "Compress old psql backups" + cd $base/backup/ + find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm + + find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' -mmin +720 | + while read dumpname; do + echo "Compressing $dumpname" + bzip2 -9fv "$dumpname" + done + find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' -mmin +720 | + while read dumpname; do + echo "Compressing $dumpname" + bzip2 -9fv "$dumpname" + done + finddup -l -d $base/backup +} + +function logstats() { + $masterdir/tools/logs.py "$1" +} + +# save timestamp when we start +function savetimestamp() { + NOW=`date "+%Y.%m.%d-%H:%M:%S"` + echo ${NOW} > "${dbdir}/dinstallstart" +} + +function maillogfile() { + cat "$LOGFILE" | mail -s "Log for dinstall run of ${NOW}" cron@ftp-master.debian.org +} + +function renamelogfile() { + if [ -f "${dbdir}/dinstallstart" ]; then + NOW=$(cat "${dbdir}/dinstallstart") +# maillogfile + mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" +# logstats "$logdir/dinstall_${NOW}.log" + bzip2 -9 "$logdir/dinstall_${NOW}.log" + else + error "Problem, I don't know when dinstall started, unable to do log statistics." + NOW=`date "+%Y.%m.%d-%H:%M:%S"` +# maillogfile + mv "$LOGFILE" "$logdir/dinstall_${NOW}.log" + bzip2 -9 "$logdir/dinstall_${NOW}.log" + fi +} + +function testingsourcelist() { + dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list +} + +# do a last run of process-unchecked before dinstall is on. +function process_unchecked() { + log "Processing the unchecked queue" + UNCHECKED_WITHOUT_LOCK="-p" + do_unchecked + sync_debbugs +} + +# do a run of newstage only before dinstall is on. +function newstage() { + log "Processing the newstage queue" + UNCHECKED_WITHOUT_LOCK="-p" + do_newstage +} + +# Function to update a "statefile" telling people what we are doing +# (more or less). +# +# This should be called with the argument(s) +# - Status name we want to show. +# +function state() { + RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")" + cat >"${DINSTALLSTATE}" <