From 9e8a49aac094ee37db8abf38e46a42d7135437de Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Thu, 30 Apr 2009 00:39:08 +0200 Subject: [PATCH] unchecked/dinstall unchecked stuff now also in functions. call a few functions from within dinstall, right before we run accepted. Otherwise, with the new way of NEW, stuff from (o-)p-u-new would take two dinstall runs before it is from there into the archive proper... Signed-off-by: Joerg Jaspert --- config/debian/common | 45 +++++++++++++ config/debian/cron.dinstall | 17 +++++ config/debian/cron.unchecked | 121 +++++++++++++++++------------------ 3 files changed, 122 insertions(+), 61 deletions(-) diff --git a/config/debian/common b/config/debian/common index 3655d9d2..090f2ab7 100644 --- a/config/debian/common +++ b/config/debian/common @@ -42,3 +42,48 @@ function make_buildd_dir () { ln -sfT tree/${STAMP} ${incoming}/builddweb find ./tree -mindepth 1 -maxdepth 1 -not -name "${STAMP}" -type d -print0 | xargs --no-run-if-empty -0 rm -rf } + +# move accepted NEW packages from stagedir into queue/accepted +function acceptnew () { + cd $newstage + for file in $(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs); do + sed '1,/Files:/d' "${file}" | sed '/BEGIN PGP SIGNATURE/,$d' \ + | while read notused1 notused2 notused3 notused4 NAME; do + if [ -z "${NAME}" ]; then + # Sometimes there is a newline after the Files:, ignore it + continue + fi + if [ -f "${NAME}" ]; then + mv --target-directory="${accepted}" "${NAME}" + else + log_error "Error, couldn't find file ${NAME} to move to ${accepted}" + exit 2 + fi + done + mv --target-directory="${accepted}" "${file}" "${file%%.changes}.dak" + done +} + +# Do the unchecked processing, in case we have files. +function do_unchecked () { + cd $unchecked + + changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) + report=$queuedir/REPORT + timestamp=$(date "+%Y-%m-%d %H:%M") + + echo "$timestamp": "$changes" >> $report + dak process-unchecked -a -d "$unchecked" >> $report + echo "--" >> $report +} + +function sync_debbugs () { + # sync with debbugs + rsync -aq -e "ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30" --remove-source-files $queuedir/bts_version_track/ bugs-sync:/org/bugs.debian.org/versions/queue/ftp-master/ 2>/dev/null && touch $lockdir/synced_bts_version || true + NOW=$(date +%s) + TSTAMP=$(stat -c %Y $lockdir/synced_bts_version) + DIFF=$(( NOW - TSTAMP )) + if [ $DIFF -ge 259200 ]; then + log "Kids, you tried your best and you failed miserably. The lesson is, never try. (Homer Simpson)" + fi +} diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall index 332aa036..dfd9e916 100755 --- a/config/debian/cron.dinstall +++ b/config/debian/cron.dinstall @@ -400,6 +400,14 @@ function renamelogfile() { function testingsourcelist() { dak ls -s testing -f heidi -r .| egrep 'source$' > ${webdir}/testing.list } + +# do a last run of process-unchecked before dinstall is on. +function process_unchecked() { + acceptnew + do_unchecked + sync_debbugs +} + ######################################################################## ######################################################################## @@ -594,6 +602,15 @@ stage $GO lockfile "$LOCK_ACCEPTED" +GO=( + FUNC="process_unchecked" + TIME="" + ARGS="" + ERR="" +) +stage $GO + + GO=( FUNC="accepted" TIME="accepted" diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked index 0006f80e..77b7d88f 100755 --- a/config/debian/cron.unchecked +++ b/config/debian/cron.unchecked @@ -1,18 +1,43 @@ -#! /bin/bash +#!/bin/bash +# No way I try to deal with a crippled sh just for POSIX foo. +# Copyright (C) 2009 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# exit on errors set -e +# make sure to only use defined variables set -u + +# import the general variable set. export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars . $SCRIPTVARS -# common functions are "outsourced" -. "${configdir}/common" - LOCKDAILY="" LOCKFILE="$lockdir/unchecked.lock" NOTICE="$lockdir/daily.lock" -if [ -e $NOTICE ]; then exit 0; fi +if [ -e $NOTICE ]; then + exit 0; +fi + +######################################################################## +# Functions # +######################################################################## +# common functions are "outsourced" +. "${configdir}/common" STAMP=$(date "+%Y%m%d%H%M") @@ -23,66 +48,40 @@ cleanup() { fi } -# in case there is anything NEW processed, move it over into the accepted queue -cd $newstage -for file in $(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs); do - sed '1,/Files:/d' "${file}" | sed '/BEGIN PGP SIGNATURE/,$d' \ - | while read notused1 notused2 notused3 notused4 NAME; do - if [ -z "${NAME}" ]; then - # Sometimes there is a newline after the Files:, ignore it - continue - fi - if [ -f "${NAME}" ]; then - mv --target-directory="${accepted}" "${NAME}" - else - log_error "Error, couldn't find file ${NAME} to move to ${accepted}" - exit 2 - fi - done - mv --target-directory="${accepted}" "${file}" "${file%%.changes}.dak" -done - -# only run one cron.unchecked -if lockfile -r3 $LOCKFILE; then - trap cleanup 0 - cd $unchecked +function do_buildd () { + if lockfile -r3 $NOTICE; then + LOCKDAILY="YES" + psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE queue = 0 AND suite = 5 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list + cd $overridedir + dak make-overrides &>/dev/null + rm -f override.sid.all3 override.sid.all3.src + for i in main contrib non-free main.debian-installer; do + cat override.sid.$i >> override.sid.all3 + if [ "$i" != "main.debian-installer" ]; then + cat override.sid.$i.src >> override.sid.all3.src + fi + done + make_buildd_dir - changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs) - report=$queuedir/REPORT - timestamp=$(date "+%Y-%m-%d %H:%M") + . $configdir/cron.buildd + fi +} - if [ ! -z "$changes" ]; then - echo "$timestamp": "$changes" >> $report - dak process-unchecked -a -d "$unchecked" >> $report - echo "--" >> $report +######################################################################## +# the actual unchecked functions follow # +######################################################################## - # sync with debbugs - rsync -aq -e "ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30" --remove-source-files $queuedir/bts_version_track/ bugs-sync:/org/bugs.debian.org/versions/queue/ftp-master/ 2>/dev/null && touch $lockdir/synced_bts_version || true +acceptnew - NOW=$(date +%s) - TSTAMP=$(stat -c %Y $lockdir/synced_bts_version) - DIFF=$(( NOW - TSTAMP )) - if [ $DIFF -ge 259200 ]; then - echo "Kids, you tried your best and you failed miserably. The lesson is, never try. (Homer Simpson)" - fi +# only run one cron.unchecked +lockfile -r3 $LOCKFILE || exit 0 +trap cleanup 0 - if lockfile -r3 $NOTICE; then - LOCKDAILY="YES" - psql projectb -A -t -q -c "SELECT filename FROM queue_build WHERE queue = 0 AND suite = 5 AND in_queue = true AND filename ~ 'd(sc|eb)$'" > $dbdir/dists/unstable_accepted.list - cd $overridedir - dak make-overrides &>/dev/null - rm -f override.sid.all3 override.sid.all3.src - for i in main contrib non-free main.debian-installer; do - cat override.sid.$i >> override.sid.all3 - if [ "$i" != "main.debian-installer" ]; then - cat override.sid.$i.src >> override.sid.all3.src - fi - done - make_buildd_dir +do_unchecked - . $configdir/cron.buildd - fi - else - echo "$timestamp": Nothing to do >> $report - fi +if [ ! -z "$changes" ]; then + sync_debbugs + do_buildd +else + echo "$timestamp": Nothing to do >> $report fi -- 2.39.5