From 01c3e3976cf234ca463ded32f09ea238251b0155 Mon Sep 17 00:00:00 2001 From: Ansgar Burchardt Date: Sun, 7 Oct 2012 15:10:29 +0200 Subject: [PATCH] push mirror on ries instead of having ries pull from franck --- config/debian/cron.hourly | 2 +- config/debian/dinstall.functions | 2 +- config/homedir/ssh/ftpmaster-authorized_keys | 9 -- config/homedir/ssh/ftpmaster-config | 9 +- config/homedir/syncdd.sh | 127 ------------------- scripts/debian/sync-dd | 109 ++++++++++++++++ 6 files changed, 118 insertions(+), 140 deletions(-) delete mode 100755 config/homedir/syncdd.sh create mode 100755 scripts/debian/sync-dd diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index e1036550..cbc687fb 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -42,7 +42,7 @@ $base/dak/tools/removals.pl $configdir/removalsrss.rc > $webdir/rss/removals.rss # Tell ries to sync its tree -ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org sync +${scriptsdir}/sync-dd ries-sync ries-sync1 ries-sync2 sync $scriptsdir/generate-di diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions index 69a725c3..c32733cf 100644 --- a/config/debian/dinstall.functions +++ b/config/debian/dinstall.functions @@ -414,7 +414,7 @@ function bts() { function ddaccess() { # Tell our dd accessible mirror to sync itself up. Including ftp dir. log "Trigger dd accessible parts sync including ftp dir" - ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30 -2 -i ${base}/s3kr1t/pushddmirror dak@ries.debian.org pool + ${scriptsdir}/sync-dd ries-sync ries-sync1 ries-sync2 pool } function mirrorpush() { diff --git a/config/homedir/ssh/ftpmaster-authorized_keys b/config/homedir/ssh/ftpmaster-authorized_keys index c04539f7..b3c653fc 100644 --- a/config/homedir/ssh/ftpmaster-authorized_keys +++ b/config/homedir/ssh/ftpmaster-authorized_keys @@ -1,12 +1,3 @@ -# projectb to flotow -command="rsync --server --sender -vlogDtprz --delete . /srv/ftp-master.debian.org/backup/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="67.192.254.200" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAn4p26q4kcoqfHjbR4CXMOOppS9zR3RtCAroL1fVeWvE8U7CVowC1MP/0pq8UAaTfdflVfIYpKbl0xMxBspyxG/fOa90TaPDT9JJrZbkQj0tMfTWFVCMg5mScT0T9wPnTkXvANU28QwDSfudvwokqA0aF2jIsBZakqtULmx6r3BED02iBNZQbbc2Sf/MvfHnpgz7yGfU/NCZzdQU0/mTbL1DqVSgbmebt6MvRfYhnxm/Tw+gfLTpG0PTKTDU5NnJBG5tPgHC2vf2jqHDn1cMu9siNjPB52sG/n+KO3Deq3dXMKMjt+9VxXC2gfND6RVnZRCfwm9QByMw5eqVejEW7iw== dak@flotow - projectb 2008-11-29 -# non-s3kr1t stuff to flotow -command="rsync --server --sender -vlogDtprz --delete . /srv/ftp-master.debian.org/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="67.192.254.200" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1viRQWCL/LNE+7Kfc/Ao+COZM4x/7TzpXtVTRgzBtaPUf6xPiec3ieYHriLNpxblNRlRrzgSu978jmNAs9yWnmaG/QvV9CMTGMyt3ZC+z7HFX6YwSz+hJOMl55myVNsWbwOCfnTmem5YFG8yJZcTREWtMW31GfSfiv64p4ths5fJCyNBGh6E3TDg1Z9PafshETXogZjn7Ff+OXvGPo/oDW+0gEGzaNK1gvIdJNkrDAzb3UGmIZ4qcKVMtJ/Oc+R0G3NBDJUlxe48ocuzu3YUernTiZgvGAmE0vNlLAeJaXvh4YRV1mxirNvPtmKX+HZfJbrq9Pmsawdt/Yl2yR9K8w== ftpmaster test machine sync - -# syncing ries -command="rsync --server --sender -vlHogDtpre.Lsf --timeout=3600 . /srv/ftp-master.debian.org/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.103" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPmVHveaikQYpiufc+fkZgD+r3HJ8wCr9MCwk4kkpA5VZ5cO89+yYupZcKtyrsz34W2IB5igo/RbXrEaonAKQTVbaJPE4RAWpLlpXx6PqSsbGd1QonXXXbk3HQSjesqOEiw7KLU/m1grk5Ad9xdhY5mA2dzgZBD76JzUo8FISO4Mb4CGcWxj2n+lw0mhOftXP5WSRt28F7UFTbY+ogC8RgvXAudTC5zhZm4APcqob+kgVjneMy2xJKF+1KOW+bAtEKlKe+yMDU1zDC3etgzKYR70oiOpKIkjjYCWE17lFiVEOlXwW3rzg6U2oZF8U3NE8sTcCe7XzpCOI+bVSLm5jL ftpmaster dd access sync ries 2011-03-21 -command="rsync --server --sender -vlHogDtpre.Lsf --timeout=3600 . /srv/ftp.debian.org/",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.103" ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF612GP1J9UZVH0G9EnLTgTfzmlNYvaCdYt9OUqzMnqFZhNjfZpaAM7m2n+f0j55j8ktS6jgmC8bpKXdwFsWjZVi4AJ3toHyfvFtOv0ec0j1p+5RqqxdvsbhAYDwGGUk0Wldc8d6g/uIy//gKuqyWuo5tOmuxXIYpG1SR7MzQNdgLRhcJK/ZsR302geQ4kbjyk8DnbVZUhWxQFELZ4cKLFETxXitr45TiUGon006MLmxWribwZVwYl7ZcTJlefK7Z4VOA39YEgacFUt9LtRmV4dipPli/I6z4DTrjaMPH8VgkMCZtSM/igoXod3/ExS2yrzZHJ/NDMLiR6hha7GuiV ftpmaster dd access sync public ries 2011-03-21 - # whenever we have to read a new dataset for testing/squeeze-updates this runs command="/srv/ftp-master.debian.org/dak/scripts/debian/import_dataset.sh testing",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.3,franck.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAqvcRf4LLH9WLz3YGg/vj62I6aMihd9eF8tEYIMvRUNIqcI95YQP6nPpnIovom30RI9l5vJP+xpd0ABoiVxGDr0fw4hfp137BxpOL2WDHoqYX0KWP5mdWpA2PV2HVOJ4xp0q18pZ0DIdhxAGDd1QRrkR2yD9CH4dhRNcYRN8TA970y5Tweesh19Ba583f25NrSv0+A1200qiSdMbn9KIQYwC0Gc9xcKS1/Tygf2Sz3ekVrODog/nACPLbHRxO+mPcHJVBb9Sf8l393l5eln7ZfmSD0wZD6X/2M9+rRoXtVycLbmISxJV8zdady/3HCX33fcWCI7xCfOsikcVWDzygtQ== release@ries command="/srv/ftp-master.debian.org/dak/scripts/debian/import_dataset.sh squeeze-updates",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="128.148.34.3,franck.debian.org" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAs0CFETy4E2rR7sH5kl5tgPVltcimtdmkpWSYLO+AJrrTvN447KjL0GhAc9raWv/wp6UeGw9zhOOxH6UGGD2DKI+lIZKW2PraLnQMs9g67B7Q/7MH7rHIzKue1niOANgPZppQ18rdiexagWyj+E8z/A1cFqpfaIIupi543eXZ4yZV3fjrHIE6zTvIzoTzlAZ5IaCOYyFT8wx6Ql53aEZfMk6S1FvXou24wFBD08CArTjRMf2eYo/aPqWbJs955eZwNqp1kS4jtJKwc7DCKpY7elHCyIqfR7YZxTUOBEGpoaAIfjIitgEedZnuMmBl8IUi1jQ0HvM7HDb4n4NVR/hbew== release@franck diff --git a/config/homedir/ssh/ftpmaster-config b/config/homedir/ssh/ftpmaster-config index 045ae29a..a4d5d18c 100644 --- a/config/homedir/ssh/ftpmaster-config +++ b/config/homedir/ssh/ftpmaster-config @@ -22,10 +22,15 @@ Host morgue-sync User dak IdentityFile /srv/ftp-master.debian.org/s3kr1t/push_morgue -Host ries-sync +Host ries-sync1 Hostname ries.debian.org User dak - IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/id_franck + IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/push_dd1 + +Host ries-sync2 + Hostname ries.debian.org + User dak + IdentityFile /srv/ftp-master.debian.org/scripts/s3kr1t/ssh/push_dd2 Host external-security Hostname chopin.debian.org diff --git a/config/homedir/syncdd.sh b/config/homedir/syncdd.sh deleted file mode 100755 index 07c4bc51..00000000 --- a/config/homedir/syncdd.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -# Copyright (C) 2011 Joerg Jaspert -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; version 2. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - - -set -e -set -u -set -E - -export LANG=C -export LC_ALL=C - -export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars -. $SCRIPTVARS - -EXTRA="" - -check_commandline() { - while [ $# -gt 0 ]; do - case "$1" in - sync) - EXTRA="--exclude ftp/" - ;; - pool) - ;; - *) - echo "Unknown option ${1} ignored" - ;; - esac - shift # Check next set of parameters. - done -} - -if [ $# -gt 0 ]; then - ORIGINAL_COMMAND=$* -else - ORIGINAL_COMMAND="" -fi - -SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} -if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then - set "nothing" "${SSH_ORIGINAL_COMMAND}" - shift - check_commandline $* -fi - -if [ -n "${ORIGINAL_COMMAND}" ]; then - set ${ORIGINAL_COMMAND} - check_commandline $* -fi - - -cleanup() { - rm -f "${HOME}/sync.lock" -} -trap cleanup EXIT TERM HUP INT QUIT - -# not using $lockdir as thats inside the rsync dir, and --delete would -# kick the lock away. Yes we could exclude it, but wth bother? -# -# Also, NEVER use --delete-excluded! -if lockfile -r3 ${HOME}/sync.lock; then - cd $base/ - rsync -aH -B8192 \ - --exclude backup/*.xz \ - --exclude backup/dump* \ - --exclude database/\*.db \ - ${EXTRA} \ - --exclude mirror \ - --exclude morgue/ \ - --exclude=lost+found/ \ - --exclude .da-backup.trace \ - --exclude lock/ \ - --exclude queue/holding/ \ - --exclude queue/newstage/ \ - --exclude queue/unchecked/ \ - --exclude tmp/ \ - --delete \ - --delete-after \ - --timeout 3600 \ - -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ - ftpmaster-sync:/srv/ftp-master.debian.org/ . - - cd $public/ - rsync -aH -B8192 \ - --exclude mirror \ - --exclude rsync/ \ - --exclude=lost+found/ \ - --exclude .da-backup.trace \ - --exclude web-users/ \ - --delete \ - --delete-after \ - --timeout 3600 \ - -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ - ftpmaster-sync2:/srv/ftp.debian.org/ . - -else - echo "Couldn't get the lock, not syncing" - exit 0 -fi - - -## ftpmaster-sync is defined in .ssh/config as: -# Host ftpmaster-sync -# Hostname franck.debian.org -# User dak -# IdentityFile ~dak/.ssh/syncftpmaster -# ForwardX11 no -# ForwardAgent no -# StrictHostKeyChecking yes -# PasswordAuthentication no -# BatchMode yes - -## ftpmaster-sync2 is the same, just a second ssh key diff --git a/scripts/debian/sync-dd b/scripts/debian/sync-dd new file mode 100755 index 00000000..5554623f --- /dev/null +++ b/scripts/debian/sync-dd @@ -0,0 +1,109 @@ +#! /bin/bash + +# Copyright (C) 2011, Joerg Jaspert +# Copyright (C) 2012, Ansgar Burchardt +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +set -e +set -u +set -E + +export LANG=C +export LC_ALL=C + +export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars +. $SCRIPTVARS + +usage() { + echo "usage: $0 sync|pool" + echo + echo "sync dd-accessible copy of the archive" + echo + echo "arguments:" + echo " lock: file used for locking" + echo " host1: hostname for syncing /srv/ftp-master.debian.org" + echo " host2: hostname for syncing /srv/ftp.debian.org" + echo " sync|pool: sync excludes ftp/, pool syncs ftp/ too" + exit ${1:-0} +} + +if [ $# -ne 4 ]; then + usage 1 +fi + +lockfile="${lockdir}/${1}" +host1="${2}" +host2="${3}" +mode="${4}" + +# extra options for rsync of /srv/ftp-master.debian.org +extra1="" + +case "${mode}" in + sync) + extra1="${extra1} --exclude /ftp/" + ;; + pool) + ;; + *) + echo "Unknown mode ${mode}." >&2 + exit 1 + ;; +esac + +cleanup() { + rm -f "${lockfile}" +} +trap cleanup EXIT TERM HUP INT QUIT + +# Also, NEVER use --delete-excluded! +if lockfile -r3 "${lockfile}"; then + rsync -aH -B8192 + ${extra1} + --exclude "/backup/*.xz" \ + --exclude "/backup/dump*" \ + --exclude "/database/*.db" \ + --exclude ".da-backup.trace" \ + --exclude "lost+found" \ + --exclude "/lock/" \ + --exclude "/mirror/" \ + --exclude "/morgue/" \ + --exclude "/queue/unchecked/" \ + --exclude "/s3kr1t" \ + --exclude "/scripts/s3kr1t" \ + --exclude "/tmp/" \ + --delete --delete-after \ + --timeout 3600 \ + -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ + /srv/ftp-master.debian.org/ "${host1}:/srv/ftp-master.debian.org/" + # command for the remote side: + # command="rsync --server -lHogDtpre.iLsf -B8192 --timeout=3600 --delete-after . /srv/ftp-master.debian.org/" + + rsync -avH -B8192 \ + --exclude mirror \ + --exclude rsync/ \ + --exclude lost+found \ + --exclude .da-backup.trace \ + --exclude web-users/ \ + --delete --delete-after \ + --timeout 3600 \ + -e 'ssh -o ConnectTimeout=30 -o SetupTimeout=30' \ + /srv/ftp.debian.org/ "${host2}:/srv/ftp.debian.org/" + # command for the remote side: + # command="rsync --server -lHogDtpre.iLsf -B8192 --timeout=3600 --delete-after . /srv/ftp.debian.org/" +else + echo "Couldn't get the lock, not syncing" + exit 0 +fi -- 2.39.5