database backup foo

no longer dump the full database on each dinstall run. instead we just
save a transaction id. DSA is doing the backup dumps in whatever cycle,
and keeping the WAL stuff around, so we can tell them to get us to
whatever point in our db which we might need, ie. the specific transaction
id.

as those are just 9 bytes per file, and not 4gigabytes, we now also save
wth we have been before and after unchecked runs, which makes our backup
be nearer to actual filesystem state than ever.

Signed-off-by: Joerg Jaspert <joerg@debian.org>
Signed-off-by: Joerg Jaspert <joerg@debian.org>

diff --git a/config/debian/common b/config/debian/common
index f611c19e55b22dffe4ee5f60cdeee3805819d889..f5ec8ae630ae8a42696c7e5d158c5db2c2607ac0 100644 (file)
--- a/config/debian/common
+++ b/config/debian/common
@@ -104,3 +104,9 @@ function reports() {
     dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
     cat $webdir/cruft-report-daily.txt | mail -a "X-Debian: DAK" -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
 }
+
+function pg_timestamp() {
+    tsname=${1:-"unknown"}
+    log "Saving postgres transaction id for ${tsname}"
+    psql -tAc 'select txid_current();' > $base/backup/txid_${tsname}_$(date +%Y.%m.%d-%H:%M:%S)
+}

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index 17064a19095d72c4ac59318de33cc1e4b40bb431..40451c8138484b5e6db55b9848909f4753e3645e 100755 (executable)
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -180,9 +180,9 @@ GO=(
 stage $GO &
 
 GO=(
-    FUNC="pgdump_pre"
+    FUNC="pg_timestamp"
     TIME="pg_dump1"
-    ARGS=""
+    ARGS="predinstall"
     ERR=""
 )
 stage $GO
@@ -397,12 +397,12 @@ GO=(
 stage $GO &
 
 GO=(
-    FUNC="pgdump_post"
+    FUNC="pg_timestamp"
     TIME="pg_dump2"
-    ARGS=""
+    ARGS="postdinstall"
     ERR=""
 )
-stage $GO &
+stage $GO
 
 GO=(
     FUNC="expire"
@@ -487,12 +487,12 @@ GO=(
 stage $GO &
 
 GO=(
-    FUNC="compress"
-    TIME="compress"
+    FUNC="cleantransactions"
+    TIME=""
     ARGS=""
     ERR=""
 )
-stage $GO &
+stage $GO
 
 GO=(
     FUNC="aptftpcleanup"

diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked
index 0d180a184b366c52bff8ebf6cbea755a0718b36a..7b81e511e2878843ae34ef733ace8ca24463b4b9 100755 (executable)
--- a/config/debian/cron.unchecked
+++ b/config/debian/cron.unchecked
@@ -95,6 +95,8 @@ if ! lockfile -r8 $LOCKFILE; then
 fi
 trap cleanup 0
 
+
+pg_timestamp preunchecked
 do_newstage
 do_unchecked
 
@@ -105,3 +107,4 @@ if [ ! -z "$changes" ]; then
 fi
 
 dak contents -l 10000 scan
+pg_timestamp postunchecked

diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index 5f8221044c2392687574aaf24216cb01c03eeaeb..afbfce167be939db564201cdccd94fdffa0b571f 100644 (file)
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -502,17 +502,10 @@ function aptftpcleanup() {
     apt-ftparchive -q clean apt.conf
 }
 
-function compress() {
-    log "Compress old psql backups"
+function cleantransactions() {
+    log "Cleanup transaction ids older than 3 months"
     cd $base/backup/
-    find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
-
-    find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 -print0 \
-        | xargs --no-run-if-empty -0 -P4 -n1 xz -9v
-
-    find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 \
-        | xargs --no-run-if-empty -0 -P4 -n1 xz -9v
-    finddup -l -d $base/backup
+    find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
 }
 
 function logstats() {

diff --git a/scripts/debian/expire_dumps b/scripts/debian/expire_dumps
index 2907ba28c55737bbc82eff789bac5c5978862ae6..b7b20655da25baa7d07798163987fb6ba8ba709c 100755 (executable)
--- a/scripts/debian/expire_dumps
+++ b/scripts/debian/expire_dumps
@@ -59,7 +59,7 @@ def all_files(pattern, search_path, pathsep=os.pathsep):
 def parse_file_dates(list):
     out = []
     # dump_2006.05.02-11:52:01.bz2
-    p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$')
+    p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.xz)?$')
     for file in list:
         m = p.search(file)
         if m: