From: Joerg Jaspert <joerg@debian.org>
Date: Mon, 21 Mar 2011 09:39:05 +0000 (+0100)
Subject: database backup foo
X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b6cf40a404bec1867ad0e5298ce08857dabb16f9;p=dak

database backup foo

no longer dump the full database on each dinstall run. instead we just
save a transaction id. DSA is doing the backup dumps in whatever cycle,
and keeping the WAL stuff around, so we can tell them to get us to
whatever point in our db which we might need, ie. the specific transaction
id.

as those are just 9 bytes per file, and not 4gigabytes, we now also save
wth we have been before and after unchecked runs, which makes our backup
be nearer to actual filesystem state than ever.

Signed-off-by: Joerg Jaspert <joerg@debian.org>

Signed-off-by: Joerg Jaspert <joerg@debian.org>
---

diff --git a/config/debian/common b/config/debian/common
index f611c19e..f5ec8ae6 100644
--- a/config/debian/common
+++ b/config/debian/common
@@ -104,3 +104,9 @@ function reports() {
     dak cruft-report -s experimental >> $webdir/cruft-report-daily.txt
     cat $webdir/cruft-report-daily.txt | mail -a "X-Debian: DAK" -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org -- -F "Debian FTP Masters" -f ftpmaster@ftp-master.debian.org
 }
+
+function pg_timestamp() {
+    tsname=${1:-"unknown"}
+    log "Saving postgres transaction id for ${tsname}"
+    psql -tAc 'select txid_current();' > $base/backup/txid_${tsname}_$(date +%Y.%m.%d-%H:%M:%S)
+}
diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index 17064a19..40451c81 100755
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -180,9 +180,9 @@ GO=(
 stage $GO &
 
 GO=(
-    FUNC="pgdump_pre"
+    FUNC="pg_timestamp"
     TIME="pg_dump1"
-    ARGS=""
+    ARGS="predinstall"
     ERR=""
 )
 stage $GO
@@ -397,12 +397,12 @@ GO=(
 stage $GO &
 
 GO=(
-    FUNC="pgdump_post"
+    FUNC="pg_timestamp"
     TIME="pg_dump2"
-    ARGS=""
+    ARGS="postdinstall"
     ERR=""
 )
-stage $GO &
+stage $GO
 
 GO=(
     FUNC="expire"
@@ -487,12 +487,12 @@ GO=(
 stage $GO &
 
 GO=(
-    FUNC="compress"
-    TIME="compress"
+    FUNC="cleantransactions"
+    TIME=""
     ARGS=""
     ERR=""
 )
-stage $GO &
+stage $GO
 
 GO=(
     FUNC="aptftpcleanup"
diff --git a/config/debian/cron.unchecked b/config/debian/cron.unchecked
index 0d180a18..7b81e511 100755
--- a/config/debian/cron.unchecked
+++ b/config/debian/cron.unchecked
@@ -95,6 +95,8 @@ if ! lockfile -r8 $LOCKFILE; then
 fi
 trap cleanup 0
 
+
+pg_timestamp preunchecked
 do_newstage
 do_unchecked
 
@@ -105,3 +107,4 @@ if [ ! -z "$changes" ]; then
 fi
 
 dak contents -l 10000 scan
+pg_timestamp postunchecked
diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index 5f822104..afbfce16 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -502,17 +502,10 @@ function aptftpcleanup() {
     apt-ftparchive -q clean apt.conf
 }
 
-function compress() {
-    log "Compress old psql backups"
+function cleantransactions() {
+    log "Cleanup transaction ids older than 3 months"
     cd $base/backup/
-    find -maxdepth 1 -mindepth 1 -type f -name 'dump_pre_*' -mtime +2 -print0 | xargs -0 --no-run-if-empty rm
-
-    find -maxdepth 1 -mindepth 1 -type f -name 'dump_*' \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 -print0 \
-        | xargs --no-run-if-empty -0 -P4 -n1 xz -9v
-
-    find -maxdepth 1 -mindepth 1 -type f -name "dumpall_*" \! -name '*.bz2' \! -name '*.gz' \! -name '*.xz' -mmin +720 \
-        | xargs --no-run-if-empty -0 -P4 -n1 xz -9v
-    finddup -l -d $base/backup
+    find -maxdepth 1 -mindepth 1 -type f -name 'txid_*' -mtime +90 -print0 | xargs -0 --no-run-if-empty rm
 }
 
 function logstats() {
diff --git a/scripts/debian/expire_dumps b/scripts/debian/expire_dumps
index 2907ba28..b7b20655 100755
--- a/scripts/debian/expire_dumps
+++ b/scripts/debian/expire_dumps
@@ -59,7 +59,7 @@ def all_files(pattern, search_path, pathsep=os.pathsep):
 def parse_file_dates(list):
     out = []
     # dump_2006.05.02-11:52:01.bz2
-    p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.bz2)?$')
+    p = re.compile('^\./dump_([0-9]{4})\.([0-9]{2})\.([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})(.xz)?$')
     for file in list:
         m = p.search(file)
         if m: