From 8ed0b27e8295d5666adcf3b5a68264ededfd5f2f Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 16 Dec 2008 22:33:09 +0100 Subject: [PATCH] logs.py Added script from thomas Signed-off-by: Joerg Jaspert --- ChangeLog | 4 ++ tools/logs.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100755 tools/logs.py diff --git a/ChangeLog b/ChangeLog index c14f7c53..390c1fc3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2008-12-16 Joerg Jaspert + + * tools/logs.py: Added script from Thomas. + 2008-12-15 Joerg Jaspert * config/debian/cron.hourly: Call the removals.pl diff --git a/tools/logs.py b/tools/logs.py new file mode 100755 index 00000000..336a68af --- /dev/null +++ b/tools/logs.py @@ -0,0 +1,126 @@ +#!/usr/bin/python +# (c) 2008 Thomas Viehmann +# Free software licensed under the GPL version 2 or later + + +import os,re,datetime, sys +import tempfile, time + +ITEMS_TO_KEEP = 10 +CACHE_FILE = os.path.join(os.path.dirname(sys.argv[0]),'time_cache') +GRAPH_DIR = '/home/tviehmann/public_html/stat' + +LINE = re.compile(r'(?:|.*/)dinstall_(\d{4})\.(\d{2})\.(\d{2})-(\d{2}):(\d{2}):(\d{2})\.log\.bz2:'+ + r'Archive maintenance timestamp \d+ \(([^\)]*)\): (\d{2}):(\d{2}):(\d{2})$') +UNSAFE = re.compile(r'[^a-zA-Z/\._:0-9\- ]') + +graphs = {"dinstall1": {"keystolist":["pg_dump1", "i18n 1", "accepted", "make-suite-file-list", "apt-ftparchive", + "pdiff", "release files", "w-b", "i18n 2", "apt-ftparchive cleanup"], + "showothers":True}, + "dinstall2": {"keystolist":['External Updates', 'p-u-new', 'o-p-u-new', 'cruft', 'import-keyring', 'overrides', 'cleanup', 'scripts', 'mirror hardlinks', 'stats', 'compress'], + "showothers":False}, + "totals":{"keystolist":[],"showothers":True}} + +#'mirror hardlinks', 'apt-ftparchive', 'logremove', 'startup', 'import-keyring', 'release files', 'accepted', 'stats', 'o-p-u-new', 'i18n 2', 'locked part finished', 'i18n 1', 'cruft', 'pdiff', 'init', 'cleanup', , 'p-u-new', 'run-parts', 'compress', 'scripts', 'expire_dumps', 'removed', 'make-suite-file-list', 'pg_dump1', 'pg_dump2', 'overrides', 'reports', 'merkel projectb push', 'buildd', 'apt-ftparchive cleanup', 'w-b' + +wantkeys = set() +for tmp in graphs.values(): + wantkeys |= set(tmp["keystolist"]) + +d = {} +kl = [] +ks = set() +if os.path.exists(CACHE_FILE): + for l in open(CACHE_FILE): + dt, l = l.split('\t',1) + l = map(lambda x: (lambda y: (y[0],float(y[1])))(x.split(':',1)), l.split('\t')) + newk = [x[0] for x in l if x[0] not in ks] + kl += newk + ks |= set(newk) + d[dt] = dict(l) + +olddt = None +args = sys.argv[1:] +m = UNSAFE.search(' '.join(args)) +if m: + raise Exception("I don't like command line arguments including char '%s'"%m.group(0)) + +if args: + for l in os.popen('bzgrep -H "Archive maintenance timestamp" "'+'" "'.join(args)+'"'): + m = LINE.match(l) + if not m: + raise Exception("woops '%s'"%l) + g = map(lambda x: (not x.isdigit() and x) or int(x), m.groups()) + dt = datetime.datetime(*g[:6]) + if olddt != dt: + oldsecs = 0 + olddt = dt + dt2 = datetime.datetime(*(g[:3]+g[-3:])) + secs = (dt2-dt).seconds + assert secs >= 0 # should add 24*60*60 + k = g[6] + d.setdefault(str(dt),{})[k] = (secs-oldsecs)/60.0 + oldsecs = secs + if k not in ks: + ks.add(k) + kl.append(k) + +if (wantkeys-ks): + print >> sys.stderr, "warning, requested keys not found in any log: "+' '.join(wantkeys-ks) + +datakeys = d.keys() +datakeys.sort() + +datakeys = datakeys[-ITEMS_TO_KEEP:] +f = open(CACHE_FILE+".tmp","w") +for dk in datakeys: + print >> f, dk+'\t'+'\t'.join( + ["%s:%s"%(k,str(d[dk][k])) for k in kl if k in d[dk]]) +f.close() +os.rename(CACHE_FILE+".tmp", CACHE_FILE) + +def dump_file(outfn,keystolist, showothers): + showothers = (showothers and 1) or 0 + # careful, outfn is NOT ESCAPED + f = tempfile.NamedTemporaryFile() + otherkeys = ks-set(keystolist) + print >>f, '\t'.join(keystolist+showothers*['other']) + for k in datakeys: + v = d[k] + others = sum(map(lambda x: v.get(x,0),otherkeys)) + print >>f, k+'\t'+'\t'.join(map(lambda x: str(v.get(x,0)), keystolist)+showothers*[str(others)]) + f.flush() + n = f.name + + p = os.popen("R --vanilla --slave > /dev/null","w") + p.write(""" + d = read.table("%(datafile)s", sep = "\t") + #d[["ts"]] <- as.POSIXct(d[["timestamp"]]) + k = setdiff(names(d),c("ts","timestamp")) + #palette(rainbow(max(length(k),2))) + palette(c("midnightblue", "gold", "turquoise", "cyan", "black", "red", "OrangeRed", "green3", "blue", + "magenta", "tomato4", + "violetred2","thistle4", "steelblue2", "springgreen4", + "salmon","gray")) + #plot(d[["runtime"]],d[["compress"]],type="l",col="blue") + #lines(d[["runtime"]],d[["logremove"]],type="l",col="red") + #legend(as.POSIXct("2008-12-05"),9500,"logremove",col="red",lty=1) + bitmap(file = "%(outfile)s", type="png16m",width=16.9,height=11.8) + #plot(d[["ts"]],d[["compress"]],type="l",col="blue") + #lines(d[["ts"]],d[["logremove"]],type="l",col="red") + barplot(t(d[,k]), col=palette(), xlab="date",ylab="time/minutes" + ) + par(xpd = TRUE) + legend(xinch(-1.2),par("usr")[4]+yinch(1),legend=k, + ncol=3,fill=1:15) #,xjust=1,yjust=1) + text(xinch(10),par("usr")[4]+yinch(.5),"%(title)s", cex=2) + + dev.off() + q() + """%{'datafile':n,'outfile':outfn, + 'title':((not showothers)*"partial ")+"dinstall times"}) + p.flush() + assert not p.close() + +for afn,params in graphs.items(): + dump_file(os.path.join(GRAPH_DIR,afn+'.png'), **params) -- 2.39.5