From c6af7014864c9a2e5d13fc44d0a63f445e8ccce6 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 21 Feb 2011 22:21:58 +0800 Subject: [PATCH] Add and enable data collection and graphing for the DEFERRED queues. --- config/backports/cron.hourly | 4 +- config/debian/cron.hourly | 4 +- dak/graph.py | 61 +++++++++++++++++++++++++++- dak/show_deferred.py | 78 ++++++++++++++++++++++++++++++++++-- 4 files changed, 138 insertions(+), 9 deletions(-) diff --git a/config/backports/cron.hourly b/config/backports/cron.hourly index e8116d18..24b8f90f 100755 --- a/config/backports/cron.hourly +++ b/config/backports/cron.hourly @@ -11,8 +11,8 @@ export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars dak import-users-from-passwd dak queue-report -n > $webdir/new.html dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat -#dak show-deferred > ${webdir}/deferred.html -dak graph -n new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates +dak show-deferred -r $webdir/stat > ${webdir}/deferred.html +dak graph -n new,byhand,proposedupdates,oldproposedupdates,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates dak show-new > /dev/null # cd $webdir diff --git a/config/debian/cron.hourly b/config/debian/cron.hourly index 0b9afd79..73ba6afc 100755 --- a/config/debian/cron.hourly +++ b/config/debian/cron.hourly @@ -11,8 +11,8 @@ export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars dak import-users-from-passwd dak queue-report -n > $webdir/new.html dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat -dak show-deferred > ${webdir}/deferred.html -dak graph -n new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates +dak show-deferred -r $webdir/stat > ${webdir}/deferred.html +dak graph -n new,byhand,proposedupdates,oldproposedupdates,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates # do not run show-new and other stuff in parallel LOCKFILE="$lockdir/unchecked.lock" diff --git a/dak/graph.py b/dak/graph.py index d6c290e4..022e98c9 100755 --- a/dak/graph.py +++ b/dak/graph.py @@ -22,6 +22,7 @@ import os import sys +import coloursys import rrdtool import apt_pkg @@ -49,7 +50,65 @@ Graphs the number of packages in queue directories (usually new and byhand). ################################################################################ -def graph(rrd_dir, image_dir, name, extra_args, graph, title, start, year_lines=False): +def graph(*args): + if args[2] == "deferred": + graph_deferred(*args) + else: + graph_normal(*args) + +def deferred_colours(): + colours = [0]*16 + for i in range(0,16): + colours[i] = colorsys.hsv_to_rgb(i/16.0, 1.0, 0.5+i/32.0) + colours[i] = ''.join(['%02X' % (c*255) for c in colours[i]]) + return colours + +colours = deferred_colours() + +def graph_deferred(rrd_dir, image_dir, name, extra_args, graph, title, start, year_lines=False): + image_file = os.path.join(image_dir, "%s-%s.png" % (name, graph)) + rrd_file = os.path.join(rrd_dir, "%s.rrd" % name) + rrd_args = [image_file, "--start", start] + rrd_args += (""" +--end +now +--width +600 +--height +150 +--vertical-label +changes +--title +%s changes count for the last %s +--lower-limit +0 +-E +""" % (name.upper(), title) ).strip().split("\n") + + if year_lines: + rrd_args += ["--x-grid", "MONTH:1:YEAR:1:YEAR:1:31536000:%Y"] + + for i in range(0,16): + rrd_args += (""" +DEF:d%i=%s:day%i:AVERAGE +AREA:d%i#%s:%i-day changes count:STACK +VDEF:ld%i=d%i,LAST +VDEF:mind%i=d%i,MINIMUM +VDEF:maxd%i=d%i,MAXIMUM +VDEF:avgd%i=d%i,AVERAGE +GPRINT:ld%i:cur\\: %%.0lf +GPRINT:mind%i:min\\: %%.0lf +GPRINT:maxd%i:max\\: %%.0lf +GPRINT:avgd%i:avg\\: %%.0lf\\j +""" % ((i, rrd_file, i, i, colours[i])+(i,)*13)).strip().split("\n") + + rrd_args += extra_args + try: + ret = rrdtool.graph(*rrd_args) + except rrdtool.error, e: + print('warning: graph: rrdtool error, skipping %s-%s.png: %s' % (name, graph, e)) + +def graph_normal(rrd_dir, image_dir, name, extra_args, graph, title, start, year_lines=False): image_file = os.path.join(image_dir, "%s-%s.png" % (name, graph)) rrd_file = os.path.join(rrd_dir, "%s.rrd" % name) rrd_args = [image_file, "--start", start] diff --git a/dak/show_deferred.py b/dak/show_deferred.py index dd1a748f..6307bd38 100755 --- a/dak/show_deferred.py +++ b/dak/show_deferred.py @@ -22,6 +22,7 @@ import sys, os, re, time import apt_pkg +import rrdtool try: # starting with squeeze @@ -124,6 +125,59 @@ def table_row(changesname, delay, changed_by, closes): row_number+=1 return res +def update_graph_database(rrd_dir, *counts): + if not rrd_dir: + return + + rrd_file = os.path.join(rrd_dir, 'deferred.rrd') + counts = [str(count) for count in counts] + update = [rrd_file, "N:"+":".join(counts)] + + try: + rrdtool.update(*update) + except rrdtool.error: + create = [rrd_file]+""" +--step +300 +--start +0 +DS:day0:GAUGE:7200:0:1000 +DS:day1:GAUGE:7200:0:1000 +DS:day2:GAUGE:7200:0:1000 +DS:day3:GAUGE:7200:0:1000 +DS:day4:GAUGE:7200:0:1000 +DS:day5:GAUGE:7200:0:1000 +DS:day6:GAUGE:7200:0:1000 +DS:day7:GAUGE:7200:0:1000 +DS:day8:GAUGE:7200:0:1000 +DS:day9:GAUGE:7200:0:1000 +DS:day10:GAUGE:7200:0:1000 +DS:day11:GAUGE:7200:0:1000 +DS:day12:GAUGE:7200:0:1000 +DS:day13:GAUGE:7200:0:1000 +DS:day14:GAUGE:7200:0:1000 +DS:day15:GAUGE:7200:0:1000 +RRA:AVERAGE:0.5:1:599 +RRA:AVERAGE:0.5:6:700 +RRA:AVERAGE:0.5:24:775 +RRA:AVERAGE:0.5:288:795 +RRA:MIN:0.5:1:600 +RRA:MIN:0.5:6:700 +RRA:MIN:0.5:24:775 +RRA:MIN:0.5:288:795 +RRA:MAX:0.5:1:600 +RRA:MAX:0.5:6:700 +RRA:MAX:0.5:24:775 +RRA:MAX:0.5:288:795 +""".strip().split("\n") + try: + rc = rrdtool.create(*create) + ru = rrdtool.update(*update) + except rrdtool.error, e: + print('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e)) + except NameError: + pass + def get_upload_data(changesfn): achanges = deb822.Changes(file(changesfn)) changesname = os.path.basename(changesfn) @@ -134,6 +188,7 @@ def get_upload_data(changesfn): remainingtime = (delaydays>0)*max(0,24*60*60+os.stat(changesfn).st_mtime-time.time()) delay = "%d days %02d:%02d" %(max(delaydays-1,0), int(remainingtime/3600),int(remainingtime/60)%60) else: + delaydays = 0 remainingtime = 0 uploader = achanges.get('changed-by') @@ -160,9 +215,9 @@ def get_upload_data(changesfn): if os.path.exists(qfn): os.symlink(qfn,lfn) os.chmod(qfn, 0644) - return (max(delaydays-1,0)*24*60*60+remainingtime, changesname, delay, uploader, achanges.get('closes','').split(),achanges) + return (max(delaydays-1,0)*24*60*60+remainingtime, changesname, delay, uploader, achanges.get('closes','').split(),achanges, delaydays) -def list_uploads(filelist): +def list_uploads(filelist, rrd_dir): uploads = map(get_upload_data, filelist) uploads.sort() # print the summary page @@ -179,7 +234,9 @@ def list_uploads(filelist): fn = os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp') f = open(fn,"w") try: + counts = [0]*16 for u in uploads: + counts[u[6]] += 1 print >> f, "Changes-file: %s"%u[1] fields = """Location: DEFERRED Delayed-Until: %s @@ -191,6 +248,7 @@ Delay-Remaining: %s"""%(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time f.close() os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp'), os.path.join(Cnf["Show-Deferred::LinkPath"],'status')) + update_graph_database(rrd_dir, *counts) except: os.unlink(fn) raise @@ -204,6 +262,7 @@ def usage (exit_code=0): -h, --help show this help and exit. -p, --link-path [path] override output directory. -d, --deferred-queue [path] path to the deferred queue + -r, --rrd=key Directory where rrd files to be updated are stored """ sys.exit(exit_code) @@ -212,7 +271,8 @@ def init(): Cnf = utils.get_conf() Arguments = [('h',"help","Show-Deferred::Options::Help"), ("p","link-path","Show-Deferred::LinkPath","HasArg"), - ("d","deferred-queue","Show-Deferred::DeferredQueue","HasArg")] + ("d","deferred-queue","Show-Deferred::DeferredQueue","HasArg"), + ('r',"rrd","Show-Deferred::Options::Rrd", "HasArg")] args = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv) for i in ["help"]: if not Cnf.has_key("Show-Deferred::Options::%s" % (i)): @@ -236,11 +296,18 @@ def main(): if len(args)!=0: usage(1) + if Cnf.has_key("Show-Deferred::Options::Rrd"): + rrd_dir = Cnf["Show-Deferred::Options::Rrd"] + elif Cnf.has_key("Dir::Rrd"): + rrd_dir = Cnf["Dir::Rrd"] + else: + rrd_dir = None + filelist = [] for r,d,f in os.walk(Cnf["Show-Deferred::DeferredQueue"]): filelist += map (lambda x: os.path.join(r,x), filter(lambda x: x.endswith('.changes'), f)) - list_uploads(filelist) + list_uploads(filelist, rrd_dir) available_changes = set(map(os.path.basename,filelist)) if Cnf.has_key("Show-Deferred::LinkPath"): @@ -251,3 +318,6 @@ def main(): if (not os.path.exists(afp) or (af.endswith('.changes') and af not in available_changes)): os.unlink(afp) + +if __name__ == '__main__': + main() -- 2.39.5