]> err.no Git - dak/commitdiff
Add and enable data collection and graphing for the DEFERRED queues.
authorPaul Wise <pabs@debian.org>
Mon, 21 Feb 2011 14:21:58 +0000 (22:21 +0800)
committerJoerg Jaspert <joerg@debian.org>
Thu, 3 Mar 2011 21:18:47 +0000 (22:18 +0100)
config/backports/cron.hourly
config/debian/cron.hourly
dak/graph.py
dak/show_deferred.py

index e8116d1841ed6533e9dbc6b783e049dc66537515..24b8f90f530a1e47008d363aa263e4ff03f6a1f7 100755 (executable)
@@ -11,8 +11,8 @@ export SCRIPTVARS=/srv/backports-master.debian.org/dak/config/backports/vars
 dak import-users-from-passwd
 dak queue-report -n > $webdir/new.html
 dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat
-#dak show-deferred > ${webdir}/deferred.html
-dak graph -n new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates
+dak show-deferred -r $webdir/stat > ${webdir}/deferred.html
+dak graph -n new,byhand,proposedupdates,oldproposedupdates,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates
 dak show-new > /dev/null
 
 # cd $webdir
index 0b9afd7936ff0d9a0240d9819374e0e96a596961..73ba6afc4c9a77eda342fb11e5242819c828eb93 100755 (executable)
@@ -11,8 +11,8 @@ export SCRIPTVARS=/srv/ftp-master.debian.org/dak/config/debian/vars
 dak import-users-from-passwd
 dak queue-report -n > $webdir/new.html
 dak queue-report -8 -d new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat
-dak show-deferred > ${webdir}/deferred.html
-dak graph -n new,byhand,proposedupdates,oldproposedupdates -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates
+dak show-deferred -r $webdir/stat > ${webdir}/deferred.html
+dak graph -n new,byhand,proposedupdates,oldproposedupdates,deferred -r $webdir/stat -i $webdir/stat -x $scriptsdir/rrd-release-freeze-dates
 
 # do not run show-new and other stuff in parallel
 LOCKFILE="$lockdir/unchecked.lock"
index d6c290e40f0c0fc68ae8f71152a4a8aedd053308..022e98c94c83d6ab61bb9dfede316cab38798b3c 100755 (executable)
@@ -22,6 +22,7 @@
 
 import os
 import sys
+import coloursys
 
 import rrdtool
 import apt_pkg
@@ -49,7 +50,65 @@ Graphs the number of packages in queue directories (usually new and byhand).
 
 ################################################################################
 
-def graph(rrd_dir, image_dir, name, extra_args, graph, title, start, year_lines=False):
+def graph(*args):
+    if args[2] == "deferred":
+        graph_deferred(*args)
+    else:
+        graph_normal(*args)
+
+def deferred_colours():
+    colours = [0]*16
+    for i in range(0,16):
+        colours[i] = colorsys.hsv_to_rgb(i/16.0, 1.0, 0.5+i/32.0)
+        colours[i] = ''.join(['%02X' % (c*255) for c in colours[i]])
+    return colours
+
+colours = deferred_colours()
+
+def graph_deferred(rrd_dir, image_dir, name, extra_args, graph, title, start, year_lines=False):
+    image_file = os.path.join(image_dir, "%s-%s.png" % (name, graph))
+    rrd_file = os.path.join(rrd_dir, "%s.rrd" % name)
+    rrd_args = [image_file, "--start", start]
+    rrd_args += ("""
+--end
+now
+--width
+600
+--height
+150
+--vertical-label
+changes
+--title
+%s changes count for the last %s
+--lower-limit
+0
+-E
+""" % (name.upper(), title) ).strip().split("\n")
+
+    if year_lines:
+        rrd_args += ["--x-grid", "MONTH:1:YEAR:1:YEAR:1:31536000:%Y"]
+
+    for i in range(0,16):
+        rrd_args += ("""
+DEF:d%i=%s:day%i:AVERAGE
+AREA:d%i#%s:%i-day changes count:STACK
+VDEF:ld%i=d%i,LAST
+VDEF:mind%i=d%i,MINIMUM
+VDEF:maxd%i=d%i,MAXIMUM
+VDEF:avgd%i=d%i,AVERAGE
+GPRINT:ld%i:cur\\: %%.0lf
+GPRINT:mind%i:min\\: %%.0lf
+GPRINT:maxd%i:max\\: %%.0lf
+GPRINT:avgd%i:avg\\: %%.0lf\\j
+""" % ((i, rrd_file, i, i, colours[i])+(i,)*13)).strip().split("\n")
+
+    rrd_args += extra_args
+    try:
+        ret = rrdtool.graph(*rrd_args)
+    except rrdtool.error, e:
+        print('warning: graph: rrdtool error, skipping %s-%s.png: %s' % (name, graph, e))
+
+def graph_normal(rrd_dir, image_dir, name, extra_args, graph, title, start, year_lines=False):
     image_file = os.path.join(image_dir, "%s-%s.png" % (name, graph))
     rrd_file = os.path.join(rrd_dir, "%s.rrd" % name)
     rrd_args = [image_file, "--start", start]
index dd1a748f7b1946c72f4f6322426eeee062820b00..6307bd384f59b1d1d8ee5bac66ff768733db6878 100755 (executable)
@@ -22,6 +22,7 @@
 
 import sys, os, re, time
 import apt_pkg
+import rrdtool
 
 try:
     # starting with squeeze
@@ -124,6 +125,59 @@ def table_row(changesname, delay, changed_by, closes):
     row_number+=1
     return res
 
+def update_graph_database(rrd_dir, *counts):
+    if not rrd_dir:
+        return
+
+    rrd_file = os.path.join(rrd_dir, 'deferred.rrd')
+    counts = [str(count) for count in counts]
+    update = [rrd_file, "N:"+":".join(counts)]
+
+    try:
+        rrdtool.update(*update)
+    except rrdtool.error:
+        create = [rrd_file]+"""
+--step
+300
+--start
+0
+DS:day0:GAUGE:7200:0:1000
+DS:day1:GAUGE:7200:0:1000
+DS:day2:GAUGE:7200:0:1000
+DS:day3:GAUGE:7200:0:1000
+DS:day4:GAUGE:7200:0:1000
+DS:day5:GAUGE:7200:0:1000
+DS:day6:GAUGE:7200:0:1000
+DS:day7:GAUGE:7200:0:1000
+DS:day8:GAUGE:7200:0:1000
+DS:day9:GAUGE:7200:0:1000
+DS:day10:GAUGE:7200:0:1000
+DS:day11:GAUGE:7200:0:1000
+DS:day12:GAUGE:7200:0:1000
+DS:day13:GAUGE:7200:0:1000
+DS:day14:GAUGE:7200:0:1000
+DS:day15:GAUGE:7200:0:1000
+RRA:AVERAGE:0.5:1:599
+RRA:AVERAGE:0.5:6:700
+RRA:AVERAGE:0.5:24:775
+RRA:AVERAGE:0.5:288:795
+RRA:MIN:0.5:1:600
+RRA:MIN:0.5:6:700
+RRA:MIN:0.5:24:775
+RRA:MIN:0.5:288:795
+RRA:MAX:0.5:1:600
+RRA:MAX:0.5:6:700
+RRA:MAX:0.5:24:775
+RRA:MAX:0.5:288:795
+""".strip().split("\n")
+        try:
+            rc = rrdtool.create(*create)
+            ru = rrdtool.update(*update)
+        except rrdtool.error, e:
+            print('warning: queue_report: rrdtool error, skipping %s.rrd: %s' % (type, e))
+    except NameError:
+        pass
+
 def get_upload_data(changesfn):
     achanges = deb822.Changes(file(changesfn))
     changesname = os.path.basename(changesfn)
@@ -134,6 +188,7 @@ def get_upload_data(changesfn):
         remainingtime = (delaydays>0)*max(0,24*60*60+os.stat(changesfn).st_mtime-time.time())
         delay = "%d days %02d:%02d" %(max(delaydays-1,0), int(remainingtime/3600),int(remainingtime/60)%60)
     else:
+        delaydays = 0
         remainingtime = 0
 
     uploader = achanges.get('changed-by')
@@ -160,9 +215,9 @@ def get_upload_data(changesfn):
                 if os.path.exists(qfn):
                     os.symlink(qfn,lfn)
                     os.chmod(qfn, 0644)
-    return (max(delaydays-1,0)*24*60*60+remainingtime, changesname, delay, uploader, achanges.get('closes','').split(),achanges)
+    return (max(delaydays-1,0)*24*60*60+remainingtime, changesname, delay, uploader, achanges.get('closes','').split(),achanges, delaydays)
 
-def list_uploads(filelist):
+def list_uploads(filelist, rrd_dir):
     uploads = map(get_upload_data, filelist)
     uploads.sort()
     # print the summary page
@@ -179,7 +234,9 @@ def list_uploads(filelist):
         fn = os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp')
         f = open(fn,"w")
         try:
+            counts = [0]*16
             for u in uploads:
+                counts[u[6]] += 1
                 print >> f, "Changes-file: %s"%u[1]
                 fields = """Location: DEFERRED
 Delayed-Until: %s
@@ -191,6 +248,7 @@ Delay-Remaining: %s"""%(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time
             f.close()
             os.rename(os.path.join(Cnf["Show-Deferred::LinkPath"],'.status.tmp'),
                       os.path.join(Cnf["Show-Deferred::LinkPath"],'status'))
+            update_graph_database(rrd_dir, *counts)
         except:
             os.unlink(fn)
             raise
@@ -204,6 +262,7 @@ def usage (exit_code=0):
   -h, --help                    show this help and exit.
   -p, --link-path [path]        override output directory.
   -d, --deferred-queue [path]   path to the deferred queue
+  -r, --rrd=key                 Directory where rrd files to be updated are stored
   """
     sys.exit(exit_code)
 
@@ -212,7 +271,8 @@ def init():
     Cnf = utils.get_conf()
     Arguments = [('h',"help","Show-Deferred::Options::Help"),
                  ("p","link-path","Show-Deferred::LinkPath","HasArg"),
-                 ("d","deferred-queue","Show-Deferred::DeferredQueue","HasArg")]
+                 ("d","deferred-queue","Show-Deferred::DeferredQueue","HasArg"),
+                 ('r',"rrd","Show-Deferred::Options::Rrd", "HasArg")]
     args = apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv)
     for i in ["help"]:
         if not Cnf.has_key("Show-Deferred::Options::%s" % (i)):
@@ -236,11 +296,18 @@ def main():
     if len(args)!=0:
         usage(1)
 
+    if Cnf.has_key("Show-Deferred::Options::Rrd"):
+        rrd_dir = Cnf["Show-Deferred::Options::Rrd"]
+    elif Cnf.has_key("Dir::Rrd"):
+        rrd_dir = Cnf["Dir::Rrd"]
+    else:
+        rrd_dir = None
+
     filelist = []
     for r,d,f  in os.walk(Cnf["Show-Deferred::DeferredQueue"]):
         filelist += map (lambda x: os.path.join(r,x),
                          filter(lambda x: x.endswith('.changes'), f))
-    list_uploads(filelist)
+    list_uploads(filelist, rrd_dir)
 
     available_changes = set(map(os.path.basename,filelist))
     if Cnf.has_key("Show-Deferred::LinkPath"):
@@ -251,3 +318,6 @@ def main():
                 if (not os.path.exists(afp) or
                     (af.endswith('.changes') and af not in available_changes)):
                     os.unlink(afp)
+
+if __name__ == '__main__':
+    main()