From f6386a74a3399e05382412c0eb23d9b301f6e6d4 Mon Sep 17 00:00:00 2001 From: James Troup Date: Thu, 2 Jan 2003 18:06:19 +0000 Subject: [PATCH] Initial import --- docs/README.first | 105 +++++++++++++++++++++ docs/cindy.1.sgml | 61 ++++++++++++ docs/lisa.1.sgml | 95 +++++++++++++++++++ saffron | 234 ++++++++++++++++++++++++++++++++++++++++++++++ saffron.R | 6 ++ 5 files changed, 501 insertions(+) create mode 100644 docs/README.first create mode 100644 docs/cindy.1.sgml create mode 100644 docs/lisa.1.sgml create mode 100755 saffron create mode 100644 saffron.R diff --git a/docs/README.first b/docs/README.first new file mode 100644 index 00000000..07a1e3f9 --- /dev/null +++ b/docs/README.first @@ -0,0 +1,105 @@ + Notes + ===== + +o Please be careful: katie sends out lots of emails and if not + configured properly will happily send them to lots of people who + probably didn't want those emails. + +o Don't use katie.conf, apt.conf, cron.* etc. as starting points for + your own configuration files, they're the configuration files for + auric (aka ftp-master.debian.org) and are highly Debian specific. + Start from scratch and refer to the security.debian.org config files + (-security) as they're a better example for a private archive. + +o Don't be put off by the names, see doc/README.names for a mapping of + name -> what the script does. + + What do all these scripts do? + ============================= + +Generic and generally useful +---------------------------- + +o To process queue/: + + * jennifer - processes queue/unchecked + * kelly - processes queue/accepted + * lisa - processes queue/new and queue/byhand + +o To generate indices files: + + * jenna - generates file lists for apt-ftparchive and removes + obsolete packages from suites + * ziyi - generates Release + +o To clean things up: + + * rhona - to remove obsolete files from the pool + * shania - to remove obsolete/stray files from the queue + * melanie - to remove package(s) from suite(s) + +o Information display: + + * madison - shows information about package(s) + * helena - shows information about package(s) in queue/ + +Generic and useful, but only for those with existing archives +------------------------------------------------------------- + +o catherine - migrates packages from legacy locations to the pool +o neve - initializes a projectb database from an exisiting archive + +Generic but not overly useful (in normal use) +--------------------------------------------- + +o ashley - dumps info in .katie files +o julia - sync PostgreSQL users with system users +o rene - check for obsolete or duplicated packages +o rose - directory creation in the initial setup of an archive +o tea - various sanity checks of the database and archive +o natalie - manpiulates/list override entries +o heidi - removes/adds/lists package(s) from/to/for a suite +o saffron - produces various statistics + +Semi-generic +------------ + +To generate less-used indices files: + +o charisma - generates Maintainers file +o denise - generates override. files + +Mostly Debian(.org) specific +---------------------------- + +o amber - wrapper for Debian security team +o halle - removes obsolete .changes files from proposed-updates +o jeri - basic dependency checking for proposed-updates + +Very Incomplete or otherwise not generally useful +------------------------------------------------- + +o alyson - currently only initializes a DB from a katie.conf config file +o andrea - looks for version descrepancies that shouldn't exist in many + archives +o cindy - override cruft checker that doesn't work well with New Incoming + +Scripts invoked by other scripts +-------------------------------- + +o fernanda - invoked by lisa to "check" NEW packages +o claire - invoked by catherine to determine packages still in legacy locations +o katie - common code used by lisa, jennifer, kelly and others + + How do I get started? + ===================== + +[Very incomplete - FIXME] + +o Write your own katie.conf and apt.conf files +o Run rose +o If you have an existing archive: + * Run neve + otherwise: + * Run alyson + diff --git a/docs/cindy.1.sgml b/docs/cindy.1.sgml new file mode 100644 index 00000000..44b98a21 --- /dev/null +++ b/docs/cindy.1.sgml @@ -0,0 +1,61 @@ + + +%katieent; + +]> + + + &katie-docinfo; + + + cindy + 1 + + + + + cindy + Utility to alter or display the contents of a suite + + + + + + cindy + + + + + Description</> + <para> + <command>cindy</command> is a cruft checker for overrides. + </PARA> + </REFSECT1> + + <RefSect1><Title>Options</> + + <VariableList> + <VarListEntry><term><option>-h/--help</option></> + <ListItem> + <Para>Show help and then exit.</PARA> + </LISTITEM> + </VarListEntry> + + </VariableList> + </RefSect1> + + <RefSect1><Title>Notes</> + + <Para>cindy is not a good idea with New Incoming as she doesn't take into account queue/accepted. You can minimize the impact of this by running her immediately after kelly but that's still racy because lisa doesn't lock with elly. A better long term fix is the evil plan for accepted to be in the DB.</> + + <RefSect1><Title>Diagnostics</> + <para> + <command>cindy</command> returns zero on normal operation, non-zero on error. + </PARA> + </RefSect1> + + &manauthor; + +</refentry> diff --git a/docs/lisa.1.sgml b/docs/lisa.1.sgml new file mode 100644 index 00000000..a89234ec --- /dev/null +++ b/docs/lisa.1.sgml @@ -0,0 +1,95 @@ +<!-- -*- mode: sgml; mode: fold -*- --> +<!doctype refentry PUBLIC "-//OASIS//DTD DocBook V3.1//EN" [ + +<!ENTITY % katieent SYSTEM "katie.ent"> +%katieent; + +]> + +<refentry> + &katie-docinfo; + + <refmeta> + <refentrytitle>lisa</> + <manvolnum>1</> + </refmeta> + + <!-- Man page title --> + <refnamediv> + <refname>lisa</> + <refpurpose>Processes BYHAND and NEW packages</> + </refnamediv> + + <!-- Arguments --> + <refsynopsisdiv> + <cmdsynopsis> + <command>lisa</> + <arg><option><replaceable>options</replaceable></></arg> + <arg choice="plain"><replaceable>changes_file</replaceable></arg> + <arg><option><replaceable>...</replaceable></option></arg> + </cmdsynopsis> + </refsynopsisdiv> + + <RefSect1><Title>Description</> + <para> + <command>lisa</command> is the program which installs packages from the accepted directory into the distribution. + </PARA></REFSECT1> + + <RefSect1><Title>Options</> + + <VariableList> + + <varlistentry> + <term><option>-a/--automatic</option></term> + <listitem> + <para>Run automatically; i.e. perform the default action if it's possible to do so without user interaction. Intend for use in cron jobs and the like.</para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-h/--help</option></term> + <listitem> + <para>Display usage help and then exit.</para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-m/--manual-reject=<replaceable>message</replaceable></option></term> + <listitem> + <para>Perform a manual rejection of the package. The <replaceable>message</replaceable> is included in the rejection notice sent to the uploader. If no <replaceable>message</replaceable> is given, an editor will be spawned so one can be added to the rejection notice. + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-n/--no-action</option></term> + <listitem> + <para>Don't actually install anything; just show what would be done.</para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-p/--no-lock</option></term> + <listitem> + <para>Don't check the lockfile. Obviously dangerous and should only be used for cron jobs (if at all).</para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-v/--version</option></term> + <listitem> + <para>Display the version number and then exit.</para> + </listitem> + </varlistentry> + + </VariableList> + </RefSect1> + + <RefSect1><Title>Diagnostics</> + <para> + <command>lisa</command> returns zero on normal operation, non-zero on error. + </PARA> + </RefSect1> + + &manauthor; + +</refentry> diff --git a/saffron b/saffron new file mode 100755 index 00000000..a9e759b1 --- /dev/null +++ b/saffron @@ -0,0 +1,234 @@ +#!/usr/bin/env python + +# Various statistical pr0nography fun and games +# Copyright (C) 2000, 2001, 2002 James Troup <james@nocrew.org> +# $Id: saffron,v 1.1 2003-01-02 18:06:19 troup Exp $ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import pg, sys; +import utils; +import apt_pkg; + +################################################################################ + +Cnf = None; +projectB = None; + +################################################################################ + +def usage(exit_code=0): + print """Usage: saffron STAT +Print various stats. + + -h, --help show this help and exit. + +The following STAT modes are available: + + arch-space - displays space used by each architecture + pkg-nums - displays the number of packages by suite/architecture + daily-install - displays daily install stats suitable for graphing +""" + sys.exit(exit_code) + +################################################################################ + +def per_arch_space_use(): + q = projectB.query(""" +SELECT a.arch_string as Architecture, sum(f.size) + FROM files f, binaries b, architecture a + WHERE a.id=b.architecture AND f.id=b.file + GROUP BY a.arch_string"""); + print q; + +################################################################################ + +def daily_install_stats(): + stats = {}; + file = utils.open_file("2001-11"); + for line in file.readlines(): + split = line.strip().split('~'); + program = split[1]; + if program != "katie": + continue; + action = split[2]; + if action != "installing changes" and action != "installed": + continue; + date = split[0][:8]; + if not stats.has_key(date): + stats[date] = {}; + stats[date]["packages"] = 0; + stats[date]["size"] = 0.0; + if action == "installing changes": + stats[date]["packages"] += 1; + elif action == "installed": + stats[date]["size"] += float(split[5]); + + dates = stats.keys(); + dates.sort(); + for date in dates: + packages = stats[date]["packages"] + size = int(stats[date]["size"] / 1024.0 / 1024.0) + print "%s %s %s" % (date, packages, size); + +################################################################################ + +def longest(list): + longest = 0; + for i in list: + l = len(i); + if l > longest: + longest = l; + return longest; + +def suite_sort(a, b): + a_priority = int(Cnf["Suite::%s::Priority" % (a)]); + b_priority = int(Cnf["Suite::%s::Priority" % (b)]); + return cmp(a_priority, b_priority); + +def output_format(suite): + output_suite = []; + for word in suite.split("-"): + output_suite.append(word[0]); + return output_suite.join("-"); + +# Obvious query with GROUP BY and mapped names -> 50 seconds +# GROUP BY but ids instead of suite/architecture names -> 28 seconds +# Simple query -> 14 seconds +# Simple query into large dictionary + processing -> 21 seconds +# Simple query into large pre-created dictionary + processing -> 18 seconds + +def number_of_packages(): + arches = {}; + arch_ids = {}; + suites = {}; + suite_ids = {}; + d = {}; + # Build up suite mapping + q = projectB.query("SELECT id, suite_name FROM suite"); + suite_ql = q.getresult(); + for i in suite_ql: + (id, name) = i; + suites[id] = name; + suite_ids[name] = id; + # Build up architecture mapping + q = projectB.query("SELECT id, arch_string FROM architecture"); + for i in q.getresult(): + (id, name) = i; + arches[id] = name; + arch_ids[name] = id; + # Pre-create the dictionary + for suite_id in suites.keys(): + d[suite_id] = {}; + for arch_id in arches.keys(): + d[suite_id][arch_id] = 0; + # Get the raw data for binaries + q = projectB.query(""" +SELECT ba.suite, b.architecture + FROM binaries b, bin_associations ba + WHERE b.id = ba.bin"""); + # Simultate 'GROUP by suite, architecture' with a dictionary + for i in q.getresult(): + (suite_id, arch_id) = i; + d[suite_id][arch_id] = d[suite_id][arch_id] + 1; + # Get the raw data for source + arch_id = arch_ids["source"]; + q = projectB.query(""" +SELECT suite, count(suite) FROM src_associations GROUP BY suite;"""); + for i in q.getresult(): + (suite_id, count) = i; + d[suite_id][arch_id] = d[suite_id][arch_id] + count; + ## Print the results + # Setup + suite_list = suites.values(); + suite_list.sort(suite_sort); + suite_id_list = []; + suite_arches = {}; + for suite in suite_list: + suite_id = suite_ids[suite]; + suite_arches[suite_id] = {}; + for arch in Cnf.ValueList("Suite::%s::Architectures" % (suite)): + suite_arches[suite_id][arch] = ""; + suite_id_list.append(suite_id); + output_list = map(lambda x: output_format(x), suite_list); + longest_suite = longest(output_list); + arch_list = arches.values(); + arch_list.sort(); + longest_arch = longest(arch_list); + # Header + output = (" "*longest_arch) + " |" + for suite in output_list: + output = output + suite.center(longest_suite)+" |"; + output = output + "\n"+(len(output)*"-")+"\n"; + # per-arch data + arch_list = arches.values(); + arch_list.sort(); + longest_arch = longest(arch_list); + for arch in arch_list: + arch_id = arch_ids[arch]; + output = output + arch.center(longest_arch)+" |"; + for suite_id in suite_id_list: + if suite_arches[suite_id].has_key(arch): + count = repr(d[suite_id][arch_id]); + else: + count = "-"; + output = output + count.rjust(longest_suite)+" |"; + output = output + "\n"; + print output; + +################################################################################ + +def main (): + global Cnf, projectB; + + Cnf = utils.get_conf(); + Arguments = [('h',"help","Saffron::Options::Help")]; + for i in [ "help" ]: + if not Cnf.has_key("Saffron::Options::%s" % (i)): + Cnf["Saffron::Options::%s" % (i)] = ""; + + args = apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv); + + Options = Cnf.SubTree("Saffron::Options") + if Options["Help"]: + usage(); + + if len(args) < 1: + utils.warn("saffron requires at least one argument"); + usage(1); + elif len(args) > 1: + utils.warn("saffron accepts only one argument"); + usage(1); + mode = args[0].lower(); + + projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])); + + if mode == "arch-space": + per_arch_space_use(); + elif mode == "pkg-nums": + number_of_packages(); + elif mode == "daily-install": + daily_install_stats(); + else: + utils.warn("unknown mode '%s'" % (mode)); + usage(1); + +################################################################################ + +if __name__ == '__main__': + main() + diff --git a/saffron.R b/saffron.R new file mode 100644 index 00000000..df12847e --- /dev/null +++ b/saffron.R @@ -0,0 +1,6 @@ +x <- read.table("x.1",row.names=1,col.names=c("Packages", "Sizes")) +y <- t(x) +postscript(file="x4.png") +barplot(y, beside=TRUE, col = c("red", "green"), main="Daily dinstall run size", legend = colnames(x), xlab="Date", ylab="Packages/Size (Mb)") +axis(4) +dev.off() -- 2.39.5