| build faster than I and others can fix it.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+(Note that the above is a gross oversimplification, and ignores issues
+including but not necessarily limited to subarchitectures, and quality
+of hardware coverage within certian architectures. It contains forward
+looking statements, and may cause cancer in lab animals.)
+
+Joey Hess in <20040317065216.GA29816@kitenet.net>
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+<jdub> now there's a thought
+<jdub> DD trading cards
+<mdz> official joeyh action figure, with rapid-fire upload action
+<jdub> lamont with pump-action NMU flame-thrower!
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#!/usr/bin/env python
# Dump variables from a .katie file to stdout
-# Copyright (C) 2001, 2002 James Troup <james@nocrew.org>
-# $Id: ashley,v 1.8 2003-01-02 18:10:02 troup Exp $
+# Copyright (C) 2001, 2002, 2004 James Troup <james@nocrew.org>
+# $Id: ashley,v 1.9 2004-04-01 17:14:25 troup Exp $
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
dsc = k.pkg.dsc;
print " Dsc:";
- for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
+ for i in [ "source", "version", "maintainer", "fingerprint", "uploaders",
+ "bts changelog" ]:
if dsc.has_key(i):
print " %s: %s" % (i.capitalize(), dsc[i]);
del dsc[i];
Done, Holding, New, Reject, Unchecked. An explanation of the function of
these directores can be found in README.new-incoming.
+Queue::BTSVersionTrack (optional): this directory holds the DebBugs
+Version Tracking support files.
+
================================================================================
Suite
* Create the table structure. init_pool.sql contains all SQL statements
which are needed for this. After changing all occurences of "projectb"
to the name of your database (as defined in DB::Name) you can run:
- psql DB::Name < init_pool.sql
- * Run alyson: it will populate your database with the values from
- katie.conf and apt.conf.
- * Copy all templates from the "templates" directory to to the directory
- specified in Dir::Templates, and adapt them to your distribution.
+ psql <DB::Name> < init_pool.sql
+o Run alyson: it will populate your database with the values from
+ katie.conf and apt.conf.
+o Copy all templates from the "templates" directory to to the directory
+ specified in Dir::Templates, and adapt them to your distribution.
+o Create an 'ftpmaster' group in postgres.
+o Run 'psql <DB::Name> < add_constraints.sql'.
# Checks Debian packages from Incoming
# Copyright (C) 2000, 2001, 2002, 2003, 2004 James Troup <james@nocrew.org>
-# $Id: jennifer,v 1.45 2004-03-11 00:20:51 troup Exp $
+# $Id: jennifer,v 1.46 2004-04-01 17:14:25 troup Exp $
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
################################################################################
-import errno, fcntl, gzip, os, re, shutil, stat, sys, time, traceback;
+import commands, errno, fcntl, os, re, shutil, stat, sys, time, tempfile, traceback;
import apt_inst, apt_pkg;
import db_access, katie, logging, utils;
re_is_changes = re.compile(r"(.+?)_(.+?)_(.+?)\.changes$");
re_valid_version = re.compile(r"^([0-9]+:)?[0-9A-Za-z\.\-\+:]+$");
re_valid_pkg_name = re.compile(r"^[\dA-Za-z][\dA-Za-z\+\-\.]+$");
+re_changelog_versions = re.compile(r"^\w[-+0-9a-z.]+ \([^\(\) \t]+\)");
+re_strip_revision = re.compile(r"-([^-]+)$");
################################################################################
# Globals
-jennifer_version = "$Revision: 1.45 $";
+jennifer_version = "$Revision: 1.46 $";
Cnf = None;
Options = None;
###############################################################################
-def check_dsc ():
+def check_dsc():
global reprocess;
+ # Ensure there is source to check
+ if not changes["architecture"].has_key("source"):
+ return;
+
# Find the .dsc
dsc_filename = None;
for file in files.keys():
else:
dsc_filename = file;
- # If there isn't one, we have nothing to do...
+ # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
if not dsc_filename:
return;
################################################################################
-# dpkg-source broke .diff.gz generation in dpkg 1.8.x; detect the
-# resulting bad source packages and reject them.
+def get_changelog_versions(source_dir):
+ """Extracts a the source package and (optionally) grabs the
+ version history out of debian/changelog for the BTS."""
-def check_diff ():
- for filename in files.keys():
- if files[filename]["type"] == "diff.gz":
- file = gzip.GzipFile(filename, 'r');
- for line in file.readlines():
- if re_bad_diff.search(line):
- reject("%s: invalid .diff.gz produced by a broken version of dpkg-dev 1.8.x." % (filename));
- break;
+ # Find the .dsc (again)
+ dsc_filename = None;
+ for file in files.keys():
+ if files[file]["type"] == "dsc":
+ dsc_filename = file;
+
+ # If there isn't one, we have nothing to do. (We have reject()ed the upload already)
+ if not dsc_filename:
+ return;
+
+ # Create a symlink mirror of the source files in our temporary directory
+ for f in files.keys():
+ m = utils.re_issource.match(f);
+ if m:
+ src = os.path.join(source_dir, f);
+ # If a file is missing for whatever reason, give up.
+ if not os.path.exists(src):
+ return;
+ type = m.group(3);
+ if type == "orig.tar.gz" and pkg.orig_tar_gz:
+ continue;
+ else:
+ dest = os.path.join(os.getcwd(), f);
+ os.symlink(src, dest);
+
+ # If the orig.tar.gz is not a part of the upload, create a symlink to the
+ # existing copy.
+ if pkg.orig_tar_gz:
+ dest = os.path.join(os.getcwd(), os.path.basename(pkg.orig_tar_gz));
+ os.symlink(pkg.orig_tar_gz, dest);
+
+ # Extract the source
+ cmd = "dpkg-source -sn -x %s" % (dsc_filename);
+ (result, output) = commands.getstatusoutput(cmd);
+ if (result != 0):
+ reject("'dpkg-source -x' failed for %s [return code: %s]." % (dsc_filename, result));
+ reject(utils.prefix_multi_line_string(output, " [dpkg-source output:] "), "");
+ return;
+
+ if not Cnf.Find("Dir::Queue::BTSVersionTrack"):
+ return;
+
+ # Get the upstream version
+ upstr_version = utils.re_no_epoch.sub('', dsc["version"]);
+ if re_strip_revision.search(upstr_version):
+ upstr_version = re_strip_revision.sub('', upstr_version);
+
+ # Ensure the changelog file exists
+ changelog_filename = "%s-%s/debian/changelog" % (dsc["source"], upstr_version);
+ if not os.path.exists(changelog_filename):
+ reject("%s: debian/changelog not found in extracted source." % (dsc_filename));
+ return;
+
+ # Parse the changelog
+ dsc["bts changelog"] = "";
+ changelog_file = utils.open_file(changelog_filename);
+ for line in changelog_file.readlines():
+ m = re_changelog_versions.match(line);
+ if m:
+ dsc["bts changelog"] += line;
+ changelog_file.close();
+
+ # Check we found at least one revision in the changelog
+ if not dsc["bts changelog"]:
+ reject("%s: changelog format not recognised (empty version tree)." % (dsc_filename));
+
+########################################
+
+def check_source():
+ # Bail out if:
+ # a) there's no source
+ # or b) reprocess is 2 - we will do this check next time when orig.tar.gz is in 'files'
+ # or c) the orig.tar.gz is MIA
+ if not changes["architecture"].has_key("source") or reprocess == 2 \
+ or pkg.orig_tar_gz == -1:
+ return;
+
+ # Create a temporary directory to extract the source into
+ if Options["No-Action"]:
+ tmpdir = tempfile.mktemp();
+ else:
+ # We're in queue/holding and can create a random directory.
+ tmpdir = "%s" % (os.getpid());
+ os.mkdir(tmpdir);
+
+ # Move into the temporary directory
+ cwd = os.getcwd();
+ os.chdir(tmpdir);
+
+ # Get the changelog version history
+ get_changelog_versions(cwd);
+
+ # Move back and cleanup the temporary tree
+ os.chdir(cwd);
+ shutil.rmtree(tmpdir);
################################################################################
check_distributions();
check_files();
check_dsc();
- check_diff();
+ check_source();
check_md5sums();
check_urgency();
check_timestamps();
if accept_count:
sets = "set"
if accept_count > 1:
- sets = "sets"
+ sets = "sets";
print "Accepted %d package %s, %s." % (accept_count, sets, utils.size_type(int(accept_bytes)));
Logger.log(["total",accept_count,accept_bytes]);
New "/org/ftp.debian.org/queue/new/";
Reject "/org/ftp.debian.org/queue/reject/";
Unchecked "/org/ftp.debian.org/queue/unchecked/";
+ BTSVersionTrack "/org/ftp.debian.org/queue/bts_version_track/";
};
};
# Utility functions for katie
# Copyright (C) 2001, 2002, 2003, 2004 James Troup <james@nocrew.org>
-# $Id: katie.py,v 1.44 2004-02-27 20:07:40 troup Exp $
+# $Id: katie.py,v 1.45 2004-04-01 17:14:25 troup Exp $
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
###############################################################################
-import cPickle, errno, os, pg, re, stat, string, sys, tempfile, time;
+import cPickle, errno, os, pg, re, stat, string, sys, time;
import utils, db_access;
import apt_inst, apt_pkg;
exec "self.pkg.%s.clear();" % (i);
self.pkg.orig_tar_id = None;
self.pkg.orig_tar_location = "";
+ self.pkg.orig_tar_gz = None;
###########################################################################
if changes.has_key(i):
d_changes[i] = changes[i];
## dsc
- for i in [ "source", "version", "maintainer", "fingerprint", "uploaders" ]:
+ for i in [ "source", "version", "maintainer", "fingerprint", "uploaders",
+ "bts changelog" ]:
if dsc.has_key(i):
d_dsc[i] = dsc[i];
## dsc_files
Cnf = self.Cnf;
Subst = self.Subst;
files = self.pkg.files;
+ changes = self.pkg.changes;
+ changes_file = self.pkg.changes_file;
+ dsc = self.pkg.dsc;
print "Accepting."
- self.Logger.log(["Accepting changes",self.pkg.changes_file]);
+ self.Logger.log(["Accepting changes",changes_file]);
self.dump_vars(Cnf["Dir::Queue::Accepted"]);
# Move all the files into the accepted directory
- utils.move(self.pkg.changes_file, Cnf["Dir::Queue::Accepted"]);
+ utils.move(changes_file, Cnf["Dir::Queue::Accepted"]);
file_keys = files.keys();
for file in file_keys:
utils.move(file, Cnf["Dir::Queue::Accepted"]);
utils.send_mail(mail_message)
self.announce(short_summary, 1)
- # Special support to enable clean auto-building of accepted packages
+
+ ## Helper stuff for DebBugs Version Tracking
+ if Cnf.Find("Dir::Queue::BTSVersionTrack"):
+ # ??? once queue/* is cleared on *.d.o and/or reprocessed
+ # the conditionalization on dsc["bts changelog"] should be
+ # dropped.
+
+ # Write out the version history from the changelog
+ if changes["architecture"].has_key("source") and \
+ dsc.has_key("bts changelog"):
+
+ temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
+ dotprefix=1, perms=0644);
+ version_history = utils.open_file(temp_filename, 'w');
+ version_history.write(dsc["bts changelog"]);
+ version_history.close();
+ filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
+ changes_file[:-8]+".versions");
+ os.rename(temp_filename, filename);
+
+ # Write out the binary -> source mapping.
+ temp_filename = utils.temp_filename(Cnf["Dir::Queue::BTSVersionTrack"],
+ dotprefix=1, perms=0644);
+ debinfo = utils.open_file(temp_filename, 'w');
+ for file in file_keys:
+ f = files[file];
+ if f["type"] == "deb":
+ line = " ".join([f["package"], f["version"],
+ f["architecture"], f["source package"],
+ f["source version"]]);
+ debinfo.write(line+"\n");
+ debinfo.close();
+ filename = "%s/%s" % (Cnf["Dir::Queue::BTSVersionTrack"],
+ changes_file[:-8]+".debinfo");
+ os.rename(temp_filename, filename);
+
+ ## Special support to enable clean auto-building of accepted packages
self.projectB.query("BEGIN WORK");
- for suite in self.pkg.changes["distribution"].keys():
+ for suite in changes["distribution"].keys():
if suite not in Cnf.ValueList("Dinstall::AcceptedAutoBuildSuites"):
continue;
suite_id = db_access.get_suite_id(suite);
# If we weren't given a manual rejection message, spawn an
# editor so the user can add one in...
if manual and not reject_message:
- temp_filename = tempfile.mktemp();
- fd = os.open(temp_filename, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0700);
- os.close(fd);
+ temp_filename = utils.temp_filename();
editor = os.environ.get("EDITOR","vi")
answer = 'E';
while answer == 'E':
maps = self.Cnf.ValueList("SuiteMappings")[:]
maps.reverse()
maps = [ m.split() for m in maps ]
- maps = [ (x[1], x[2]) for x in maps
+ maps = [ (x[1], x[2]) for x in maps
if x[0] == "map" or x[0] == "silent-map" ]
s = [suite]
for x in maps:
if x[1] in s and x[0] not in s:
s.append(x[0])
-
+
que = "SELECT s.version FROM source s JOIN src_associations sa ON (s.id = sa.source) JOIN suite su ON (sa.suite = su.id) WHERE s.source = '%s' AND (%s)" % (package, string.join(["su.suite_name = '%s'" % a for a in s], " OR "));
q = self.projectB.query(que)
files = self.pkg.files;
dsc_files = self.pkg.dsc_files;
legacy_source_untouchable = self.pkg.legacy_source_untouchable;
- orig_tar_gz = None;
+ self.pkg.orig_tar_gz = None;
# Try and find all files mentioned in the .dsc. This has
# to work harder to cope with the multiple possible
# Strip out anything that isn't '%s' or '/%s$'
for i in ql:
if i[2] != dsc_file and i[2][-(len(dsc_file)+1):] != '/'+dsc_file:
- self.Logger.log(["check_dsc_against_db",i[2],dsc_file]);
ql.remove(i);
# "[katie] has not broken them. [katie] has fixed a
# Strip out anything that isn't '%s' or '/%s$'
for i in ql:
if i[1] != dsc_file and i[1][-(len(dsc_file)+1):] != '/'+dsc_file:
- self.Logger.log(["check_dsc_against_db",i[1],dsc_file]);
ql.remove(i);
if ql:
- # Unfortunately, we make get more than one
- # match here if, for example, the package was
- # in potato but had a -sa upload in woody. So
- # we need to choose the right one.
+ # Unfortunately, we may get more than one match here if,
+ # for example, the package was in potato but had an -sa
+ # upload in woody. So we need to choose the right one.
x = ql[0]; # default to something sane in case we don't match any or have only one
dsc_files[dsc_file]["files id"] = x[3]; # need this for updating dsc_files in install()
# See install() in katie...
self.pkg.orig_tar_id = x[3];
+ self.pkg.orig_tar_gz = old_file;
if suite_type == "legacy" or suite_type == "legacy-mixed":
self.pkg.orig_tar_location = "legacy";
else:
actual_md5 = apt_pkg.md5sum(utils.open_file(in_otherdir));
actual_size = os.stat(in_otherdir)[stat.ST_SIZE];
found = in_otherdir;
+ self.pkg.orig_tar_gz = in_otherdir;
if not found:
self.reject("%s refers to %s, but I can't find it in the queue or in the pool." % (file, dsc_file));
+ self.pkg.orig_tar_gz = -1;
continue;
else:
self.reject("%s refers to %s, but I can't find it in the queue." % (file, dsc_file));
if actual_size != int(dsc_files[dsc_file]["size"]):
self.reject("size for %s doesn't match %s." % (found, file));
- return (self.reject_message, orig_tar_gz);
+ return (self.reject_message, None);
def do_query(self, q):
sys.stderr.write("query: \"%s\" ... " % (q));
projects / dak / commitdiff