export SCRIPTVARS=/org/security.debian.org/dak/config/debian-security/vars
. $SCRIPTVARS
-if [ ! -e $ftpdir/Archive_Maintenance_In_Progress ]; then
- cd $masterdir
- for d in $DISTS; do
- eval SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
- eval PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
- done
- apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
- dists=
- for d in $DISTS; do
- eval NEW_SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
- eval NEW_PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
- old=SOURCES_$d
- new=NEW_$old
- if [ ${!new} -gt ${!old} ]; then
- if [ -z "$dists" ]; then
- dists="$d"
- else
- dists="$dists $d"
- fi
- continue
+if [ -e $ftpdir/Archive_Maintenance_In_Progress ]; then
+ exit 0
+fi
+
+cd $masterdir
+for d in $DISTS; do
+ eval SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
+ eval PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
+done
+
+apt-ftparchive -qq -o APT::FTPArchive::Contents=off generate apt.conf.buildd
+dists=
+for d in $DISTS; do
+ eval NEW_SOURCES_$d=`stat -c "%Y" $base/buildd/$d/Sources.gz`
+ eval NEW_PACKAGES_$d=`stat -c "%Y" $base/buildd/$d/Packages.gz`
+ old=SOURCES_$d
+ new=NEW_$old
+ if [ ${!new} -gt ${!old} ]; then
+ if [ -z "$dists" ]; then
+ dists="$d"
+ else
+ dists="$dists $d"
fi
- old=PACKAGES_$d
- new=NEW_$old
- if [ ${!new} -gt ${!old} ]; then
- if [ -z "$dists" ]; then
- dists="$d"
- else
- dists="$dists $d"
- fi
- continue
+ continue
+ fi
+ old=PACKAGES_$d
+ new=NEW_$old
+ if [ ${!new} -gt ${!old} ]; then
+ if [ -z "$dists" ]; then
+ dists="$d"
+ else
+ dists="$dists $d"
fi
+ continue
+ fi
+done
+
+if [ ! -z "$dists" ]; then
+ # setup ssh master process
+ ssh buildd@buildd -S $SSH_SOCKET -MN 2> /dev/null &
+ SSH_PID=$!
+ while [ ! -S $SSH_SOCKET ]; do
+ sleep 1
done
- if [ ! -z "$dists" ]; then
- # setup ssh master process
- ssh buildd@buildd -S $SSH_SOCKET -MN 2> /dev/null &
- SSH_PID=$!
- while [ ! -S $SSH_SOCKET ]; do
- sleep 1
- done
- trap 'kill -TERM $SSH_PID' 0
- for d in $dists; do
- archs=ARCHS_$d
- ARCHS=${!archs}
- cd /org/security.debian.org/buildd/$d
+ trap 'kill -TERM $SSH_PID' 0
+ for d in $dists; do
+ archs=ARCHS_$d
+ ARCHS=${!archs}
+ cd /org/security.debian.org/buildd/$d
+ if [ "$d" != "oldstable" ]; then
+ # disabled for oldstable-security by ajt 2008-01-01
for a in $ARCHS; do
quinn-diff -a /org/security.debian.org/buildd/Packages-arch-specific -A $a 2>/dev/null | ssh buildd@buildd -S $SSH_SOCKET wanna-build -d $d-security -b $a/build-db --merge-partial-quinn
+
ssh buildd@buildd -S $SSH_SOCKET wanna-build -d $d-security -A $a -b $a/build-db --merge-packages < Packages
done
- if [ "$d" = "oldstable" ]; then
- ssh buildd@bester.farm.ftbfs.de -i ~/.ssh/id_bester sleep 1
- fi
- done
- fi
+ else
+ ssh buildd@bester.farm.ftbfs.de -i ~/.ssh/id_bester sleep 1
+ fi
+ done
fi
# Fix overrides
-rsync -ql ftp-master::indices/override\* $overridedir
+# disabled by ajt 2008-01-01: requires auth
+rsync --password-file /srv/non-us.debian.org/s3kr1t/rsync-password -ql security-master@ftp-master::indices/override\* $overridedir
cd $overridedir
find . -name override\*.gz -type f -maxdepth 1 -mindepth 1 | xargs gunzip -f
-find . -type l -maxdepth 1 -mindepth 1 | xargs rm
+find . -type l -maxdepth 1 -mindepth 1 | xargs --no-run-if-empty rm
for suite in $suites; do
case $suite in
dak control-overrides -q -S -t $override_type -s $suite -c updates/$component < override.$override_suite.$component$type
fi
else
- dak control-overrides -q -S -t $override_type -s $suite -c updates/$component < override.$override_suite.$component$type
+ # XXX removed 2007-08-16 ajt
+ #dak control-overrides -q -S -t $override_type -s $suite -c updates/$component < override.$override_suite.$component$type
+ true
fi
case $suite in
oldstable)
Dinstall
{
- PGPKeyring "/org/keyring.debian.org/keyrings/debian-keyring.pgp";
- GPGKeyring "/org/keyring.debian.org/keyrings/debian-keyring.gpg";
+ GPGKeyring {
+ "/org/keyring.debian.org/keyrings/debian-keyring.gpg";
+ "/org/keyring.debian.org/keyrings/debian-keyring.pgp";
+ };
SigningKeyring "/org/non-us.debian.org/s3kr1t/dot-gnupg/secring.gpg";
SigningPubKeyring "/org/non-us.debian.org/s3kr1t/dot-gnupg/pubring.gpg";
SigningKeyIds "6070D3A1";
Reject "/org/security.debian.org/queue/reject/";
Unchecked "/org/security.debian.org/queue/unchecked/";
ProposedUpdates "/does/not/exist/"; // XXX fixme
+ OldProposedUpdates "/does/not/exist/"; // XXX fixme
Embargoed "/org/security.debian.org/queue/embargoed/";
Unembargoed "/org/security.debian.org/queue/unembargoed/";
import daklib.queue, daklib.logging, daklib.utils, daklib.database
import apt_pkg, os, sys, pwd, time, re, commands
-re_taint_free = re.compile(r"^['/;\-\+\.\s\w]+$");
+re_taint_free = re.compile(r"^['/;\-\+\.~\s\w]+$");
Cnf = None
Options = None
('n', "no-action", "Security-Install::Options::No-Action"),
('s', "sudo", "Security-Install::Options::Sudo"),
(' ', "no-upload", "Security-Install::Options::No-Upload"),
+ ('u', "fg-upload", "Security-Install::Options::Foreground-Upload"),
(' ', "drop-advisory", "Security-Install::Options::Drop-Advisory"),
('A', "approve", "Security-Install::Options::Approve"),
('R', "reject", "Security-Install::Options::Reject"),
daklib.utils.fubar("Process what?")
Upload = daklib.queue.Upload(Cnf)
+ if Options["No-Action"]:
+ Options["Sudo"] = ""
if not Options["Sudo"] and not Options["No-Action"]:
Logger = Upload.Logger = daklib.logging.Logger(Cnf, "new-security-install")
def do_upload():
if Options["No-Upload"]:
print "Not uploading as requested"
- return
+ elif Options["Foreground-Upload"]:
+ actually_upload(changes)
+ else:
+ child = os.fork()
+ if child == 0:
+ actually_upload(changes)
+ os._exit(0)
+ print "Uploading in the background"
+
+def actually_upload(changes_files):
+ file_list = ""
+ suites = {}
+ component_mapping = {}
+ for component in Cnf.SubTree("Security-Install::ComponentMappings").List():
+ component_mapping[component] = Cnf["Security-Install::ComponentMappings::%s" % (component)]
+ uploads = {}; # uploads[uri] = file_list
+ changesfiles = {}; # changesfiles[uri] = file_list
+ package_list = {} # package_list[source_name][version]
+ changes_files.sort(daklib.utils.changes_compare)
+ for changes_file in changes_files:
+ changes_file = daklib.utils.validate_changes_file_arg(changes_file)
+ # Reset variables
+ components = {}
+ upload_uris = {}
+ file_list = []
+ Upload.init_vars()
+ # Parse the .dak file for the .changes file
+ Upload.pkg.changes_file = changes_file
+ Upload.update_vars()
+ files = Upload.pkg.files
+ changes = Upload.pkg.changes
+ dsc = Upload.pkg.dsc
+ # We have the changes, now return if its amd64, to not upload them to ftp-master
+ if changes["distribution"].has_key("oldstable-security") and changes["architecture"].has_key("amd64"):
+ print "Not uploading amd64 oldstable-security changes to ftp-master\n"
+ continue
+ # Build the file list for this .changes file
+ for file in files.keys():
+ poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"],
+ daklib.utils.poolify(changes["source"], files[file]["component"]),
+ file)
+ file_list.append(poolname)
+ orig_component = files[file].get("original component", files[file]["component"])
+ components[orig_component] = ""
+ # Determine the upload uri for this .changes file
+ for component in components.keys():
+ upload_uri = component_mapping.get(component)
+ if upload_uri:
+ upload_uris[upload_uri] = ""
+ num_upload_uris = len(upload_uris.keys())
+ if num_upload_uris == 0:
+ daklib.utils.fubar("%s: No valid upload URI found from components (%s)."
+ % (changes_file, ", ".join(components.keys())))
+ elif num_upload_uris > 1:
+ daklib.utils.fubar("%s: more than one upload URI (%s) from components (%s)."
+ % (changes_file, ", ".join(upload_uris.keys()),
+ ", ".join(components.keys())))
+ upload_uri = upload_uris.keys()[0]
+ # Update the file list for the upload uri
+ if not uploads.has_key(upload_uri):
+ uploads[upload_uri] = []
+ uploads[upload_uri].extend(file_list)
+ # Update the changes list for the upload uri
+ if not changesfiles.has_key(upload_uri):
+ changesfiles[upload_uri] = []
+ changesfiles[upload_uri].append(changes_file)
+ # Remember the suites and source name/version
+ for suite in changes["distribution"].keys():
+ suites[suite] = ""
+ # Remember the source name and version
+ if changes["architecture"].has_key("source") and \
+ changes["distribution"].has_key("testing"):
+ if not package_list.has_key(dsc["source"]):
+ package_list[dsc["source"]] = {}
+ package_list[dsc["source"]][dsc["version"]] = ""
+
+ for uri in uploads.keys():
+ uploads[uri].extend(changesfiles[uri])
+ (host, path) = uri.split(":")
+ file_list = " ".join(uploads[uri])
+ print "Uploading files to %s..." % (host)
+ spawn("lftp -c 'open %s; cd %s; put %s'" % (host, path, file_list))
- print "Would upload to ftp-master" # XXX
+ if not Options["No-Action"]:
+ filename = "%s/testing-processed" % (Cnf["Dir::Log"])
+ file = daklib.utils.open_file(filename, 'a')
+ for source in package_list.keys():
+ for version in package_list[source].keys():
+ file.write(" ".join([source, version])+'\n')
+ file.close()
def generate_advisory(template):
global changes, advisory
adv = daklib.utils.TemplateSubst(Subst, template)
return adv
-
def spawn(command):
if not re_taint_free.match(command):
daklib.utils.fubar("Invalid character in \"%s\"." % (command))
if advisory == None:
daklib.utils.fubar("Must set advisory name")
os.spawnl(os.P_WAIT, "/usr/bin/sudo", "/usr/bin/sudo", "-u", "dak", "-H",
- "/usr/local/bin/dak new-security-install", "-"+arg, "--", advisory)
+ "/usr/local/bin/dak", "new-security-install", "-"+arg, "--", advisory)
else:
fn()
if exit:
for c in changes:
daklib.utils.copy(c, os.path.join(dest, c))
os.unlink(c)
- k = c[:8] + ".dak"
+ k = c[:-8] + ".dak"
daklib.utils.copy(k, os.path.join(dest, k))
os.unlink(k)
aborted = Upload.do_reject()
if not aborted:
- os.unlink(c[:-8]+".katie")
+ os.unlink(c[:-8]+".dak")
for f in files:
Upload.projectB.query(
"DELETE FROM queue_build WHERE filename = '%s'" % (f))
os.unlink(f)
print "Updating buildd information..."
- spawn("/org/security.debian.org/katie/cron.buildd-security")
+ spawn("/org/security.debian.org/dak/config/debian-security/cron.buildd")
adv_file = "./advisory.%s" % (advisory)
if os.path.exists(adv_file):
and not Upload.source_exists(source_package, source_version, Upload.pkg.changes["distribution"].keys()):
source_epochless_version = daklib.utils.re_no_epoch.sub('', source_version)
dsc_filename = "%s_%s.dsc" % (source_package, source_epochless_version)
- if not os.path.exists(Cnf["Dir::Queue::Accepted"] + '/' + dsc_filename):
+ found = 0
+ for q in ["Accepted", "Embargoed", "Unembargoed"]:
+ if os.path.exists(Cnf["Dir::Queue::%s" % (q)] + '/' + dsc_filename):
+ found = 1
+ if not found:
reject("no source found for %s %s (%s)." % (source_package, source_version, file))
# Version and file overwrite checks
elif files[file]["type"] == "dsc":
reject(Upload.check_source_against_db(file))
(reject_msg, is_in_incoming) = Upload.check_dsc_against_db(file)
- reject(reject_msg)
+ reject(reject_msg, "")
- if reject_message:
+ if reject_message.find("Rejected") != -1:
answer = "XXX"
if Options["No-Action"] or Options["Automatic"]:
answer = 'S'
################################################################################
-def determine_new (changes, files):
- new = {}
-
- # Build up a list of potentially new things
- for file in files.keys():
- f = files[file]
- # Skip byhand elements
- if f["type"] == "byhand":
- continue
- pkg = f["package"]
- priority = f["priority"]
- section = f["section"]
- # FIXME: unhardcode
- if section == "non-US/main":
- section = "non-US"
- type = get_type(f)
- component = f["component"]
-
- if type == "dsc":
- priority = "source"
- if not new.has_key(pkg):
- new[pkg] = {}
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"] = []
- else:
- old_type = new[pkg]["type"]
- if old_type != type:
- # source gets trumped by deb or udeb
- if old_type == "dsc":
- new[pkg]["priority"] = priority
- new[pkg]["section"] = section
- new[pkg]["type"] = type
- new[pkg]["component"] = component
- new[pkg]["files"].append(file)
- if f.has_key("othercomponents"):
- new[pkg]["othercomponents"] = f["othercomponents"]
-
- for suite in changes["suite"].keys():
- suite_id = daklib.database.get_suite_id(suite)
- for pkg in new.keys():
- component_id = daklib.database.get_component_id(new[pkg]["component"])
- type_id = daklib.database.get_override_type_id(new[pkg]["type"])
- q = projectB.query("SELECT package FROM override WHERE package = '%s' AND suite = %s AND component = %s AND type = %s" % (pkg, suite_id, component_id, type_id))
- ql = q.getresult()
- if ql:
- for file in new[pkg]["files"]:
- if files[file].has_key("new"):
- del files[file]["new"]
- del new[pkg]
-
- if changes["suite"].has_key("stable"):
- print "WARNING: overrides will be added for stable!"
- if changes["suite"].has_key("oldstable"):
- print "WARNING: overrides will be added for OLDstable!"
- for pkg in new.keys():
- if new[pkg].has_key("othercomponents"):
- print "WARNING: %s already present in %s distribution." % (pkg, new[pkg]["othercomponents"])
-
- return new
-
-################################################################################
-
def indiv_sg_compare (a, b):
"""Sort by source name, source, version, 'have source', and
finally by filename."""
################################################################################
-def check_valid (new):
- for pkg in new.keys():
- section = new[pkg]["section"]
- priority = new[pkg]["priority"]
- type = new[pkg]["type"]
- new[pkg]["section id"] = daklib.database.get_section_id(section)
- new[pkg]["priority id"] = daklib.database.get_priority_id(new[pkg]["priority"])
- # Sanity checks
- if (section == "debian-installer" and type != "udeb") or \
- (section != "debian-installer" and type == "udeb"):
- new[pkg]["section id"] = -1
- if (priority == "source" and type != "dsc") or \
- (priority != "source" and type == "dsc"):
- new[pkg]["priority id"] = -1
-
-################################################################################
-
def print_new (new, indexed, file=sys.stdout):
- check_valid(new)
+ daklib.queue.check_valid(new)
broken = 0
index = 0
for pkg in new.keys():
################################################################################
-def get_type (f):
- # Determine the type
- if f.has_key("dbtype"):
- type = f["dbtype"]
- elif f["type"] == "orig.tar.gz" or f["type"] == "tar.gz" or f["type"] == "diff.gz" or f["type"] == "dsc":
- type = "dsc"
- else:
- daklib.utils.fubar("invalid type (%s) for new. Dazed, confused and sure as heck not continuing." % (type))
-
- # Validate the override type
- type_id = daklib.database.get_override_type_id(type)
- if type_id == -1:
- daklib.utils.fubar("invalid type (%s) for new. Say wha?" % (type))
-
- return type
-
-################################################################################
-
def index_range (index):
if index == 1:
return "1"
done = 0
while not done:
# Find out what's new
- new = determine_new(changes, files)
+ new = daklib.queue.determine_new(changes, files, projectB)
if not new:
break
Arguments = [('a',"automatic","Process-New::Options::Automatic"),
('h',"help","Process-New::Options::Help"),
+ ('C',"comments-dir","Process-New::Options::Comments-Dir", "HasArg"),
('m',"manual-reject","Process-New::Options::Manual-Reject", "HasArg"),
('n',"no-action","Process-New::Options::No-Action")]
- for i in ["automatic", "help", "manual-reject", "no-action", "version"]:
+ for i in ["automatic", "help", "manual-reject", "no-action", "version", "comments-dir"]:
if not Cnf.has_key("Process-New::Options::%s" % (i)):
Cnf["Process-New::Options::%s" % (i)] = ""
################################################################################
+def get_accept_lock():
+ retry = 0
+ while retry < 10:
+ try:
+ lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
+ retry = 10
+ except OSError, e:
+ if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
+ retry += 1
+ if (retry >= 10):
+ daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
+ else:
+ print("Unable to get accepted lock (try %d of 10)" % retry)
+ time.sleep(60)
+ else:
+ raise
+
+def move_to_dir (dest, perms=0660, changesperms=0664):
+ daklib.utils.move (Upload.pkg.changes_file, dest, perms=changesperms)
+ file_keys = Upload.pkg.files.keys()
+ for file in file_keys:
+ daklib.utils.move (file, dest, perms=perms)
+
def do_accept():
print "ACCEPT"
if not Options["No-Action"]:
- retry = 0
- while retry < 10:
- try:
- lock_fd = os.open(Cnf["Process-New::AcceptedLockFile"], os.O_RDONLY | os.O_CREAT | os.O_EXCL)
- retry = 10
- except OSError, e:
- if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EEXIST':
- retry += 1
- if (retry >= 10):
- daklib.utils.fubar("Couldn't obtain lock; assuming 'dak process-unchecked' is already running.")
- else:
- print("Unable to get accepted lock (try %d of 10)" % retry)
- time.sleep(60)
- else:
- raise
+ get_accept_lock()
(summary, short_summary) = Upload.build_summaries()
- Upload.accept(summary, short_summary)
+ if Cnf.FindB("Dinstall::SecurityQueueHandling"):
+ Upload.dump_vars(Cnf["Dir::Queue::Embargoed"])
+ move_to_dir(Cnf["Dir::Queue::Embargoed"])
+ Upload.queue_build("embargoed", Cnf["Dir::Queue::Embargoed"])
+ # Check for override disparities
+ Upload.Subst["__SUMMARY__"] = summary
+ else:
+ Upload.accept(summary, short_summary)
os.unlink(Upload.pkg.changes_file[:-8]+".dak")
os.unlink(Cnf["Process-New::AcceptedLockFile"])
################################################################################
+def do_comments(dir, opref, npref, line, fn):
+ for comm in [ x for x in os.listdir(dir) if x.startswith(opref) ]:
+ lines = open("%s/%s" % (dir, comm)).readlines()
+ if len(lines) == 0 or lines[0] != line + "\n": continue
+ changes_files = [ x for x in os.listdir(".") if x.startswith(comm[7:]+"_")
+ and x.endswith(".changes") ]
+ changes_files = sort_changes(changes_files)
+ for f in changes_files:
+ f = daklib.utils.validate_changes_file_arg(f, 0)
+ if not f: continue
+ print "\n" + f
+ fn(f, "".join(lines[1:]))
+
+ if opref != npref and not Options["No-Action"]:
+ newcomm = npref + comm[len(opref):]
+ os.rename("%s/%s" % (dir, comm), "%s/%s" % (dir, newcomm))
+
+################################################################################
+
+def comment_accept(changes_file, comments):
+ Upload.pkg.changes_file = changes_file
+ Upload.init_vars()
+ Upload.update_vars()
+ Upload.update_subst()
+ files = Upload.pkg.files
+
+ if not recheck():
+ return # dak wants to REJECT, crap
+
+ (new, byhand) = check_status(files)
+ if not new and not byhand:
+ do_accept()
+
+################################################################################
+
+def comment_reject(changes_file, comments):
+ Upload.pkg.changes_file = changes_file
+ Upload.init_vars()
+ Upload.update_vars()
+ Upload.update_subst()
+ files = Upload.pkg.files
+
+ if not recheck():
+ pass # dak has its own reasons to reject as well, which is fine
+
+ reject(comments)
+ print "REJECT\n" + reject_message,
+ if not Options["No-Action"]:
+ Upload.do_reject(0, reject_message)
+ os.unlink(Upload.pkg.changes_file[:-8]+".dak")
+
+################################################################################
+
def main():
changes_files = init()
if len(changes_files) > 50:
else:
Upload.Subst["__BCC__"] = bcc
- for changes_file in changes_files:
- changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
- if not changes_file:
- continue
- print "\n" + changes_file
- do_pkg (changes_file)
+ commentsdir = Cnf.get("Process-New::Options::Comments-Dir","")
+ if commentsdir:
+ if changes_files != []:
+ sys.stderr.write("Can't specify any changes files if working with comments-dir")
+ sys.exit(1)
+ do_comments(commentsdir, "ACCEPT.", "ACCEPTED.", "OK", comment_accept)
+ do_comments(commentsdir, "REJECT.", "REJECTED.", "NOTOK", comment_reject)
+ else:
+ for changes_file in changes_files:
+ changes_file = daklib.utils.validate_changes_file_arg(changes_file, 0)
+ if not changes_file:
+ continue
+ print "\n" + changes_file
+ do_pkg (changes_file)
end()
if changes["architecture"].has_key("amd64"):
print "Not uploading amd64 part to ftp-master\n"
continue
- if changes["distribution"].has_key("oldstable"):
- print "Not uploading oldstable-security changes to ftp-master\n"
- continue
# Build the file list for this .changes file
for file in files.keys():
poolname = os.path.join(Cnf["Dir::Root"], Cnf["Dir::PoolRoot"],
def main():
+ print "Disabled. See your team@security email, and/or contact aj on OFTC."
+ sys.exit(1)
+
(advisory_number, changes_files) = init()
if not Options["No-Action"]:
os.chdir(Cnf["Dir::Queue::Accepted"])
print "Installing packages into the archive..."
- spawn("dak process-accepted -pa %s" % (Cnf["Dir::Dak"], " ".join(changes_files)))
+ spawn("dak process-accepted -pa %s" % (" ".join(changes_files)))
os.chdir(Cnf["Dir::Dak"])
print "Updating file lists for apt-ftparchive..."
spawn("dak make-suite-file-list")
projects / dak / commitdiff