From dca650055f023509bf797c760c11e01f03d894bb Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Wed, 5 Dec 2007 07:27:13 +0000 Subject: [PATCH] changes for DM support --- config/debian/cron.daily | 2 + config/debian/dak.conf | 8 ++++ dak/clean_suites.py | 7 ++- dak/dak.py | 2 + dak/import_ldap_fingerprints.py | 18 +++++--- dak/process_accepted.py | 13 ++++++ dak/process_unchecked.py | 76 ++++++++++++++++++++++++++++++++- daklib/queue.py | 2 +- scripts/debian/byhand-dm | 70 ++++++++++++++++++++++++++++++ scripts/debian/dm-monitor | 61 ++++++++++++++++++++++++++ 10 files changed, 249 insertions(+), 10 deletions(-) create mode 100755 scripts/debian/byhand-dm create mode 100755 scripts/debian/dm-monitor diff --git a/config/debian/cron.daily b/config/debian/cron.daily index 059020eb..b1af5312 100755 --- a/config/debian/cron.daily +++ b/config/debian/cron.daily @@ -169,6 +169,8 @@ dak queue-report | mail -e -s "NEW and BYHAND on $(date +%D)" ftpmaster@ftp-mast # and one on crufty packages dak cruft-report | tee $webdir/cruft-report-daily.txt | mail -e -s "Debian archive cruft report for $(date +%D)" ftpmaster@ftp-master.debian.org +$scriptsdir/dm-monitor >$webdir/dm-uploaders.html + ################################################################################ # Run mirror-split diff --git a/config/debian/dak.conf b/config/debian/dak.conf index 5e8f206d..b266d77b 100644 --- a/config/debian/dak.conf +++ b/config/debian/dak.conf @@ -3,6 +3,7 @@ Dinstall GPGKeyring { "/srv/keyring.debian.org/keyrings/debian-keyring.gpg"; "/srv/keyring.debian.org/keyrings/debian-keyring.pgp"; + "/srv/ftp.debian.org/keyrings/debian-maintainers.gpg"; }; SigningKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/secring.gpg"; SigningPubKeyring "/srv/ftp.debian.org/s3kr1t/dot-gnupg/pubring.gpg"; @@ -636,6 +637,13 @@ AutomaticByHandPackages { Script "/srv/ftp.debian.org/dak/scripts/debian/byhand-di"; }; + "debian-maintainers" { + Source "debian-maintainers"; + Section "raw-keyring"; + Extension "gpg"; + Script "/srv/ftp.debian.org/dak/scripts/debian/byhand-dm"; + }; + "tag-overrides" { Source "tag-overrides"; Section "byhand"; diff --git a/dak/clean_suites.py b/dak/clean_suites.py index c9a81539..a59d6e3a 100755 --- a/dak/clean_suites.py +++ b/dak/clean_suites.py @@ -199,6 +199,7 @@ def clean(): before = time.time() sys.stdout.write("[Deleting from source table... ") projectB.query("DELETE FROM dsc_files WHERE EXISTS (SELECT 1 FROM source s, files f, dsc_files df WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = df.source AND df.id = dsc_files.id)" % (delete_date)) + projectB.query("DELETE FROM src_uploaders WHERE EXISTS (SELECT 1 FROM source s, files f WHERE f.last_used <= '%s' AND s.file = f.id AND s.id = src_uploaders.source)" % (delete_date)) projectB.query("DELETE FROM source WHERE EXISTS (SELECT 1 FROM files WHERE source.file = files.id AND files.last_used <= '%s')" % (delete_date)) sys.stdout.write("done. (%d seconds)]\n" % (int(time.time()-before))) @@ -249,7 +250,8 @@ def clean_maintainers(): q = projectB.query(""" SELECT m.id FROM maintainer m WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.maintainer = m.id) - AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id)""") + AND NOT EXISTS (SELECT 1 FROM source s WHERE s.maintainer = m.id) + AND NOT EXISTS (SELECT 1 FROM src_uploaders u WHERE u.maintainer = m.id)""") ql = q.getresult() count = 0 @@ -271,7 +273,8 @@ def clean_fingerprints(): q = projectB.query(""" SELECT f.id FROM fingerprint f - WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id) + WHERE f.keyring IS NULL + AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.sig_fpr = f.id) AND NOT EXISTS (SELECT 1 FROM source s WHERE s.sig_fpr = f.id)""") ql = q.getresult() diff --git a/dak/dak.py b/dak/dak.py index c4d76be8..06e923d1 100755 --- a/dak/dak.py +++ b/dak/dak.py @@ -90,6 +90,8 @@ def init(): "Check for users with no packages in the archive"), ("import-archive", "Populate SQL database based from an archive tree"), + ("import-keyring", + "Populate fingerprint/uid table based on a new/updated keyring"), ("import-ldap-fingerprints", "Syncs fingerprint and uid tables with Debian LDAP db"), ("import-users-from-passwd", diff --git a/dak/import_ldap_fingerprints.py b/dak/import_ldap_fingerprints.py index 22ee2ddc..eda3710c 100755 --- a/dak/import_ldap_fingerprints.py +++ b/dak/import_ldap_fingerprints.py @@ -149,9 +149,13 @@ SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id if not existing_uid: q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)) print "Assigning %s to 0x%s." % (uid, fingerprint) + elif existing_uid == uid: + pass + elif existing_uid[:3] == "dm:": + q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)) + print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid, fingerprint) else: - if existing_uid != uid: - daklib.utils.fubar("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid)) + daklib.utils.warn("%s has %s in LDAP, but projectB says it should be %s." % (uid, fingerprint, existing_uid)) # Try to update people who sign with non-primary key q = projectB.query("SELECT fingerprint, id FROM fingerprint WHERE uid is null") @@ -168,10 +172,11 @@ SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id primary_key = m.group(1) primary_key = primary_key.replace(" ","") if not ldap_fin_uid_id.has_key(primary_key): - daklib.utils.fubar("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint)) - (uid, uid_id) = ldap_fin_uid_id[primary_key] - q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)) - print "Assigning %s to 0x%s." % (uid, fingerprint) + daklib.utils.warn("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint)) + else: + (uid, uid_id) = ldap_fin_uid_id[primary_key] + q = projectB.query("UPDATE fingerprint SET uid = %s WHERE id = %s" % (uid_id, fingerprint_id)) + print "Assigning %s to 0x%s." % (uid, fingerprint) else: extra_keyrings = "" for keyring in Cnf.ValueList("Import-LDAP-Fingerprints::ExtraKeyrings"): @@ -204,6 +209,7 @@ SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id guess_uid = "???" name = " ".join(output.split('\n')[0].split()[3:]) print "0x%s -> %s -> %s" % (fingerprint, name, guess_uid) + # FIXME: make me optionally non-interactive # FIXME: default to the guessed ID uid = None diff --git a/dak/process_accepted.py b/dak/process_accepted.py index 18b5b056..7dca0d4d 100755 --- a/dak/process_accepted.py +++ b/dak/process_accepted.py @@ -309,6 +309,19 @@ def install (): files_id = daklib.database.set_files_id (filename, dsc_files[dsc_file]["size"], dsc_files[dsc_file]["md5sum"], dsc_location_id) projectB.query("INSERT INTO dsc_files (source, file) VALUES (currval('source_id_seq'), %d)" % (files_id)) + # Add the src_uploaders to the DB + if dsc.get("dm-upload-allowed", "no") == "yes": + uploader_ids = [] + if dsc.has_key("uploaders"): + uploader_ids = [ + daklib.database.get_or_set_maintainer_id( u.strip() ) + for u in dsc["uploaders"].split(",") + ] + uploader_ids.append(maintainer_id) + for u in uploader_ids: + projectB.query("INSERT INTO src_uploaders (source, maintainer) VALUES (currval('source_id_seq'), %d)" % (u)) + + # Add the .deb files to the DB for file in files.keys(): if files[file]["type"] == "deb": diff --git a/dak/process_unchecked.py b/dak/process_unchecked.py index 412ad64a..5cb9124c 100755 --- a/dak/process_unchecked.py +++ b/dak/process_unchecked.py @@ -1016,10 +1016,84 @@ def check_timestamps(): ################################################################################ +def lookup_uid_from_fingerprint(fpr): + q = Upload.projectB.query("SELECT u.uid, u.name FROM fingerprint f, uid u WHERE f.uid = u.id AND f.fingerprint = '%s'" % (fpr)) + qs = q.getresult() + if len(qs) == 0: + return (None, None) + else: + return qs[0] + def check_signed_by_key(): """Ensure the .changes is signed by an authorized uploader.""" - # We only check binary-only uploads right now + (uid, uid_name) = lookup_uid_from_fingerprint(changes["fingerprint"]) + if uid_name == None: + uid_name = "" + + # match claimed name with actual name: + if uid == None: + uid, uid_email = changes["fingerprint"], uid + may_nmu, may_sponsor = 1, 1 + # XXX by default new dds don't have a fingerprint/uid in the db atm, + # and can't get one in there if we don't allow nmu/sponsorship + elif uid[:3] == "dm:": + uid_email = uid[3:] + may_nmu, may_sponsor = 0, 0 + else: + uid_email = "%s@debian.org" % (uid) + may_nmu, may_sponsor = 1, 1 + + if uid_email in [changes["maintaineremail"], changes["changedbyemail"]]: + sponsored = 0 + elif uid_name in [changes["maintainername"], changes["changedbyname"]]: + sponsored = 0 + if uid_name == "": sponsored = 1 + else: + sponsored = 1 + + if sponsored and not may_sponsor: + reject("%s is not authorised to sponsor uploads" % (uid)) + + if not sponsored and not may_nmu: + source_ids = [] + check_suites = changes["distribution"].keys() + if "unstable" not in check_suites: check_suites.append("unstable") + for suite in check_suites: + suite_id = daklib.database.get_suite_id(suite) + q = Upload.projectB.query("SELECT s.id FROM source s JOIN src_associations sa ON (s.id = sa.source) WHERE s.source = '%s' AND sa.suite = %d" % (changes["source"], suite_id)) + for si in q.getresult(): + if si[0] not in source_ids: source_ids.append(si[0]) + + print "source_ids: %s" % (",".join([str(x) for x in source_ids])) + + is_nmu = 1 + for si in source_ids: + is_nmu = 1 + q = Upload.projectB.query("SELECT m.name FROM maintainer m WHERE m.id IN (SELECT maintainer FROM src_uploaders WHERE src_uploaders.source = %s)" % (si)) + for m in q.getresult(): + (rfc822, rfc2047, name, email) = daklib.utils.fix_maintainer(m[0]) + if email == uid_email or name == uid_name: + is_nmu=0 + break + if is_nmu: + reject("%s may not upload/NMU source package %s" % (uid, changes["source"])) + + for b in changes["binary"].keys(): + for suite in changes["distribution"].keys(): + suite_id = daklib.database.get_suite_id(suite) + q = Upload.projectB.query("SELECT DISTINCT s.source FROM source s JOIN binaries b ON (s.id = b.source) JOIN bin_associations ba On (b.id = ba.bin) WHERE b.package = '%s' AND ba.suite = %s" % (b, suite_id)) + for s in q.getresult(): + if s[0] != changes["source"]: + reject("%s may not hijack %s from source package %s in suite %s" % (uid, b, s, suite)) + + for file in files.keys(): + if files[file].has_key("byhand"): + reject("%s may not upload BYHAND file %s" % (uid, file)) + if files[file].has_key("new"): + reject("%s may not upload NEW file %s" % (uid, file)) + + # The remaining checks only apply to binary-only uploads right now if changes["architecture"].has_key("source"): return diff --git a/daklib/queue.py b/daklib/queue.py index ff06f217..d77f5f35 100644 --- a/daklib/queue.py +++ b/daklib/queue.py @@ -134,7 +134,7 @@ class Upload: d_changes[i] = changes[i] ## dsc for i in [ "source", "version", "maintainer", "fingerprint", - "uploaders", "bts changelog" ]: + "uploaders", "bts changelog", "dm-upload-allowed" ]: if dsc.has_key(i): d_dsc[i] = dsc[i] ## dsc_files diff --git a/scripts/debian/byhand-dm b/scripts/debian/byhand-dm new file mode 100755 index 00000000..a410f703 --- /dev/null +++ b/scripts/debian/byhand-dm @@ -0,0 +1,70 @@ +#!/bin/sh -e + +BYHAND="$1" +VERSION="$2" +ARCH="$3" +CHANGES="$4" + +KEYRING=/srv/keyring.debian.org/keyrings/debian-keyring.gpg + +DESTKR=/srv/ftp.debian.org/keyrings/debian-maintainers.gpg + +get_id () { + echo "SELECT U.name, U.uid FROM fingerprint F JOIN uid U ON (F.uid = U.id) WHERE F.fingerprint = '$1';" | + psql projectb -At | + sed 's/|\(.*\)/ <\1@debian.org>/' +} + +is_allowed () { + echo "SELECT M.name from src_uploaders U join source S on (U.source = S.id) join maintainer M on (U.maintainer = M.id) WHERE S.source = 'debian-maintainers';" | + psql projectb -At | + while read ALLOWED; do + if [ "$1" = "$ALLOWED" ]; then + echo yes + break + fi + done +} + +FPRINT=$(gpgv --keyring "$KEYRING" --status-fd 3 3>&1 >/dev/null 2>&1 "$CHANGES" | + cut -d\ -f2,3 | grep ^VALIDSIG | head -n1 | cut -d\ -f2) + +ID="$(get_id "$FPRINT")" + +if [ "$(is_allowed "$ID")" != "yes" ]; then + echo "Unauthorised upload by $ID" + exit 1 +fi + +echo "Authorised upload by $ID, copying into place" + +OUT=$(mktemp) + +cp "$BYHAND" "$DESTKR" +dak import-keyring --generate-users "dm:%s" "$DESTKR" >$OUT + +if [ -s "$OUT" ]; then + /usr/sbin/sendmail -odq -oi -t < +Subject: Updated Debian Maintainers Keyring +Content-Type: text/plain; charset=utf-8 +MIME-Version: 1.0 + +With the upload of debian-maintainers version $VERSION, the following +changes to the keyring have been made: + +$(cat $OUT) + +A summary of all the changes in this upload follows. + +Debian distribution maintenance software, +on behalf of, +$ID + +$(cat $CHANGES) +EOF +fi +rm -f "$OUT" + +exit 0 diff --git a/scripts/debian/dm-monitor b/scripts/debian/dm-monitor new file mode 100755 index 00000000..d4616433 --- /dev/null +++ b/scripts/debian/dm-monitor @@ -0,0 +1,61 @@ +#!/bin/sh + +echo "Known debian maintainers:" + +psql --html projectb <') + WHERE u.uid LIKE 'dm:%' AND sa.suite = 5 +ORDER BY u.uid, s.source, s.version; +EOF + +echo "Source packages in the pool uploaded by debian maintainers:" + +psql --html projectb <