From a0b2b1aafec94c872ddcf2cd21dada1f0ab9240c Mon Sep 17 00:00:00 2001 From: James Troup Date: Sun, 28 Jan 2001 09:06:44 +0000 Subject: [PATCH] .dsc validation stuff --- katie | 23 +++++++++--- melanie | 4 +- shania | 6 +-- tea | 62 ++++++++++++++++++++++--------- test/001/1.dsc | 22 +++++++++++ test/001/2.dsc | 21 +++++++++++ test/001/3.dsc | 21 +++++++++++ test/001/4.dsc | 19 ++++++++++ test/001/5.dsc | 23 ++++++++++++ test/001/6.dsc | 23 ++++++++++++ test/001/test.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ utils.py | 58 +++++++++++++++++++++++++++-- 12 files changed, 345 insertions(+), 32 deletions(-) create mode 100644 test/001/1.dsc create mode 100644 test/001/2.dsc create mode 100644 test/001/3.dsc create mode 100644 test/001/4.dsc create mode 100644 test/001/5.dsc create mode 100644 test/001/6.dsc create mode 100755 test/001/test.py diff --git a/katie b/katie index 271e1a06..13edbd85 100755 --- a/katie +++ b/katie @@ -2,7 +2,7 @@ # Installs Debian packaes # Copyright (C) 2000 James Troup -# $Id: katie,v 1.24 2001-01-27 03:51:14 troup Exp $ +# $Id: katie,v 1.25 2001-01-28 09:06:44 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -161,7 +161,7 @@ def check_changes(filename): # Parse the .changes field into a dictionary [FIXME - need to trap errors, pass on to reject_message etc.] try: - changes = utils.parse_changes(filename) + changes = utils.parse_changes(filename, 0) except utils.cant_open_exc: reject_message = "Rejected: can't read changes file '%s'.\n" % (filename) return 0; @@ -182,9 +182,6 @@ def check_changes(filename): reject_message = "Rejected: Missing field `%s' in changes file.\n" % (i) return 0 # Avoid errors during later tests - # Fix the Maintainer: field to be RFC822 compatible - (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"]) - # Override the Distribution: field if appropriate if Cnf["Dinstall::Options::Override-Distribution"] != "": reject_message = reject_message + "Warning: Distribution was overriden from %s to %s.\n" % (changes["distribution"], Cnf["Dinstall::Options::Override-Distribution"]) @@ -199,6 +196,17 @@ def check_changes(filename): for j in string.split(o): changes[i][j] = 1 + # Fix the Maintainer: field to be RFC822 compatible + (changes["maintainer822"], changes["maintainername"], changes["maintaineremail"]) = utils.fix_maintainer (changes["maintainer"]) + + # Fix the Changed-By: field to be RFC822 compatible; if it exists. + (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]) = utils.fix_maintainer(changes.get("changed-by","")); + + # For source uploads the Changed-By field wins; otherwise Maintainer wins. + if changes["architecture"].has_key("source"): + changes["uploader822"] = "To: %s\nCc: %s" % (changes["changedby822"], changes["maintainer822"]); + # changes["uploadername"], changes["uploaderemail"]) = (changes["changedby822"], changes["changedbyname"], changes["changedbyemail"]); + # Ensure all the values in Closes: are numbers if changes.has_key("closes"): for i in changes["closes"].keys(): @@ -450,13 +458,16 @@ def check_dsc (): for file in files.keys(): if files[file]["type"] == "dsc": try: - dsc = utils.parse_changes(file) + dsc = utils.parse_changes(file, 1) except utils.cant_open_exc: reject_message = reject_message + "Rejected: can't read changes file '%s'.\n" % (filename) return 0; except utils.changes_parse_error_exc, line: reject_message = reject_message + "Rejected: error parsing changes file '%s', can't grok: %s.\n" % (filename, line) return 0; + except utils.invalid_dsc_format_exc, line: + reject_message = reject_message + "Rejected: syntax error in .dsc file '%s', line %s.\n" % (filename, line) + return 0; try: dsc_files = utils.build_file_list(dsc, 1) except utils.no_files_exc: diff --git a/melanie b/melanie index dfaaa12e..c76d7015 100755 --- a/melanie +++ b/melanie @@ -2,7 +2,7 @@ # General purpose archive tool for ftpmaster # Copyright (C) 2000 James Troup -# $Id: melanie,v 1.2 2001-01-16 21:52:37 troup Exp $ +# $Id: melanie,v 1.3 2001-01-28 09:06:44 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -184,7 +184,7 @@ def main (): for i in source_packages.keys(): filename = string.join(source_packages[i], '/'); try: - dsc = utils.parse_changes(filename); + dsc = utils.parse_changes(filename, 0); except utils.cant_open_exc: sys.stderr.write("W: couldn't open '%s'.\n" % (filename)); continue; diff --git a/shania b/shania index b12fb0ab..7a988901 100755 --- a/shania +++ b/shania @@ -2,7 +2,7 @@ # Clean incoming of old unused files # Copyright (C) 2000 James Troup -# $Id: shania,v 1.1 2000-12-13 03:18:50 troup Exp $ +# $Id: shania,v 1.2 2001-01-28 09:06:44 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -93,7 +93,7 @@ def main (): # Proces all .changes and .dsc files. for changes_filename in changes_files: try: - changes = utils.parse_changes(changes_filename) + changes = utils.parse_changes(changes_filename, 0) except: continue; try: @@ -105,7 +105,7 @@ def main (): for file in files.keys(): if re_isdsc.search(file) != None: try: - dsc = utils.parse_changes(file) + dsc = utils.parse_changes(file, 0) except: continue; try: diff --git a/tea b/tea index dc8aade0..ff29e8d2 100755 --- a/tea +++ b/tea @@ -2,7 +2,7 @@ # Sanity check the database # Copyright (C) 2000 James Troup -# $Id: tea,v 1.3 2001-01-16 21:52:37 troup Exp $ +# $Id: tea,v 1.4 2001-01-28 09:06:44 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -51,24 +51,9 @@ def process_dir (arg, dirname, filenames): if os.path.isfile(filename) and not os.path.islink(filename) and not db_files.has_key(filename) and not excluded.has_key(filename): waste = waste + os.stat(filename)[stat.ST_SIZE]; print filename - ################################################################################ -def main (): - global Cnf, projectB, db_files, waste, excluded; - - apt_pkg.init(); - - Cnf = apt_pkg.newConfiguration(); - apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); - - Arguments = [('d',"debug","Tea::Options::Debug", "IntVal"), - ('h',"help","Tea::Options::Help"), - ('v',"version","Tea::Options::Version")]; - - apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); - projectB = pg.connect('projectb', 'localhost'); - db_access.init(Cnf, projectB); +def check_files(): print "Building list of Database files..."; @@ -94,6 +79,49 @@ def main (): print print "%s wasted..." % (utils.size_type(waste)); +################################################################################ + +def check_dscs(): + count = 0; + suite = 'unstable'; + for component in Cnf.SubTree("Component").List(): + if component == "mixed": + continue; + component = string.lower(component); + list_filename = '%s%s_%s_source.list' % (Cnf["Dir::ListsDir"], suite, component); + list_file = utils.open_file(list_filename, 'r'); + for line in list_file.readlines(): + file = line[:-1]; + try: + utils.parse_changes(file, 1); + except utils.invalid_dsc_format_exc, line: + sys.stderr.write("E: syntax error in .dsc file '%s', line %s.\n" % (file, line)); + count = count + 1; + + if count: + sys.stderr.write("Found %s invalid .dsc files.\n" % (count)); + +################################################################################ + +def main (): + global Cnf, projectB, db_files, waste, excluded; + + apt_pkg.init(); + + Cnf = apt_pkg.newConfiguration(); + apt_pkg.ReadConfigFileISC(Cnf,utils.which_conf_file()); + + Arguments = [('d',"debug","Tea::Options::Debug", "IntVal"), + ('h',"help","Tea::Options::Help"), + ('v',"version","Tea::Options::Version")]; + + apt_pkg.ParseCommandLine(Cnf,Arguments,sys.argv); + projectB = pg.connect('projectb', 'localhost'); + db_access.init(Cnf, projectB); + + check_dscs(); + #check_files(); + ####################################################################################### if __name__ == '__main__': diff --git a/test/001/1.dsc b/test/001/1.dsc new file mode 100644 index 00000000..dfdd92f7 --- /dev/null +++ b/test/001/1.dsc @@ -0,0 +1,22 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +Format: 1.0 +Source: amaya +Version: 3.2.1-1 +Binary: amaya +Maintainer: Steve Dunham +Architecture: any +Standards-Version: 2.4.0.0 +Files: + 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz + da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.2 (GNU/Linux) +Comment: For info see http://www.gnupg.org + +iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ +rhYnRmVuNMa8oYSvL4hl/Yw= +=EFAA +-----END PGP SIGNATURE----- diff --git a/test/001/2.dsc b/test/001/2.dsc new file mode 100644 index 00000000..a6c9d85a --- /dev/null +++ b/test/001/2.dsc @@ -0,0 +1,21 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +Format: 1.0 +Source: amaya +Version: 3.2.1-1 +Binary: amaya +Maintainer: Steve Dunham +Architecture: any +Standards-Version: 2.4.0.0 +Files: + 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz + da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.2 (GNU/Linux) +Comment: For info see http://www.gnupg.org + +iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ +rhYnRmVuNMa8oYSvL4hl/Yw= +=EFAA +-----END PGP SIGNATURE----- diff --git a/test/001/3.dsc b/test/001/3.dsc new file mode 100644 index 00000000..211340ed --- /dev/null +++ b/test/001/3.dsc @@ -0,0 +1,21 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 +Format: 1.0 +Source: amaya +Version: 3.2.1-1 +Binary: amaya +Maintainer: Steve Dunham +Architecture: any +Standards-Version: 2.4.0.0 +Files: + 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz + da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.2 (GNU/Linux) +Comment: For info see http://www.gnupg.org + +iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ +rhYnRmVuNMa8oYSvL4hl/Yw= +=EFAA +-----END PGP SIGNATURE----- diff --git a/test/001/4.dsc b/test/001/4.dsc new file mode 100644 index 00000000..91e361f7 --- /dev/null +++ b/test/001/4.dsc @@ -0,0 +1,19 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 +Format: 1.0 +Source: amaya +Version: 3.2.1-1 +Binary: amaya +Maintainer: Steve Dunham +Architecture: any +Standards-Version: 2.4.0.0 +Files: + 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz + da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.2 (GNU/Linux) +Comment: For info see http://www.gnupg.org +iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ +rhYnRmVuNMa8oYSvL4hl/Yw= +=EFAA +-----END PGP SIGNATURE----- diff --git a/test/001/5.dsc b/test/001/5.dsc new file mode 100644 index 00000000..db9d8d30 --- /dev/null +++ b/test/001/5.dsc @@ -0,0 +1,23 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +Format: 1.0 +Source: amaya +Version: 3.2.1-1 +Binary: amaya +Maintainer: Steve Dunham +Architecture: any +Standards-Version: 2.4.0.0 +Files: + 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz + da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz + + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.2 (GNU/Linux) +Comment: For info see http://www.gnupg.org + +iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ +rhYnRmVuNMa8oYSvL4hl/Yw= +=EFAA +-----END PGP SIGNATURE----- diff --git a/test/001/6.dsc b/test/001/6.dsc new file mode 100644 index 00000000..ae36d643 --- /dev/null +++ b/test/001/6.dsc @@ -0,0 +1,23 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + + +Format: 1.0 +Source: amaya +Version: 3.2.1-1 +Binary: amaya +Maintainer: Steve Dunham +Architecture: any +Standards-Version: 2.4.0.0 +Files: + 07f95f92b7cb0f12f7cf65ee5c5fbde2 4532418 amaya_3.2.1.orig.tar.gz + da06b390946745d9efaf9e7df8e05092 4817 amaya_3.2.1-1.diff.gz + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.2 (GNU/Linux) +Comment: For info see http://www.gnupg.org + +iD8DBQE5j091iPgEjVqvb1kRAvFtAJ0asUAaac6ebfR3YeaH16HjL7F3GwCfV+AQ +rhYnRmVuNMa8oYSvL4hl/Yw= +=EFAA +-----END PGP SIGNATURE----- diff --git a/test/001/test.py b/test/001/test.py new file mode 100755 index 00000000..ffd7055b --- /dev/null +++ b/test/001/test.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +# Check utils.parse_changes()'s .dsc file validation +# Copyright (C) 2000 James Troup +# $Id: test.py,v 1.1 2001-01-28 09:06:44 troup Exp $ + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import os, sys + +sys.path.append(os.path.abspath('../../')); + +import utils + +################################################################################ + +def fail(message): + sys.stderr.write("%s\n" % (message)); + sys.exit(1); + +################################################################################ + +def main (): + # Valid .dsc + utils.parse_changes('1.dsc',1); + + # Missing blank line before signature body + try: + utils.parse_changes('2.dsc',1); + except utils.invalid_dsc_format_exc, line: + if line != 14: + fail("Incorrect line number ('%s') for test #2." % (line)); + else: + fail("Test #2 wasn't recognised as invalid."); + + # Missing blank line after signature header + try: + utils.parse_changes('3.dsc',1); + except utils.invalid_dsc_format_exc, line: + if line != 14: + fail("Incorrect line number ('%s') for test #3." % (line)); + else: + fail("Test #3 wasn't recognised as invalid."); + + # No blank lines at all + try: + utils.parse_changes('4.dsc',1); + except utils.invalid_dsc_format_exc, line: + if line != 19: + fail("Incorrect line number ('%s') for test #4." % (line)); + else: + fail("Test #4 wasn't recognised as invalid."); + + # Extra blank line before signature body + try: + utils.parse_changes('5.dsc',1); + except utils.invalid_dsc_format_exc, line: + if line != 15: + fail("Incorrect line number ('%s') for test #5." % (line)); + else: + fail("Test #5 wasn't recognised as invalid."); + + # Extra blank line after signature header + try: + utils.parse_changes('6.dsc',1); + except utils.invalid_dsc_format_exc, line: + if line != 5: + fail("Incorrect line number ('%s') for test #6." % (line)); + else: + fail("Test #6 wasn't recognised as invalid."); + + # Valid .dsc ; ignoring errors + utils.parse_changes('1.dsc', 0); + + # Invalid .dsc ; ignoring errors + utils.parse_changes('2.dsc', 0); + +################################################################################ + +if __name__ == '__main__': + main() diff --git a/utils.py b/utils.py index 8058d3be..2f42cb39 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,6 @@ # Utility functions # Copyright (C) 2000 James Troup -# $Id: utils.py,v 1.12 2001-01-25 06:00:07 troup Exp $ +# $Id: utils.py,v 1.13 2001-01-28 09:06:44 troup Exp $ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,6 +25,7 @@ re_arch_from_filename = re.compile(r"/binary-[^/]+/") re_extract_src_version = re.compile (r"(\S+)\s*\((.*)\)") changes_parse_error_exc = "Can't parse line in .changes file"; +invalid_dsc_format_exc = "Invalid .dsc file"; nk_format_exc = "Unknown Format: in .changes file"; no_files_exc = "No Files: field in .dsc file."; cant_open_exc = "Can't read file."; @@ -77,15 +78,58 @@ def extract_component_from_section(section): ###################################################################################### -def parse_changes(filename): +# dsc_whitespace_rules turns on strict format checking to avoid +# allowing in source packages which are unextracable by the +# inappropriately fragile dpkg-source. +# +# The rules are: +# +# +# o The PGP header consists of "-----BEGIN PGP SIGNED MESSAGE-----" +# followed by any PGP header data and must end with a blank line. +# +# o The data section must end with a blank line and must be followed by +# "-----BEGIN PGP SIGNATURE-----". + +def parse_changes(filename, dsc_whitespace_rules): changes_in = open_file(filename,'r'); - error = "" + error = ""; changes = {}; lines = changes_in.readlines(); + + # Reindex by line number so we can easily verify the format of + # .dsc files... + index = 0; + indexed_lines = {}; for line in lines: + index = index + 1; + indexed_lines[index] = line[:-1]; + + inside_signature = 0; + + indices = indexed_lines.keys() + index = 0; + while index < max(indices): + index = index + 1; + line = indexed_lines[index]; + if line == "": + if dsc_whitespace_rules: + index = index + 1; + if index > max(indices): + raise invalid_dsc_format_exc, index; + line = indexed_lines[index]; + if not re.match('^-----BEGIN PGP SIGNATURE', line): + raise invalid_dsc_format_exc, index; + inside_signature = 0; + break; if re.match('^-----BEGIN PGP SIGNATURE', line): break; - if re.match(r'^\s*$|^-----BEGIN PGP SIGNED MESSAGE', line): + if re.match(r'^-----BEGIN PGP SIGNED MESSAGE', line): + if dsc_whitespace_rules: + inside_signature = 1; + while index < max(indices) and line != "": + index = index + 1; + line = indexed_lines[index]; continue; slf = re.match(r'^(\S*)\s*:\s*(.*)', line); if slf: @@ -101,10 +145,16 @@ def parse_changes(filename): changes[field] = changes[field] + mlf.groups()[0] + '\n'; continue; error = error + line; + + if dsc_whitespace_rules and inside_signature: + raise invalid_dsc_format_exc, index; + changes_in.close(); changes["filecontents"] = string.join (lines, ""); + if error != "": raise changes_parse_error_exc, error; + return changes; ###################################################################################### -- 2.39.5