Improve dak/import.py.

diff --git a/dak/import.py b/dak/import.py
index 0e11647dbd2655ba83306253f7235a51bfcb8b8d..c9722933d55d56583cc3cb1fed7d73cb54f10791 100644 (file)
--- a/dak/import.py
+++ b/dak/import.py
@@ -23,6 +23,7 @@ import sys
 from daklib.dbconn import *
 import daklib.archive
 import daklib.config
+import daklib.daklog
 import daklib.upload
 import daklib.regexes
 
@@ -30,33 +31,44 @@ def usage():
     print """Usage:
 
 dak import <suite> <component> <files...>
+dak import -D|--dump <file> <suite> <component>
+dak import -E|--export-dump <suite> <component>
+
+WARNING: This command does no sanity checks. Only use it on trusted packages.
 
 Options:
   -h, --help:             show this help message
   -a, --add-overrides:    add missing overrides automatically
   -c, --changed-by:       Changed-By for imported source packages
                           (default: maintainer)
+  -D, --dump <file>:      Import all files listed in <file>. The format
+                          is described below.
+  -E, --export-dump:      Export list of files in the format required
+                          by dak import --dump.
   -s, --ignore-signature: ignore signature for imported source packages
 
-WARNING: This command does no sanity checks. Only use it on trusted packages.
+File format used by --dump:
+
+  <filename>:<md5>:<sha1>:<sha256>:[<fingerprint>]:[<changed-by>]
 """
 
-def import_source(transaction, suite, component, directory, filename,
-                  changed_by=None, keyrings=None, require_signature=True,
-                  add_overrides=False):
+def import_source(log, transaction, suite, component, directory, hashed_file,
+                  fingerprint=None, changed_by=None,
+                  keyrings=None, require_signature=True, add_overrides=False):
     if keyrings is None:
         keyrings = []
+    filename = hashed_file.filename
     session = transaction.session
 
-    source = daklib.upload.Source.from_file(directory, filename, keyrings, require_signature)
-    fingerprint = None
+    source = daklib.upload.Source(directory, [hashed_file], keyrings, require_signature)
     if source.valid_signature:
-        fingerprint = session.query(Fingerprint).filter_by(fingerprint=source.primary_fingerprint).one()
+        fingerprint = session.query(Fingerprint).filter_by(fingerprint=source.primary_fingerprint).first()
     if changed_by is None:
         changed_by = source.dsc['Maintainer']
     db_changed_by = get_or_set_maintainer(changed_by, session)
 
     transaction.install_source(directory, source, suite, component, db_changed_by, fingerprint=fingerprint)
+    log.log(['import-source', suite.suite_name, component.component_name, filename])
 
     if add_overrides and not session.query(Override).filter_by(suite=suite.get_overridesuite(), component=component, package=source.dsc['Source']).join(OverrideType).filter(OverrideType.overridetype == 'dsc').first():
         overridetype = session.query(OverrideType).filter_by(overridetype='dsc').one()
@@ -70,12 +82,15 @@ def import_source(transaction, suite, component, directory, filename,
         override = Override(package=control.dsc['Source'], suite=overridesuite, component=component,
                             section=section, priority=priority, overridetype=overridetype)
         session.add(override)
+        log.log(['add-source-override', suite.suite_name, component.component_name, control.dsc['Source'], section.section, priority.priority])
 
-def import_binary(transaction, suite, component, directory, filename, add_overrides=False):
+def import_binary(log, transaction, suite, component, directory, hashed_file, fingerprint=None, add_overrides=False):
+    filename = hashed_file.filename
     session = transaction.session
 
-    binary = daklib.upload.Binary.from_file(directory, filename)
-    transaction.install_binary(directory, binary, suite, component)
+    binary = daklib.upload.Binary(directory, hashed_file)
+    transaction.install_binary(directory, binary, suite, component, fingerprint=fingerprint)
+    log.log(['import-binary', suite.suite_name, component.component_name, filename])
 
     if add_overrides and not session.query(Override).filter_by(suite=suite.get_overridesuite(), component=component, package=binary.control['Package']).join(OverrideType).filter(OverrideType.overridetype == binary.type).first():
         overridetype = session.query(OverrideType).filter_by(overridetype=binary.type).one()
@@ -86,22 +101,80 @@ def import_binary(transaction, suite, component, directory, filename, add_overri
         override = Override(package=binary.control['Package'], suite=overridesuite, component=component,
                             section=section, priority=priority, overridetype=overridetype)
         session.add(override)
+        log.log(['add-binary-override', suite.suite_name, component.component_name, binary.control['Package'], section.section, priority.priority])
 
-def import_file(transaction, suite, component, directory, filename,
-                changed_by=None, keyrings=None, require_signature=False,
+def import_file(log, transaction, suite, component, directory, hashed_file,
+                fingerprint=None, changed_by=None, keyrings=None, require_signature=True,
                 add_overrides = False):
-    if keyrings is None:
-        keyrings = []
-
+    filename = hashed_file.filename
     if daklib.regexes.re_file_binary.match(filename):
-        import_binary(transaction, suite, component, directory, filename, add_overrides)
+        import_binary(log, transaction, suite, component, directory, hashed_file,
+                      fingerprint=fingerprint, add_overrides=add_overrides)
     elif daklib.regexes.re_file_dsc.match(filename):
-        import_source(transaction, suite, component, directory, filename,
-                      changed_by=changed_by, keyrings=keyrings,
+        import_source(log, transaction, suite, component, directory, hashed_file,
+                      fingerprint=fingerprint, changed_by=changed_by, keyrings=keyrings,
                       require_signature=require_signature, add_overrides=add_overrides)
     else:
         raise Exception('File is neither source nor binary package: {0}'.format(filename))
 
+def import_dump(log, transaction, suite, component, fh,
+                keyrings=None, require_signature=True, add_overrides=False):
+    session = transaction.session
+    for line in fh:
+        path, size, md5, sha1, sha256, fpr, changed_by = line.strip().split(':', 6)
+
+        if not changed_by:
+            changed_by = None
+        fingerprint = None
+        if fpr:
+            fingerprint = session.query(Fingerprint).filter_by(fingerprint=fpr).first()
+            if fingerprint is None:
+                print 'W: {0}: unknown fingerprint {1}'.format(filename, fpr)
+
+        directory, filename = os.path.split(os.path.abspath(path))
+        hashed_file = daklib.upload.HashedFile(filename, long(size), md5, sha1, sha256)
+        hashed_file.check(directory)
+
+        import_file(log, transaction, suite, component, directory, hashed_file,
+                    fingerprint=fingerprint, changed_by=changed_by,
+                    keyrings=keyrings, require_signature=require_signature, add_overrides=add_overrides)
+
+_export_query = r"""
+WITH
+tmp AS
+  (SELECT 1 AS order, s.file AS file_id, s.sig_fpr AS fingerprint_id, s.changedby AS changed_by, sa.suite AS suite_id
+     FROM source s
+     JOIN src_associations sa ON sa.source = s.id
+   UNION
+   SELECT 2 AS order, b.file AS file_id, b.sig_fpr AS fingerprint_id, NULL, ba.suite AS suite_id
+     FROM binaries b
+     JOIN bin_associations ba ON ba.bin = b.id
+  )
+
+SELECT
+  f.filename, f.size::TEXT, f.md5sum, f.sha1sum, f.sha256sum, COALESCE(fpr.fingerprint, ''), COALESCE(m.name, '')
+FROM files f
+JOIN tmp ON f.id = tmp.file_id
+JOIN suite ON suite.id = tmp.suite_id
+JOIN files_archive_map fam ON fam.file_id = f.id AND fam.archive_id = suite.archive_id
+LEFT JOIN fingerprint fpr ON fpr.id = tmp.fingerprint_id
+LEFT JOIN maintainer m ON m.id = tmp.changed_by
+
+WHERE
+  suite.id = :suite_id
+  AND fam.component_id = :component_id
+
+ORDER BY tmp.order, f.filename;
+"""
+
+def export_dump(transaction, suite, component):
+    session = transaction.session
+    query = session.execute(_export_query,
+                            {'suite_id': suite.suite_id,
+                             'component_id': component.component_id})
+    for row in query:
+        print ":".join(row)
+
 def main(argv=None):
     if argv is None:
         argv = sys.argv
@@ -110,6 +183,8 @@ def main(argv=None):
         ('h', 'help', 'Import::Options::Help'),
         ('a', 'add-overrides', 'Import::Options::AddOverrides'),
         ('c', 'changed-by', 'Import::Options::ChangedBy', 'HasArg'),
+        ('D', 'dump', 'Import::Options::Dump', 'HasArg'),
+        ('E', 'export-dump', 'Import::Options::Export'),
         ('s', 'ignore-signature', 'Import::Options::IgnoreSignature'),
         ]
 
@@ -118,18 +193,19 @@ def main(argv=None):
     argv = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
     options = cnf.subtree('Import::Options')
 
-    if 'Help' in options or len(argv) < 3:
+    if 'Help' in options or len(argv) < 2:
         usage()
         sys.exit(0)
 
     suite_name = argv[0]
     component_name = argv[1]
-    files = argv[2:]
 
     add_overrides = options.find_b('AddOverrides')
     require_signature = not options.find_b('IgnoreSignature')
     changed_by = options.find('ChangedBy') or None
 
+    log = daklib.daklog.Logger('import')
+
     with daklib.archive.ArchiveTransaction() as transaction:
         session = transaction.session
         suite = session.query(Suite).filter_by(suite_name=suite_name).one()
@@ -137,14 +213,27 @@ def main(argv=None):
         keyrings = session.query(Keyring).filter_by(active=True).order_by(Keyring.priority)
         keyring_files = [ k.keyring_name for k in keyrings ]
 
-        for f in files:
-            directory, filename = os.path.split(os.path.abspath(f))
-            import_file(transaction, suite, component, directory, filename,
-                        changed_by=changed_by,
-                        keyrings=keyring_files, require_signature=require_signature,
-                        add_overrides=add_overrides)
-
-        transaction.commit()
+        dump = options.find('Dump') or None
+        if options.find_b('Export'):
+            export_dump(transaction, suite, component)
+            transaction.rollback()
+        elif dump is not None:
+            with open(dump, 'r') as fh:
+                import_dump(log, transaction, suite, component, fh, keyring_files,
+                            require_signature=require_signature, add_overrides=add_overrides)
+            transaction.commit()
+        else:
+            files = argv[2:]
+            for f in files:
+                directory, filename = os.path.split(os.path.abspath(f))
+                hashed_file = daklib.upload.HashedFile.from_file(directory, filename)
+                import_file(log, transaction, suite, component, directory, hashed_file,
+                            changed_by=changed_by,
+                            keyrings=keyring_files, require_signature=require_signature,
+                            add_overrides=add_overrides)
+            transaction.commit()
+
+    log.close()
 
 if __name__ == '__main__':
     main()