From 383c2803acef3821323048259119ef2cf01e5d83 Mon Sep 17 00:00:00 2001 From: Raphael Hertzog Date: Tue, 18 Mar 2008 11:55:11 +0100 Subject: [PATCH] Allow inclusion of binary files outside of the debian subdirectory * scripts/Dpkg/ErrorHandling.pm (errormsg): New function displaying an error message to stderr without dying. * scripts/Dpkg/Source/Patch.pm (add_diff_file): Add a hook in the handling of modified binary files. Don't fail unconditionnaly when a binary file is detected. Return a value indicating if the diff call succeeded. (add_diff_directory): Don't display some warnings (that apply only to files whose changes can be represented in diff) for modified binary files. (_fail_with_msg, _fail_not_same_type): Use new errormsg() function. * scripts/Dpkg/Source/Package/V2_0.pm (do_extract): Extract the debian tarball "in-place" so that it can contain other files besides the debian sub-directory. (do_build): Handle modified binary files. They are added to the debian tarball if --include-binaries is used, or if they are whitelisted in debian/source/include-binaries. The usage of --include-binaries will modify debian/source/include-binaries if needed. * man/dpkg-source.1: Update manual page accordingly. --- man/dpkg-source.1 | 24 ++++++++--- scripts/Dpkg/ErrorHandling.pm | 11 +++-- scripts/Dpkg/Source/Package/V2_0.pm | 64 ++++++++++++++++++++++++++--- scripts/Dpkg/Source/Patch.pm | 52 ++++++++++++++--------- 4 files changed, 117 insertions(+), 34 deletions(-) diff --git a/man/dpkg-source.1 b/man/dpkg-source.1 index f03f4f88..f9a30ec0 100644 --- a/man/dpkg-source.1 +++ b/man/dpkg-source.1 @@ -323,8 +323,11 @@ tarballs (\fB.orig-\fP\fIcomponent\fP\fB.tar.\fP\fIext\fP). The main original tarball is extracted first, then all additional original tarballs are extracted in subdirectories named after the \fIcomponent\fP part of their filename (any pre-existing directory is replaced). The -debian tarball is extracted in a \fBdebian\fP subdirectory (and replaces -any pre-existing \fBdebian\fP directory). +debian tarball is extracted on top of the source directory after prior +removal of any pre-existing \fBdebian\fP directory). Note that the +debian tarball must contain a \fBdebian\fP sub-directory but it +can also contain binary files outside of that directory (see +\fB\-\-include\-binaries\fP option). .PP All patches in \fBdebian/patches\fP matching the perl regular expression \fB[\\w\-]+\fP are then applied in alphabetical order. The timestamp of @@ -339,8 +342,14 @@ debian directory is copied over in the temporary directory, and all patches except \fBzz_debian-diff-auto\fP are applied. The temporary directory is compared to the source package directory and the diff (if non-empty) is stored in -\fBdebian/patches/zz_debian-diff-auto\fP. The updated debian directory is -then used to regenerate the debian tarball. +\fBdebian/patches/zz_debian-diff-auto\fP. Any change on a binary file +is not representable in a diff and will thus lead to a failure unless +the maintainer deliberately decided to include that modified binary +file in the debian tarball (by listing it in +\fBdebian/source/include-binaries\fP). + +The updated debian directory and the list of modified binaries is then +used to regenerate the debian tarball. The automatically generated diff doesn't include changes on VCS specific files as well as many temporary files (see default value associated to @@ -364,12 +373,17 @@ in the 1.0 format. It supports all compression methods and will ignore by default any VCS specific files and directories as well as many temporary files (see default value associated to \fB-I\fP option in the \fB\-\-help\fP output). +.TP +.B \-\-include\-binaries +Add all modified binaries in the debian tarball. Also add them to +\fBdebian/source/include-binaries\fP: they will be added by default +in subsequent builds and this option is thus no more needed. . .SS Format: 3.0 (quilt) This is a variant of the 2.0 format. The differences concern the management of the patches. This format uses an explicit list of patches contained in \fBdebian/patches/debian.series\fP or -\fBdebian/patches/series\fP. The patches can remove files. +\fBdebian/patches/series\fP. .PP .B Extracting .PP diff --git a/scripts/Dpkg/ErrorHandling.pm b/scripts/Dpkg/ErrorHandling.pm index 8710abd9..2cb7bb3b 100644 --- a/scripts/Dpkg/ErrorHandling.pm +++ b/scripts/Dpkg/ErrorHandling.pm @@ -4,9 +4,9 @@ use Dpkg; use Dpkg::Gettext; use base qw(Exporter); -our @EXPORT_OK = qw(warning warnerror error failure unknown syserr internerr - subprocerr usageerr syntaxerr report info - $warnable_error $quiet_warnings); +our @EXPORT_OK = qw(warning warnerror error errormsg failure unknown + syserr internerr subprocerr usageerr syntaxerr report + info $warnable_error $quiet_warnings); our $warnable_error = 1; our $quiet_warnings = 0; @@ -54,6 +54,11 @@ sub error($;@) die report(_g("error"), @_); } +sub errormsg($;@) +{ + print STDERR report(_g("error"), @_); +} + sub internerr($;@) { die report(_g("internal error"), @_); diff --git a/scripts/Dpkg/Source/Package/V2_0.pm b/scripts/Dpkg/Source/Package/V2_0.pm index 08df0f1e..0a24da2b 100644 --- a/scripts/Dpkg/Source/Package/V2_0.pm +++ b/scripts/Dpkg/Source/Package/V2_0.pm @@ -23,7 +23,7 @@ use base 'Dpkg::Source::Package'; use Dpkg; use Dpkg::Gettext; -use Dpkg::ErrorHandling qw(error syserr warning usageerr subprocerr info); +use Dpkg::ErrorHandling qw(error errormsg syserr warning usageerr subprocerr info); use Dpkg::Compression; use Dpkg::Source::Archive; use Dpkg::Source::Patch; @@ -44,6 +44,8 @@ sub init_options { unless exists $self->{'options'}{'include_removal'}; $self->{'options'}{'include_timestamp'} = 0 unless exists $self->{'options'}{'include_timestamp'}; + $self->{'options'}{'include_binaries'} = 0 + unless exists $self->{'options'}{'include_binaries'}; } sub parse_cmdline_option { @@ -54,6 +56,9 @@ sub parse_cmdline_option { } elsif ($opt =~ /^--include-timestamp$/) { $self->{'options'}{'include_timestamp'} = 1; return 1; + } elsif ($opt =~ /^--include-binaries$/) { + $self->{'options'}{'include_binaries'} = 1; + return 1; } return 0; } @@ -96,7 +101,7 @@ sub do_extract { # Extract main tarball info(_g("unpacking %s"), $tarfile); my $tar = Dpkg::Source::Archive->new(filename => "$dscdir$tarfile"); - $tar->extract($newdirectory); + $tar->extract($newdirectory, no_fixperms => 1); # Extract additional orig tarballs foreach my $subdir (keys %origtar) { @@ -107,14 +112,14 @@ sub do_extract { erasedir("$newdirectory/$subdir"); } $tar = Dpkg::Source::Archive->new(filename => "$dscdir$file"); - $tar->extract("$newdirectory/$subdir"); + $tar->extract("$newdirectory/$subdir", no_fixperms => 1); } # Extract debian tarball after removing the debian directory info(_g("unpacking %s"), $debianfile); erasedir("$newdirectory/debian"); $tar = Dpkg::Source::Archive->new(filename => "$dscdir$debianfile"); - $tar->extract("$newdirectory/debian"); + $tar->extract($newdirectory, in_place => 1); # Apply patches (in a separate method as it might be overriden) $self->apply_patches($newdirectory); @@ -176,6 +181,7 @@ sub do_build { my ($self, $dir) = @_; my @argv = @{$self->{'options'}{'ARGV'}}; my @tar_ignore = map { "--exclude=$_" } @{$self->{'options'}{'tar_ignore'}}; + my $include_binaries = $self->{'options'}{'include_binaries'}; my ($dirname, $updir) = fileparse($dir); if (scalar(@argv)) { @@ -232,6 +238,36 @@ sub do_build { # Apply all patches except the last automatic one $self->apply_patches($tmp, 1); + # Prepare handling of binary files + my %auth_bin_files; + my $incbin_file = File::Spec->catfile($dir, "debian", "source", "include-binaries"); + if (-f $incbin_file) { + open(INC, "<", $incbin_file) || syserr(_g("can't read %s"), $incbin_file); + while(defined($_ = )) { + chomp; s/^\s*//; s/\s*$//; + next if /^#/; + $auth_bin_files{$_} = 1; + } + close(INC); + } + my @binary_files; + my $handle_binary = sub { + my ($self, $old, $new) = @_; + my $relfn = File::Spec->abs2rel($new, $dir); + # Include binaries if they are whitelisted or if + # --include-binaries has been given + if ($include_binaries or $auth_bin_files{$relfn}) { + push @binary_files, $relfn; + } else { + errormsg(_g("cannot represent change to %s: %s"), $new, + _g("binary file contents changed")); + errormsg(_g("add %s in debian/source/include-binaries if you want" . + " to store the modified binary in the debian tarball"), + $relfn); + $self->register_error(); + } + }; + # Create a patch my ($difffh, $tmpdiff) = tempfile("$basenamerev.diff.XXXXXX", DIR => $updir, UNLINK => 0); @@ -240,11 +276,12 @@ sub do_build { compression => "none"); $diff->create(); $diff->add_diff_directory($tmp, $dir, basedirname => $basedirname, - %{$self->{'diff_options'}}); + %{$self->{'diff_options'}}, handle_binary_func => $handle_binary); error(_g("unrepresentable changes to source")) if not $diff->finish(); # The previous auto-patch must be removed, it has not been used and it # will be recreated if it's still needed - my $autopatch = "$dir/debian/patches/" . $self->get_autopatch_name(); + my $autopatch = File::Spec->catfile($dir, "debian", "patches", + $self->get_autopatch_name()); if (-e $autopatch) { unlink($autopatch) || syserr(_g("cannot remove %s"), $autopatch); } @@ -263,12 +300,27 @@ sub do_build { erasedir($tmp); pop @Dpkg::Exit::handlers; + # Update debian/source/include-binaries if needed + if (scalar(@binary_files) and $include_binaries) { + mkpath(File::Spec->catdir($dir, "debian", "source")); + open(INC, ">>", $incbin_file) || syserr(_g("can't write %s"), $incbin_file); + foreach my $binary (@binary_files) { + unless ($auth_bin_files{$binary}) { + print INC "$binary\n"; + info(_g("adding %s to %s"), $binary, "debian/source/include-binaries"); + } + } + close(INC); + } # Create the debian.tar my $debianfile = "$basenamerev.debian.tar." . $self->{'options'}{'comp_ext'}; info(_g("building %s in %s"), $sourcepackage, $debianfile); $tar = Dpkg::Source::Archive->new(filename => $debianfile); $tar->create(options => \@tar_ignore, 'chdir' => $dir); $tar->add_directory("debian"); + foreach my $binary (@binary_files) { + $tar->add_file($binary); + } $tar->finish(); $self->add_file($debianfile); diff --git a/scripts/Dpkg/Source/Patch.pm b/scripts/Dpkg/Source/Patch.pm index 79e87b84..33bd559b 100644 --- a/scripts/Dpkg/Source/Patch.pm +++ b/scripts/Dpkg/Source/Patch.pm @@ -25,7 +25,7 @@ use Dpkg::Source::Compressor; use Dpkg::Compression; use Dpkg::Gettext; use Dpkg::IPC; -use Dpkg::ErrorHandling qw(error syserr warning subprocerr); +use Dpkg::ErrorHandling qw(error errormsg syserr warning subprocerr); use POSIX; use File::Find; @@ -59,6 +59,10 @@ sub create { sub add_diff_file { my ($self, $old, $new, %opts) = @_; $opts{"include_timestamp"} = 0 unless exists $opts{"include_timestamp"}; + my $handle_binary = $opts{"handle_binary_func"} || sub { + my ($self, $old, $new) = @_; + $self->_fail_with_msg($new, _g("binary file contents changed")); + }; # Default diff options my @options; if ($opts{"options"}) { @@ -94,9 +98,11 @@ sub add_diff_file { ); # Check diff and write it in patch file my $difflinefound = 0; + my $binary = 0; while (<$diffgen>) { if (m/^binary/i) { - $self->_fail_with_msg($new,_g("binary file contents changed")); + $binary = 1; + &$handle_binary($self, $old, $new); last; } elsif (m/^[-+\@ ]/) { $difflinefound++; @@ -115,11 +121,12 @@ sub add_diff_file { cmdline => "diff -u @options -- $old $new"); # Verify diff process ended successfully # Exit code of diff: 0 => no difference, 1 => diff ok, 2 => error + # Ignore error if binary content detected my $exit = WEXITSTATUS($?); - unless (WIFEXITED($?) && ($exit == 0 || $exit == 1)) { + unless (WIFEXITED($?) && ($exit == 0 || $exit == 1 || $binary)) { subprocerr(_g("diff on %s"), $new); } - return $exit; + return ($exit == 0 || $exit == 1); } sub add_diff_directory { @@ -163,6 +170,17 @@ sub add_diff_directory { $! == ENOENT || syserr(_g("cannot stat file %s"), "$old/$fn"); $old_file = '/dev/null'; + } elsif (not -f _) { + $self->_fail_not_same_type("$old/$fn", "$new/$fn"); + return; + } + + my $success = $self->add_diff_file($old_file, "$new/$fn", + label_old => "$basedir.orig/$fn", + label_new => "$basedir/$fn", + %opts); + + if ($success and ($old_file eq "/dev/null")) { if (not $size) { warning(_g("newly created empty file '%s' will not " . "be represented in diff"), $fn); @@ -177,15 +195,7 @@ sub add_diff_directory { "be represented in diff"), $mode, $fn); } } - } elsif (not -f _) { - $self->_fail_not_same_type("$old/$fn", "$new/$fn"); - return; } - - $self->add_diff_file($old_file, "$new/$fn", - label_old => "$basedir.orig/$fn", - label_new => "$basedir/$fn", - %opts); } elsif (-p _) { unless (-p "$old/$fn") { $self->_fail_not_same_type("$old/$fn", "$new/$fn"); @@ -240,21 +250,23 @@ sub finish { return not $self->{'errors'}; } +sub register_error { + my ($self) = @_; + $self->{'errors'}++; +} sub _fail_with_msg { my ($self, $file, $msg) = @_; - printf(STDERR _g("%s: cannot represent change to %s: %s")."\n", - $progname, $file, $msg); - $self->{'errors'}++; + errormsg(_g("cannot represent change to %s: %s"), $file, $msg); + $self->register_error(); } sub _fail_not_same_type { my ($self, $old, $new) = @_; my $old_type = get_type($old); my $new_type = get_type($new); - printf(STDERR _g("%s: cannot represent change to %s:\n". - "%s: new version is %s\n". - "%s: old version is %s\n"), - $progname, $new, $progname, $old_type, $progname, $new_type); - $self->{'errors'}++; + errormsg(_g("cannot represent change to %s:"), $new); + errormsg(_g(" new version is %s"), $old_type); + errormsg(_g(" old version is %s"), $new_type); + $self->register_error(); } # check diff for sanity, find directories to create as a side effect -- 2.39.5