Merge branch 'devel' of https://github.com/upx/upx into devel

.travis.yml

@@ -6,30 +6,106 @@ branches:
     - devel
     - travis
 
+matrix:
+  include:
+    - os: linux
+      compiler: clang
+      env: C=clang-m32 B=debug
+      addons:
+        apt:
+          packages:
+            - g++-multilib
+            - zlib1g-dev:i386
+    - os: linux
+      compiler: clang
+      env: C=clang-m64 B=debug
+    - os: linux
+      compiler: clang
+      env: C=clang-m32
+      addons:
+        apt:
+          packages:
+            - g++-multilib
+            - zlib1g-dev:i386
+    - os: linux
+      compiler: clang
+      env: C=clang-m64
+    - os: linux
+      compiler: gcc
+      env: C=gcc-m32
+      addons:
+        apt:
+          packages:
+            - g++-multilib
+            - zlib1g-dev:i386
+    - os: linux
+      compiler: gcc
+      env: C=gcc-m64
+    - os: linux
+      compiler: gcc-5
+      env: C=gcc-5-m32
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-5-multilib
+            - zlib1g-dev:i386
+    - os: linux
+      compiler: gcc-5
+      env: C=gcc-5-m64
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-5
+    - os: linux
+      compiler: gcc-6
+      env: C=gcc-6-m32
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-6-multilib
+            - zlib1g-dev:i386
+    - os: linux
+      compiler: gcc-6
+      env: C=gcc-6-m64
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-6
+    - os: linux
+      compiler: gcc-6
+      # FIXME: linking fails - need updated binutils?
+      env: C=gcc-6-m64 B=sanitize ALLOW_FAIL=1
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-6
+    - os: osx
+      compiler: clang
+      env: C=clang-m64 B=debug
+      #osx_image: xcode7
+    - os: osx
+      compiler: clang
+      # FIXME: linking fails with "ld: internal error: atom not found"
+      env: C=clang-m64 B=release ALLOW_FAIL=1
+      #osx_image: xcode7
+
 install:
   - DEPS_DIR="$TRAVIS_BUILD_DIR/deps"
   - mkdir -p "$DEPS_DIR" && cd "$DEPS_DIR"
   - wget --no-check-certificate -q -O - https://download.freenas.org/distfiles/ucl-1.03.tar.gz | tar -xz
+  - git clone https://github.com/upx/upx-testsuite
 
-os:
-  - linux
-#  - osx
-
-compiler:
-  - clang
-  - gcc
-
-env:
-  global:
-    - B=
-  matrix:
-    - B=make/debug
-    - B=make/release
-#    - B=make/scan-build
-
-before_script:
-  - export BUILD_DIR="$TRAVIS_BUILD_DIR/build"
-  - export BUILD_METHOD_AND_BUILD_TYPE="$B"
-  - export UPX_UCLDIR="$TRAVIS_BUILD_DIR/deps/ucl-1.03"
 script:
   - bash "$TRAVIS_BUILD_DIR/.travis_build.sh"
+
+# vim:set ts=2 sw=2 et:

.travis_build.sh

@@ -2,7 +2,26 @@
 ## vim:set ts=4 sw=4 et:
 set -e; set -o pipefail
 
-echo "BUILD_METHOD_AND_BUILD_TYPE='$BUILD_METHOD_AND_BUILD_TYPE'"
+cd /; cd "$TRAVIS_BUILD_DIR" || exit 1
+
+if test "X$B" = "X"; then B=release; fi
+BUILD_METHOD="$B"
+
+case $C in
+    clang-m32) CC="clang -m32"; CXX="clang++ -m32" ;;
+    clang-m64) CC="clang -m64"; CXX="clang++ -m64" ;;
+    gcc-m32) CC="gcc -m32"; CXX="g++ -m32" ;;
+    gcc-m64) CC="gcc -m64"; CXX="g++ -m64" ;;
+    gcc-5-m32) CC="gcc-5 -m32"; CXX="g++-5 -m32" ;;
+    gcc-5-m64) CC="gcc-5 -m64"; CXX="g++-5 -m64" ;;
+    gcc-6-m32) CC="gcc-6 -m32 -std=gnu90"; CXX="g++-6 -m32" ;;
+    gcc-6-m64) CC="gcc-6 -m64 -std=gnu90"; CXX="g++-6 -m64" ;;
+esac
+export CC CXX
+
+export UPX_UCLDIR="$TRAVIS_BUILD_DIR/deps/ucl-1.03"
+
+echo "BUILD_METHOD='$BUILD_METHOD'"
 echo "CC='$CC'"
 echo "CXX='$CXX'"
 echo "CPPFLAGS='$CPPFLAGS'"
@@ -14,34 +33,129 @@ echo "BUILD_DIR='$BUILD_DIR'"
 echo "UPX_UCLDIR='$UPX_UCLDIR'"
 #env | LC_ALL=C sort
 
-# build UCL
+echo "$CC --version"; $CC --version
-cd /
+echo "$CXX --version"; $CXX --version
+
+# whitespace
+if test "X$TRAVIS_OS_NAME" = "Xlinux"; then
+echo "Checking source code for whitespace violations..."
+find . \
+    -type d -name '.git' -prune -o \
+    -type d -name 'deps' -prune -o \
+    -type f -iname '*.bat' -prune -o \
+    -type f -iname '*.exe' -prune -o \
+    -type f -iname '*.pdf' -prune -o \
+    -type f -print0 | LC_ALL=C sort -z | \
+xargs -0r perl -n -e '
+    if (m,[\r\x1a],) { print "ERROR: DOS EOL detected $ARGV: $_"; exit(1); }
+    if (m,([ \t]+)$,) {
+        # allow exactly two trailing spaces for GitHub flavoured Markdown in .md files
+        if ($1 ne "  " || $ARGV !~ m,\.md$,) {
+            print "ERROR: trailing whitespace detected $ARGV: $_"; exit(1);
+        }
+    }
+    if (m,\t,) {
+       if ($ARGV =~ m,(^|/)\.gitmodules$,) { }
+       elsif ($ARGV =~ m,(^|/)make(file|vars),i) { }
+       elsif ($ARGV =~ m,/tmp/.*\.(disasm|dump)$,) { }
+       elsif ($ARGV =~ m,/src/stub/src/arch/.*\.S$,) { }
+       else { print "ERROR: hard TAB detected $ARGV: $_"; exit(1); }
+    }
+' || exit 1
+echo "  Done."
+fi # linux
+
 set -x
-cd "$UPX_UCLDIR"
+BUILD_DIR="$TRAVIS_BUILD_DIR/build"
+mkdir -p "$BUILD_DIR"
+
+# build UCL
+cd /; cd "$UPX_UCLDIR" || exit 1
 ./configure --enable-static --disable-shared
 make
 
-cd /
+# build UPX
-set -x
+cd /; cd "$BUILD_DIR" || exit 1
-mkdir -p "$BUILD_DIR"
+f="EXTRA_CPPFLAGS=-DUCL_NO_ASM"
-cd "$BUILD_DIR"
+make="make -f $TRAVIS_BUILD_DIR/src/Makefile $f"
-
+if test "X$ALLOW_FAIL" = "X1"; then
-case $BUILD_METHOD_AND_BUILD_TYPE in
+    echo "ALLOW_FAIL=$ALLOW_FAIL"
-make/debug)
+    set +e
-    make -f "$TRAVIS_BUILD_DIR/src/Makefile" USE_DEBUG=1
+fi
-    ;;
+case $BUILD_METHOD in
-make/release)
+debug)
-    make -f "$TRAVIS_BUILD_DIR/src/Makefile"
+    $make USE_DEBUG=1 ;;
-    ;;
+debug+sanitize)
-make/scan-build)
+    $make USE_DEBUG=1 USE_SANITIZE=1 ;;
-    if test "$CC" = "clang"; then
+release)
-        scan-build make -f "$TRAVIS_BUILD_DIR/src/Makefile"
+    $make ;;
-    else
+sanitize)
-        make -f "$TRAVIS_BUILD_DIR/src/Makefile" USE_SANITIZE=1
+    $make USE_SANITIZE=1 ;;
-    fi
+scan-build)
-    ;;
+    scan-build $make ;;
 *)
-    echo "ERROR: invalid build '$BUILD_METHOD_AND_BUILD_TYPE'"
+    echo "ERROR: invalid BUILD_METHOD '$BUILD_METHOD'"
     exit 1
     ;;
 esac
+ls -l upx.out
+size upx.out
+file upx.out
+
+exit_code=0
+
+# very first version of the upx-testsuite
+if test -x $PWD/upx.out; then
+checksum=sha256sum
+if test "X$TRAVIS_OS_NAME" = "Xosx"; then
+    checksum=true # TODO: travis-osx does not have md5sum and friends?
+fi
+upx="$PWD/upx.out"
+upx_391=false
+if test "X$TRAVIS_OS_NAME" = "Xlinux"; then
+    cp "$TRAVIS_BUILD_DIR/deps/upx-testsuite/files/packed/amd64-linux.elf/upx-3.91" upx391.out
+    upx_391="$PWD/upx391.out"
+fi
+$upx --help
+cd /; cd "$TRAVIS_BUILD_DIR/deps/upx-testsuite/files" || exit 1
+$upx -l packed/*/upx-3.91*
+$upx --file-info packed/*/upx-3.91*
+for f in packed/*/upx-3.91*; do
+    echo "===== $f"
+    if test "X$TRAVIS_OS_NAME" = "Xlinux"; then
+        $upx_391 -d $f -o v391.tmp
+        $upx     -d $f -o v392.tmp
+        $checksum v391.tmp v392.tmp
+        cmp -s v391.tmp v392.tmp
+        $upx_391 --lzma --fake-stub-version=3.92 --fake-stub-year=2016 v391.tmp -o v391_packed.tmp
+        $upx     --lzma                                                v392.tmp -o v392_packed.tmp
+        $checksum v391_packed.tmp v392_packed.tmp
+    else
+        $upx     -d $f -o v392.tmp
+        $checksum v392.tmp
+        $upx     --lzma                                                v392.tmp -o v392_packed.tmp
+        $checksum v392_packed.tmp
+    fi
+    $upx -d v392_packed.tmp -o v392_decompressed.tmp
+    # after the first compression+decompression step the exe should be
+    # canonicalized so that further compression+decompression runs
+    # should yield identical results
+    if ! cmp -s v392.tmp v392_decompressed.tmp; then
+        # UPX 3.91 and 3.92 differ; run one more compression+decompression
+        ls -l v392.tmp v392_decompressed.tmp
+        echo "UPX-WARNING: $f"
+        $upx v392_decompressed.tmp -o v392_packed_2.tmp
+        $upx -d v392_packed_2.tmp -o v392_decompressed_2.tmp
+        if ! cmp -s v392_decompressed.tmp v392_decompressed_2.tmp; then
+            ls -l v392_decompressed.tmp v392_decompressed_2.tmp
+            echo "UPX-ERROR: $f"
+            exit_code=1
+        fi
+    fi
+    rm *.tmp
+done
+fi
+
+exit $exit_code
+
+# vim:set ts=4 sw=4 et:

Makefile

@@ -17,12 +17,12 @@ top_srcdir = .
 include $(wildcard $(top_srcdir)/Makevars.global ./Makevars.local)
 
 
-# info: src/stub needs special build tools
+# info: src/stub needs special build tools from https://github.com/upx/upx-stubtools
 BUILD_STUB = 0
-ifneq ($(wildcard $(HOME)/local/bin/bin-upx/.),)
+ifneq ($(wildcard $(HOME)/local/bin/bin-upx/upx-stubtools-check-version),)
 BUILD_STUB = 1
 endif
-ifneq ($(wildcard $(HOME)/bin/bin-upx/.),)
+ifneq ($(wildcard $(HOME)/bin/bin-upx/upx-stubtools-check-version),)
 BUILD_STUB = 1
 endif
 
@@ -40,3 +40,4 @@ endif
 
 .PHONY: default all mostlyclean clean distclean maintainer-clean
 
+# vim:set ts=8 sw=8 noet:

src/Makefile

@@ -4,12 +4,10 @@
 # Copyright (C) 1996-2016 Markus Franz Xaver Johannes Oberhumer
 #
 
-# In GNU Make 4.0 this "MAKEFLAGS += -rR" is not executed immediately.
+# build configuration options
-# Instead, it is executed later, and we cannot rely on sequentiality
+#USE_DEBUG = 1
-# with respect to builtin variables.  So if we want to have no
+#USE_SANITIZE = 1
-# dependencies on the default values of builtin variables,
+
-# then we cannot use builtin variables at all.
-# See: http://lists.gnu.org/archive/html/help-make/2016-08/msg00011.html
 MAKEFLAGS += -r
 ifneq ($(upx_CXX),)
 CXX = $(upx_CXX)
@@ -20,10 +18,6 @@ else
 CXXLD = $(CXX)
 endif
 
-# build configuration options
-#USE_DEBUG = 1
-#USE_SANITIZE = 1
-
 .SUFFIXES:
 export SHELL = /bin/sh
 override e = $($1) $(EXTRA_$1) $(upx_$1) $($(basename $(notdir $@)).$1)
@@ -104,7 +98,7 @@ ifeq (1,1)
 	@echo "Updating $@"
 	@$(strip $(CXX) $(call e,CPPFLAGS) -MM) $(filter %.cpp,$^) > $@
 else
-	touch $@
+	@echo "timestamp" > $@
 endif
 
 
@@ -137,3 +131,5 @@ CLANG_FORMAT_FILES += ui.cpp ui.h util.h
 clang-format:
 	$(top_srcdir)/src/stub/scripts/upx-clang-format -i $(addprefix $(top_srcdir)/src/,$(CLANG_FORMAT_FILES))
 .PHONY: clang-format
+
+# vim:set ts=8 sw=8 noet:

src/conf.h

@@ -66,11 +66,6 @@
 #define ACC_WANT_ACC_LIB_H 1
 #define ACC_WANT_ACC_CXX_H 1
 #include "miniacc.h"
-#if (ACC_OS_CYGWIN || ACC_OS_DOS16 || ACC_OS_DOS32 || ACC_OS_EMX || ACC_OS_OS2 || ACC_OS_OS216 || ACC_OS_WIN16 || ACC_OS_WIN32 || ACC_OS_WIN64)
-#  if defined(INVALID_HANDLE_VALUE) || defined(MAKEWORD) || defined(RT_CURSOR)
-#    error "something pulled in <windows.h>"
-#  endif
-#endif
 
 /* intergral types */
 typedef acc_int8_t      upx_int8_t;
@@ -89,7 +84,7 @@ typedef acc_uintptr_t   upx_uintptr_t;
 #define UPX_INT64_C     ACC_INT64_C
 #define UPX_UINT64_C    ACC_UINT64_C
 
-#define upx_byte        unsigned char
+typedef unsigned char   upx_byte;
 #define upx_bytep       upx_byte *
 
 
@@ -615,9 +610,6 @@ struct upx_compress_result_t
 **************************************************************************/
 
 #include "snprintf.h"
-
-#if defined(__cplusplus)
-
 #include "stdcxx.h"
 #include "options.h"
 #include "except.h"
@@ -707,9 +699,6 @@ int upx_test_overlap       ( const upx_bytep buf,
                              const upx_compress_result_t *cresult );
 
 
-#endif /* __cplusplus */
-
-
 #if (ACC_OS_CYGWIN || ACC_OS_DOS16 || ACC_OS_DOS32 || ACC_OS_EMX || ACC_OS_OS2 || ACC_OS_OS216 || ACC_OS_WIN16 || ACC_OS_WIN32 || ACC_OS_WIN64)
 #  if defined(INVALID_HANDLE_VALUE) || defined(MAKEWORD) || defined(RT_CURSOR)
 #    error "something pulled in <windows.h>"

src/packer.cpp

@@ -1116,27 +1116,29 @@ void Packer::initLoader(const void *pdata, int plen, int small)
 
 
 #define C const char *
+#define N ACC_STATIC_CAST(void *, 0)
 void Packer::addLoader(C a)
-{ addLoaderVA(a, NULL); }
+{ addLoaderVA(a, N); }
 void Packer::addLoader(C a, C b)
-{ addLoaderVA(a, b, NULL); }
+{ addLoaderVA(a, b, N); }
 void Packer::addLoader(C a, C b, C c)
-{ addLoaderVA(a, b, c, NULL); }
+{ addLoaderVA(a, b, c, N); }
 void Packer::addLoader(C a, C b, C c, C d)
-{ addLoaderVA(a, b, c, d, NULL); }
+{ addLoaderVA(a, b, c, d, N); }
 void Packer::addLoader(C a, C b, C c, C d, C e)
-{ addLoaderVA(a, b, c, d, e, NULL); }
+{ addLoaderVA(a, b, c, d, e, N); }
 void Packer::addLoader(C a, C b, C c, C d, C e, C f)
-{ addLoaderVA(a, b, c, d, e, f, NULL); }
+{ addLoaderVA(a, b, c, d, e, f, N); }
 void Packer::addLoader(C a, C b, C c, C d, C e, C f, C g)
-{ addLoaderVA(a, b, c, d, e, f, g, NULL); }
+{ addLoaderVA(a, b, c, d, e, f, g, N); }
 void Packer::addLoader(C a, C b, C c, C d, C e, C f, C g, C h)
-{ addLoaderVA(a, b, c, d, e, f, g, h, NULL); }
+{ addLoaderVA(a, b, c, d, e, f, g, h, N); }
 void Packer::addLoader(C a, C b, C c, C d, C e, C f, C g, C h, C i)
-{ addLoaderVA(a, b, c, d, e, f, g, h, i, NULL); }
+{ addLoaderVA(a, b, c, d, e, f, g, h, i, N); }
 void Packer::addLoader(C a, C b, C c, C d, C e, C f, C g, C h, C i, C j)
-{ addLoaderVA(a, b, c, d, e, f, g, h, i, j, NULL); }
+{ addLoaderVA(a, b, c, d, e, f, g, h, i, j, N); }
 #undef C
+#undef N
 
 void __acc_cdecl_va Packer::addLoaderVA(const char *s, ...)
 {

src/stub/Makefile

@@ -35,10 +35,10 @@ endif
 include $(wildcard $(top_srcdir)/Makevars.global ./Makevars.local)
 
 # update $PATH for our special stub build tools
-ifneq ($(wildcard $(HOME)/local/bin/bin-upx/.),)
+ifneq ($(wildcard $(HOME)/local/bin/bin-upx/upx-stubtools-check-version),)
 export PATH := $(HOME)/local/bin/bin-upx:$(PATH)
 endif
-ifneq ($(wildcard $(HOME)/bin/bin-upx/.),)
+ifneq ($(wildcard $(HOME)/bin/bin-upx/upx-stubtools-check-version),)
 export PATH := $(HOME)/bin/bin-upx:$(PATH)
 endif
 
@@ -1317,9 +1317,7 @@ powerpc64le-darwin.macho-entry.h : $(srcdir)/src/$$T.S
 	$(call tc,bin2h) tmp/$T.bin $@
 
 powerpc64le-darwin.macho-fold.h : tmp/$$T.o tmp/powerpc64le-darwin.macho-main.o
-# FIXME: undefined reference to `_savegpr0_28' etc.
 	$(call tc,ld) --no-warn-mismatch --strip-all --oformat binary -Map tmp/$T.map $(filter %.o,$^) -o tmp/$T.bin
-#	echo "FIXME" > tmp/$T.bin
 	chmod a-x tmp/$T.bin
 	$(call tc,bin2h) tmp/$T.bin $@