Fix --brute for p_lx_elf.cpp, which compresses multiple pieces

Force all pieces to use the same de-compressor.
(Future: allow each PT_LOAD to choose its own.)
Has minor wobbles due to page alignment, and size of de-compressor.
64-bit only for now.
https://github.com/upx/upx/issues/570
https://github.com/upx/upx/issues/297
	modified:   p_lx_elf.cpp
	modified:   p_unix.cpp
	modified:   packer.cpp
	modified:   packer.h

src/p_lx_elf.cpp

@@ -1266,7 +1266,7 @@ PackLinuxElf64::buildLinuxLoader(
     {
     unsigned h_sz_cpr = h.sz_cpr;
     int r = upx_compress(uncLoader, h.sz_unc, sizeof(h) + cprLoader, &h_sz_cpr,
-        nullptr, ph.method, 10, nullptr, nullptr );
+        nullptr, forced_method(ph.method), 10, nullptr, nullptr );
     h.sz_cpr = h_sz_cpr;
     if (r != UPX_E_OK || h.sz_cpr >= h.sz_unc)
         throwInternalError("loader compression failed");
@@ -2808,7 +2808,8 @@ proceed: ;
     exetype = 0;
 
     // set options
-    // .blocksize: avoid over-allocating
+    // this->blocksize: avoid over-allocating.
+    // (file_size - max_offset): debug info, non-globl symbols, etc.
     opt->o_unix.blocksize = blocksize = UPX_MAX(max_LOADsz, file_size - max_offset);
     return true;
 }
@@ -3687,7 +3688,7 @@ void PackLinuxElf64ppc::pack1(OutputFile *fo, Filter &ft)
     generateElfHdr(fo, stub_powerpc64_linux_elf_fold, getbrk(phdri, e_phnum) );
 }
 
-void PackLinuxElf64::pack1(OutputFile *fo, Filter & /*ft*/)
+void PackLinuxElf64::pack1(OutputFile *fo, Filter &ft)
 {
     fi->seek(0, SEEK_SET);
     fi->readx(&ehdri, sizeof(ehdri));
@@ -3698,16 +3699,102 @@ void PackLinuxElf64::pack1(OutputFile *fo, Filter & /*ft*/)
 // that are not covered by any PT_LOAD), but currently at run time there can be
 // only one decompressor method.
 // Therefore we must plan ahead because Packer::compressWithFilters tries
-// to find the smallest result among the available methods.
+// to find the smallest result among the available methods, for one piece only.
-// In the future we may allow more than one decompression method at run time,
+// In the future we may allow more than one decompression method at run time.
-// but for now we must choose just one, and force compressWithFilters to use it.
+// For now we must choose only one, and force PackUnix::packExtent
+// (==> compressWithFilters) to use it.
+    int nfilters = 0;
+    {
+        int const *fp = getFilters();
+        while (FT_END != *fp++) {
+            ++nfilters;
+        }
+    }
+    {
+        int npieces = 1;  // tail after highest PT_LOAD
+        Elf64_Phdr *phdr = phdri;
+        for (unsigned j=0; j < e_phnum; ++phdr, ++j) {
+            if (PT_LOAD64 == get_te32(&phdr->p_type)) {
+                unsigned const  flags = get_te32(&phdr->p_flags);
+                unsigned       offset = get_te64(&phdr->p_offset);
+                if (!xct_off  // not shlib
+                  // new-style shlib: PT_LOAD[0] has symbol table
+                  // which must not be compressed, but also lacks PF_X
+                ||    (Elf64_Phdr::PF_X & flags)
+                  // Read-only, non-first PT_LOAD is _assumed_ to be compressible
+                ||  (!(Elf64_Phdr::PF_W & flags) && 0!=offset))
+                {
+                    ++npieces;  // will attempt compression of this PT_LOAD
+                }
+            }
+        }
+        uip->ui_total_passes += npieces;
+    }
     int methods[256];
-    int nmethods = prepareMethods(methods, ph.method, getCompressionMethods(M_ALL, ph.level));
+    unsigned nmethods = prepareMethods(methods, ph.method, getCompressionMethods(M_ALL, ph.level));
-    if (1 < nmethods) {
+    if (1 < nmethods) { // Many are available, but we must choose only one
+        uip->ui_total_passes += 1;  // the batch for output
+        uip->ui_total_passes *= nmethods * (1+ nfilters);  // finding smallest total
+        PackHeader orig_ph = ph;
+        Filter orig_ft = ft;
+        unsigned max_offset = 0;
+        unsigned sz_best= ~0u;
+        int method_best = 0;
+        for (unsigned k = 0; k < nmethods; ++k) { // FIXME: parallelize; cost: working space
+            unsigned sz_this = 0;
+            Elf64_Phdr *phdr = phdri;
+            for (unsigned j=0; j < e_phnum; ++phdr, ++j) {
+                if (PT_LOAD64 == get_te32(&phdr->p_type)) {
+                    unsigned const  flags = get_te32(&phdr->p_flags);
+                    unsigned       offset = get_te64(&phdr->p_offset);
+                    unsigned       filesz = get_te64(&phdr->p_filesz);
+                    max_offset = UPX_MAX(max_offset, filesz + offset);
+                    if (!xct_off  // not shlib
+                      // new-style shlib: PT_LOAD[0] has symbol table
+                      // which must not be compressed, but also lacks PF_X
+                    ||    (Elf64_Phdr::PF_X & flags)
+                      // Read-only, non-first PT_LOAD is _assumed_ to be compressible
+                    ||  (!(Elf64_Phdr::PF_W & flags) && 0!=offset))
+                    {
+                        if (xct_off && 0==offset) { // old-style shlib
+                            offset  = xct_off;
+                            filesz -= xct_off;
+                        }
+                        fi->seek(offset, SEEK_SET);
+                        fi->readx(ibuf, filesz);
+                        ft = orig_ft;
+                        ph = orig_ph;
+                        ph.method = force_method(methods[k]);
+                        ph.u_len = filesz;
+                        compressWithFilters(&ft, OVERHEAD, NULL_cconf, 10, true);
+                        sz_this += ph.c_len;
+                    }
+                }
+            }
+            unsigned const sz_tail = file_size - max_offset;  // debuginfo, etc.
+            if (sz_tail) {
+                fi->seek(max_offset, SEEK_SET);
+                fi->readx(ibuf, sz_tail);
+                ft = orig_ft;
+                ph = orig_ph;
+                ph.method = force_method(methods[k]);
+                ph.u_len = sz_tail;
+                compressWithFilters(&ft, OVERHEAD, NULL_cconf, 10, true);
+                sz_this += ph.c_len;
+            }
+            // FIXME: loader size also depends on method
+            if (sz_best > sz_this) {
+                sz_best = sz_this;
+                method_best = methods[k];
+            }
+        }
+        ft = orig_ft;
+        ph = orig_ph;
+        ph.method = force_method(method_best);
     }
 
-    Elf64_Phdr *phdr = phdri;
     note_size = 0;
+    Elf64_Phdr *phdr = phdri;
     for (unsigned j=0; j < e_phnum; ++phdr, ++j) {
         if (PT_NOTE64 == get_te32(&phdr->p_type)) {
             note_size += up4(get_te64(&phdr->p_filesz));

src/p_unix.cpp

@@ -394,7 +394,7 @@ void PackUnix::packExtent(
             MemBuffer hdr_obuf;
             hdr_obuf.allocForCompression(hdr_u_len);
             int r = upx_compress(hdr_ibuf, hdr_u_len, hdr_obuf, &hdr_c_len, nullptr,
-                ph.method, 10, nullptr, nullptr);
+                forced_method(ph.method), 10, nullptr, nullptr);
             if (r != UPX_E_OK)
                 throwInternalError("header compression failed");
             if (hdr_c_len >= hdr_u_len)
@@ -407,7 +407,7 @@ void PackUnix::packExtent(
             memset(&tmp, 0, sizeof(tmp));
             set_te32(&tmp.sz_unc, hdr_u_len);
             set_te32(&tmp.sz_cpr, hdr_c_len);
-            tmp.b_method = (unsigned char) ph.method;
+            tmp.b_method = (unsigned char) forced_method(ph.method);
             tmp.b_extra = b_extra;
             fo->write(&tmp, sizeof(tmp));
             b_len += sizeof(b_info);

src/packer.cpp

@@ -159,12 +159,31 @@ bool ph_skipVerify(const PackHeader &ph) {
     return true;
 }
 
+int force_method(int method)  // mark as forced
+{
+    return (0x80ul<<24) | method;
+}
+
+int is_forced_method(int method)  // predicate
+{
+    return -0x80 == (method >> 24);
+}
+
+int forced_method(int method)  // extract the forced method
+{
+    if (is_forced_method(method))
+        method &= ~(0x80ul<<24);
+    assert(method > 0);
+    return method;
+}
+
 /*************************************************************************
 // compress - wrap call to low-level upx_compress()
 **************************************************************************/
 
 bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr,
-                      const upx_compress_config_t *cconf_parm) {
+                      const upx_compress_config_t *cconf_parm)
+{
     ph.u_len = i_len;
     ph.c_len = 0;
     assert(ph.level >= 1);
@@ -185,7 +204,8 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr,
     if (cconf_parm)
         cconf = *cconf_parm;
     // cconf options
-    if (M_IS_NRV2B(ph.method) || M_IS_NRV2D(ph.method) || M_IS_NRV2E(ph.method)) {
+    int method = forced_method(ph.method);
+    if (M_IS_NRV2B(method) || M_IS_NRV2D(method) || M_IS_NRV2E(method)) {
         if (opt->crp.crp_ucl.c_flags != -1)
             cconf.conf_ucl.c_flags = opt->crp.crp_ucl.c_flags;
         if (opt->crp.crp_ucl.p_level != -1)
@@ -203,14 +223,14 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr,
             step = 0;
 #endif
     }
-    if (M_IS_LZMA(ph.method)) {
+    if (M_IS_LZMA(method)) {
         oassign(cconf.conf_lzma.pos_bits, opt->crp.crp_lzma.pos_bits);
         oassign(cconf.conf_lzma.lit_pos_bits, opt->crp.crp_lzma.lit_pos_bits);
         oassign(cconf.conf_lzma.lit_context_bits, opt->crp.crp_lzma.lit_context_bits);
         oassign(cconf.conf_lzma.dict_size, opt->crp.crp_lzma.dict_size);
         oassign(cconf.conf_lzma.num_fast_bytes, opt->crp.crp_lzma.num_fast_bytes);
     }
-    if (M_IS_DEFLATE(ph.method)) {
+    if (M_IS_DEFLATE(method)) {
         oassign(cconf.conf_zlib.mem_level, opt->crp.crp_zlib.mem_level);
         oassign(cconf.conf_zlib.window_bits, opt->crp.crp_zlib.window_bits);
         oassign(cconf.conf_zlib.strategy, opt->crp.crp_zlib.strategy);
@@ -223,7 +243,7 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr,
     // OutputFile::dump("data.raw", in, ph.u_len);
 
     // compress
-    int r = upx_compress(i_ptr, ph.u_len, o_ptr, &ph.c_len, uip->getCallback(), ph.method, ph.level,
+    int r = upx_compress(i_ptr, ph.u_len, o_ptr, &ph.c_len, uip->getCallback(), method, ph.level,
                          &cconf, &ph.compress_result);
 
     // uip->finalCallback(ph.u_len, ph.c_len);
@@ -234,7 +254,7 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr,
     if (r != UPX_E_OK)
         throwInternalError("compression failed");
 
-    if (M_IS_NRV2B(ph.method) || M_IS_NRV2D(ph.method) || M_IS_NRV2E(ph.method)) {
+    if (M_IS_NRV2B(method) || M_IS_NRV2D(method) || M_IS_NRV2E(method)) {
         const ucl_uint *res = ph.compress_result.result_ucl.result;
         // ph.min_offset_found = res[0];
         ph.max_offset_found = res[1];
@@ -251,7 +271,7 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr,
         }
     }
 
-    // printf("\nPacker::compress: %d/%d: %7d -> %7d\n", ph.method, ph.level, ph.u_len, ph.c_len);
+    // printf("\nPacker::compress: %d/%d: %7d -> %7d\n", method, ph.level, ph.u_len, ph.c_len);
     if (!checkCompressionRatio(ph.u_len, ph.c_len))
         return false;
     // return in any case if not compressible
@@ -264,10 +284,10 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr,
     if (!ph_skipVerify(ph)) {
         // decompress
         unsigned new_len = ph.u_len;
-        r = upx_decompress(o_ptr, ph.c_len, i_ptr, &new_len, ph.method, &ph.compress_result);
+        r = upx_decompress(o_ptr, ph.c_len, i_ptr, &new_len, method, &ph.compress_result);
         if (r == UPX_E_OUT_OF_MEMORY)
             throwOutOfMemoryException();
-        // printf("%d %d: %d %d %d\n", ph.method, r, ph.c_len, ph.u_len, new_len);
+        // printf("%d %d: %d %d %d\n", method, r, ph.c_len, ph.u_len, new_len);
         if (r != UPX_E_OK)
             throwInternalError("decompression failed");
         if (new_len != ph.u_len)
@@ -338,7 +358,7 @@ void ph_decompress(PackHeader &ph, const upx_bytep in, upx_bytep out, bool verif
         throwCantUnpack("header corrupted");
     }
     unsigned new_len = ph.u_len;
-    int r = upx_decompress(in, ph.c_len, out, &new_len, ph.method, &ph.compress_result);
+    int r = upx_decompress(in, ph.c_len, out, &new_len, forced_method(ph.method), &ph.compress_result);
     if (r == UPX_E_OUT_OF_MEMORY)
         throwOutOfMemoryException();
     if (r != UPX_E_OK || new_len != ph.u_len)
@@ -382,8 +402,8 @@ static bool ph_testOverlappingDecompression(const PackHeader &ph, const upx_byte
 
     unsigned src_off = ph.u_len + overlap_overhead - ph.c_len;
     unsigned new_len = ph.u_len;
-    int r = upx_test_overlap(buf - src_off, tbuf, src_off, ph.c_len, &new_len, ph.method,
+    int r = upx_test_overlap(buf - src_off, tbuf, src_off, ph.c_len, &new_len,
-                             &ph.compress_result);
+           forced_method(ph.method), &ph.compress_result);
     if (r == UPX_E_OUT_OF_MEMORY)
         throwOutOfMemoryException();
     return (r == UPX_E_OK && new_len == ph.u_len);
@@ -1140,8 +1160,8 @@ void Packer::relocateLoader() {
 
 int Packer::prepareMethods(int *methods, int ph_method, const int *all_methods) const {
     int nmethods = 0;
-    if (!opt->all_methods || all_methods == nullptr) {
+    if (!opt->all_methods || all_methods == nullptr || (-0x80 == (ph_method>>24))) {
-        methods[nmethods++] = ph_method;
+        methods[nmethods++] = forced_method(ph_method);
         return nmethods;
     }
     for (int mm = 0; all_methods[mm] != M_END; ++mm) {
@@ -1252,19 +1272,21 @@ void Packer::compressWithFilters(
     assert(nfilters < 256);
 #if 0
     printf("compressWithFilters: m(%d):", nmethods);
-    for (int i = 0; i < nmethods; i++) printf(" %d", methods[i]);
+    for (int i = 0; i < nmethods; i++) printf(" %#x", methods[i]);
     printf(" f(%d):", nfilters);
-    for (int i = 0; i < nfilters; i++) printf(" %d", filters[i]);
+    for (int i = 0; i < nfilters; i++) printf(" %#x", filters[i]);
     printf("\n");
 #endif
 
     // update total_passes; previous (ui_total_passes > 0) means incremental
-    if (uip->ui_total_passes > 0)
+    if (!is_forced_method(ph.method)) {
-        uip->ui_total_passes -= 1;
+        if (uip->ui_total_passes > 0)
-    if (filter_strategy < 0)
+            uip->ui_total_passes -= 1;
-        uip->ui_total_passes += nmethods;
+        if (filter_strategy < 0)
-    else
+            uip->ui_total_passes += nmethods;
-        uip->ui_total_passes += nfilters * nmethods;
+        else
+            uip->ui_total_passes += nfilters * nmethods;
+    }
 
     // Working buffer for compressed data. Don't waste memory and allocate as needed.
     upx_bytep o_tmp = o_ptr;
@@ -1274,6 +1296,9 @@ void Packer::compressWithFilters(
     int nfilters_success_total = 0;
     for (int mm = 0; mm < nmethods; mm++) // for all methods
     {
+#if 0  //{
+        printf("\nmethod %d (%d of %d)\n", methods[mm], 1+ mm, nmethods);
+#endif  //}
         assert(isValidCompressionMethod(methods[mm]));
         unsigned hdr_c_len = 0;
         if (hdr_ptr != nullptr && hdr_len) {
@@ -1319,7 +1344,7 @@ void Packer::compressWithFilters(
             }
             // filter success
 #if 0
-            printf("filter: id 0x%02x size %6d, calls %5d/%5d/%3d/%5d/%5d, cto 0x%02x\n",
+            printf("\nfilter: id 0x%02x size %6d, calls %5d/%5d/%3d/%5d/%5d, cto 0x%02x\n",
                    ft.id, ft.buf_len, ft.calls, ft.noncalls, ft.wrongcalls, ft.firstcall, ft.lastcall, ft.cto);
 #endif
             if (nfilters_success_total != 0 && o_tmp == o_ptr) {

src/packer.h

@@ -341,6 +341,10 @@ private:
     Packer &operator=(const Packer &) = delete;
 };
 
+int     force_method(int method); // (0x80ul<<24)|method
+int    forced_method(int method); // (0x80ul<<24)|method ==> method
+int is_forced_method(int method); // predicate
+
 #endif /* already included */
 
 /* vim:set ts=4 sw=4 et: */