From e7ca5c54ff8db85d4e4c730527766395166599c2 Mon Sep 17 00:00:00 2001 From: John Reiser Date: Tue, 22 Mar 2022 11:46:25 -0700 Subject: [PATCH] Fix --brute for p_lx_elf.cpp, which compresses multiple pieces Force all pieces to use the same de-compressor. (Future: allow each PT_LOAD to choose its own.) Has minor wobbles due to page alignment, and size of de-compressor. 64-bit only for now. https://github.com/upx/upx/issues/570 https://github.com/upx/upx/issues/297 modified: p_lx_elf.cpp modified: p_unix.cpp modified: packer.cpp modified: packer.h --- src/p_lx_elf.cpp | 105 +++++++++++++++++++++++++++++++++++++++++++---- src/p_unix.cpp | 4 +- src/packer.cpp | 71 +++++++++++++++++++++----------- src/packer.h | 4 ++ 4 files changed, 150 insertions(+), 34 deletions(-) diff --git a/src/p_lx_elf.cpp b/src/p_lx_elf.cpp index dc7e3e22..2f61a5d9 100644 --- a/src/p_lx_elf.cpp +++ b/src/p_lx_elf.cpp @@ -1266,7 +1266,7 @@ PackLinuxElf64::buildLinuxLoader( { unsigned h_sz_cpr = h.sz_cpr; int r = upx_compress(uncLoader, h.sz_unc, sizeof(h) + cprLoader, &h_sz_cpr, - nullptr, ph.method, 10, nullptr, nullptr ); + nullptr, forced_method(ph.method), 10, nullptr, nullptr ); h.sz_cpr = h_sz_cpr; if (r != UPX_E_OK || h.sz_cpr >= h.sz_unc) throwInternalError("loader compression failed"); @@ -2808,7 +2808,8 @@ proceed: ; exetype = 0; // set options - // .blocksize: avoid over-allocating + // this->blocksize: avoid over-allocating. + // (file_size - max_offset): debug info, non-globl symbols, etc. opt->o_unix.blocksize = blocksize = UPX_MAX(max_LOADsz, file_size - max_offset); return true; } @@ -3687,7 +3688,7 @@ void PackLinuxElf64ppc::pack1(OutputFile *fo, Filter &ft) generateElfHdr(fo, stub_powerpc64_linux_elf_fold, getbrk(phdri, e_phnum) ); } -void PackLinuxElf64::pack1(OutputFile *fo, Filter & /*ft*/) +void PackLinuxElf64::pack1(OutputFile *fo, Filter &ft) { fi->seek(0, SEEK_SET); fi->readx(&ehdri, sizeof(ehdri)); @@ -3698,16 +3699,102 @@ void PackLinuxElf64::pack1(OutputFile *fo, Filter & /*ft*/) // that are not covered by any PT_LOAD), but currently at run time there can be // only one decompressor method. // Therefore we must plan ahead because Packer::compressWithFilters tries -// to find the smallest result among the available methods. -// In the future we may allow more than one decompression method at run time, -// but for now we must choose just one, and force compressWithFilters to use it. +// to find the smallest result among the available methods, for one piece only. +// In the future we may allow more than one decompression method at run time. +// For now we must choose only one, and force PackUnix::packExtent +// (==> compressWithFilters) to use it. + int nfilters = 0; + { + int const *fp = getFilters(); + while (FT_END != *fp++) { + ++nfilters; + } + } + { + int npieces = 1; // tail after highest PT_LOAD + Elf64_Phdr *phdr = phdri; + for (unsigned j=0; j < e_phnum; ++phdr, ++j) { + if (PT_LOAD64 == get_te32(&phdr->p_type)) { + unsigned const flags = get_te32(&phdr->p_flags); + unsigned offset = get_te64(&phdr->p_offset); + if (!xct_off // not shlib + // new-style shlib: PT_LOAD[0] has symbol table + // which must not be compressed, but also lacks PF_X + || (Elf64_Phdr::PF_X & flags) + // Read-only, non-first PT_LOAD is _assumed_ to be compressible + || (!(Elf64_Phdr::PF_W & flags) && 0!=offset)) + { + ++npieces; // will attempt compression of this PT_LOAD + } + } + } + uip->ui_total_passes += npieces; + } int methods[256]; - int nmethods = prepareMethods(methods, ph.method, getCompressionMethods(M_ALL, ph.level)); - if (1 < nmethods) { + unsigned nmethods = prepareMethods(methods, ph.method, getCompressionMethods(M_ALL, ph.level)); + if (1 < nmethods) { // Many are available, but we must choose only one + uip->ui_total_passes += 1; // the batch for output + uip->ui_total_passes *= nmethods * (1+ nfilters); // finding smallest total + PackHeader orig_ph = ph; + Filter orig_ft = ft; + unsigned max_offset = 0; + unsigned sz_best= ~0u; + int method_best = 0; + for (unsigned k = 0; k < nmethods; ++k) { // FIXME: parallelize; cost: working space + unsigned sz_this = 0; + Elf64_Phdr *phdr = phdri; + for (unsigned j=0; j < e_phnum; ++phdr, ++j) { + if (PT_LOAD64 == get_te32(&phdr->p_type)) { + unsigned const flags = get_te32(&phdr->p_flags); + unsigned offset = get_te64(&phdr->p_offset); + unsigned filesz = get_te64(&phdr->p_filesz); + max_offset = UPX_MAX(max_offset, filesz + offset); + if (!xct_off // not shlib + // new-style shlib: PT_LOAD[0] has symbol table + // which must not be compressed, but also lacks PF_X + || (Elf64_Phdr::PF_X & flags) + // Read-only, non-first PT_LOAD is _assumed_ to be compressible + || (!(Elf64_Phdr::PF_W & flags) && 0!=offset)) + { + if (xct_off && 0==offset) { // old-style shlib + offset = xct_off; + filesz -= xct_off; + } + fi->seek(offset, SEEK_SET); + fi->readx(ibuf, filesz); + ft = orig_ft; + ph = orig_ph; + ph.method = force_method(methods[k]); + ph.u_len = filesz; + compressWithFilters(&ft, OVERHEAD, NULL_cconf, 10, true); + sz_this += ph.c_len; + } + } + } + unsigned const sz_tail = file_size - max_offset; // debuginfo, etc. + if (sz_tail) { + fi->seek(max_offset, SEEK_SET); + fi->readx(ibuf, sz_tail); + ft = orig_ft; + ph = orig_ph; + ph.method = force_method(methods[k]); + ph.u_len = sz_tail; + compressWithFilters(&ft, OVERHEAD, NULL_cconf, 10, true); + sz_this += ph.c_len; + } + // FIXME: loader size also depends on method + if (sz_best > sz_this) { + sz_best = sz_this; + method_best = methods[k]; + } + } + ft = orig_ft; + ph = orig_ph; + ph.method = force_method(method_best); } - Elf64_Phdr *phdr = phdri; note_size = 0; + Elf64_Phdr *phdr = phdri; for (unsigned j=0; j < e_phnum; ++phdr, ++j) { if (PT_NOTE64 == get_te32(&phdr->p_type)) { note_size += up4(get_te64(&phdr->p_filesz)); diff --git a/src/p_unix.cpp b/src/p_unix.cpp index e26d0d92..504e21a6 100644 --- a/src/p_unix.cpp +++ b/src/p_unix.cpp @@ -394,7 +394,7 @@ void PackUnix::packExtent( MemBuffer hdr_obuf; hdr_obuf.allocForCompression(hdr_u_len); int r = upx_compress(hdr_ibuf, hdr_u_len, hdr_obuf, &hdr_c_len, nullptr, - ph.method, 10, nullptr, nullptr); + forced_method(ph.method), 10, nullptr, nullptr); if (r != UPX_E_OK) throwInternalError("header compression failed"); if (hdr_c_len >= hdr_u_len) @@ -407,7 +407,7 @@ void PackUnix::packExtent( memset(&tmp, 0, sizeof(tmp)); set_te32(&tmp.sz_unc, hdr_u_len); set_te32(&tmp.sz_cpr, hdr_c_len); - tmp.b_method = (unsigned char) ph.method; + tmp.b_method = (unsigned char) forced_method(ph.method); tmp.b_extra = b_extra; fo->write(&tmp, sizeof(tmp)); b_len += sizeof(b_info); diff --git a/src/packer.cpp b/src/packer.cpp index dd9a74d3..9c06147e 100644 --- a/src/packer.cpp +++ b/src/packer.cpp @@ -159,12 +159,31 @@ bool ph_skipVerify(const PackHeader &ph) { return true; } +int force_method(int method) // mark as forced +{ + return (0x80ul<<24) | method; +} + +int is_forced_method(int method) // predicate +{ + return -0x80 == (method >> 24); +} + +int forced_method(int method) // extract the forced method +{ + if (is_forced_method(method)) + method &= ~(0x80ul<<24); + assert(method > 0); + return method; +} + /************************************************************************* // compress - wrap call to low-level upx_compress() **************************************************************************/ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr, - const upx_compress_config_t *cconf_parm) { + const upx_compress_config_t *cconf_parm) +{ ph.u_len = i_len; ph.c_len = 0; assert(ph.level >= 1); @@ -185,7 +204,8 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr, if (cconf_parm) cconf = *cconf_parm; // cconf options - if (M_IS_NRV2B(ph.method) || M_IS_NRV2D(ph.method) || M_IS_NRV2E(ph.method)) { + int method = forced_method(ph.method); + if (M_IS_NRV2B(method) || M_IS_NRV2D(method) || M_IS_NRV2E(method)) { if (opt->crp.crp_ucl.c_flags != -1) cconf.conf_ucl.c_flags = opt->crp.crp_ucl.c_flags; if (opt->crp.crp_ucl.p_level != -1) @@ -203,14 +223,14 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr, step = 0; #endif } - if (M_IS_LZMA(ph.method)) { + if (M_IS_LZMA(method)) { oassign(cconf.conf_lzma.pos_bits, opt->crp.crp_lzma.pos_bits); oassign(cconf.conf_lzma.lit_pos_bits, opt->crp.crp_lzma.lit_pos_bits); oassign(cconf.conf_lzma.lit_context_bits, opt->crp.crp_lzma.lit_context_bits); oassign(cconf.conf_lzma.dict_size, opt->crp.crp_lzma.dict_size); oassign(cconf.conf_lzma.num_fast_bytes, opt->crp.crp_lzma.num_fast_bytes); } - if (M_IS_DEFLATE(ph.method)) { + if (M_IS_DEFLATE(method)) { oassign(cconf.conf_zlib.mem_level, opt->crp.crp_zlib.mem_level); oassign(cconf.conf_zlib.window_bits, opt->crp.crp_zlib.window_bits); oassign(cconf.conf_zlib.strategy, opt->crp.crp_zlib.strategy); @@ -223,7 +243,7 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr, // OutputFile::dump("data.raw", in, ph.u_len); // compress - int r = upx_compress(i_ptr, ph.u_len, o_ptr, &ph.c_len, uip->getCallback(), ph.method, ph.level, + int r = upx_compress(i_ptr, ph.u_len, o_ptr, &ph.c_len, uip->getCallback(), method, ph.level, &cconf, &ph.compress_result); // uip->finalCallback(ph.u_len, ph.c_len); @@ -234,7 +254,7 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr, if (r != UPX_E_OK) throwInternalError("compression failed"); - if (M_IS_NRV2B(ph.method) || M_IS_NRV2D(ph.method) || M_IS_NRV2E(ph.method)) { + if (M_IS_NRV2B(method) || M_IS_NRV2D(method) || M_IS_NRV2E(method)) { const ucl_uint *res = ph.compress_result.result_ucl.result; // ph.min_offset_found = res[0]; ph.max_offset_found = res[1]; @@ -251,7 +271,7 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr, } } - // printf("\nPacker::compress: %d/%d: %7d -> %7d\n", ph.method, ph.level, ph.u_len, ph.c_len); + // printf("\nPacker::compress: %d/%d: %7d -> %7d\n", method, ph.level, ph.u_len, ph.c_len); if (!checkCompressionRatio(ph.u_len, ph.c_len)) return false; // return in any case if not compressible @@ -264,10 +284,10 @@ bool Packer::compress(upx_bytep i_ptr, unsigned i_len, upx_bytep o_ptr, if (!ph_skipVerify(ph)) { // decompress unsigned new_len = ph.u_len; - r = upx_decompress(o_ptr, ph.c_len, i_ptr, &new_len, ph.method, &ph.compress_result); + r = upx_decompress(o_ptr, ph.c_len, i_ptr, &new_len, method, &ph.compress_result); if (r == UPX_E_OUT_OF_MEMORY) throwOutOfMemoryException(); - // printf("%d %d: %d %d %d\n", ph.method, r, ph.c_len, ph.u_len, new_len); + // printf("%d %d: %d %d %d\n", method, r, ph.c_len, ph.u_len, new_len); if (r != UPX_E_OK) throwInternalError("decompression failed"); if (new_len != ph.u_len) @@ -338,7 +358,7 @@ void ph_decompress(PackHeader &ph, const upx_bytep in, upx_bytep out, bool verif throwCantUnpack("header corrupted"); } unsigned new_len = ph.u_len; - int r = upx_decompress(in, ph.c_len, out, &new_len, ph.method, &ph.compress_result); + int r = upx_decompress(in, ph.c_len, out, &new_len, forced_method(ph.method), &ph.compress_result); if (r == UPX_E_OUT_OF_MEMORY) throwOutOfMemoryException(); if (r != UPX_E_OK || new_len != ph.u_len) @@ -382,8 +402,8 @@ static bool ph_testOverlappingDecompression(const PackHeader &ph, const upx_byte unsigned src_off = ph.u_len + overlap_overhead - ph.c_len; unsigned new_len = ph.u_len; - int r = upx_test_overlap(buf - src_off, tbuf, src_off, ph.c_len, &new_len, ph.method, - &ph.compress_result); + int r = upx_test_overlap(buf - src_off, tbuf, src_off, ph.c_len, &new_len, + forced_method(ph.method), &ph.compress_result); if (r == UPX_E_OUT_OF_MEMORY) throwOutOfMemoryException(); return (r == UPX_E_OK && new_len == ph.u_len); @@ -1140,8 +1160,8 @@ void Packer::relocateLoader() { int Packer::prepareMethods(int *methods, int ph_method, const int *all_methods) const { int nmethods = 0; - if (!opt->all_methods || all_methods == nullptr) { - methods[nmethods++] = ph_method; + if (!opt->all_methods || all_methods == nullptr || (-0x80 == (ph_method>>24))) { + methods[nmethods++] = forced_method(ph_method); return nmethods; } for (int mm = 0; all_methods[mm] != M_END; ++mm) { @@ -1252,19 +1272,21 @@ void Packer::compressWithFilters( assert(nfilters < 256); #if 0 printf("compressWithFilters: m(%d):", nmethods); - for (int i = 0; i < nmethods; i++) printf(" %d", methods[i]); + for (int i = 0; i < nmethods; i++) printf(" %#x", methods[i]); printf(" f(%d):", nfilters); - for (int i = 0; i < nfilters; i++) printf(" %d", filters[i]); + for (int i = 0; i < nfilters; i++) printf(" %#x", filters[i]); printf("\n"); #endif // update total_passes; previous (ui_total_passes > 0) means incremental - if (uip->ui_total_passes > 0) - uip->ui_total_passes -= 1; - if (filter_strategy < 0) - uip->ui_total_passes += nmethods; - else - uip->ui_total_passes += nfilters * nmethods; + if (!is_forced_method(ph.method)) { + if (uip->ui_total_passes > 0) + uip->ui_total_passes -= 1; + if (filter_strategy < 0) + uip->ui_total_passes += nmethods; + else + uip->ui_total_passes += nfilters * nmethods; + } // Working buffer for compressed data. Don't waste memory and allocate as needed. upx_bytep o_tmp = o_ptr; @@ -1274,6 +1296,9 @@ void Packer::compressWithFilters( int nfilters_success_total = 0; for (int mm = 0; mm < nmethods; mm++) // for all methods { +#if 0 //{ + printf("\nmethod %d (%d of %d)\n", methods[mm], 1+ mm, nmethods); +#endif //} assert(isValidCompressionMethod(methods[mm])); unsigned hdr_c_len = 0; if (hdr_ptr != nullptr && hdr_len) { @@ -1319,7 +1344,7 @@ void Packer::compressWithFilters( } // filter success #if 0 - printf("filter: id 0x%02x size %6d, calls %5d/%5d/%3d/%5d/%5d, cto 0x%02x\n", + printf("\nfilter: id 0x%02x size %6d, calls %5d/%5d/%3d/%5d/%5d, cto 0x%02x\n", ft.id, ft.buf_len, ft.calls, ft.noncalls, ft.wrongcalls, ft.firstcall, ft.lastcall, ft.cto); #endif if (nfilters_success_total != 0 && o_tmp == o_ptr) { diff --git a/src/packer.h b/src/packer.h index ad166279..15df2304 100644 --- a/src/packer.h +++ b/src/packer.h @@ -341,6 +341,10 @@ private: Packer &operator=(const Packer &) = delete; }; +int force_method(int method); // (0x80ul<<24)|method +int forced_method(int method); // (0x80ul<<24)|method ==> method +int is_forced_method(int method); // predicate + #endif /* already included */ /* vim:set ts=4 sw=4 et: */