Reduce usage of patchDecompressor()

src/p_lx_exc.cpp

@@ -83,14 +83,9 @@ const int *PackLinuxI386::getFilters() const
 {
     static const int filters[] = {
         0x49, 0x46,
-// FIXME 2002-11-11: We use stub/fold_exec86.asm, which calls the
-// decompressor multiple times, and unfilter is independent of decompress.
-// Currently only filters 0x49, 0x46, 0x80..0x87 can handle this;
-// and 0x80..0x87 are regarded as "untested".
-#if 0
         0x26, 0x24, 0x11, 0x14, 0x13, 0x16, 0x25, 0x15, 0x12,
-#endif
 #if 0
+// 0x80..0x87 are regarded as "untested".
         0x83, 0x86, 0x80, 0x84, 0x87, 0x81, 0x82, 0x85,
         0x24, 0x16, 0x13, 0x14, 0x11, 0x25, 0x15, 0x12,
 #endif
@@ -136,6 +131,10 @@ PackLinuxI386::generateElfHdr(
     assert(h2->ehdr.e_phentsize == sizeof(Elf32_Phdr));
     assert(h2->ehdr.e_shnum     == 0);
 
+#if 0  //{
+    unsigned identsize;
+    char const *const ident = getIdentstr(&identsize);
+#endif  //}
     h2->phdr[0].p_filesz = sizeof(*h2);  // + identsize;
     h2->phdr[0].p_memsz  = h2->phdr[0].p_filesz;
 
@@ -334,6 +333,10 @@ PackLinuxI386::buildLinuxLoader(
         }
     }
     addLoader("LEXEC010", NULL);
+    linker->defineSymbol("filter_cto", ft->cto);
+    linker->defineSymbol("filter_length",
+                         (ft->id & 0xf) % 3 == 0 ? ft->calls :
+                         ft->lastcall - ft->calls * 4);
     addLoader(getDecompressorSections(), NULL);
     addLoader("LEXEC015", NULL);
     if (ft->id) {
@@ -360,18 +363,20 @@ PackLinuxI386::buildLinuxLoader(
     addLoader("LEXEC020", NULL);
     addLoader("FOLDEXEC", NULL);
     freezeLoader();
-    //addLinkerSymbols(ft);
+    if (ph.method == M_LZMA) {
-    linker->relocate();
+        const lzma_compress_result_t *res = &ph.compress_result.result_lzma;
-
+        unsigned const properties = // lc, lp, pb, dummy
-    upx_byte *ptr_cto = getLoader();
+            (res->lit_context_bits << 0) |
-    int sz_cto = getLoaderSize();
+            (res->lit_pos_bits << 8) |
-    if (0x20==(ft->id & 0xF0) || 0x30==(ft->id & 0xF0)) {  // push byte '?'  ; cto8
+            (res->pos_bits << 16);
-        patch_le16(ptr_cto, sz_cto, "\x6a?", 0x6a + (ft->cto << 8));
+        linker->defineSymbol("UPXd", properties);
-        checkPatch(NULL, 0, 0, 0);  // reset
+        // -2 for properties
+        linker->defineSymbol("UPXc", ph.c_len - 2);
+        linker->defineSymbol("UPXb", ph.u_len);
+        unsigned const stack = getDecompressorWrkmemSize();
+        linker->defineSymbol("UPXa", 0u - stack);
     }
-    // PackHeader and overlay_offset at the end of the output file,
+    linker->relocate();
-    // after the compressed data.
-
     return getLoaderSize();
 }
 

src/p_unix.cpp

@@ -241,7 +241,6 @@ void PackUnix::pack3(OutputFile *fo, Filter &ft)
     upx_byte *p = getLoader();
     lsize = getLoaderSize();
     patchFilter32(p, lsize, &ft);
-    patchDecompressor(p, lsize);
     updateLoader(fo);
     patchLoaderChecksum();
     fo->write(p, lsize);

src/packer_f.cpp

@@ -370,25 +370,6 @@ bool Packer::patchFilter32(void *loader, int lsize, const Filter *ft)
         return false;
     assert(ft->calls > 0);
 
-    if (ft->id < 0x80) {
-        if (0x40 <= ft->id && ft->id < 0x50
-        && (  UPX_F_LINUX_i386   ==ph.format
-           || UPX_F_VMLINUX_i386 ==ph.format
-           || UPX_F_VMLINUZ_i386 ==ph.format
-           || UPX_F_BVMLINUZ_i386==ph.format ) ) {
-            // "push byte '?'"
-            patch_le16(loader, lsize, "\x6a?", 0x6a + (ft->cto << 8));
-            checkPatch(NULL, 0, 0, 0);  // reset
-        }
-        if (0x20 <= ft->id && ft->id < 0x40) {
-            // 077==modr/m of "cmp [edi], byte '?'" (compare immediate 8 bits)
-            patch_le16(loader, lsize, "\077?", 077 + (ft->cto << 8));
-        }
-        if (ft->id < 0x40) {
-            patch_le32(loader, lsize, "TEXL", (ft->id & 0xf) % 3 == 0 ? ft->calls :
-                    ft->lastcall - ft->calls * 4);
-        }
-    }
     if (0x80==(ft->id & 0xF0)) {
         int const mru = ph.n_mru ? 1+ ph.n_mru : 0;
         if (mru && mru!=256) {

src/stub/src/arch/i386/macros.S

@@ -197,6 +197,83 @@ section         CALLTR13
 ctend:
 .endm
 
+// cit32 is same as cjt32 except esi captures 'addvalue' at the beginning
+.macro          cit32   addvalue
+section         CALLTR00
+                mov     ecx, offset filter_length
+                .ifnc   \addvalue, 0
+                mov     esi, \addvalue
+                .endif
+calltrickloop:
+                mov     al, [edi]
+                inc     edi
+                sub     al, 0xE8
+ct1:
+                cmp     al, 1
+                ja      calltrickloop
+section         CTCLEVE1
+                cmpb    [edi], offset filter_cto
+                jnzs    calltrickloop
+section         CALLTR01
+                mov     eax, [edi]
+                mov     bl, [edi + 4]
+section         CTBSHR01
+                shr     ax, 8
+section         CTBROR01
+                xchg    ah, al
+section         CTBSWA01
+                rol     eax, 16
+                xchg    ah, al
+section         CALLTR02
+                sub     eax, edi
+                sub     bl, 0xE8
+
+                .ifnc   \addvalue, 0
+                add     eax, esi
+                .endif
+
+                mov     [edi], eax
+                add     edi, 5
+                mov     al, bl
+                loop    ct1
+
+section         CALLTR10
+//;; 32-bit call XOR jump trick
+                mov     ecx, offset filter_length
+ctloop1:
+section         CALLTRE8
+                mov     al,0xE8
+section         CALLTRE9
+                mov     al,0xE9
+section         CALLTR11
+ctloop2:
+                repnz
+                scasb
+                jnzs    ctend
+section         CTCLEVE2
+                cmpb    [edi], offset filter_cto
+                jnzs    ctloop2
+section         CALLTR12
+                mov     eax, [edi]
+section         CTBSHR11
+                shr     ax, 8
+section         CTBROR11
+                xchg    ah, al
+section         CTBSWA11
+                rol     eax, 16
+                xchg    ah, al
+section         CALLTR13
+                sub     eax, edi
+
+                .ifnc   \addvalue, 0
+                add     eax, esi
+                .endif
+
+                stosd
+                jmps    ctloop1
+ctend:
+.endm
+
 /*
 ;;  call/jump/jcc trick; also used more than once (and/or optionally), so
 ;;  ecx has byte count (not count of applied instances), and
@@ -227,7 +304,7 @@ section         CKLLTR20
                 cmp     al, 0xE9 - 0xE8
                 jas     ckcount
 ckmark:
-                cmp     [edi], \cto8
+                cmpb    [edi], \cto8
                 jnzs    ckcount
                 mov     eax, [edi]
 

src/stub/src/i386-bsd.elf-entry.S

@@ -85,7 +85,6 @@ section LEXEC009
         //  empty section for commonality with l_lx_exec86.asm
 section LEXEC010
                 pusha
-                push    '?'  // cto8 (sign extension does not matter)
                 // cld
 
                 mov     esi, INP
@@ -98,7 +97,6 @@ section LEXEC010
 #include      "arch/i386/nrv2d_d32.S"
 #include      "arch/i386/nrv2e_d32.S"
 #include      "arch/i386/lzma_d.S"
-                cjt32 0
 
 section LEXEC015
                 // eax is 0 from decompressor code
@@ -117,14 +115,13 @@ section LEXEC015
                 mov     edx, OUTS
                 mov     [edx], edi
 
-                pop edx  // cto8
-
                 mov [7*4 + esp], eax
                 popa
                 ret
 
                 ctojr32
                 ckt32   edi, dl
+                cit32   edi
 section LEXEC017
                 popa
                 ret

src/stub/src/i386-bsd.elf.execve-entry.S

@@ -86,7 +86,7 @@ section LEXEC009
                 mov     eax, offset NMRU  // free slot in following 'pusha'
 section LEXEC010
                 pusha
-                push '?'  // cto8 (sign extension does not matter)
+                .byte   0x6a, filter_cto  // push byte
                 // cld
 
                 mov     esi, INP
@@ -99,7 +99,6 @@ section LEXEC010
 #include      "arch/i386/nrv2d_d32.S"
 #include      "arch/i386/nrv2e_d32.S"
 #include      "arch/i386/lzma_d.S"
-                cjt32 0
 
 section LEXEC015
                 // eax is 0 from decompressor code
@@ -137,6 +136,7 @@ section LEXEC016  // bug in APP: jmp and target must be in same .asx
 
                 ctojr32
                 ckt32   edi, dl
+                cit32   edi
 section LEXEC017
                 popa
                 ret

src/stub/src/i386-linux.elf.execve-entry.S

@@ -93,7 +93,7 @@ section LEXEC009
                 mov     eax, offset NMRU  // free slot in following 'pusha'
 section LEXEC010
                 pusha
-                push '?'  // cto8 (sign extension does not matter)
+                .byte   0x6a, filter_cto  // push byte
                 // cld
 
                 mov     esi, INP
@@ -106,7 +106,6 @@ section LEXEC010
 #include      "arch/i386/nrv2d_d32.S"
 #include      "arch/i386/nrv2e_d32.S"
 #include      "arch/i386/lzma_d.S"
-                cjt32 0
 
 section LEXEC015
                 // eax is 0 from decompressor code
@@ -144,6 +143,7 @@ section LEXEC016  // bug in APP: jmp and target must be in same .asx
 
                 ctojr32
                 ckt32   edi, dl
+                cit32   edi
 section LEXEC017
                 popa
                 ret