Fixing support for inject_x64 injection inside WOW64 processes

core/heap.c

@@ -1901,8 +1901,13 @@ vmm_heap_init()
     if (DYNAMO_OPTION(vm_reserve)) {
         vmm_heap_unit_init(&heapmgt->vmcode, DYNAMO_OPTION(vm_size), true, "vmcode");
         if (!REACHABLE_HEAP()) {
-            vmm_heap_unit_init(&heapmgt->vmheap, DYNAMO_OPTION(vmheap_size), false,
-                               "vmheap");
+            if (DYNAMO_OPTION(inject_x64)){
+                vmm_heap_unit_init(&heapmgt->vmheap, DYNAMO_OPTION(vmheap_size_wow64), false,
+                                   "vmheap");
+            } else {
+                vmm_heap_unit_init(&heapmgt->vmheap, DYNAMO_OPTION(vmheap_size), false,
+                                   "vmheap");
+            }
         }
     }
 }

core/optionsx.h

@@ -1595,7 +1595,12 @@ OPTION_DEFAULT(uint_size, vmheap_size, IF_X64_ELSE(8192ULL, 128) * 1024 * 1024,
                 * for which we need more than 256MB.
                 */
                "capacity of virtual memory region reserved for unreachable heap")
-
+OPTION_DEFAULT(uint_size, vmheap_size_wow64, 128 * 1024 * 1024,
+               /* XXX: default value is currently not good enough for 32-bit sqlserver,
+                * for which we need more than 256MB.
+                */
+               "capacity of virtual memory region reserved for unreachable heap "
+               "on WoW64 processes")
 /* We hardcode an address in the mmap_text region here, but verify via
  * in vmk_init().
  * For Linux we start higher to avoid limiting the brk (i#766), but with our

core/win32/inject.c

@@ -1065,7 +1065,6 @@ inject_gencode_at_ldr(HANDLE phandle, char *dynamo_path, uint inject_location,
  * process, mode_switch_buf_sz is maximum size for switch code, and
  * mode_switch_data is the address where the app stack pointer is stored.
  */
-
 static size_t
 generate_switch_mode_jmp_to_hook(HANDLE phandle, byte *local_code_buf,
                                  byte *mode_switch_buf, byte *hook_location,
@@ -1083,20 +1082,26 @@ generate_switch_mode_jmp_to_hook(HANDLE phandle, byte *local_code_buf,
     instr_t *restore_esp =
         INSTR_CREATE_mov_ld(GDC, opnd_create_reg(REG_ESP),
                             OPND_CREATE_MEM32(REG_NULL, (int)(size_t)mode_switch_data));
+    const byte *eax_saved_offset = (byte *)((size_t)mode_switch_data) + 4;
+    // Restoring eax back, which is storing routine address that needs to be passed to
+    // RtlUserStartThread
+    instr_t *restore_eax = INSTR_CREATE_mov_ld(GDC, opnd_create_reg(REG_EAX),
+                            OPND_CREATE_MEM32(REG_NULL, (int)(size_t)eax_saved_offset));
 
     instr_set_x86_mode(jmp, true);
     instr_set_x86_mode(restore_esp, true);
+    instr_set_x86_mode(restore_eax, true);
     instrlist_init(&ilist);
     /* We patch the 0 with the correct target location in this function */
     APP(&ilist, INSTR_CREATE_push_imm(GDC, OPND_CREATE_INT32(0)));
     APP(&ilist,
         INSTR_CREATE_mov_st(GDC, OPND_CREATE_MEM16(REG_RSP, 4),
                             OPND_CREATE_INT16((ushort)CS32_SELECTOR)));
-
     APP(&ilist,
         INSTR_CREATE_jmp_far_ind(GDC,
                                  opnd_create_base_disp(REG_RSP, REG_NULL, 0, 0, OPSZ_6)));
     APP(&ilist, restore_esp);
+    APP(&ilist, restore_eax);
     APP(&ilist, jmp);
 
     pc = instrlist_encode_to_copy(GDC, &ilist, local_code_buf, mode_switch_buf,
@@ -1108,7 +1113,7 @@ generate_switch_mode_jmp_to_hook(HANDLE phandle, byte *local_code_buf,
      * to x86 mode
      */
     sz = (size_t)(pc - local_code_buf - instr_length(GDC, jmp) -
-                  instr_length(GDC, restore_esp));
+                  instr_length(GDC, restore_esp) - instr_length(GDC, restore_eax));
     instrlist_clear(GDC, &ilist);
     /* For x86 code the address must be 32 bit */
     ASSERT_TRUNCATE(target, uint, (size_t)mode_switch_buf);
@@ -1185,7 +1190,10 @@ inject_gencode_mapped_helper(HANDLE phandle, char *dynamo_path, uint64 hook_loca
     byte *mode_switch_buf = NULL;
     byte *mode_switch_data = NULL;
     size_t switch_code_sz = PAGE_SIZE;
-    static const size_t switch_data_sz = SWITCH_MODE_DATA_SIZE;
+    size_t switch_data_sz = SWITCH_MODE_DATA_SIZE;
+    if (x86_code && DYNAMO_OPTION(inject_x64)) {
+        switch_data_sz += 4; // we need space for ESP and EAX
+    }
 #endif
     size_t num_bytes_out;
     uint old_prot;
@@ -1251,21 +1259,34 @@ inject_gencode_mapped_helper(HANDLE phandle, char *dynamo_path, uint64 hook_loca
         /* Mode Switch from 32 bit to 64 bit.
          * Forward align stack.
          */
+        const byte *eax_saved_offset = mode_switch_data + 4;
+        // mov dword ptr[mode_switch_data] , esp
         *cur_local_pos++ = MOV_REG32_2_RM32;
         *cur_local_pos++ = 0x24;
         *cur_local_pos++ = 0x25;
         RAW_INSERT_INT32(cur_local_pos, mode_switch_data);
+
+        // mov dword ptr[mode_switch_data+4], eax
+        *cur_local_pos++ = MOV_REG32_2_RM32;
+        *cur_local_pos++ = MOV_IMM_RM_ABS;
+        RAW_INSERT_INT32(cur_local_pos, eax_saved_offset);
+
         /* Far jmp to next instr. */
         const int far_jmp_len = 7;
         byte *pre_jmp = cur_local_pos;
+        uint64 cur_remote_pos_tmp =
+            remote_code_buf + (cur_local_pos - local_code_buf + switch_code_sz);
+
         *cur_local_pos++ = JMP_FAR_DIRECT;
-        RAW_INSERT_INT32(cur_local_pos, pre_jmp + far_jmp_len);
+        RAW_INSERT_INT32(cur_local_pos, cur_remote_pos_tmp + far_jmp_len);
         RAW_INSERT_INT16(cur_local_pos, CS64_SELECTOR);
         ASSERT(cur_local_pos == pre_jmp + far_jmp_len);
+
         /* Align stack. */
-        *cur_local_pos++ = AND_RM32_IMM32;
+        // and    rsp,0xfffffffffffffff0
+        *cur_local_pos++ = 0x83;
         *cur_local_pos++ = 0xe4;
-        RAW_INSERT_INT32(cur_local_pos, -8);
+        *cur_local_pos++ = 0xf0;
     }
 #endif
     /* Save xax, which we clobber below.  It is live for INJECT_LOCATION_ThreadStart.

core/win32/syscall.c

@@ -1744,7 +1744,10 @@ propagate_options_via_env_vars(dcontext_t *dcontext, HANDLE process_handle,
      * model without requiring setting up config files for children.
      */
     uint64 peb;
-    bool peb_is_32 = is_32bit_process(process_handle);
+    bool peb_is_32 = is_32bit_process(process_handle)
+	    // if x64 client targeting WOW64 we need to 
+	    // target x64 PEB
+	    IF_X64(&& !DYNAMO_OPTION(inject_x64));
     size_t sz_read;
     union {
         uint64 ptr_64;
@@ -2961,7 +2964,13 @@ pre_system_call(dcontext_t *dcontext)
     reg_t *param_base = pre_system_call_param_base(mc);
     dr_where_am_i_t old_whereami = dcontext->whereami;
     dcontext->whereami = DR_WHERE_SYSCALL_HANDLER;
-    IF_X64(ASSERT_TRUNCATE(sysnum, int, mc->xax));
+    /* FIXME
+     * Commenting this ASSERT since x64 client injected in
+     * WOW64 binaries will be using mc->rax as comparaison
+     * thus making the assert fail, no matter if the
+     * syscall is correct or not.
+     */
+    // IF_X64(ASSERT_TRUNCATE(sysnum, int, mc->xax)); 
     DODEBUG(dcontext->expect_last_syscall_to_fail = false;);
 
     KSTART(pre_syscall);