From 54fb46dbb0fe6effc94d6b6e533c534827f2fe88 Mon Sep 17 00:00:00 2001 From: Derek Bruening Date: Fri, 16 Apr 2021 13:01:00 -0400 Subject: [PATCH] i#4316 a64 rseq: Port rseq mangling to AArch64 (#4854) Ports rseq mangling to AArch64. This requires implementing patch_mov_immed_arch() (done by leveraging existing raw encoding code from exit stub support) and spilling extra scratch registers in multiple places. Expands translation support to cover the new mangling. This includes adding proper recognition of an mcontext base load, which was incorrectly identified as an indirect branch target load on x86. Ports the heuristic for finding the app's rseq TLS offset on attach (when we did not see an rseq syscall) for AArch64 to look forward instead of backward as on x86 where negative segment offsets are used. Enables the rseq tests for AArch64. Tested all 6 rseq tests manually on a machine with a newer kernel. The Jenkins machine is too old and does not have rseq. 3 rseq tests do run on QEMU so those are enabled but until Jenkins is upgraded we will not have perfect regression tests. Updates the rseq docs to state that AArch64 is supported. One final step is to support stores with writeback, which are seen in real rseq sequences. That will be done separately. This completes porting the x86 support to AArch64. Issue: #4316 --- api/docs/bt.dox | 2 +- core/arch/aarch64/emit_utils.c | 13 ++-- core/arch/aarchxx/mangle.c | 25 +++++++- core/arch/arch.h | 8 +++ core/arch/mangle_shared.c | 107 +++++++++++++++++++++++++++------ core/ir/aarchxx/ir_utils.c | 3 +- core/ir/instr.h | 4 ++ core/translate.c | 51 +++++++++++++--- core/unix/rseq_linux.c | 30 ++++++--- suite/tests/CMakeLists.txt | 13 +++- suite/tests/linux/rseq.c | 6 +- 11 files changed, 206 insertions(+), 56 deletions(-) diff --git a/api/docs/bt.dox b/api/docs/bt.dox index 3a251950f77..dc227642b81 100644 --- a/api/docs/bt.dox +++ b/api/docs/bt.dox @@ -1312,7 +1312,7 @@ commit. This run-twice approach is subject to the following limitations: -- Only x86 is supported for now (no arm or aarch64 support yet). +- Only x86 and aarch64 are supported for now, and 32-bit x86 is not as well-tested. - The application must store an rseq_cs struct for each rseq region in a section of its binary named "__rseq_cs", optionally with an "__rseq_cs_ptr_array" section of pointers into the __rseq_cs section, per established conventions. diff --git a/core/arch/aarch64/emit_utils.c b/core/arch/aarch64/emit_utils.c index 0b7d93dcfab..66f3b194ad1 100644 --- a/core/arch/aarch64/emit_utils.c +++ b/core/arch/aarch64/emit_utils.c @@ -43,7 +43,6 @@ #define PRE instrlist_meta_preinsert #define OPREG opnd_create_reg -#define NOP_INST 0xd503201f #define BR_X1_INST (0xd61f0000 | 1 << 5) /* br x1 */ /***************************************************************************/ @@ -149,12 +148,12 @@ get_fcache_return_tls_offs(dcontext_t *dcontext, uint flags) /* Generate move (immediate) of a 64-bit value using at most 4 instructions. * pc must be a writable (vmcode) pc. */ -static uint * +uint * insert_mov_imm(uint *pc, reg_id_t dst, ptr_int_t val) { uint rt = dst - DR_REG_X0; ASSERT(rt < 31); - *pc++ = 0xd2800000 | rt | (val & 0xffff) << 5; /* mov x(rt), #x */ + *pc++ = 0xd2800000 | rt | (val & 0xffff) << 5; /* movz x(rt), #x */ if ((val >> 16 & 0xffff) != 0) *pc++ = 0xf2a00000 | rt | (val >> 16 & 0xffff) << 5; /* movk x(rt), #x, lsl #16 */ @@ -211,7 +210,7 @@ insert_exit_stub_other_flags(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, * lots of places expect the stub size to be fixed. */ for (uint j = 0; j < num_nops_needed; j++) - *pc++ = NOP_INST; + *pc++ = RAW_NOP_INST; /* The final slot is a data slot, which will hold the address of either * the fcache-return routine or the linked fragment. We reserve 12 bytes * and use the 8-byte aligned region of 8 bytes within it. @@ -248,7 +247,7 @@ insert_exit_stub_other_flags(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, * lots of places expect the stub size to be fixed. */ for (uint j = 0; j < num_nops_needed; j++) - *pc++ = NOP_INST; + *pc++ = RAW_NOP_INST; } return (int)((byte *)pc - (byte *)write_stub_pc); @@ -404,7 +403,7 @@ static uint * get_stub_branch(uint *pc) { /* Skip NOP instructions backwards. */ - while (*pc == NOP_INST) + while (*pc == RAW_NOP_INST) pc--; /* The First non-NOP instruction must be the branch. */ ASSERT(*pc == BR_X1_INST); @@ -1047,6 +1046,6 @@ fill_with_nops(dr_isa_mode_t isa_mode, byte *addr, size_t size) return false; } for (pc = addr; pc < addr + size; pc += 4) - *(uint *)pc = NOP_INST; /* nop */ + *(uint *)pc = RAW_NOP_INST; /* nop */ return true; } diff --git a/core/arch/aarchxx/mangle.c b/core/arch/aarchxx/mangle.c index f14a1378236..e226f3b2ad9 100644 --- a/core/arch/aarchxx/mangle.c +++ b/core/arch/aarchxx/mangle.c @@ -1201,11 +1201,34 @@ mangle_reinstate_it_blocks(dcontext_t *dcontext, instrlist_t *ilist, instr_t *st #endif /* !AARCH64 */ +/* This is *not* a hot-patchable patch: i.e., it is subject to races. */ void patch_mov_immed_arch(dcontext_t *dcontext, ptr_int_t val, byte *pc, instr_t *first, instr_t *last) { - ASSERT_NOT_IMPLEMENTED(false); /* FIXME i#1551, i#1569 */ +#ifdef AARCH64 + uint *write_pc = (uint *)vmcode_get_writable_addr(pc); + ASSERT(first != NULL && last != NULL); + /* We expect OP_movz followed by up to 3 OP_movk. */ + ASSERT(instr_get_opcode(first) == OP_movz && opnd_is_reg(instr_get_dst(first, 0))); + reg_id_t dst_reg = opnd_get_reg(instr_get_dst(first, 0)); + int instr_count = 1; + for (instr_t *inst = instr_get_next(first); inst != NULL; + inst = instr_get_next(inst)) { + ++instr_count; + ASSERT(instr_get_opcode(inst) == OP_movk && opnd_is_reg(instr_get_dst(inst, 0))); + if (inst == last) + break; + } + uint *end_pc = insert_mov_imm(write_pc, dst_reg, val); + ASSERT(end_pc - write_pc <= instr_count); + while (end_pc - write_pc < instr_count) { + *end_pc = RAW_NOP_INST; + ++end_pc; + } +#else + ASSERT_NOT_IMPLEMENTED(false); /* TODO i#1551: NYI */ +#endif } /* Used for fault translation */ diff --git a/core/arch/arch.h b/core/arch/arch.h index d02fd124619..51d990eb705 100644 --- a/core/arch/arch.h +++ b/core/arch/arch.h @@ -1201,6 +1201,14 @@ emit_do_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc, byte *fcache_return_pc, bool thread_shared, int interrupt, uint *syscall_offs /*OUT*/); +#ifdef AARCH64 +/* Generate move (immediate) of a 64-bit value using at most 4 instructions. + * pc must be a writable (vmcode) pc. + */ +uint * +insert_mov_imm(uint *pc, reg_id_t dst, ptr_int_t val); +#endif + #ifdef AARCHXX byte * emit_fcache_enter_gonative(dcontext_t *dcontext, generated_code_t *code, byte *pc); diff --git a/core/arch/mangle_shared.c b/core/arch/mangle_shared.c index f3509985346..939b73f12cd 100644 --- a/core/arch/mangle_shared.c +++ b/core/arch/mangle_shared.c @@ -947,7 +947,8 @@ mangle_rseq_insert_call_sequence(dcontext_t *dcontext, instrlist_t *ilist, instr ilist, next_instr, XINST_CREATE_add(dcontext, opnd_create_reg(DR_REG_RSP), OPND_CREATE_INT32(8))); # else - /* TODO i#2350: Add non-x86 support. We need to pay particular attention + /* TODO i#2350: Given that we plan to deprecate -rseq_assume_call, it may not be + * worth implementing non-x86 support. We'd need to pay particular attention * to the stolen register. If we do a local copy (with no callouts) we could * mangle it. We also cannot do an indirect call through anything but a * register and thus need a dead register for the call-return approach, but @@ -955,11 +956,12 @@ mangle_rseq_insert_call_sequence(dcontext_t *dcontext, instrlist_t *ilist, instr */ REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), get_application_pid(), - "Rseq is not yet supported for non-x86"); + "-rseq_assume_call is not supported for non-x86"); ASSERT_NOT_REACHED(); # endif } +/* scratch_reg is *not* spilled on entry. */ static void mangle_rseq_write_exit_reason(dcontext_t *dcontext, instrlist_t *ilist, instr_t *insert_at, reg_id_t scratch_reg) @@ -976,11 +978,25 @@ mangle_rseq_write_exit_reason(dcontext_t *dcontext, instrlist_t *ilist, opnd_create_reg(scratch_reg), ilist, insert_at, NULL, NULL); } +# ifdef AARCHXX + /* We need a 2nd scratch for our immediate. */ + ASSERT(SCRATCH_ALWAYS_TLS()); + reg_id_t scratch2 = + (scratch_reg == DR_REG_START_GPR) ? DR_REG_START_GPR + 1 : DR_REG_START_GPR; + PRE(ilist, insert_at, instr_create_save_to_tls(dcontext, scratch2, TLS_REG2_SLOT)); + insert_mov_immed_ptrsz(dcontext, EXIT_REASON_RSEQ_ABORT, opnd_create_reg(scratch2), + ilist, insert_at, NULL, NULL); +# endif PRE(ilist, insert_at, - XINST_CREATE_store(dcontext, - opnd_create_dcontext_field_via_reg_sz( - dcontext, scratch_reg, EXIT_REASON_OFFSET, OPSZ_2), - OPND_CREATE_INT16(EXIT_REASON_RSEQ_ABORT))); + XINST_CREATE_store_2bytes(dcontext, + opnd_create_dcontext_field_via_reg_sz( + dcontext, scratch_reg, EXIT_REASON_OFFSET, OPSZ_2), + IF_X86_ELSE(OPND_CREATE_INT16(EXIT_REASON_RSEQ_ABORT), + opnd_create_reg(scratch2)))); +# ifdef AARCHXX + PRE(ilist, insert_at, + instr_create_restore_from_tls(dcontext, scratch2, TLS_REG2_SLOT)); +# endif if (SCRATCH_ALWAYS_TLS()) { PRE(ilist, insert_at, instr_create_restore_from_tls(dcontext, scratch_reg, TLS_REG1_SLOT)); @@ -1120,27 +1136,45 @@ mangle_rseq_insert_native_sequence(dcontext_t *dcontext, instrlist_t *ilist, * decode_fragment() and even disassembly. */ instr_t *immed_first, *immed_last; - insert_mov_immed_ptrsz(dcontext, (ptr_int_t)INT_MAX IF_X64(+1), - opnd_create_reg(scratch_reg), ilist, insert_at, &immed_first, - &immed_last); + insert_mov_immed_ptrsz(dcontext, (ptr_int_t)-1, opnd_create_reg(scratch_reg), ilist, + insert_at, &immed_first, &immed_last); ASSERT(immed_first != NULL); IF_X86(ASSERT(immed_last == NULL)); + int immed_count = 1; + for (instr_t *immed_inst = immed_first; + immed_last != NULL && immed_inst != immed_last; + immed_inst = instr_get_next(immed_inst)) { + ++immed_count; + } instr_t *label_rseq_cs = - mangle_rseq_create_label(dcontext, DR_RSEQ_LABEL_CS, immed_last == NULL ? 1 : 2); + mangle_rseq_create_label(dcontext, DR_RSEQ_LABEL_CS, immed_count); PRE(ilist, immed_first /*prior to immeds*/, label_rseq_cs); - /* We need to mangle this segment ref, and all of the subsequent local copy. */ # ifdef X86 + /* We need to mangle this segment ref, and all of the subsequent local copy. */ instr_t *start_mangling = XINST_CREATE_store( dcontext, opnd_create_far_base_disp(LIB_SEG_TLS, DR_REG_NULL, DR_REG_NULL, 0, rseq_get_tls_ptr_offset(), OPSZ_PTR), opnd_create_reg(scratch_reg)); + instrlist_preinsert(ilist, insert_at, start_mangling); # else - /* TODO i#2350: Construct an app TLS access instruction for aarchxx. */ - ASSERT_NOT_IMPLEMENTED(false); - instr_t *start_mangling = INSTR_CREATE_label(dcontext); /* So it compiles. */ -# endif + /* We need another scratch reg to write to TLS. */ + ASSERT(SCRATCH_ALWAYS_TLS()); + reg_id_t scratch2 = + (scratch_reg == DR_REG_START_GPR) ? DR_REG_START_GPR + 1 : DR_REG_START_GPR; + PRE(ilist, insert_at, instr_create_save_to_tls(dcontext, scratch2, TLS_REG2_SLOT)); + /* We need to mangle this segment ref, and the local copy below. */ + instr_t *start_mangling = INSTR_CREATE_mrs(dcontext, opnd_create_reg(scratch2), + opnd_create_reg(LIB_SEG_TLS)); instrlist_preinsert(ilist, insert_at, start_mangling); + PRE(ilist, insert_at, + XINST_CREATE_store(dcontext, + opnd_create_base_disp(scratch2, DR_REG_NULL, 0, + rseq_get_tls_ptr_offset(), OPSZ_PTR), + opnd_create_reg(scratch_reg))); + PRE(ilist, insert_at, + instr_create_restore_from_tls(dcontext, scratch2, TLS_REG2_SLOT)); +# endif /* Restore scratch_reg. */ if (SCRATCH_ALWAYS_TLS()) { @@ -1240,6 +1274,7 @@ mangle_rseq_insert_native_sequence(dcontext_t *dcontext, instrlist_t *ilist, } generic_hash_destroy(dcontext, pc2instr); /* Now mangle from this point. */ + ASSERT(start_mangling != NULL); *next_instr = start_mangling; /* Clear the rseq ptr on exit to avoid problems if we free the rseq_cs and @@ -1256,8 +1291,28 @@ mangle_rseq_insert_native_sequence(dcontext_t *dcontext, instrlist_t *ilist, rseq_get_tls_ptr_offset(), OPSZ_PTR), OPND_CREATE_INT32(0))); # else - /* TODO i#2350: Construct an app TLS access instruction for aarchxx. */ - ASSERT_NOT_IMPLEMENTED(false); + PRE(ilist, insert_at, instr_create_save_to_tls(dcontext, scratch2, TLS_REG2_SLOT)); + instrlist_preinsert(ilist, insert_at, + INSTR_CREATE_mrs(dcontext, opnd_create_reg(scratch2), + opnd_create_reg(LIB_SEG_TLS))); +# ifdef ARM /* No zero register. */ + PRE(ilist, insert_at, instr_create_save_to_tls(dcontext, scratch_reg, TLS_REG1_SLOT)); + PRE(ilist, insert_at, + XINST_CREATE_load_int(dcontext, opnd_create_reg(scratch_reg), + OPND_CREATE_INT(0))); +# endif + instrlist_preinsert( + ilist, insert_at, + XINST_CREATE_store(dcontext, + opnd_create_base_disp(scratch2, DR_REG_NULL, 0, + rseq_get_tls_ptr_offset(), OPSZ_PTR), + opnd_create_reg(IF_AARCH64_ELSE(DR_REG_XZR, scratch_reg)))); +# ifdef ARM /* No zero register. */ + PRE(ilist, insert_at, + instr_create_restore_from_tls(dcontext, scratch_reg, TLS_REG1_SLOT)); +# endif + PRE(ilist, insert_at, + instr_create_restore_from_tls(dcontext, scratch2, TLS_REG2_SLOT)); # endif DOLOG(4, LOG_INTERP, { @@ -1277,6 +1332,9 @@ mangle_rseq(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr, bool *reg_written; int reg_written_size; reg_id_t scratch_reg = DR_REG_START_GPR; +# ifdef ARM + ASSERT_NOT_TESTED(); +# endif if (!rseq_get_region_info(pc, &start, &end, &handler, ®_written, ®_written_size)) { ASSERT_NOT_REACHED(); /* Caller was supposed to check for overlap */ @@ -1424,16 +1482,23 @@ mangle_rseq_finalize(dcontext_t *dcontext, instrlist_t *ilist, fragment_t *f) case DR_RSEQ_LABEL_CS: immed_start_pc = pc; immed_first = instr_get_next(instr); - if (label_data->data[1] > 1) + ptr_int_t immed_count = label_data->data[1]; + /* For A64 we should have 4 immeds to handle any address. */ + IF_AARCH64(ASSERT(immed_count == 4)); + if (immed_count > 1) { immed_last = instr_get_next(immed_first); + --immed_count; + while (immed_count > 1) { + immed_last = instr_get_next(immed_last); + --immed_count; + } + } break; default: ASSERT_NOT_REACHED(); } } pc += instr_length(dcontext, instr); } - LOG(THREAD, LOG_INTERP, 4, "%s: start=" PFX ", end=" PFX ", abort=" PFX "\n", - __FUNCTION__, rseq_start, rseq_end, rseq_abort); ASSERT(rseq_start != NULL && rseq_end != NULL && rseq_abort != NULL); byte *rseq_cs_alloc, *rseq_cs; @@ -1445,6 +1510,8 @@ mangle_rseq_finalize(dcontext_t *dcontext, instrlist_t *ilist, fragment_t *f) rseq_cs_alloc = rseq_get_rseq_cs_alloc(&rseq_cs); rseq_record_rseq_cs(rseq_cs_alloc, f, rseq_start, rseq_end, rseq_abort); ASSERT(immed_start_pc != NULL && immed_first != NULL); + LOG(THREAD, LOG_INTERP, 4, "%s: start=%p, end=%p, abort=%p stored @%p\n", + __FUNCTION__, rseq_start, rseq_end, rseq_abort, rseq_cs); patch_mov_immed_ptrsz(dcontext, (ptr_int_t)rseq_cs, immed_start_pc, immed_first, immed_last); } diff --git a/core/ir/aarchxx/ir_utils.c b/core/ir/aarchxx/ir_utils.c index 8408db94377..0b10cbaf2f1 100644 --- a/core/ir/aarchxx/ir_utils.c +++ b/core/ir/aarchxx/ir_utils.c @@ -1,5 +1,5 @@ /* ********************************************************** - * Copyright (c) 2014-2020 Google, Inc. All rights reserved. + * Copyright (c) 2014-2021 Google, Inc. All rights reserved. * Copyright (c) 2016 ARM Limited. All rights reserved. * **********************************************************/ @@ -162,6 +162,7 @@ convert_to_near_rel_arch(dcontext_t *dcontext, instrlist_t *ilist, instr_t *inst #endif } +/* Keep this in sync with patch_mov_immed_arch(). */ void insert_mov_immed_arch(dcontext_t *dcontext, instr_t *src_inst, byte *encode_estimate, ptr_int_t val, opnd_t dst, instrlist_t *ilist, instr_t *instr, diff --git a/core/ir/instr.h b/core/ir/instr.h index ee45bcf813d..7833c1ee270 100644 --- a/core/ir/instr.h +++ b/core/ir/instr.h @@ -865,6 +865,10 @@ enum { CBZ_BYTE_A = 0xb1, /* this assumes the top bit of the disp is 0 */ CBNZ_BYTE_A = 0xb9, /* this assumes the top bit of the disp is 0 */ }; +#elif defined(AARCH64) +enum { + RAW_NOP_INST = 0xd503201f, +}; #endif #include "instr_inline_api.h" diff --git a/core/translate.c b/core/translate.c index 8dcd35764b0..5b85dd70b6c 100644 --- a/core/translate.c +++ b/core/translate.c @@ -161,15 +161,17 @@ instr_is_seg_ref_load(dcontext_t *dcontext, instr_t *inst) } static inline bool -instr_is_rseq_load(dcontext_t *dcontext, instr_t *inst) +instr_is_rseq_mangling(dcontext_t *dcontext, instr_t *inst) { - /* TODO i#2350: Add non-x86 support. */ -# if defined(LINUX) && defined(X86) +# ifdef LINUX /* This won't fault but we don't want it marked as unsupported. */ if (!instr_is_our_mangling(inst)) return false; + if (vmvector_empty(d_r_rseq_areas)) + return false; /* XXX: Keep this consistent with mangle_rseq_* in mangle_shared.c. */ - if (instr_get_opcode(inst) == OP_mov_ld && opnd_is_reg(instr_get_dst(inst, 0)) && + if (instr_get_opcode(inst) == IF_X86_ELSE(OP_mov_ld, OP_ldr) && + opnd_is_reg(instr_get_dst(inst, 0)) && opnd_is_base_disp(instr_get_src(inst, 0))) { reg_id_t dst = opnd_get_reg(instr_get_dst(inst, 0)); opnd_t memref = instr_get_src(inst, 0); @@ -181,6 +183,19 @@ instr_is_rseq_load(dcontext_t *dcontext, instr_t *inst) sizeof(reg_t) * (dst - DR_REG_START_GPR)) return true; } +# ifdef AARCH64 + if (instr_get_opcode(inst) == OP_mrs && + opnd_get_reg(instr_get_src(inst, 0)) == LIB_SEG_TLS) + return true; + if (instr_get_opcode(inst) == OP_movz || instr_get_opcode(inst) == OP_movk) + return true; + if (instr_get_opcode(inst) == OP_strh && opnd_is_base_disp(instr_get_dst(inst, 0)) && + opnd_get_disp(instr_get_dst(inst, 0)) == EXIT_REASON_OFFSET) + return true; + if (instr_get_opcode(inst) == OP_str && opnd_is_base_disp(instr_get_dst(inst, 0)) && + opnd_get_disp(instr_get_dst(inst, 0)) == rseq_get_tls_ptr_offset()) + return true; +# endif # endif return false; } @@ -222,6 +237,15 @@ instr_is_mov_PC_immed(dcontext_t *dcontext, instr_t *inst) } #endif +static bool +instr_is_load_mcontext_base(instr_t *inst) +{ + if (instr_get_opcode(inst) != OP_load || !opnd_is_base_disp(instr_get_src(inst, 0))) + return false; + return opnd_get_disp(instr_get_src(inst, 0)) == + os_tls_offset((ushort)TLS_DCONTEXT_SLOT); +} + #ifdef X86 /* FIXME i#3329: add support for ARM/AArch64. */ @@ -470,13 +494,16 @@ translate_walk_track_post_instr(dcontext_t *tdcontext, instr_t *inst, /* We don't support restoring a fault in the middle, but we * identify here to avoid "unsupported mangle instr" message */ + } else if (instr_is_load_mcontext_base(inst)) { + LOG(THREAD_GET, LOG_INTERP, 4, "\tmcontext base load\n"); + /* nothing to do */ } #ifdef UNIX else if (instr_is_inline_syscall_jmp(tdcontext, inst)) { /* nothing to do */ } else if (instr_is_seg_ref_load(tdcontext, inst)) { /* nothing to do */ - } else if (instr_is_rseq_load(tdcontext, inst)) { + } else if (instr_is_rseq_mangling(tdcontext, inst)) { /* nothing to do */ } #endif @@ -896,12 +923,11 @@ recreate_app_state_from_ilist(dcontext_t *tdcontext, instrlist_t *ilist, byte *s /* Case 4531, 4344: raw instructions being up-decoded can have * their translation fields clobbered so we don't want any of those. - * (We used to have raw jecxz and nop instrs.) - * FIXME: if bb associated with this instr was hot patched, then - * the inserted raw instructions can trigger this assert. Part of - * fix for case 5981. In that case, this would be harmless. + * (We used to have raw jecxz and nop instrs.) But we do have cases + * of !instr_operands_valid() (rseq signature instr-as-data; or + * if the bb associated with this instr was hot patched, then + * the inserted raw instructions can trigger this assert). */ - ASSERT_CURIOSITY(instr_operands_valid(inst)); /* PR 332437: skip label instrs. Nobody should expect setting * a label's translation field to have any effect, and we @@ -1005,6 +1031,11 @@ recreate_app_state_from_ilist(dcontext_t *tdcontext, instrlist_t *ilist, byte *s LOG(THREAD_GET, LOG_INTERP, 2, "recreate_app -- found valid state pc " PFX "\n", answer); } else { + LOG(THREAD_GET, LOG_INTERP, 2, + "recreate_app -- invalid state: unsup=%d in-mangle=%d xl8=%p " + "walk=%p\n", + walk.unsupported_mangle, walk.in_mangle_region, answer, + walk.translation); #ifdef X86 int op = instr_get_opcode(inst); if (TEST(FRAG_SELFMOD_SANDBOXED, flags) && diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index bb75142c16a..310018e0705 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -583,6 +583,9 @@ rseq_process_module(module_area_t *ma, bool at_map) return res; } +/* If we did not observe the app invoke SYS_rseq (because we attached mid-run) + * we must search for its TLS location. + */ static int rseq_locate_tls_offset(void) { @@ -592,20 +595,26 @@ rseq_locate_tls_offset(void) * struct using heuristics, because the system call was poorly designed and will not * let us replace the app's. Alternatives of no local copy have worse problems. */ - /* Static TLS is at a negative offset from the app library segment base. We simply - * search all possible aligned slots. Typically there are <64 possible slots. + /* We simply search all possible aligned slots. Typically there + * are <64 possible slots. */ int offset = 0; byte *addr = get_app_segment_base(LIB_SEG_TLS); byte *seg_bottom; - if (addr > 0 && get_memory_info(addr, &seg_bottom, NULL, NULL)) { + size_t seg_size; + if (addr > 0 && get_memory_info(addr, &seg_bottom, &seg_size, NULL)) { LOG(GLOBAL, LOG_LOADER, 3, "rseq within static TLS " PFX " - " PFX "\n", seg_bottom, addr); /* struct rseq_cs is aligned to 32. */ int alignment = __alignof(struct rseq_cs); int i; - for (i = 0; addr - i * alignment >= seg_bottom; i++) { - byte *try_addr = addr - i * alignment; + /* For x86, static TLS is at a negative offset from the app library segment + * base, while for aarchxx it is positive. + */ + for (i = 0; IF_X86_ELSE(addr - i * alignment >= seg_bottom, + addr + i * alignment < seg_bottom + seg_size); + i++) { + byte *try_addr = IF_X86_ELSE(addr - i * alignment, addr + i * alignment); ASSERT(try_addr >= seg_bottom); /* For loop guarantees this. */ /* Our strategy is to check all of the aligned static TLS addresses to * find the registered one. Our caller is not supposed to call here @@ -629,9 +638,9 @@ rseq_locate_tls_offset(void) if (res == -EPERM) { /* Found it! */ LOG(GLOBAL, LOG_LOADER, 2, - "Found struct rseq @ " PFX " for thread => %s:-0x%x\n", try_addr, - get_register_name(LIB_SEG_TLS), i * alignment); - offset = -i * alignment; + "Found struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_addr, + get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), i * alignment); + offset = IF_X86_ELSE(-i * alignment, i * alignment); } break; } @@ -653,8 +662,9 @@ rseq_process_syscall(dcontext_t *dcontext) SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); constant_offset = (prior == 0 || prior == offset); LOG(GLOBAL, LOG_LOADER, 2, - "Observed struct rseq @ " PFX " for thread => %s:-0x%x\n", app_addr, - get_register_name(LIB_SEG_TLS), -rseq_tls_offset); + "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, + get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), + IF_X86_ELSE(-rseq_tls_offset, rseq_tls_offset)); } else constant_offset = (seg_base + rseq_tls_offset == app_addr); if (!constant_offset) { diff --git a/suite/tests/CMakeLists.txt b/suite/tests/CMakeLists.txt index 76b66d3db7a..5b9688c9f72 100644 --- a/suite/tests/CMakeLists.txt +++ b/suite/tests/CMakeLists.txt @@ -3490,7 +3490,7 @@ if (BUILD_CLIENTS) unset(tool.drcacheoff.func_view_heap_rawtemp) # use preprocessor endif () - if (LINUX AND X86 AND X64 AND HAVE_RSEQ) + if (LINUX AND X64 AND HAVE_RSEQ) torunonly_drcacheoff(rseq linux.rseq "" "@-test_mode" "") endif () @@ -4138,9 +4138,9 @@ if (UNIX) # when running tests in parallel: have to generate pcaches first set(linux.persist-use_FLAKY_depends linux.persist_FLAKY) - if (LINUX AND X86 AND X64 AND HAVE_RSEQ) + if (LINUX AND X64 AND HAVE_RSEQ) # The rseq kernel feature is Linux-only. - # TODO i#2350: Port the assembly in the test to 32-bit, ARM, AArch64. + # TODO i#2350: Port the assembly in the test to 32-bit x86 and to ARM. tobuild(linux.rseq linux/rseq.c) # Test the other sections. Unfortunately we need a separate binary for each. tobuild(linux.rseq_table linux/rseq.c) @@ -4715,6 +4715,13 @@ if (NOT ANDROID AND AARCHXX) code_api|sample.memtrace_simple code_api|sample.opcodes PROPERTIES LABELS RUNS_ON_QEMU) + if (LINUX AND X64 AND HAVE_RSEQ) + set_tests_properties( + code_api|linux.rseq + code_api|linux.rseq_table + code_api|linux.rseq_noarray + PROPERTIES LABELS RUNS_ON_QEMU) + endif () endif () if (ARM) set_tests_properties( diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index bd978707033..7d148a31868 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -215,7 +215,7 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o "cbz x0, 7f\n\t" "mov x0, #1\n\t" "prfm pldl3keep, [x0]\n\t" /* See above: annotation for trace_invariants. */ - "udf #0\n\t" + ".word 0\n\t" /* udf */ "7:\n\t" "ldr x1, %[completions]\n\t" "add x1, x1, #1\n\t" @@ -376,7 +376,7 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "cbz x0, 7f\n\t" "mov x0, #1\n\t" "prfm pldl3keep, [x0]\n\t" /* See above: annotation for trace_invariants. */ - "udf #0\n\t" + ".word 0\n\t" /* udf */ "7:\n\t" "ldr x1, %[completions]\n\t" "add x1, x1, #1\n\t" @@ -519,7 +519,7 @@ test_rseq_native_fault(void) "cmp x0, #2\n\t" "b.ne 11f\n\t" /* Raise a signal on the native run. */ - "udf #0\n\t" + ".word 0\n\t" /* udf */ "11:\n\t" "nop\n\t"