diff --git a/linker/linker.cpp b/linker/linker.cpp index 869c2b2b4..bd5e3f49c 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -2936,6 +2936,7 @@ static soinfo_list_t g_empty_list; bool soinfo::prelink_image() { if (flags_ & FLAG_PRELINKED) return true; + DEBUG("si->load_bias %p si->base = %p si->flags = 0x%08x", reinterpret_cast(load_bias), reinterpret_cast(base), flags_); /* Extract dynamic section */ ElfW(Word) dynamic_flags = 0; phdr_table_get_dynamic_section(phdr, phnum, load_bias, &dynamic, &dynamic_flags); diff --git a/linker/linker_block_allocator.cpp b/linker/linker_block_allocator.cpp index dd61db225..5d1e7c686 100644 --- a/linker/linker_block_allocator.cpp +++ b/linker/linker_block_allocator.cpp @@ -126,7 +126,7 @@ void LinkerBlockAllocator::create_new_page() { "Invalid sizeof(LinkerBlockAllocatorPage)"); LinkerBlockAllocatorPage* page = reinterpret_cast( - mmap(nullptr, kAllocateSize, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)); + mmap(nullptr, kAllocateSize, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)); if (page == MAP_FAILED) { abort(); // oom diff --git a/linker/linker_mapped_file_fragment.cpp b/linker/linker_mapped_file_fragment.cpp index d4f3f9fd6..83f846f9f 100644 --- a/linker/linker_mapped_file_fragment.cpp +++ b/linker/linker_mapped_file_fragment.cpp @@ -35,6 +35,18 @@ #include #include +#if defined(__APPLE__) && defined(__aarch64__) +#define page_start __page_start +#define page_offset __page_offset +static off64_t __page_start(off64_t offset) { + return offset & ~static_cast(4*4096-1); +} + +static size_t __page_offset(off64_t offset) { + return static_cast(offset & (4*4096-1)); +} +#endif + MappedFileFragment::MappedFileFragment() : map_start_(nullptr), map_size_(0), data_(nullptr), size_ (0) { } diff --git a/linker/linker_phdr.cpp b/linker/linker_phdr.cpp index 741f8ccc0..051b993d7 100644 --- a/linker/linker_phdr.cpp +++ b/linker/linker_phdr.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include "linker.h" #include "linker_dlwarning.h" @@ -42,6 +43,11 @@ #include "linker_debug.h" #include "linker_utils.h" +#if defined(__APPLE__) && defined(__aarch64__) +#include +#include +#endif + #include "private/CFIShadow.h" // For kLibraryAlignment static int GetTargetElfMachine() { @@ -484,11 +490,13 @@ bool ElfReader::ReadDynamicSection() { */ size_t phdr_table_get_load_size(const ElfW(Phdr)* phdr_table, size_t phdr_count, ElfW(Addr)* out_min_vaddr, - ElfW(Addr)* out_max_vaddr) { + ElfW(Addr)* out_max_vaddr, + void** writeableAfterExec) { ElfW(Addr) min_vaddr = UINTPTR_MAX; ElfW(Addr) max_vaddr = 0; bool found_pt_load = false; + bool prevExec = false; for (size_t i = 0; i < phdr_count; ++i) { const ElfW(Phdr)* phdr = &phdr_table[i]; @@ -497,6 +505,13 @@ size_t phdr_table_get_load_size(const ElfW(Phdr)* phdr_table, size_t phdr_count, } found_pt_load = true; + if(prevExec && phdr->p_flags & PF_W) { + if(writeableAfterExec) { + *writeableAfterExec = (void*) phdr->p_vaddr; + } + } + prevExec = phdr->p_flags & PF_X; + if (phdr->p_vaddr < min_vaddr) { min_vaddr = phdr->p_vaddr; } @@ -524,6 +539,31 @@ size_t phdr_table_get_load_size(const ElfW(Phdr)* phdr_table, size_t phdr_count, // Reserve a virtual address range such that if it's limits were extended to the next 2**align // boundary, it would not overlap with any existing mappings. static void* ReserveAligned(size_t size, size_t align) { +#if defined(__APPLE__) && defined(__aarch64__) + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT; + + // Allocate enough space so that the end of the desired region aligned up is still inside the + // mapping. + size_t mmap_size = align_up(size, align) + align - PAGE_SIZE; + uint8_t* mmap_ptr = + reinterpret_cast(mmap(nullptr, mmap_size, PROT_READ | PROT_WRITE | PROT_EXEC, mmap_flags, -1, 0)); + if (mmap_ptr == MAP_FAILED) { + return nullptr; + } + + uint8_t* first = align_up(mmap_ptr, align); + uint8_t* last = align_down(mmap_ptr + mmap_size, align) - size; + + // arc4random* is not available in first stage init because /dev/urandom hasn't yet been + // created. Don't randomize then. + size_t n = 0;//is_first_stage_init() ? 0 : arc4random_uniform((last - first) / PAGE_SIZE + 1); + uint8_t* start = first + n * PAGE_SIZE; + + //munmap(mmap_ptr, start - mmap_ptr); + //munmap(start + size, mmap_ptr + mmap_size - (start + size)); + INFO("Allocated RWX MAP_JIT segment @ %p size %lld", reinterpret_cast(start), (long long)size); + return start; +#else int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS; if (align == PAGE_SIZE) { void* mmap_ptr = mmap(nullptr, size, PROT_NONE, mmap_flags, -1, 0); @@ -552,14 +592,16 @@ static void* ReserveAligned(size_t size, size_t align) { munmap(mmap_ptr, start - mmap_ptr); munmap(start + size, mmap_ptr + mmap_size - (start + size)); return start; +#endif } // Reserve a virtual address range big enough to hold all loadable // segments of a program header table. This is done by creating a // private anonymous mmap() with PROT_NONE. bool ElfReader::ReserveAddressSpace(address_space_params* address_space) { + void* writeableAfterExec; ElfW(Addr) min_vaddr; - load_size_ = phdr_table_get_load_size(phdr_table_, phdr_num_, &min_vaddr); + load_size_ = phdr_table_get_load_size(phdr_table_, phdr_num_, &min_vaddr, nullptr, &writeableAfterExec); if (load_size_ == 0) { DL_ERR("\"%s\" has no loadable segments", name_.c_str()); return false; @@ -590,10 +632,57 @@ bool ElfReader::ReserveAddressSpace(address_space_params* address_space) { load_start_ = start; load_bias_ = reinterpret_cast(start) - addr; +#if defined(__APPLE__) && defined(__aarch64__) + INFO("[ Reserved Memory before extra align ] load_start_=%p load_bias_=%p", load_start_, load_bias_); + if(writeableAfterExec) { + auto __load_bias_2 = load_bias_ + 0x4000 - (((intptr_t)load_bias_ + (intptr_t)writeableAfterExec ) & 0x3000); + load_start_ = reinterpret_cast(start) + (__load_bias_2 - load_bias_); + load_bias_ = __load_bias_2; + } + INFO("[ Reserved Memory after extra align ] load_start_=%p load_bias_=%p", load_start_, load_bias_); +#endif return true; } bool ElfReader::LoadSegments() { +#if defined(__APPLE__) && defined(__aarch64__) + INFO("[macOS/m1] pthread_jit_write_protect_np(0)"); + pthread_jit_write_protect_np(0); + + auto scope_guard = android::base::make_scope_guard([&]() { + INFO("[macOS/m1] pthread_jit_write_protect_np(1)"); + pthread_jit_write_protect_np(1); + }); + + // Due to overlapping pages do this before copy the data + for (size_t i = 0; i < phdr_num_; ++i) { + const ElfW(Phdr)* phdr = &phdr_table_[i]; + + if (phdr->p_type != PT_LOAD) { + continue; + } + + // Segment addresses in memory. + ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_; + ElfW(Addr) seg_end = seg_start + phdr->p_memsz; + + ElfW(Addr) seg_page_start = PAGE_START(seg_start); + ElfW(Addr) seg_page_end = PAGE_END(seg_end); + + int prot = PFLAGS_TO_PROT(phdr->p_flags); + if(prot & PROT_WRITE) { + auto my_seg_page_start = seg_page_start & ~(4 * 4096 - 1); + auto myseg_page_end = (seg_page_end + 4 * 4096) & ~(4 * 4096 - 1); + size_t seg_size = myseg_page_end - my_seg_page_start; + void* seg_addr = mmap(reinterpret_cast(my_seg_page_start), seg_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (seg_addr == MAP_FAILED) { + DL_ERR("couldn't remap \"%s\" segment %zd: %s, seg_page_addr=%lld, seg_size=%lld, load_bias_=%lld to be RW", name_.c_str(), i, strerror(errno), (long long)(intptr_t)reinterpret_cast(my_seg_page_start), (long long)(intptr_t)seg_size, (long long)(intptr_t)load_bias_); + return false; + } + INFO("remapped \"%s\" segment %zd seg_page_addr=%lld, seg_size=%lld, load_bias_=%lld to be RW", name_.c_str(), i, (long long)(intptr_t)reinterpret_cast(my_seg_page_start), (long long)(intptr_t)seg_size, (long long)(intptr_t)load_bias_); + } + } +#endif for (size_t i = 0; i < phdr_num_; ++i) { const ElfW(Phdr)* phdr = &phdr_table_[i]; @@ -630,8 +719,49 @@ bool ElfReader::LoadSegments() { reinterpret_cast(file_end), file_size_); return false; } + int prot = PFLAGS_TO_PROT(phdr->p_flags); +#if defined(__APPLE__) && defined(__aarch64__) + void* seg_addr = reinterpret_cast(seg_page_start); + + if (file_length != 0) { + auto seekoffset = lseek(fd_, file_offset_ + file_page_start, SEEK_SET); + if(seekoffset != (file_offset_ + file_page_start)) { + DL_ERR("couldn't lseek \"%s\" segment %zd: %s, (file_offset_ + file_page_start)=%lld, seekoffset=%lld", name_.c_str(), i, strerror(errno), (long long)(file_offset_ + file_page_start), (long long)seekoffset); + return false; + } + + auto remainingBytes = file_length; + auto dest = (unsigned char*)seg_addr; + unsigned char buffer[4096]; + while(remainingBytes > 0) { + auto expectedreadsize = remainingBytes < 4096 ? remainingBytes : 4096; + auto readsize = read(fd_, buffer, expectedreadsize); + if(readsize != expectedreadsize) { + DL_ERR("couldn't read \"%s\" segment %zd: %s, expectedreadsize=%lld, readsize=%lld, remainingBytes=%lld, file_length=%lld", name_.c_str(), i, strerror(errno), (long long)expectedreadsize, (long long)readsize, (long long)remainingBytes, (long long)file_length); + return false; + } + memcpy(dest, buffer, readsize); + dest += readsize; + remainingBytes -= readsize; + } + } + // Patch instructions for macOS/m1 + if(prot & PROT_EXEC) { + INFO("[macOS/m1] Patch instructions"); + auto c = (uint32_t*)seg_start; + auto e = (uint32_t*)seg_end; + for(; (e - c) > 0; c++) { + // Replace "mrs %0, tpidr_el0" with "mrs %0, tpidrro_el0" + if((*c & 0xffffffe0) == 0xd53bd040) { + *c = 0xd53bd060 | (*c & 0x0000001f); + } + } + INFO("[macOS/m1] sys_icache_invalidate"); + sys_icache_invalidate(reinterpret_cast(seg_page_start), seg_page_end - seg_page_start); + } + INFO("[LoadedSegement] %s %zd @ %p file_length=%lld seg_start=%p seg_end=%p load_bias_=%p", name_.c_str(), i, seg_addr, file_length, seg_start, seg_end, load_bias_); +#else if (file_length != 0) { - int prot = PFLAGS_TO_PROT(phdr->p_flags); #if 0 if ((prot & (PROT_EXEC | PROT_WRITE)) == (PROT_EXEC | PROT_WRITE)) { // W + E PT_LOAD segments are not allowed in O. @@ -674,7 +804,7 @@ bool ElfReader::LoadSegments() { size_t zeromap_size = seg_page_end - seg_file_end; void* zeromap = mmap(reinterpret_cast(seg_file_end), zeromap_size, - PFLAGS_TO_PROT(phdr->p_flags) | PROT_WRITE, + prot | PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); @@ -685,6 +815,7 @@ bool ElfReader::LoadSegments() { } + #endif } return true; } @@ -1025,6 +1156,7 @@ void phdr_table_get_dynamic_section(const ElfW(Phdr)* phdr_table, size_t phdr_co for (size_t i = 0; i(load_bias), reinterpret_cast(phdr.p_vaddr), i); *dynamic = reinterpret_cast(load_bias + phdr.p_vaddr); if (dynamic_flags) { *dynamic_flags = phdr.p_flags; diff --git a/linker/linker_phdr.h b/linker/linker_phdr.h index 5d1cfc2bd..ed1dd2644 100644 --- a/linker/linker_phdr.h +++ b/linker/linker_phdr.h @@ -107,7 +107,7 @@ class ElfReader { }; size_t phdr_table_get_load_size(const ElfW(Phdr)* phdr_table, size_t phdr_count, - ElfW(Addr)* min_vaddr = nullptr, ElfW(Addr)* max_vaddr = nullptr); + ElfW(Addr)* min_vaddr = nullptr, ElfW(Addr)* max_vaddr = nullptr, void** writeableAfterExec = nullptr); int phdr_table_protect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count, ElfW(Addr) load_bias); diff --git a/linker/linker_soinfo.cpp b/linker/linker_soinfo.cpp index 603480861..60b2ac635 100644 --- a/linker/linker_soinfo.cpp +++ b/linker/linker_soinfo.cpp @@ -547,6 +547,7 @@ void soinfo::call_constructors() { if (constructors_called || g_is_ldd) { return; } + INFO("call_constructors [ base %p load_bias %p for '%s' ]", reinterpret_cast(base), reinterpret_cast(load_bias), get_realpath()); // We set constructors_called before actually calling the constructors, otherwise it doesn't // protect against recursive constructor calls. One simple example of constructor recursion