From 8a0bd4b33189eb2c109c4e15020d75ea7dc8af57 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Fri, 9 Sep 2022 18:24:15 +0200 Subject: [PATCH] Added some aarch64 atomic access emitter (not finished) Removed public Imports::import_shared_memory and try to auto-initialize shared memory directly Fixed clippy Added Atomic Add/Sub/And/Or/Xor operator to Singlepass/AArch64 backend Added atomic_xchg support for Singlepass/AArch64 backend Finished all atomic access operator for Singlepass/Aarch64 backend --- lib/api/src/js/imports.rs | 6 +- lib/api/src/sys/imports.rs | 38 +- lib/api/src/sys/instance.rs | 2 +- lib/cli/src/commands/run/wasi.rs | 3 +- lib/compiler-singlepass/src/emitter_arm64.rs | 124 + lib/compiler-singlepass/src/machine_arm64.rs | 3587 ++++++++++++++---- lib/vm/src/lib.rs | 4 +- tests/ignores.txt | 1 - 8 files changed, 3110 insertions(+), 655 deletions(-) diff --git a/lib/api/src/js/imports.rs b/lib/api/src/js/imports.rs index a668b96d5fd..fb82e9885c1 100644 --- a/lib/api/src/js/imports.rs +++ b/lib/api/src/js/imports.rs @@ -133,7 +133,11 @@ impl Imports { /// Resolve and return a vector of imports in the order they are defined in the `module`'s source code. /// /// This means the returned `Vec` might be a subset of the imports contained in `self`. - pub fn imports_for_module(&self, module: &Module) -> Result, LinkError> { + pub fn imports_for_module( + &self, + module: &Module, + _store: &mut impl AsStoreMut, + ) -> Result, LinkError> { let mut ret = vec![]; for import in module.imports() { if let Some(imp) = self diff --git a/lib/api/src/sys/imports.rs b/lib/api/src/sys/imports.rs index b6864e5ee2f..18eef148f2c 100644 --- a/lib/api/src/sys/imports.rs +++ b/lib/api/src/sys/imports.rs @@ -114,11 +114,11 @@ impl Imports { /// Imports (any) shared memory into the imports. /// (if the module does not import memory then this function is ignored) - pub fn import_shared_memory( - &mut self, + pub(crate) fn import_shared_memory( + &self, module: &Module, store: &mut impl AsStoreMut, - ) -> Option { + ) -> Self { // Determine if shared memory needs to be created and imported let shared_memory = module .imports() @@ -130,16 +130,21 @@ impl Imports { VMSharedMemory::new(&ty, &style).unwrap() }); + let mut ret = self.clone(); if let Some(memory) = shared_memory { - self.define( - "env", - "memory", - Memory::new_from_existing(store, memory.clone().into()), - ); - Some(memory) - } else { - None - } + // if the memory has already be defined, don't redefine it! + if !self + .map + .contains_key(&("env".to_string(), "memory".to_string())) + { + ret.define( + "env", + "memory", + Memory::new_from_existing(store, memory.into()), + ); + } + }; + ret } /// Returns the contents of a namespace as an `Exports`. @@ -162,10 +167,15 @@ impl Imports { /// Resolve and return a vector of imports in the order they are defined in the `module`'s source code. /// /// This means the returned `Vec` might be a subset of the imports contained in `self`. - pub fn imports_for_module(&self, module: &Module) -> Result, LinkError> { + pub fn imports_for_module( + &self, + module: &Module, + store: &mut impl AsStoreMut, + ) -> Result, LinkError> { let mut ret = vec![]; + let imports = self.import_shared_memory(module, store); for import in module.imports() { - if let Some(imp) = self + if let Some(imp) = imports .map .get(&(import.module().to_string(), import.name().to_string())) { diff --git a/lib/api/src/sys/instance.rs b/lib/api/src/sys/instance.rs index ab8e9d5c293..315191128e6 100644 --- a/lib/api/src/sys/instance.rs +++ b/lib/api/src/sys/instance.rs @@ -116,7 +116,7 @@ impl Instance { imports: &Imports, ) -> Result { let imports = imports - .imports_for_module(module) + .imports_for_module(module, store) .map_err(InstantiationError::Link)?; let mut handle = module.instantiate(store, &imports)?; let exports = module diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index d913e5119c7..ffc70c42036 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -104,8 +104,7 @@ impl Wasi { is_wasix_module(module), std::sync::atomic::Ordering::Release, ); - let mut import_object = import_object_for_all_wasi_versions(store, &wasi_env.env); - import_object.import_shared_memory(module, store); + let import_object = import_object_for_all_wasi_versions(store, &wasi_env.env); let instance = Instance::new(store, module, &import_object)?; let memory = instance.exports.get_memory("memory")?; wasi_env.data_mut(store).set_memory(memory.clone()); diff --git a/lib/compiler-singlepass/src/emitter_arm64.rs b/lib/compiler-singlepass/src/emitter_arm64.rs index bdf010a38b2..074ae0acd00 100644 --- a/lib/compiler-singlepass/src/emitter_arm64.rs +++ b/lib/compiler-singlepass/src/emitter_arm64.rs @@ -153,6 +153,31 @@ pub trait EmitterARM64 { fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>; fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>; + fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>; + fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>; + fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>; + fn emit_stlxr( + &mut self, + sz: Size, + status: Location, + reg: Location, + dst: Location, + ) -> Result<(), CompileError>; + fn emit_stlxrb( + &mut self, + sz: Size, + status: Location, + reg: Location, + dst: Location, + ) -> Result<(), CompileError>; + fn emit_stlxrh( + &mut self, + sz: Size, + status: Location, + reg: Location, + dst: Location, + ) -> Result<(), CompileError>; + fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>; fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>; @@ -1059,6 +1084,105 @@ impl EmitterARM64 for Assembler { Ok(()) } + fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> { + match (sz, reg, dst) { + (Size::S32, Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + dynasm!(self ; ldaxr W(reg), [X(dst)]); + } + (Size::S64, Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + dynasm!(self ; ldaxr X(reg), [X(dst)]); + } + _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst), + } + Ok(()) + } + fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> { + match (reg, dst) { + (Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + dynasm!(self ; ldaxrb W(reg), [X(dst)]); + } + _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst), + } + Ok(()) + } + fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> { + match (reg, dst) { + (Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + dynasm!(self ; ldaxrh W(reg), [X(dst)]); + } + _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst), + } + Ok(()) + } + fn emit_stlxr( + &mut self, + sz: Size, + status: Location, + reg: Location, + dst: Location, + ) -> Result<(), CompileError> { + match (sz, status, reg, dst) { + (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + let status = status.into_index() as u32; + dynasm!(self ; stlxr W(status), W(reg), [X(dst)]); + } + (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + let status = status.into_index() as u32; + dynasm!(self ; stlxr W(status), X(reg), [X(dst)]); + } + _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst), + } + Ok(()) + } + fn emit_stlxrb( + &mut self, + _sz: Size, + status: Location, + reg: Location, + dst: Location, + ) -> Result<(), CompileError> { + match (status, reg, dst) { + (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + let status = status.into_index() as u32; + dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]); + } + _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst), + } + Ok(()) + } + fn emit_stlxrh( + &mut self, + _sz: Size, + status: Location, + reg: Location, + dst: Location, + ) -> Result<(), CompileError> { + match (status, reg, dst) { + (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => { + let reg = reg.into_index() as u32; + let dst = dst.into_index() as u32; + let status = status.into_index() as u32; + dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]); + } + _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst), + } + Ok(()) + } + fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> { match (sz, src, dst) { (Size::S64, Location::GPR(src), Location::GPR(dst)) => { diff --git a/lib/compiler-singlepass/src/machine_arm64.rs b/lib/compiler-singlepass/src/machine_arm64.rs index 806bb33eb1e..1f0c5b95ff2 100644 --- a/lib/compiler-singlepass/src/machine_arm64.rs +++ b/lib/compiler-singlepass/src/machine_arm64.rs @@ -1096,11 +1096,11 @@ impl MachineARM64 { self.release_gpr(tmp_bound); self.release_gpr(tmp_base); - let align = memarg.align; + let align = value_size as u32; if check_alignment && align != 1 { self.assembler.emit_tst( Size::S64, - Location::Imm32((align - 1).into()), + Location::Imm32(align - 1), Location::GPR(tmp_addr), )?; self.assembler @@ -3290,42 +3290,75 @@ impl Machine for MachineARM64 { } fn i32_atomic_load( &mut self, - _addr: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_load unimplemented"); + self.memory_op( + addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)), + ) } fn i32_atomic_load_8u( &mut self, - _addr: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_load_8u unimplemented"); + self.memory_op( + addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)), + ) } fn i32_atomic_load_16u( &mut self, - _addr: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_load_16u unimplemented"); + self.memory_op( + addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)), + ) } fn i32_save( &mut self, @@ -3401,361 +3434,1321 @@ impl Machine for MachineARM64 { } fn i32_atomic_save( &mut self, - _value: Location, - _memarg: &MemoryImmediate, - _target_addr: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_save unimplemented"); + self.memory_op( + target_addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)), + )?; + self.assembler.emit_dmb() } fn i32_atomic_save_8( &mut self, - _value: Location, - _memarg: &MemoryImmediate, - _target_addr: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_save_8 unimplemented"); + self.memory_op( + target_addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)), + )?; + self.assembler.emit_dmb() } fn i32_atomic_save_16( &mut self, - _value: Location, - _memarg: &MemoryImmediate, - _target_addr: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_save_16 unimplemented"); + self.memory_op( + target_addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)), + )?; + self.assembler.emit_dmb() } // i32 atomic Add with i32 fn i32_atomic_add( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_add unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Add with u8 fn i32_atomic_add_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_add_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Add with u16 fn i32_atomic_add_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_add_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Sub with i32 fn i32_atomic_sub( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_sub unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Sub with u8 fn i32_atomic_sub_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_sub_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Sub with u16 fn i32_atomic_sub_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_sub_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic And with i32 fn i32_atomic_and( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_and unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic And with u8 fn i32_atomic_and_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_and_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic And with u16 fn i32_atomic_and_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_and_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Or with i32 fn i32_atomic_or( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_or unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Or with u8 fn i32_atomic_or_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_or_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Or with u16 fn i32_atomic_or_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_or_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Xor with i32 fn i32_atomic_xor( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_xor unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Xor with u8 fn i32_atomic_xor_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_xor_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Xor with u16 fn i32_atomic_xor_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_xor_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Exchange with i32 fn i32_atomic_xchg( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_xchg unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Exchange with u8 fn i32_atomic_xchg_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_xchg_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?; + this.assembler.emit_stlxrb( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Exchange with u16 fn i32_atomic_xchg_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_xchg_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?; + this.assembler.emit_stlxrh( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Exchange with i32 fn i32_atomic_cmpxchg( &mut self, - _new: Location, - _cmp: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + new: Location, + cmp: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_cmpxchg unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + let nosame = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_relaxed_cmp(Size::S32, dst, cmp)?; + this.assembler.emit_bcond_label(Condition::Ne, nosame)?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + this.emit_label(nosame)?; + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Exchange with u8 fn i32_atomic_cmpxchg_8u( &mut self, - _new: Location, - _cmp: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + new: Location, + cmp: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_cmpxchg_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + let nosame = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?; + this.emit_relaxed_cmp(Size::S32, dst, cmp)?; + this.assembler.emit_bcond_label(Condition::Ne, nosame)?; + this.assembler.emit_stlxrb( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + this.emit_label(nosame)?; + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i32 atomic Exchange with u16 fn i32_atomic_cmpxchg_16u( &mut self, - _new: Location, - _cmp: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + new: Location, + cmp: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i32_atomic_cmpxchg_16u unimplemented"); - } + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + let nosame = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?; + this.emit_relaxed_cmp(Size::S32, dst, cmp)?; + this.assembler.emit_bcond_label(Condition::Ne, nosame)?; + this.assembler.emit_stlxrh( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + this.emit_label(nosame)?; + if dst != ret { + this.move_location(Size::S32, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) + } fn emit_call_with_reloc( &mut self, @@ -4420,55 +5413,99 @@ impl Machine for MachineARM64 { } fn i64_atomic_load( &mut self, - _addr: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_load unimplemented"); + self.memory_op( + addr, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)), + ) } fn i64_atomic_load_8u( &mut self, - _addr: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_load_8u unimplemented"); + self.memory_op( + addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)), + ) } fn i64_atomic_load_16u( &mut self, - _addr: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_load_16u unimplemented"); + self.memory_op( + addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)), + ) } fn i64_atomic_load_32u( &mut self, - _addr: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_load_32u unimplemented"); + self.memory_op( + addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)), + ) } fn i64_save( &mut self, @@ -4568,479 +5605,1759 @@ impl Machine for MachineARM64 { } fn i64_atomic_save( &mut self, - _value: Location, - _memarg: &MemoryImmediate, - _target_addr: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_save unimplemented"); + self.memory_op( + target_addr, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)), + )?; + self.assembler.emit_dmb() } fn i64_atomic_save_8( &mut self, - _value: Location, - _memarg: &MemoryImmediate, - _target_addr: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_save_8 unimplemented"); + self.memory_op( + target_addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)), + )?; + self.assembler.emit_dmb() } fn i64_atomic_save_16( &mut self, - _value: Location, - _memarg: &MemoryImmediate, - _target_addr: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_save_16 unimplemented"); + self.memory_op( + target_addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)), + )?; + self.assembler.emit_dmb() } fn i64_atomic_save_32( &mut self, - _value: Location, - _memarg: &MemoryImmediate, - _target_addr: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + target_value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_save_32 unimplemented"); + self.memory_op( + target_addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)), + )?; + self.assembler.emit_dmb() } // i64 atomic Add with i64 fn i64_atomic_add( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, - ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_add unimplemented"); - } - // i64 atomic Add with u8 - fn i64_atomic_add_8u( - &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, + ) -> Result<(), CompileError> { + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) + } + // i64 atomic Add with u8 + fn i64_atomic_add_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_add_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Add with u16 fn i64_atomic_add_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_add_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Add with u32 fn i64_atomic_add_32u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_add_32u unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Sub with i64 fn i64_atomic_sub( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_sub unimplemented"); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Sub with u8 fn i64_atomic_sub_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_sub_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Sub with u16 fn i64_atomic_sub_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_sub_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Sub with u32 fn i64_atomic_sub_32u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_sub_32u unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic And with i64 fn i64_atomic_and( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_and unimplemented"); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic And with u8 fn i64_atomic_and_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_and_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic And with u16 fn i64_atomic_and_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_and_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic And with u32 fn i64_atomic_and_32u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_and_32u unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Or with i64 fn i64_atomic_or( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_or unimplemented"); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Or with u8 fn i64_atomic_or_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_or_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Or with u16 fn i64_atomic_or_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_or_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Or with u32 fn i64_atomic_or_32u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_or_32u unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } - // i64 atomic xor with i64 + // i64 atomic Xor with i64 fn i64_atomic_xor( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xor unimplemented"); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } - // i64 atomic xor with u8 + // i64 atomic Xor with u8 fn i64_atomic_xor_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xor_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrb( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } - // i64 atomic xor with u16 + // i64 atomic Xor with u16 fn i64_atomic_xor_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xor_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?; + this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxrh( + Size::S64, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } - // i64 atomic xor with u32 + // i64 atomic Xor with u32 fn i64_atomic_xor_32u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xor_32u unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp1 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let tmp2 = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp2), + Location::GPR(tmp1), + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with i64 fn i64_atomic_xchg( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xchg unimplemented"); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?; + this.assembler.emit_stlxr( + Size::S64, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with u8 fn i64_atomic_xchg_8u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xchg_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?; + this.assembler.emit_stlxrb( + Size::S64, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with u16 fn i64_atomic_xchg_16u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xchg_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?; + this.assembler.emit_stlxrh( + Size::S64, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with u32 fn i64_atomic_xchg_32u( &mut self, - _loc: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_xchg_32u unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with i64 fn i64_atomic_cmpxchg( &mut self, - _new: Location, - _cmp: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + new: Location, + cmp: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_cmpxchg unimplemented"); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + let nosame = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?; + this.emit_relaxed_cmp(Size::S64, dst, cmp)?; + this.assembler.emit_bcond_label(Condition::Ne, nosame)?; + this.assembler.emit_stlxr( + Size::S64, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + this.emit_label(nosame)?; + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with u8 fn i64_atomic_cmpxchg_8u( &mut self, - _new: Location, - _cmp: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + new: Location, + cmp: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_cmpxchg_8u unimplemented"); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + let nosame = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?; + this.emit_relaxed_cmp(Size::S64, dst, cmp)?; + this.assembler.emit_bcond_label(Condition::Ne, nosame)?; + this.assembler.emit_stlxrb( + Size::S64, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + this.emit_label(nosame)?; + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with u16 fn i64_atomic_cmpxchg_16u( &mut self, - _new: Location, - _cmp: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + new: Location, + cmp: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_cmpxchg_16u unimplemented"); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + let nosame = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?; + this.emit_relaxed_cmp(Size::S64, dst, cmp)?; + this.assembler.emit_bcond_label(Condition::Ne, nosame)?; + this.assembler.emit_stlxrh( + Size::S64, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + this.emit_label(nosame)?; + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } // i64 atomic Exchange with u32 fn i64_atomic_cmpxchg_32u( &mut self, - _new: Location, - _cmp: Location, - _target: Location, - _memarg: &MemoryImmediate, - _ret: Location, - _need_check: bool, - _imported_memories: bool, - _offset: i32, - _heap_access_oob: Label, - _unaligned_atomic: Label, + new: Location, + cmp: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + unaligned_atomic: Label, ) -> Result<(), CompileError> { - codegen_error!("singlepass i64_atomic_cmpxchg_32u unimplemented"); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + unaligned_atomic, + |this, addr| { + let mut temps = vec![]; + let tmp = this.acquire_temp_gpr().ok_or_else(|| { + CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned()) + })?; + let dst = + this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?; + let org = + this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?; + let reread = this.get_label(); + let nosame = this.get_label(); + + this.emit_label(reread)?; + this.assembler + .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?; + this.emit_relaxed_cmp(Size::S64, dst, cmp)?; + this.assembler.emit_bcond_label(Condition::Ne, nosame)?; + this.assembler.emit_stlxr( + Size::S32, + Location::GPR(tmp), + org, + Location::GPR(addr), + )?; + this.assembler + .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?; + this.assembler.emit_dmb()?; + + this.emit_label(nosame)?; + if dst != ret { + this.move_location(Size::S64, ret, dst)?; + } + for r in temps { + this.release_gpr(r); + } + Ok(()) + }, + ) } fn f32_load( diff --git a/lib/vm/src/lib.rs b/lib/vm/src/lib.rs index 8d5602fefed..3b1abc55127 100644 --- a/lib/vm/src/lib.rs +++ b/lib/vm/src/lib.rs @@ -45,7 +45,9 @@ pub use crate::function_env::VMFunctionEnvironment; pub use crate::global::*; pub use crate::imports::Imports; pub use crate::instance::{InstanceAllocator, InstanceHandle}; -pub use crate::memory::{initialize_memory_with_data, LinearMemory, VMMemory, VMOwnedMemory, VMSharedMemory}; +pub use crate::memory::{ + initialize_memory_with_data, LinearMemory, VMMemory, VMOwnedMemory, VMSharedMemory, +}; pub use crate::mmap::Mmap; pub use crate::probestack::PROBESTACK; pub use crate::sig_registry::SignatureRegistry; diff --git a/tests/ignores.txt b/tests/ignores.txt index 08a5982bdc0..b84bbeaa442 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -24,7 +24,6 @@ llvm traps::start_trap_pretty cranelift+aarch64+macos traps::start_trap_pretty # Atomics (WIP) -singlepass+aarch64 spec::threads::atomic singlepass spec::threads::imports cranelift spec::threads::imports llvm spec::threads::imports