Merge pull request bytecodealliance#50 from dhil/wasmfx-merge

Merge with upstream

cranelift/codegen/src/egraph.rs

@@ -12,13 +12,13 @@ use crate::ir::{
     Block, DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,
 };
 use crate::loop_analysis::LoopAnalysis;
-use crate::opts::generated_code::ContextIter;
 use crate::opts::IsleContext;
 use crate::scoped_hash_map::{Entry as ScopedEntry, ScopedHashMap};
 use crate::trace;
 use crate::unionfind::UnionFind;
 use cranelift_entity::packed_option::ReservedValue;
 use cranelift_entity::SecondaryMap;
+use smallvec::SmallVec;
 use std::hash::Hasher;
 
 mod cost;
@@ -69,6 +69,9 @@ pub struct EgraphPass<'a> {
     eclasses: UnionFind<Value>,
 }
 
+// The maximum number of rewrites we will take from a single call into ISLE.
+const MATCHES_LIMIT: usize = 5;
+
 /// Context passed through node insertion and optimization.
 pub(crate) struct OptimizeCtx<'opt, 'analysis>
 where
@@ -87,6 +90,7 @@ where
     // Held locally during optimization of one node (recursively):
     pub(crate) rewrite_depth: usize,
     pub(crate) subsume_values: FxHashSet<Value>,
+    optimized_values: SmallVec<[Value; MATCHES_LIMIT]>,
 }
 
 /// For passing to `insert_pure_enode`. Sometimes the enode already
@@ -213,6 +217,7 @@ where
         // A pure node always has exactly one result.
         let orig_value = self.func.dfg.first_result(inst);
 
+        let mut optimized_values = std::mem::take(&mut self.optimized_values);
         let mut isle_ctx = IsleContext { ctx: self };
 
         // Limit rewrite depth. When we apply optimization rules, they
@@ -239,26 +244,26 @@ where
         // values produced as equivalents to this value.
         trace!("Calling into ISLE with original value {}", orig_value);
         isle_ctx.ctx.stats.rewrite_rule_invoked += 1;
-        let mut optimized_values =
-            crate::opts::generated_code::constructor_simplify(&mut isle_ctx, orig_value);
+        debug_assert!(optimized_values.is_empty());
+        crate::opts::generated_code::constructor_simplify(
+            &mut isle_ctx,
+            orig_value,
+            &mut optimized_values,
+        );
         trace!(
-            "  -> returned from ISLE, optimized values's size hint = {:?}",
-            optimized_values.size_hint()
+            "  -> returned from ISLE, generated {} optimized values",
+            optimized_values.len()
         );
+        if optimized_values.len() > MATCHES_LIMIT {
+            trace!("Reached maximum matches limit; too many optimized values, ignoring rest.");
+            optimized_values.truncate(MATCHES_LIMIT);
+        }
 
         // Create a union of all new values with the original (or
         // maybe just one new value marked as "subsuming" the
         // original, if present.)
-        let mut i = 0;
         let mut union_value = orig_value;
-        while let Some(optimized_value) = optimized_values.next(&mut isle_ctx) {
-            i += 1;
-            const MATCHES_LIMIT: u32 = 5;
-            if i > MATCHES_LIMIT {
-                trace!("Reached maximum matches limit; too many optimized values, ignoring rest.");
-                break;
-            }
-
+        for optimized_value in optimized_values.drain(..) {
             trace!(
                 "Returned from ISLE for {}, got {:?}",
                 orig_value,
@@ -305,6 +310,9 @@ where
 
         isle_ctx.ctx.rewrite_depth -= 1;
 
+        debug_assert!(isle_ctx.ctx.optimized_values.is_empty());
+        isle_ctx.ctx.optimized_values = optimized_values;
+
         union_value
     }
 
@@ -550,6 +558,7 @@ impl<'a> EgraphPass<'a> {
                             stats: &mut self.stats,
                             alias_analysis: self.alias_analysis,
                             alias_analysis_state: &mut alias_analysis_state,
+                            optimized_values: Default::default(),
                         };
 
                         if is_pure_for_egraph(ctx.func, inst) {

cranelift/codegen/src/ir/pcc.rs

@@ -1655,7 +1655,7 @@ pub fn check_vcode_facts<B: LowerBackend + TargetIsa>(
         for inst in vcode.block_insns(block).iter() {
             // Check any output facts on this inst.
             if let Err(e) = backend.check_fact(&ctx, vcode, inst, &mut flow_state) {
-                log::error!("Error checking instruction: {:?}", vcode[inst]);
+                log::info!("Error checking instruction: {:?}", vcode[inst]);
                 return Err(e);
             }
 

cranelift/codegen/src/isa/x64/pcc.rs

@@ -3,6 +3,7 @@
 use crate::ir::pcc::*;
 use crate::ir::types::*;
 use crate::ir::Type;
+use crate::isa::x64::args::AvxOpcode;
 use crate::isa::x64::inst::args::{
     AluRmiROpcode, Amode, Gpr, Imm8Reg, RegMem, RegMemImm, ShiftKind, SyntheticAmode,
     ToWritableReg, CC,
@@ -533,38 +534,69 @@ pub(crate) fn check(
             ensure_no_fact(vcode, dst.to_writable_reg().to_reg())
         }
 
+        // NOTE: it's assumed that all of these cases perform 128-bit loads, but this hasn't been
+        // verified. The effect of this will be spurious PCC failures when these instructions are
+        // involved.
         Inst::XmmRmRUnaligned { dst, ref src2, .. }
-        | Inst::XmmRmRImmVex { dst, ref src2, .. }
-        | Inst::XmmRmRVex3 {
-            dst,
-            src3: ref src2,
-            ..
-        }
-        | Inst::XmmRmRBlendVex { dst, ref src2, .. }
-        | Inst::XmmUnaryRmRVex {
+        | Inst::XmmRmREvex { dst, ref src2, .. }
+        | Inst::XmmUnaryRmRImmEvex {
             dst, src: ref src2, ..
         }
-        | Inst::XmmUnaryRmRImmVex {
+        | Inst::XmmUnaryRmRUnaligned {
             dst, src: ref src2, ..
         }
-        | Inst::XmmRmREvex { dst, ref src2, .. }
-        | Inst::XmmUnaryRmRImmEvex {
+        | Inst::XmmUnaryRmREvex {
             dst, src: ref src2, ..
         }
         | Inst::XmmRmREvex3 {
             dst,
             src3: ref src2,
             ..
+        } => {
+            match <&RegMem>::from(src2) {
+                RegMem::Mem { ref addr } => {
+                    check_load(ctx, None, addr, vcode, I8X16, 128)?;
+                }
+                RegMem::Reg { .. } => {}
+            }
+            ensure_no_fact(vcode, dst.to_writable_reg().to_reg())
+        }
+
+        Inst::XmmRmRImmVex {
+            op, dst, ref src2, ..
+        }
+        | Inst::XmmRmRVex3 {
+            op,
+            dst,
+            src3: ref src2,
+            ..
         }
-        | Inst::XmmUnaryRmRUnaligned {
-            dst, src: ref src2, ..
+        | Inst::XmmRmRBlendVex {
+            op, dst, ref src2, ..
         }
-        | Inst::XmmUnaryRmREvex {
-            dst, src: ref src2, ..
+        | Inst::XmmUnaryRmRVex {
+            op,
+            dst,
+            src: ref src2,
+            ..
+        }
+        | Inst::XmmUnaryRmRImmVex {
+            op,
+            dst,
+            src: ref src2,
+            ..
         } => {
+            let size = match op {
+                AvxOpcode::Vmovss => 32,
+                AvxOpcode::Vmovsd => 64,
+
+                // We assume all other operations happen on 128-bit values.
+                _ => 128,
+            };
+
             match <&RegMem>::from(src2) {
                 RegMem::Mem { ref addr } => {
-                    check_load(ctx, None, addr, vcode, I8X16, 128)?;
+                    check_load(ctx, None, addr, vcode, I8X16, size)?;
                 }
                 RegMem::Reg { .. } => {}
             }

cranelift/codegen/src/opts.rs

@@ -26,7 +26,7 @@ pub type ValueArray3 = [Value; 3];
 pub type ConstructorVec<T> = SmallVec<[T; 8]>;
 
 pub(crate) mod generated_code;
-use generated_code::ContextIter;
+use generated_code::{ContextIter, IntoContextIter};
 
 pub(crate) struct IsleContext<'a, 'b, 'c> {
     pub(crate) ctx: &'a mut OptimizeCtx<'b, 'c>,
@@ -38,6 +38,18 @@ pub(crate) struct InstDataEtorIter<'a, 'b, 'c> {
     _phantom2: PhantomData<&'b ()>,
     _phantom3: PhantomData<&'c ()>,
 }
+
+impl Default for InstDataEtorIter<'_, '_, '_> {
+    fn default() -> Self {
+        InstDataEtorIter {
+            stack: SmallVec::default(),
+            _phantom1: PhantomData,
+            _phantom2: PhantomData,
+            _phantom3: PhantomData,
+        }
+    }
+}
+
 impl<'a, 'b, 'c> InstDataEtorIter<'a, 'b, 'c> {
     fn new(root: Value) -> Self {
         debug_assert_ne!(root, Value::reserved_value());
@@ -84,13 +96,27 @@ where
     }
 }
 
+impl<'a, 'b, 'c> IntoContextIter for InstDataEtorIter<'a, 'b, 'c>
+where
+    'b: 'a,
+    'c: 'b,
+{
+    type Context = IsleContext<'a, 'b, 'c>;
+    type Output = (Type, InstructionData);
+    type IntoIter = Self;
+
+    fn into_context_iter(self) -> Self {
+        self
+    }
+}
+
 impl<'a, 'b, 'c> generated_code::Context for IsleContext<'a, 'b, 'c> {
     isle_common_prelude_methods!();
 
-    type inst_data_etor_iter = InstDataEtorIter<'a, 'b, 'c>;
+    type inst_data_etor_returns = InstDataEtorIter<'a, 'b, 'c>;
 
-    fn inst_data_etor(&mut self, eclass: Value) -> InstDataEtorIter<'a, 'b, 'c> {
-        InstDataEtorIter::new(eclass)
+    fn inst_data_etor(&mut self, eclass: Value, returns: &mut InstDataEtorIter<'a, 'b, 'c>) {
+        *returns = InstDataEtorIter::new(eclass);
     }
 
     fn make_inst_ctor(&mut self, ty: Type, op: &InstructionData) -> Value {

cranelift/entity/src/primary.rs

@@ -7,6 +7,7 @@ use alloc::boxed::Box;
 use alloc::vec::Vec;
 use core::iter::FromIterator;
 use core::marker::PhantomData;
+use core::mem;
 use core::ops::{Index, IndexMut};
 use core::slice;
 #[cfg(feature = "enable-serde")]
@@ -158,6 +159,41 @@ where
         unsafe { BoxedSlice::<K, V>::from_raw(Box::<[V]>::into_raw(self.elems.into_boxed_slice())) }
     }
 
+    /// Returns mutable references to many elements at once.
+    ///
+    /// Returns an error if an element does not exist, or if the same key was passed more than
+    /// once.
+    // This implementation is taken from the unstable `get_many_mut`.
+    //
+    // Once it has been stabilised we can call that method directly.
+    pub fn get_many_mut<const N: usize>(
+        &mut self,
+        indices: [K; N],
+    ) -> Result<[&mut V; N], GetManyMutError<K>> {
+        for (i, &idx) in indices.iter().enumerate() {
+            if idx.index() >= self.len() {
+                return Err(GetManyMutError::DoesNotExist(idx));
+            }
+            for &idx2 in &indices[..i] {
+                if idx == idx2 {
+                    return Err(GetManyMutError::MultipleOf(idx));
+                }
+            }
+        }
+
+        let slice: *mut V = self.elems.as_mut_ptr();
+        let mut arr: mem::MaybeUninit<[&mut V; N]> = mem::MaybeUninit::uninit();
+        let arr_ptr = arr.as_mut_ptr();
+
+        unsafe {
+            for i in 0..N {
+                let idx = *indices.get_unchecked(i);
+                *(*arr_ptr).get_unchecked_mut(i) = &mut *slice.add(idx.index());
+            }
+            Ok(arr.assume_init())
+        }
+    }
+
     /// Performs a binary search on the values with a key extraction function.
     ///
     /// Assumes that the values are sorted by the key extracted by the function.
@@ -181,6 +217,12 @@ where
     }
 }
 
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum GetManyMutError<K> {
+    DoesNotExist(K),
+    MultipleOf(K),
+}
+
 impl<K, V> Default for PrimaryMap<K, V>
 where
     K: EntityRef,
@@ -453,4 +495,22 @@ mod tests {
             assert!(*me == **ne);
         }
     }
+
+    #[test]
+    fn get_many_mut() {
+        let mut m: PrimaryMap<E, usize> = PrimaryMap::new();
+        let _0 = m.push(0);
+        let _1 = m.push(1);
+        let _2 = m.push(2);
+
+        assert_eq!([&mut 0, &mut 2], m.get_many_mut([_0, _2]).unwrap());
+        assert_eq!(
+            m.get_many_mut([_0, _0]),
+            Err(GetManyMutError::MultipleOf(_0))
+        );
+        assert_eq!(
+            m.get_many_mut([E(4)]),
+            Err(GetManyMutError::DoesNotExist(E(4)))
+        );
+    }
 }