Scheduling enclaves in the C target

core/src/integrationTest/java/org/lflang/tests/runtime/CTest.java

@@ -60,6 +60,11 @@ protected boolean supportsDockerOption() {
     return true;
   }
 
+  @Override
+  protected boolean supportsEnclaves() {
+    return true;
+  }
+
   @Test
   @Override
   public void runBasicTests() {

core/src/main/java/org/lflang/AttributeUtils.java

@@ -25,11 +25,10 @@
 
 package org.lflang;
 
-import static org.lflang.ast.ASTUtils.factory;
-
 import java.util.List;
 import java.util.Objects;
 import org.eclipse.emf.ecore.EObject;
+import org.eclipse.emf.ecore.util.EcoreUtil;
 import org.eclipse.xtext.nodemodel.ICompositeNode;
 import org.eclipse.xtext.nodemodel.util.NodeModelUtils;
 import org.eclipse.xtext.resource.XtextResource;
@@ -46,6 +45,7 @@
 import org.lflang.lf.StateVar;
 import org.lflang.lf.Timer;
 import org.lflang.util.StringUtil;
+import org.lflang.validation.AttributeSpec;
 
 /**
  * A helper class for processing attributes in the AST.
@@ -255,16 +255,32 @@ public static boolean isEnclave(Instantiation node) {
     return getEnclaveAttribute(node) != null;
   }
 
+  public static int getEnclaveNumWorkersFromAttribute(Instantiation node) {
+    Attribute enclaveAttr = getEnclaveAttribute(node);
+    if (enclaveAttr != null) {
+     for( AttrParm attrParm: enclaveAttr.getAttrParms()) {
+       if (attrParm.getName().equals(AttributeSpec.WORKERS_ATTR)) {
+         int value = Integer.valueOf(attrParm.getValue());
+         if (value > 0) {
+           return value;
+         } else {
+           return 1;
+         }
+       }
+     }
+    }
+    return 1; // Not specified
+  }
+
   /**
-   * Annotate @{code node} with enclave @attribute
-   *
-   * @param node
+   * Move the enclave attribute from source inst to target inst.
+   * @param source
+   * @param target
    */
-  public static void setEnclaveAttribute(Instantiation node) {
-    if (!isEnclave(node)) {
-      Attribute enclaveAttr = factory.createAttribute();
-      enclaveAttr.setAttrName("enclave");
-      node.getAttributes().add(enclaveAttr);
+  public static void copyEnclaveAttribute(Instantiation source, Instantiation target) {
+    Attribute enclaveAttr = EcoreUtil.copy(getEnclaveAttribute(source));
+    if (enclaveAttr != null) {
+      target.getAttributes().add(enclaveAttr);
     }
   }
 }

core/src/main/java/org/lflang/ast/ASTUtils.java

@@ -25,6 +25,8 @@
 
 package org.lflang.ast;
 
+import static org.lflang.AttributeUtils.isEnclave;
+
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
@@ -33,8 +35,10 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
@@ -63,6 +67,7 @@
 import org.lflang.generator.CodeMap;
 import org.lflang.generator.InvalidSourceException;
 import org.lflang.generator.ReactorInstance;
+import org.lflang.generator.c.CUtil;
 import org.lflang.lf.Action;
 import org.lflang.lf.Assignment;
 import org.lflang.lf.Code;
@@ -1853,4 +1858,21 @@ public static void addReactionAttribute(Reaction reaction, String name) {
     fedAttr.setAttrName(name);
     reaction.getAttributes().add(fedAttr);
   }
+
+  public static List<Instantiation> getEnclaves(Reactor top) {
+    List<Instantiation> enclaves = new ArrayList<>();
+    Queue<Reactor> queue = new LinkedList<>();
+    queue.add(top);
+
+    while (!queue.isEmpty()) {
+      Reactor inst = queue.poll();
+      for (Instantiation child: inst.getInstantiations()) {
+        if (isEnclave(child)) {
+          enclaves.add(child);
+        }
+        queue.add(ASTUtils.toDefinition(child.getReactorClass()));
+      }
+    }
+    return enclaves;
+  }
 }

core/src/main/java/org/lflang/generator/EnclaveInfo.java

@@ -6,14 +6,15 @@ public class EnclaveInfo {
   public int numShutdownReactions = 0;
   public int numTimerTriggers = 0;
   public int numResetReactions = 0;
-  public int numWorkers = 1;
+  public int numWorkers = 0;
   public int numModalReactors = 0;
   public int numModalResetStates = 0;
 
   public String traceFileName = null;
   private ReactorInstance instance;
 
-  public EnclaveInfo(ReactorInstance inst) {
+  public EnclaveInfo(ReactorInstance inst, int numWorkers) {
     instance = inst;
+    this.numWorkers = numWorkers;
   }
 }

core/src/main/java/org/lflang/generator/ReactionInstanceGraph.java

@@ -27,8 +27,10 @@
 package org.lflang.generator;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import org.lflang.generator.ReactionInstance.Runtime;
@@ -101,19 +103,36 @@ public void rebuildAndAssignDeadlines() {
     this.clear();
   }
 
+  public Integer[] getNumReactionsPerLevel() {
+    List<Integer> res = new ArrayList<>();
+    for (Map<ReactorInstance, Integer> numReactionsMap : numReactionsPerEnclavePerLevel) {
+      Integer numReactions = numReactionsMap.values().stream().mapToInt(Integer::intValue).sum();
+      res.add(numReactions);
+      }
+    return res.toArray(new Integer[0]);
+  }
+  public Integer[] getNumReactionsPerLevel(ReactorInstance enclave) {
+    List<Integer> res = new ArrayList<>();
+    for (Map<ReactorInstance, Integer> breadthMap : numReactionsPerEnclavePerLevel) {
+      var breadth = breadthMap.get(enclave);
+      if (breadth != null) {
+        res.add(breadth);
+      } else {
+        res.add(0);
+      }
+    }
+    return res.toArray(new Integer[0]);
+  }
   /*
    * Get an array of non-negative integers representing the number of reactions
    * per each level, where levels are indices of the array.
    */
-  public Integer[] getNumReactionsPerLevel() {
-    return numReactionsPerLevel.toArray(new Integer[0]);
-  }
-
   /** Return the max breadth of the reaction dependency graph */
-  public int getBreadth() {
+  public int getBreadth(ReactorInstance enclave) {
     var maxBreadth = 0;
-    for (Integer breadth : numReactionsPerLevel) {
-      if (breadth > maxBreadth) {
+    for (Map<ReactorInstance, Integer> breadthMap: numReactionsPerEnclavePerLevel) {
+      var breadth = breadthMap.get(enclave);
+      if (breadth != null && breadth > maxBreadth) {
         maxBreadth = breadth;
       }
     }
@@ -247,11 +266,9 @@ protected void addNodesAndEdges(ReactorInstance reactor) {
   ///////////////////////////////////////////////////////////
   //// Private fields
 
-  /**
-   * Number of reactions per level, represented as a list of integers where the indices are the
-   * levels.
-   */
-  private List<Integer> numReactionsPerLevel = new ArrayList<>(List.of(0));
+  // Number of reactions per level, per enclave. Used to calculate how many workers to use
+  // for each enclave
+  private List<Map<ReactorInstance, Integer>> numReactionsPerEnclavePerLevel = new ArrayList<>();
 
   ///////////////////////////////////////////////////////////
   //// Private methods
@@ -301,8 +318,9 @@ private void assignLevels() {
       // Remove visited origin.
       removeNode(origin);
 
+      ReactorInstance enclave = CUtil.getClosestEnclave(origin.getReaction().parent);
       // Update numReactionsPerLevel info
-      adjustNumReactionsPerLevel(origin.level, 1);
+      adjustNumReactionsPerLevel(origin.level, enclave);
     }
   }
 
@@ -346,21 +364,23 @@ private void assignInferredDeadlines() {
   }
 
   /**
-   * Adjust {@link #numReactionsPerLevel} at index <code>level<code> by
+   * Adjust {@link #numReactionsPerEnclavePerLevel} at index <code>level<code> by
    * adding to the previously recorded number <code>valueToAdd<code>.
    * If there is no previously recorded number for this level, then
    * create one with index <code>level</code> and value <code>valueToAdd</code>.
    * @param level The level.
-   * @param valueToAdd The value to add to the number of levels.
+   * @param enclave The enclave with which to increment the level count.
    */
-  private void adjustNumReactionsPerLevel(int level, int valueToAdd) {
-    if (numReactionsPerLevel.size() > level) {
-      numReactionsPerLevel.set(level, numReactionsPerLevel.get(level) + valueToAdd);
+  private void adjustNumReactionsPerLevel(int level, ReactorInstance enclave) {
+    while (numReactionsPerEnclavePerLevel.size() <= level) {
+      numReactionsPerEnclavePerLevel.add(new HashMap<>());
+    }
+
+    Map<ReactorInstance, Integer> numReactionsPerLevel = numReactionsPerEnclavePerLevel.get(level);
+    if (numReactionsPerLevel.containsKey(enclave)) {
+      numReactionsPerLevel.put(enclave, numReactionsPerLevel.get(enclave) + 1);
     } else {
-      while (numReactionsPerLevel.size() < level) {
-        numReactionsPerLevel.add(0);
-      }
-      numReactionsPerLevel.add(valueToAdd);
+      numReactionsPerLevel.put(enclave, 1);
     }
   }
 

core/src/main/java/org/lflang/generator/ReactorInstance.java

@@ -24,6 +24,7 @@
 
 package org.lflang.generator;
 
+import static org.lflang.AttributeUtils.getEnclaveNumWorkersFromAttribute;
 import static org.lflang.AttributeUtils.isEnclave;
 import static org.lflang.ast.ASTUtils.getLiteralTimeValue;
 
@@ -774,6 +775,7 @@ protected TriggerInstance<BuiltinTriggerVariable> getOrCreateBuiltinTrigger(
         trigger.getType(), ref -> TriggerInstance.builtinTrigger(trigger, this));
   }
 
+  // FIXME: This appears to never be used
   /** Create all the watchdog instances of this reactor instance. */
   protected void createWatchdogInstances() {
     List<Watchdog> watchdogs = ASTUtils.allWatchdogs(reactorDefinition);
@@ -789,6 +791,20 @@ protected void createWatchdogInstances() {
     }
   }
 
+  public boolean hasLocalMutex() {
+    if (watchdogs.size() > 0) {
+      return true;
+    }
+    // FIXME: How can we do this test better?
+    if (parent != null && parent.enclaveInfo != null) {
+      if (reactorDefinition.getName().equals("EnclaveConnectionReactor")) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
   ////////////////////////////////////////
   //// Private constructors
 
@@ -820,7 +836,7 @@ public ReactorInstance(
     // enclaveInfo object to track information about the enclave needed for
     // later code-generation
     if (isEnclave(definition) || this.isMainOrFederated()) {
-      enclaveInfo = new EnclaveInfo(this);
+      this.enclaveInfo = new EnclaveInfo(this, getEnclaveNumWorkersFromAttribute(definition));
     }
 
     // check for recursive instantiation