CSV secondary instance support

src/jmh/java/org/javarosa/benchmarks/BenchmarkTemplate.java

@@ -37,7 +37,7 @@ public void setUp() {
             // If your forms load external files, you'll need to prime the ResourceManager using the assetsDir
             // returned by BenchmarkUtils.prepareAssets(). You need to make sure that all external files in the
             // form are like jr://file/filename (all use the "file" hostname, all are located at the root).
-            setUpSimpleReferenceManager("file", assetsPath);
+            setUpSimpleReferenceManager(assetsPath, "file");
         }
     }
 

src/jmh/java/org/javarosa/benchmarks/BenchmarkUtils.java

@@ -146,21 +146,21 @@ public static Path getMinifiedNigeriaWardsXMLWithInternal2ndryInstance(){
 
     public static Path getNigeriaWardsXMLWithExternal2ndryInstance(){
         Path assetsPath = prepareAssets("nigeria_wards_external_2ndry_instance.xml", "lgas.xml", "wards.xml");
-        setUpSimpleReferenceManager("file", assetsPath);
+        setUpSimpleReferenceManager(assetsPath, "file");
         Path filePath = assetsPath.resolve("nigeria_wards_external_2ndry_instance.xml");
         return filePath;
     }
 
     public static Path getWardsExternalInstance(){
         Path assetsPath = prepareAssets( "wards.xml");
-        setUpSimpleReferenceManager("file", assetsPath);
+        setUpSimpleReferenceManager(assetsPath, "file");
         Path filePath = assetsPath.resolve("wards.xml");
         return filePath;
     }
 
     public static Path getLGAsExternalInstance(){
         Path assetsPath = prepareAssets( "lgas.xml");
-        setUpSimpleReferenceManager("file", assetsPath);
+        setUpSimpleReferenceManager(assetsPath, "file");
         Path filePath = assetsPath.resolve("lgas.xml");
         return filePath;
     }

src/jmh/java/org/javarosa/benchmarks/ExternalDataInstanceBuildBenchmark.java

@@ -29,7 +29,7 @@ public static class ExternalDataInstanceState {
         @Setup(Level.Trial)
         public void initialize() {
             Path assetsPath = prepareAssets( "wards.xml", "lgas.xml");
-            setUpSimpleReferenceManager("file", assetsPath);
+            setUpSimpleReferenceManager(assetsPath, "file");
         }
     }
 

src/main/java/org/javarosa/core/model/instance/CsvExternalInstance.java

@@ -1,9 +1,10 @@
 package org.javarosa.core.model.instance;
 
+import org.javarosa.core.model.data.UncastData;
+
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.IOException;
-import org.javarosa.core.model.data.UncastData;
 
 public class CsvExternalInstance {
     public static TreeElement parse(String instanceId, String path) throws IOException {
@@ -14,17 +15,20 @@ public static TreeElement parse(String instanceId, String path) throws IOExcepti
 
         if (csvLine != null) {
             String[] fieldNames = csvLine.split(",");
+            int multiplicity = 0;
 
             while ((csvLine = br.readLine()) != null) {
-                TreeElement item = new TreeElement("item", 0);
+                TreeElement item = new TreeElement("item", multiplicity);
                 String[] data = csvLine.split(",");
                 for (int i = 0; i < fieldNames.length; ++i) {
                     TreeElement field = new TreeElement(fieldNames[i], 0);
-                    field.setValue(new UncastData(data[i]));
+                    field.setValue(new UncastData(i < data.length ? data[i] : ""));
+
                     item.addChild(field);
                 }
 
                 root.addChild(item);
+                multiplicity++;
             }
         }
         return root;

src/main/java/org/javarosa/xform/parse/XFormParser.java

@@ -16,48 +16,6 @@
 
 package org.javarosa.xform.parse;
 
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
-import static java.util.Collections.unmodifiableSet;
-import static org.javarosa.core.model.Constants.CONTROL_AUDIO_CAPTURE;
-import static org.javarosa.core.model.Constants.CONTROL_FILE_CAPTURE;
-import static org.javarosa.core.model.Constants.CONTROL_IMAGE_CHOOSE;
-import static org.javarosa.core.model.Constants.CONTROL_INPUT;
-import static org.javarosa.core.model.Constants.CONTROL_OSM_CAPTURE;
-import static org.javarosa.core.model.Constants.CONTROL_RANGE;
-import static org.javarosa.core.model.Constants.CONTROL_RANK;
-import static org.javarosa.core.model.Constants.CONTROL_SECRET;
-import static org.javarosa.core.model.Constants.CONTROL_SELECT_MULTI;
-import static org.javarosa.core.model.Constants.CONTROL_SELECT_ONE;
-import static org.javarosa.core.model.Constants.CONTROL_TRIGGER;
-import static org.javarosa.core.model.Constants.CONTROL_UPLOAD;
-import static org.javarosa.core.model.Constants.CONTROL_VIDEO_CAPTURE;
-import static org.javarosa.core.model.Constants.DATATYPE_CHOICE;
-import static org.javarosa.core.model.Constants.DATATYPE_MULTIPLE_ITEMS;
-import static org.javarosa.core.model.Constants.XFTAG_UPLOAD;
-import static org.javarosa.core.services.ProgramFlow.die;
-import static org.javarosa.xform.parse.Constants.ID_ATTR;
-import static org.javarosa.xform.parse.Constants.NODESET_ATTR;
-import static org.javarosa.xform.parse.Constants.RANK;
-import static org.javarosa.xform.parse.Constants.SELECT;
-import static org.javarosa.xform.parse.Constants.SELECTONE;
-import static org.javarosa.xform.parse.RandomizeHelper.cleanNodesetDefinition;
-import static org.javarosa.xform.parse.RandomizeHelper.cleanSeedDefinition;
-import static org.javarosa.xform.parse.RangeParser.populateQuestionWithRangeAttributes;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 import org.javarosa.core.model.DataBinding;
 import org.javarosa.core.model.FormDef;
 import org.javarosa.core.model.GroupDef;
@@ -110,6 +68,49 @@
 import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static java.util.Collections.unmodifiableSet;
+import static org.javarosa.core.model.Constants.CONTROL_AUDIO_CAPTURE;
+import static org.javarosa.core.model.Constants.CONTROL_FILE_CAPTURE;
+import static org.javarosa.core.model.Constants.CONTROL_IMAGE_CHOOSE;
+import static org.javarosa.core.model.Constants.CONTROL_INPUT;
+import static org.javarosa.core.model.Constants.CONTROL_OSM_CAPTURE;
+import static org.javarosa.core.model.Constants.CONTROL_RANGE;
+import static org.javarosa.core.model.Constants.CONTROL_RANK;
+import static org.javarosa.core.model.Constants.CONTROL_SECRET;
+import static org.javarosa.core.model.Constants.CONTROL_SELECT_MULTI;
+import static org.javarosa.core.model.Constants.CONTROL_SELECT_ONE;
+import static org.javarosa.core.model.Constants.CONTROL_TRIGGER;
+import static org.javarosa.core.model.Constants.CONTROL_UPLOAD;
+import static org.javarosa.core.model.Constants.CONTROL_VIDEO_CAPTURE;
+import static org.javarosa.core.model.Constants.DATATYPE_CHOICE;
+import static org.javarosa.core.model.Constants.DATATYPE_MULTIPLE_ITEMS;
+import static org.javarosa.core.model.Constants.XFTAG_UPLOAD;
+import static org.javarosa.core.services.ProgramFlow.die;
+import static org.javarosa.xform.parse.Constants.ID_ATTR;
+import static org.javarosa.xform.parse.Constants.NODESET_ATTR;
+import static org.javarosa.xform.parse.Constants.RANK;
+import static org.javarosa.xform.parse.Constants.SELECT;
+import static org.javarosa.xform.parse.Constants.SELECTONE;
+import static org.javarosa.xform.parse.RandomizeHelper.cleanNodesetDefinition;
+import static org.javarosa.xform.parse.RandomizeHelper.cleanSeedDefinition;
+import static org.javarosa.xform.parse.RangeParser.populateQuestionWithRangeAttributes;
+
 /* droos: i think we need to start storing the contents of the <bind>s in the formdef again */
 
 /**
@@ -170,6 +171,9 @@ public class XFormParser implements IXFormParserFunctions {
     private List<String> itextKnownForms;
     private static HashMap<String, IElementHandler> actionHandlers;
 
+    /** The string IDs of all instances that are referenced in a instance() function call in the primary instance **/
+    private static Set<String> referencedInstanceIds;
+
     private final List<WarningCallback> warningCallbacks = new ArrayList<>();
     private final List<ErrorCallback> errorCallbacks = new ArrayList<>();
 
@@ -198,6 +202,8 @@ private static void staticInit() {
         initProcessingRules();
         modelPrototypes = new PrototypeFactoryDeprecated();
         submissionParsers = new ArrayList<>(1);
+
+        referencedInstanceIds = new HashSet<>();
     }
 
     private static void initProcessingRules() {
@@ -455,7 +461,10 @@ private void parseDoc(Map<String, String> namespacePrefixesByUri, String lastSav
 
         initState();
         final String defaultNamespace = _xmldoc.getRootElement().getNamespaceUri(null);
+
+        referencedInstanceIds.clear();
         parseElement(_xmldoc.getRootElement(), _f, topLevelHandlers);
+
         collapseRepeatGroups(_f);
 
         final FormInstanceParser instanceParser = new FormInstanceParser(_f, defaultNamespace,
@@ -471,25 +480,27 @@ private void parseDoc(Map<String, String> namespacePrefixesByUri, String lastSav
                 final String instanceId = instanceNodeIdStrs.get(instanceIndex);
                 final String instanceSrc = parseInstanceSrc(instance, lastSavedSrc);
 
-                // Disable jr://file-csv/ support by explicitly only supporting jr://file/
-                // until https://github.com/opendatakit/javarosa/issues/417 is addressed
-                if (instanceSrc != null && instanceSrc.toLowerCase().startsWith("jr://file/")) {
-                    final ExternalDataInstance externalDataInstance;
-                    try {
-                        externalDataInstance = ExternalDataInstance.build(instanceSrc, instanceId);
-                    } catch (IOException | UnfullfilledRequirementsException | InvalidStructureException |
+                // Only read in an secondary instance if its ID is used in the primary instance as an argument to an
+                // instance() call
+                if (referencedInstanceIds.contains(instanceId)) {
+                    if (instanceSrc != null) {
+                        final ExternalDataInstance externalDataInstance;
+                        try {
+                            externalDataInstance = ExternalDataInstance.build(instanceSrc, instanceId);
+                        } catch (IOException | UnfullfilledRequirementsException | InvalidStructureException |
                             XmlPullParserException | InvalidReferenceException e) {
-                        String msg = "Unable to parse external secondary instance";
-                        logger.error(msg, e);
-                        throw new XFormParseException(msg + ": " + e.toString(), instance);
+                            String msg = "Unable to parse external secondary instance";
+                            logger.error(msg, e);
+                            throw new XFormParseException(msg + ": " + e.toString(), instance);
+                        }
+                        _f.addNonMainInstance(externalDataInstance);
+                    } else {
+                        FormInstance fi = instanceParser.parseInstance(instance, false,
+                            instanceNodeIdStrs.get(instanceNodes.indexOf(instance)), namespacePrefixesByUri);
+                        loadNamespaces(_xmldoc.getRootElement(), fi); // same situation as below
+                        loadInstanceData(instance, fi.getRoot(), _f);
+                        _f.addNonMainInstance(fi);
                     }
-                    _f.addNonMainInstance(externalDataInstance);
-                } else {
-                    FormInstance fi = instanceParser.parseInstance(instance, false,
-                        instanceNodeIdStrs.get(instanceNodes.indexOf(instance)), namespacePrefixesByUri);
-                    loadNamespaces(_xmldoc.getRootElement(), fi); // same situation as below
-                    loadInstanceData(instance, fi.getRoot(), _f);
-                    _f.addNonMainInstance(fi);
                 }
             }
         }
@@ -578,6 +589,13 @@ private void parseElement(Element e, Object parent, HashMap<String, IElementHand
         }
     }
 
+    /**
+     * Records that the given instance ID was used as the argument to an instance() function call.
+     */
+    public static void recordInstanceFunctionCall(String instanceId) {
+        referencedInstanceIds.add(instanceId);
+    }
+
     private void parseTitle(Element e) {
         List<String> usedAtts = new ArrayList<>(); //no attributes parsed in title.
         String title = getXMLText(e, true);
@@ -805,13 +823,18 @@ private void parseSubmission(Element submission) {
     private void saveInstanceNode(Element instance) {
         Element instanceNode = null;
         String instanceId = instance.getAttributeValue("", "id");
-
-        for (int i = 0; i < instance.getChildCount(); i++) {
-            if (instance.getType(i) == Node.ELEMENT) {
-                if (instanceNode != null) {
-                    throw new XFormParseException("XForm Parse: <instance> has more than one child element", instance);
-                } else {
-                    instanceNode = instance.getElement(i);
+        String instanceSrc = instance.getAttributeValue("", "src");
+
+        // Only consider child nodes if the instance declaration does not include a source.
+        // TODO: revisit this to allow for a mix of static and dynamic data but beware of https://github.com/opendatakit/javarosa/issues/451
+        if (instanceSrc == null) {
+            for (int i = 0; i < instance.getChildCount(); i++) {
+                if (instance.getType(i) == Node.ELEMENT) {
+                    if (instanceNode != null) {
+                        throw new XFormParseException("XForm Parse: <instance> has more than one child element", instance);
+                    } else {
+                        instanceNode = instance.getElement(i);
+                    }
                 }
             }
         }

src/main/java/org/javarosa/xpath/expr/XPathFuncExpr.java

@@ -31,6 +31,7 @@
 import org.javarosa.core.util.externalizable.ExtUtil;
 import org.javarosa.core.util.externalizable.ExtWrapListPoly;
 import org.javarosa.core.util.externalizable.PrototypeFactory;
+import org.javarosa.xform.parse.XFormParser;
 import org.javarosa.xpath.IExprDataType;
 import org.javarosa.xpath.XPathArityException;
 import org.javarosa.xpath.XPathNodeset;
@@ -70,6 +71,10 @@ public XPathFuncExpr () { } //for deserialization
     public XPathFuncExpr (XPathQName id, XPathExpression[] args) {
         this.id = id;
         this.args = args;
+
+        if (id.name.equals("instance") && args[0] instanceof XPathStringLiteral) {
+            XFormParser.recordInstanceFunctionCall(((XPathStringLiteral) args[0]).s);
+        }
     }
 
     public String toString () {

src/test/java/org/javarosa/core/reference/ReferenceManagerTestUtils.java

@@ -22,19 +22,17 @@ protected Reference factory(String terminal, String URI) {
      * <p>
      * Please, be aware that this method resets the singleton ReferenceManager, which could
      * have unintended consequences for other classes using it during the same JVM session.
-     * <p>
-     * Use of this method is intended when only one scheme is to be derived. If your test
-     * form uses more than one scheme, you will have to follow a more conventional setup of
-     * having a reference factory for jr://file to a base path and some session translators
-     * that derive any other scheme (e.g. jr://audio) to a jr://file path.
      */
-    public static ReferenceManager setUpSimpleReferenceManager(String scheme, Path path) {
+    public static ReferenceManager setUpSimpleReferenceManager(Path path, String... schemes) {
         ReferenceManager refManager = ReferenceManager.instance();
         refManager.reset();
-        refManager.addReferenceFactory(buildReferenceFactory(
-            scheme,
-            path.toAbsolutePath().toString()
-        ));
+
+        for (String scheme : schemes) {
+            refManager.addReferenceFactory(buildReferenceFactory(
+                scheme,
+                path.toAbsolutePath().toString()
+            ));
+        }
         return refManager;
     }
 }