Merge pull request #522 from lognaturel/csv-external

pyxform/builder.py

@@ -115,7 +115,7 @@ def create_survey_element_from_dict(self, d):
             d = self._sections[section_name]
             full_survey = self.create_survey_element_from_dict(d)
             return full_survey.children
-        elif d["type"] == "xml-external":
+        elif d["type"] in ["xml-external", "csv-external"]:
             return ExternalInstance(**d)
         else:
             self._save_trigger_as_setvalue_and_remove_calculate(d)

pyxform/question_type_dictionary.py

@@ -369,6 +369,9 @@ def generate_new_dict():
     "xml-external": {
         # Only effect is to add an external instance.
     },
+    "csv-external": {
+        # Only effect is to add an external instance.
+    },
     "start-geopoint": {
         "control": {"tag": "action"},
         "bind": {"type": "geopoint"},

pyxform/survey.py

@@ -312,7 +312,9 @@ def _generate_static_instances(list_name, choice_list):
     def _generate_external_instances(element):
         if isinstance(element, ExternalInstance):
             name = element["name"]
-            src = "jr://file/{}.xml".format(name)
+            extension = element["type"].split("-")[0]
+            prefix = "file-csv" if extension == "csv" else "file"
+            src = "jr://{}/{}.{}".format(prefix, name, extension)
             return InstanceInfo(
                 type="external",
                 context="[type: {t}, name: {n}]".format(

pyxform/tests_v1/test_xmldata.py → pyxform/tests_v1/test_external_instances.py

@@ -2,7 +2,7 @@
 """
 Test xml-external syntax and instances generated from pulldata calls.
 
-See also test_support_external_instances
+See also test_external_instances_for_selects
 """
 from pyxform.errors import PyXFormError
 from pyxform.tests_v1.pyxform_test_case import PyxformTestCase, PyxformTestError
@@ -22,7 +22,17 @@ def test_can__output_single_external_xml_item(self):
             |        | xml-external | mydata |       |
             """,
             model__contains=['<instance id="mydata" src="jr://file/mydata.xml"/>'],
-            run_odk_validate=True,
+        )
+
+    def test_can__output_single_external_csv_item(self):
+        """Simplest possible example to include an external instance."""
+        self.assertPyxformXform(
+            md="""
+            | survey |              |        |       |
+            |        | type         | name   | label |
+            |        | csv-external | mydata |       |
+            """,
+            model__contains=['<instance id="mydata" src="jr://file-csv/mydata.csv"/>'],
         )
 
     def test_cannot__use_same_external_xml_id_in_same_section(self):
@@ -56,7 +66,21 @@ def test_can__use_unique_external_xml_in_same_section(self):
                 '<instance id="mydata" src="jr://file/mydata.xml"/>',
                 '<instance id="mydata2" src="jr://file/mydata2.xml"/>',
             ],
-            run_odk_validate=True,
+        )
+
+    def test_can__use_unique_external_csv_in_same_section(self):
+        """Two unique external instances in the same section is OK."""
+        self.assertPyxformXform(
+            md="""
+            | survey |              |         |       |
+            |        | type         | name    | label |
+            |        | csv-external | mydata  |       |
+            |        | csv-external | mydata2 |       |
+            """,
+            model__contains=[
+                '<instance id="mydata" src="jr://file-csv/mydata.csv"/>',
+                '<instance id="mydata2" src="jr://file-csv/mydata2.csv"/>',
+            ],
         )
 
     def test_cannot__use_same_external_xml_id_across_groups(self):
@@ -79,6 +103,23 @@ def test_cannot__use_same_external_xml_id_across_groups(self):
         self.assertIn("Instance names must be unique", repr(ctx.exception))
         self.assertIn("The name 'mydata' was found 3 time(s)", repr(ctx.exception))
 
+    def test_cannot__use_external_xml_and_csv_with_same_filename(self):
+        """Duplicate external instances anywhere raises an error."""
+        with self.assertRaises(PyxformTestError) as ctx:
+            self.assertPyxformXform(
+                md="""
+                | survey |              |        |       |
+                |        | type         | name   | label |
+                |        | csv-external | mydata |       |
+                |        | begin group  | g1     |       |
+                |        | xml-external | mydata |       |
+                |        | end group    | g1     |       |
+                """,
+                model__contains=[],
+            )
+        self.assertIn("Instance names must be unique", repr(ctx.exception))
+        self.assertIn("The name 'mydata' was found 2 time(s)", repr(ctx.exception))
+
     def test_can__use_unique_external_xml_across_groups(self):
         """Unique external instances anywhere is OK."""
         self.assertPyxformXform(
@@ -105,7 +146,6 @@ def test_can__use_unique_external_xml_across_groups(self):
                 '<instance id="mydata2" src="jr://file/mydata2.xml"/>',
                 '<instance id="mydata3" src="jr://file/mydata3.xml"/>',
             ],
-            run_odk_validate=True,
         )
 
     def test_cannot__use_same_external_xml_id_with_mixed_types(self):
@@ -133,6 +173,30 @@ def test_cannot__use_same_external_xml_id_with_mixed_types(self):
             )
         self.assertIn("The name 'city' was found 2 time(s)", repr(ctx.exception))
 
+    def test_can__use_same_external_csv_id_with_mixed_types(self):
+        """Multiple fields that require the same external instance result in a single instance declaration."""
+        self.assertPyxformXform(
+            md="""
+            | survey |                                      |      |       |                                             |
+            |        | type                                 | name | label | calculation                                 |
+            |        | begin group                          | g1   |       |                                             |
+            |        | text                                 | foo  | Foo   |                                             |
+            |        | csv-external                         | city |       |                                             |
+            |        | end group                            | g1   |       |                                             |
+            |        | begin group                          | g2   |       |                                             |
+            |        | select_one_from_file cities.csv      | city | City  |                                             |
+            |        | end group                            | g2   |       |                                             |
+            |        | begin group                          | g3   |       |                                             |
+            |        | select_multiple_from_file cities.csv | city | City  |                                             |
+            |        | end group                            | g3   |       |                                             |
+            |        | begin group                          | g4   |       |                                             |
+            |        | text                                 | foo  | Foo   |                                             |
+            |        | calculate                            | city | City  | pulldata('fruits', 'name', 'name', 'mango') |
+            |        | end group                            | g4   |       |                                             |
+            """,  # noqa
+            model__contains=['<instance id="city" src="jr://file-csv/city.csv"/>'],
+        )
+
     def test_can__use_all_types_together_with_unique_ids(self):
         """Unique instances with other sources present are OK."""
         self.assertPyxformXform(
@@ -178,7 +242,6 @@ def test_can__use_all_types_together_with_unique_ids(self):
       </instance>
 """,
             ],  # noqa
-            run_odk_validate=True,
         )
 
     def test_cannot__use_different_src_same_id__select_then_internal(self):
@@ -267,9 +330,7 @@ def test_can__reuse_csv__selects_then_pulldata(self):
         expected = """
       <instance id="pain_locations" src="jr://file-csv/pain_locations.csv"/>
 """  # noqa
-        self.assertPyxformXform(
-            md=md, model__contains=[expected], run_odk_validate=True
-        )
+        self.assertPyxformXform(md=md, model__contains=[expected])
         survey = self.md_to_pyxform_survey(md_raw=md)
         xml = survey._to_pretty_xml()
         self.assertEqual(1, xml.count(expected))
@@ -287,9 +348,7 @@ def test_can__reuse_csv__pulldata_then_selects(self):
             |        | select_one_from_file pain_locations.csv      | pyear  | Location of worst pain this year.  |                                                   |
             """  # noqa
         expected = """<instance id="pain_locations" src="jr://file-csv/pain_locations.csv"/>"""  # noqa
-        self.assertPyxformXform(
-            md=md, model__contains=[expected], run_odk_validate=True
-        )
+        self.assertPyxformXform(md=md, model__contains=[expected])
 
     def test_can__reuse_xml__selects_then_external(self):
         """Re-using the same xml external data source id and URI is OK."""
@@ -321,9 +380,7 @@ def test_can__reuse_xml__external_then_selects(self):
             |        | select_one_from_file pain_locations.xml      | pyear          | Location of worst pain this year.  |
             """  # noqa
         expected = """<instance id="pain_locations" src="jr://file/pain_locations.xml"/>"""  # noqa
-        self.assertPyxformXform(
-            md=md, model__contains=[expected], run_odk_validate=True
-        )
+        self.assertPyxformXform(md=md, model__contains=[expected])
         survey = self.md_to_pyxform_survey(md_raw=md)
         xml = survey._to_pretty_xml()
         self.assertEqual(1, xml.count(expected))
@@ -363,7 +420,11 @@ def test_external_instance_pulldata_readonly(self):
         """
         node = """<instance id="ID" src="jr://file-csv/ID.csv"/>"""
 
-        self.assertPyxformXform(md=md, xml__contains=[node])
+        self.assertPyxformXform(
+            md=md,
+            xml__contains=[node],
+            run_odk_validate=False,  # Validate sees self references in readonly as circular but shouldn't
+        )
 
     def test_external_instance_pulldata_required(self):
         """
@@ -376,7 +437,11 @@ def test_external_instance_pulldata_required(self):
         |        | text   | Part_ID | Participant ID | pulldata('ID', 'ParticipantID', 'ParticipantIDValue',.) |
         """
         node = """<instance id="ID" src="jr://file-csv/ID.csv"/>"""
-        self.assertPyxformXform(md=md, xml__contains=[node], debug=False)
+        self.assertPyxformXform(
+            md=md,
+            xml__contains=[node],
+            run_odk_validate=False,  # Validate sees self references in requireds as circular but shouldn't
+        )
 
     def test_external_instance_pulldata_relevant(self):
         """
@@ -389,7 +454,11 @@ def test_external_instance_pulldata_relevant(self):
         |        | text   | Part_ID | Participant ID | pulldata('ID', 'ParticipantID', 'ParticipantIDValue',.) |
         """
         node = """<instance id="ID" src="jr://file-csv/ID.csv"/>"""
-        self.assertPyxformXform(md=md, xml__contains=[node], debug=False)
+        self.assertPyxformXform(
+            md=md,
+            xml__contains=[node],
+            run_odk_validate=False,  # Validate sees self references in relevants as circular but shouldn't
+        )
 
     # This is not something that is recommended since pulldata and choice_filter both should use XPath predicates
     # behind the scenes but it should still be possible.

pyxform/tests_v1/test_support_external_instances.py → pyxform/tests_v1/test_external_instances_for_selects.py

@@ -2,7 +2,7 @@
 """
 Test external instance syntax
 
-See also test_xmldata
+See also test_external_instances
 """
 from pyxform.tests_v1.pyxform_test_case import PyxformTestCase
 

pyxform/tests_v1/test_secondary_instance_translations.py

@@ -121,7 +121,7 @@ def test_select_with_choice_filter_and_translations_generates_single_translation
         xform_md = """
         | survey |                    |      |       |               |
         |        | type               | name | label | choice_filter |
-        |        | select_one list    | foo  | Foo   | name != "     |
+        |        | select_one list    | foo  | Foo   | name != ''    |
         | choices |
         |         | list_name | name | label | image | label::French |
         |         | list      | a    | A     | a.jpg | Ah            |