Fix basic_json_schema fail with long types, closes #80

scrapinghub · May 6, 2019 · 6ca2821 · 6ca2821
1 parent 285cd21
commit 6ca2821
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 14 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -10,14 +10,18 @@ Note that the top-most release is changes in the unreleased master branch on Git
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
-## [0.3.4dev] (Work In Progress)
+## [0.3.5dev] (Work In Progress)
 ### Added
 ### Changed
 ### Fixed
 ### Removed
 
+## [0.3.4] (2019-05-06)
+### Fixed
+- basic_json_schema() fails with long `1.0` types, #80
+
 
-## [0.3.3]
+## [0.3.3] (2019-05-03)
 ### Added
 - Accept dataframes as source or target, #69
 ### Changed

diff --git a/src/arche/tools/schema.py b/src/arche/tools/schema.py
@@ -57,15 +57,15 @@ def create_json_schema(
     samples = []
     for n in item_numbers:
         items = api.get_items(source_key, start_index=n, count=1, p_bar=None)
-        samples.append(items.iloc[0].to_dict())
+        samples.extend(items.to_dict("records"))
 
     return infer_schema(samples)
 
 
 def infer_schema(samples: List[Dict[str, Any]]) -> Schema:
     builder = SchemaBuilder("http://json-schema.org/draft-07/schema#")
-    for s in samples:
-        builder.add_object(s)
+    for sample in samples:
+        builder.add_object(sample)
     builder.add_schema(extension)
 
     return builder.to_schema()

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -7,10 +7,10 @@
 
 
 default_items = [
-    {"_key": "112358/13/21/0", "_type": "NameItem", "name": "Elizabeth"},
-    {"_key": "112358/13/21/1", "_type": "NameItem", "name": "Margaret"},
-    {"_key": "112358/13/21/2", "_type": "NameItem", "name": "Yulia"},
-    {"_key": "112358/13/21/3", "_type": "NameItem", "name": "Vivien"},
+    {"_key": "112358/13/21/0", "price": 0.0, "name": "Elizabeth"},
+    {"_key": "112358/13/21/1", "name": "Margaret"},
+    {"_key": "112358/13/21/2", "price": 10.0, "name": "Yulia"},
+    {"_key": "112358/13/21/3", "price": 11.0, "name": "Vivien"},
 ]
 default_source = pd.DataFrame(default_items)
 

diff --git a/tests/tools/test_schema_tools.py b/tests/tools/test_schema_tools.py
@@ -45,25 +45,26 @@ def test_basic_json_schema(mocker):
 def test_create_json_schema(mocker, get_job, get_items):
     mocker.patch("arche.tools.api.get_job", return_value=get_job, autospec=True)
     mocker.patch("arche.tools.api.get_items", return_value=get_items, autospec=True)
+    schema_tools.create_json_schema(get_job.key, [2])
     assert schema_tools.create_json_schema(get_job.key, [2, 3]) == {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "definitions": {
-            "float": {"pattern": "^-?[0-9]+\\.[0-9]{2}$"},
+            "float": {"pattern": r"^-?[0-9]+\.[0-9]{2}$"},
             "url": {
                 "pattern": (
-                    "^https?://(www\\.)?[a-z0-9.-]*\\.[a-z]{2,}"
-                    "([^<>%\\x20\\x00-\\x1f\\x7F]|%[0-9a-fA-F]{2})*$"
+                    r"^https?://(www\.)?[a-z0-9.-]*\.[a-z]{2,}"
+                    r"([^<>%\x20\x00-\x1f\x7F]|%[0-9a-fA-F]{2})*$"
                 )
             },
         },
         "additionalProperties": False,
         "type": "object",
         "properties": {
             "_key": {"type": "string"},
-            "_type": {"type": "string"},
             "name": {"type": "string"},
+            "price": {"type": "number"},
         },
-        "required": ["_key", "_type", "name"],
+        "required": ["_key", "name", "price"],
     }