Use search to get datasets, as it is significantly faster

ckanext/datajson/build_datajsonld.py

@@ -70,10 +70,6 @@ def apply_jsonld_metadata_mapping(data, newdict):
 
         # skip fields with no mapping to RDF
         if k not in jsonld_metadata_mapping: continue
-
-        # specially handle 'keyword' which in JSON is packed in a comma-separated field
-        if k == "keyword":
-            v = v.split(",")
 
         # specially handle literal fields with datatypes
         if k in jsonld_metadata_datatypes:

ckanext/datajson/plugin.py

@@ -128,7 +128,27 @@ def validator(self):
 
 def make_json():
     # Build the data.json file.
-    packages = p.toolkit.get_action("current_package_list_with_resources")(None, {})
-    return [make_datajson_entry(pkg) for pkg in packages if pkg["type"] == "dataset"]
-
+    return [make_datajson_entry(dataset) for dataset in _get_ckan_datasets()]
+
+def _get_ckan_datasets():
+
+    n = 500
+    page = 1
+    datasets = []
+
+    while True:
+        search_data_dict = {
+            'q': '*:*',
+            'fq': 'dataset_type:dataset',
+            'sort': 'metadata_modified desc',
+            'rows': n,
+            'start': n * (page - 1),
+        }
 
+        query = p.toolkit.get_action('package_search')({}, search_data_dict)
+        if len(query['results']):
+            datasets.extend(query['results'])
+            page = page + 1
+        else:
+            break
+    return datasets