save yaml

bin/wfh-upload.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import yaml
 import os
 import sys
 import requests
@@ -18,26 +19,30 @@ def doUpload(crate_path):
         "ro_crate": (crate_path, open(crate_path, "rb")),
         "workflow[project_ids][]": (None, 63),
     }
-    headers = {"authorization": "Token " + os.environ["DEV_WFH_TOKEN"]}
+    headers = {"authorization": "Token " + os.environ["DEV_WFH_TOKEN"], 'User-Agent': 'github.com/galaxyproject/[email protected]'}
 
     response = requests.post(
         "https://dev.workflowhub.eu/workflows/submit", files=payload, headers=headers
     )
     code = response.status_code
     if code != 200:
         print(f"Error {code} uploading {crate_path}")
-        # sys.stdout.write('!')
-        return response.text
-    # sys.stdout.write('.')
+        print(response.text)
+        return None
 
     data = response.json()
     wf_id = data["data"]["id"]
-    return f"Uploaded {crate_path} as workflow {wf_id}"
+    return (topic, tutorial, workflow, wf_id)
 
 
-with multiprocessing.Pool(8) as p:
+with multiprocessing.Pool(4) as p:
     results = p.map(doUpload, crates)
 
-print()
-for result in results:
-    print(result)
+results = [x for x in results if x is not None]
+
+data = {}
+for (topic, tutorial, workflow, wf_id) in results:
+    data.setdefault(topic, {}).setdefault(tutorial, {})[workflow] = wf_id
+
+with open('metadata/workflowhub.yml', 'w') as handle:
+    yaml.dump(data, handle)