Download analysis releases for use in paper compilation

.github/workflows/makefile.yml

@@ -19,31 +19,31 @@ jobs:
     - uses: dsaltares/fetch-gh-release-asset@master
       with:
         repo: 'OA-WCVP/ipni-oa'
-        version: 'tags/v0.1.0'
+        version: 'latest'
         file: "ipni-oa-data.zip"
         target: 'downloads/ipni-oa-data.zip'
         token: ${{ secrets.PAT }}
 
     - uses: dsaltares/fetch-gh-release-asset@master
       with:
         repo: 'OA-WCVP/catalog-number-access'
-        version: 'tags/v0.1.1'
+        version: 'latest'
         file: "catalog-number-access-data.zip"
         target: 'downloads/catalog-number-access-data.zip'
         token: ${{ secrets.PAT }}
 
     - uses: dsaltares/fetch-gh-release-asset@master
       with:
         repo: 'OA-WCVP/gbif-literature'
-        version: 'tags/v0.0.1'
+        version: 'latest'
         file: "gbif-literature-data.zip"
         target: 'downloads/gbif-literature-data.zip'
         token: ${{ secrets.PAT }}
 
     - uses: dsaltares/fetch-gh-release-asset@master
       with:
         repo: 'OA-WCVP/phytotaxa-oa'
-        version: 'tags/v0.0.1'
+        version: 'latest'
         file: "phytotaxa-oa-data.zip"
         target: 'downloads/phytotaxa-oa-data.zip'
         token: ${{ secrets.PAT }}

.gitignore

@@ -1,4 +1,5 @@
 build/*
 data/*
 downloads/*
+env/*
 pandoc-filters/*

Makefile

@@ -6,37 +6,34 @@
 # dependencies of the generated outputs - so that an edit in a source text will 
 # trigger the regeneration of the compiled output.
 
+python_launch_cmd=python
+python_launch_cmd=winpty python
+
 ###############################################################################
-# Define URLs and download for archived analytical runs
+# Gather results of archived analytical runs
 ###############################################################################
-# Note - this is actually stored as part of a github release: 
-# https://github.com/OA-WCVP/catalog-number-access/releases/download/v0.1/data-20220701-122423.zip
-# Because the repo is currently private, its is not possible to automate its 
-# download (it could possibly be done with a python github client, which could 
-# authenticate to ensure access. The file has temporarily been copied to dropbox: 
-archived_analysis_url_catalog_numbers=https://www.dropbox.com/s/yqmz9qb60k25bwg/catalog-number-access-data-20220823-195702.zip?dl=0
-archived_analysis_url_ipni_oa=https://www.dropbox.com/s/i06omvj0i8jn1l1/ipni-oa-data-20220823-195714.zip?dl=0
+# These targets must authenticate to github nd therefore need an access_token 
+# which must be stored in an environment variable named "GITHUB_TOKEN"
 
-catalog_numbers_chart_catalognumbertrend=data/catalognumbertrend.png
-catalog_numbers_chart_linktrend=data/linktrend.png
+archived_analyses: downloads/ipni-oa-data.zip downloads/phytotaxa-oa-data.zip downloads/gbif-literature-data.zip downloads/catalog-number-access-data.zip
 
-downloads/catalog_numbers-analysis.zip:
+downloads/%-data.zip: util/download-artifact.py
 	mkdir -p downloads
-	wget -O $@ $(archived_analysis_url_catalog_numbers)
+	$(python_launch_cmd) util/download-artifact.py $*
 
-data/catalognumbertrend.png: downloads/catalog_numbers-analysis.zip
+data/catalognumbertrend.png: downloads/catalog-number-access-data.zip
 	mkdir -p data
 	unzip -o $^ $@
 
-data/linktrend.png: downloads/catalog_numbers-analysis.zip
+data/linktrend.png: downloads/catalog-number-access-data.zip
 	mkdir -p data
 	unzip -o $^ $@
 
 downloads/ipni-oa-analysis.zip:
 	mkdir -p downloads
 	wget -O $@ $(archived_analysis_url_ipni_oa)
 
-data/oa%.png: downloads/ipni-oa-analysis.zip
+data/oa%.png: downloads/ipni-oa-data.zip
 	mkdir -p data
 	unzip -o $^ $@
 

requirements.txt

@@ -0,0 +1 @@
+PyGithub

util/download-artifact.py

@@ -0,0 +1,34 @@
+import argparse
+from github import Github
+import requests
+from pathlib import Path   
+import os
+
+organization='OA-WCVP'
+
+def main():
+    # Read repo, version tag, and artifact name from command line
+    parser = argparse.ArgumentParser()
+    parser.add_argument("repo", type=str)
+    args = parser.parse_args()
+
+    # Create a Github instance, using an access token:
+    access_token = os.getenv('GITHUB_TOKEN')
+    g = Github(access_token)
+    repo = g.get_repo('{org}/{repo}'.format(org=organization,repo=args.repo))
+    release = repo.get_latest_release()
+    for asset in release.get_assets():
+        # Initiate a session
+        headers = {'Authorization': 'token ' + access_token,
+            'Accept': 'application/octet-stream'}
+        session = requests.Session()
+        response = session.get(asset.url, stream = True, headers=headers)
+        dest = Path() / "downloads" / asset.name
+        with open(dest, 'wb') as f:
+            for chunk in response.iter_content(1024*1024): 
+                f.write(chunk)
+
+if __name__ == '__main__':
+    main()
+
+