From 34679f3b165b385a9fc4f349ca60e2f251b12d9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 May 2021 19:18:07 +0000 Subject: [PATCH 01/47] Bump flask-cors from 3.0.6 to 3.0.9 Bumps [flask-cors](https://github.com/corydolphin/flask-cors) from 3.0.6 to 3.0.9. - [Release notes](https://github.com/corydolphin/flask-cors/releases) - [Changelog](https://github.com/corydolphin/flask-cors/blob/master/CHANGELOG.md) - [Commits](https://github.com/corydolphin/flask-cors/compare/3.0.6...3.0.9) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e7fa6d0..0a3e8e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ docopt==0.6.2 docutils==0.17.1 Flask==1.0.2 -Flask-Cors==3.0.6 +Flask-Cors==3.0.9 gevent==1.4.0 greenlet==0.4.15 html5lib==1.0.1 From 3580c5f478579549659646c73c9dbfab7c72444e Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sun, 16 May 2021 22:27:33 +0200 Subject: [PATCH 02/47] Add a warning if license is not found --- src/swagger.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/swagger.py b/src/swagger.py index 243659d..613512b 100644 --- a/src/swagger.py +++ b/src/swagger.py @@ -161,7 +161,11 @@ def build_spec(user, repo, subdir=None, query_url=None, sha=None, prov=None, ext if item: items.append(item) - # TODO: we should then return two things: list of items and list of errors + + # Add a warning if no license is found + if loader.getLicenceURL() is None: + warnings.append("Queries behind this API do not have a license. You may not be allowed to use them.") + return items, warnings From 9b1f2d76051291f8856d8404d9434c812d003fc4 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Fri, 28 May 2021 20:28:18 +0200 Subject: [PATCH 03/47] Adding support for relative urls in spec files --- src/fileLoaders.py | 6 ++++-- tests/repo/url.yml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fileLoaders.py b/src/fileLoaders.py index efde936..f39b3ca 100644 --- a/src/fileLoaders.py +++ b/src/fileLoaders.py @@ -11,6 +11,7 @@ from github.GithubObject import NotSet from github.GithubException import BadCredentialsException from configparser import ConfigParser +from urllib.parse import urljoin glogger = glogging.getGrlcLogger(__name__) @@ -304,13 +305,14 @@ def getTextFor(self, fileItem): def _getText(self, itemName): """Return the content of the specified item in the specification.""" if itemName in self.spec['files']: - itemUrl = self.spec['files'][itemName]['download_url'] headers = {'Accept' : 'text/plain'} + itemUrl = self.spec['files'][itemName]['download_url'] + itemUrl = urljoin(self.spec['url'], itemUrl) # Join with base URL if relative URL resp = requests.get(itemUrl, headers=headers) if resp.status_code == 200: return resp.text else: - raise Exception(resp.text) + raise Exception('HTTP status {} encountered while loading {}'.format(resp.status_code, itemUrl)) else: return None diff --git a/tests/repo/url.yml b/tests/repo/url.yml index df6f74e..fb7a8ad 100644 --- a/tests/repo/url.yml +++ b/tests/repo/url.yml @@ -7,3 +7,4 @@ queries: - http://example.org/test-rq.rq - http://example.org/test-sparql.sparql - http://example.org/test-tpf.tpf + - ./test-rq.rq From 514cde035fafa9c679c6cf615cef75dce924ceda Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 26 Jul 2021 23:26:29 +0200 Subject: [PATCH 04/47] Change warnings into debug messages. See #350 --- src/gquery.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/gquery.py b/src/gquery.py index b7d91ec..53aa63c 100644 --- a/src/gquery.py +++ b/src/gquery.py @@ -37,7 +37,7 @@ def guess_endpoint_uri(rq, loader): if has_request_context() and "endpoint" in request.args: endpoint = request.args['endpoint'] - glogger.info("Endpoint provided in request: " + endpoint) + glogger.debug("Endpoint provided in request: " + endpoint) return endpoint, auth # Decorator @@ -45,14 +45,14 @@ def guess_endpoint_uri(rq, loader): decorators = get_yaml_decorators(rq) endpoint = decorators['endpoint'] auth = None - glogger.info("Decorator guessed endpoint: " + endpoint) + glogger.debug("Decorator guessed endpoint: " + endpoint) except (TypeError, KeyError): # File try: endpoint_content = loader.getEndpointText() endpoint = endpoint_content.strip().splitlines()[0] auth = None - glogger.info("File guessed endpoint: " + endpoint) + glogger.debug("File guessed endpoint: " + endpoint) # TODO: except all is really ugly except: # Default @@ -60,7 +60,7 @@ def guess_endpoint_uri(rq, loader): auth = (static.DEFAULT_ENDPOINT_USER, static.DEFAULT_ENDPOINT_PASSWORD) if auth == ('none', 'none'): auth = None - glogger.warning("No endpoint specified, using default ({})".format(endpoint)) + glogger.debug("No endpoint specified, using default ({})".format(endpoint)) return endpoint, auth @@ -175,7 +175,7 @@ def get_parameters(rq, variables, endpoint, query_metadata, auth=None): if vdefault is not None: parameters[vname]['default'] = vdefault - glogger.info('Finished parsing the following parameters: {}'.format(parameters)) + glogger.debug('Finished parsing the following parameters: {}'.format(parameters)) return parameters @@ -213,7 +213,7 @@ def get_enumeration_sparql(rq, v, endpoint, auth=None): """ Returns a list of enumerated values for variable 'v' in query 'rq' """ - glogger.info('Retrieving enumeration for variable {}'.format(v)) + glogger.debug('Retrieving enumeration for variable {}'.format(v)) vcodes = [] # tpattern_matcher = re.compile(".*(FROM\s+)?(?P.*)\s+WHERE.*[\.\{][\n\t\s]*(?P.*\?" + re.escape(v) + ".*?\.).*", flags=re.DOTALL) # tpattern_matcher = re.compile(".*?((FROM\s*)(?P(\<.*\>)+))?\s*WHERE\s*\{(?P.*)\}.*", flags=re.DOTALL) @@ -346,9 +346,9 @@ def get_metadata(rq, endpoint): try: # update query - glogger.info("Trying to parse UPDATE query") + glogger.debug("Trying to parse UPDATE query") parsed_query = UpdateUnit.parseString(rq, parseAll=True) - glogger.info(parsed_query) + glogger.debug(parsed_query) query_metadata['type'] = parsed_query[0]['request'][0].name if query_metadata['type'] == 'InsertData': query_metadata['parameters'] = { @@ -357,7 +357,7 @@ def get_metadata(rq, endpoint): 'data': {'datatype': None, 'enum': [], 'lang': None, 'name': 'data', 'original': '?_data', 'required': True, 'type': 'literal'}} - glogger.info("Update query parsed with {}".format(query_metadata['type'])) + glogger.debug("Update query parsed with {}".format(query_metadata['type'])) # if query_metadata['type'] == 'InsertData': # query_metadata['variables'] = parsed_query.algebra['PV'] except: @@ -378,7 +378,7 @@ def paginate_query(query, results_per_page, get_args): split display a maximum of `results_per_page`.""" page = get_args.get('page', 1) - glogger.info("Paginating query for page {}, {} results per page".format(page, results_per_page)) + glogger.debug("Paginating query for page {}, {} results per page".format(page, results_per_page)) # If contains LIMIT or OFFSET, remove them glogger.debug("Original query: " + query) From 83b879f9105cbbe0b00c8d7a118dabb9e1acc46f Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 3 Aug 2021 21:39:16 +0200 Subject: [PATCH 05/47] Propagate exceptions raised by rdflib parsing --- src/gquery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gquery.py b/src/gquery.py index b7d91ec..711d5eb 100644 --- a/src/gquery.py +++ b/src/gquery.py @@ -360,10 +360,11 @@ def get_metadata(rq, endpoint): glogger.info("Update query parsed with {}".format(query_metadata['type'])) # if query_metadata['type'] == 'InsertData': # query_metadata['variables'] = parsed_query.algebra['PV'] - except: + except Exception as e: glogger.error("Could not parse query") glogger.error(query_metadata['query']) glogger.error(traceback.print_exc()) + raise Exception('could not parse query: {}'.format(str(e))) pass glogger.debug("Finished parsing query of type {}".format(query_metadata['type'])) From b58fa694be1d82b1c38df74271d4c36ee196766e Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 3 Aug 2021 21:49:27 +0200 Subject: [PATCH 06/47] Remove 'pass' --- src/gquery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gquery.py b/src/gquery.py index 711d5eb..6bf11ac 100644 --- a/src/gquery.py +++ b/src/gquery.py @@ -365,7 +365,6 @@ def get_metadata(rq, endpoint): glogger.error(query_metadata['query']) glogger.error(traceback.print_exc()) raise Exception('could not parse query: {}'.format(str(e))) - pass glogger.debug("Finished parsing query of type {}".format(query_metadata['type'])) glogger.debug("All parsed query metadata (from decorators and content): ") From ce088a9f0374b57e0d47fd6b4b3f92c18ad9fb63 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 3 Aug 2021 23:16:45 +0200 Subject: [PATCH 07/47] Use safe_load for loading YAML --- src/gquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gquery.py b/src/gquery.py index b7d91ec..6d122b3 100644 --- a/src/gquery.py +++ b/src/gquery.py @@ -274,7 +274,7 @@ def get_yaml_decorators(rq): query_metadata = yaml_string elif type(yaml_string) == str: try: # Invalid YAMLs will produce empty metadata - query_metadata = yaml.load(yaml_string) + query_metadata = yaml.safe_load(yaml_string) except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e: try: query_metadata = json.loads(yaml_string) From 6c2cf0a7235e1f0a3ca88b712efdd394b64d1177 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 4 Aug 2021 19:25:22 +0200 Subject: [PATCH 08/47] Update debug to info message --- src/gquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gquery.py b/src/gquery.py index 53aa63c..9278941 100644 --- a/src/gquery.py +++ b/src/gquery.py @@ -60,7 +60,7 @@ def guess_endpoint_uri(rq, loader): auth = (static.DEFAULT_ENDPOINT_USER, static.DEFAULT_ENDPOINT_PASSWORD) if auth == ('none', 'none'): auth = None - glogger.debug("No endpoint specified, using default ({})".format(endpoint)) + glogger.info("No endpoint specified, using default ({})".format(endpoint)) return endpoint, auth From 17fe6a08fce44f1a6336f0f75d892c81b20f95a6 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Thu, 12 Aug 2021 19:53:50 +0200 Subject: [PATCH 09/47] Update fileloader to extend yaml specification --- src/fileLoaders.py | 54 +++++++++++++++++++++++++++++++++------------- src/queryTypes.py | 13 +++++++++++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/fileLoaders.py b/src/fileLoaders.py index f39b3ca..724b376 100644 --- a/src/fileLoaders.py +++ b/src/fileLoaders.py @@ -1,5 +1,5 @@ import grlc.static as static -from grlc.queryTypes import qType +from grlc.queryTypes import qType, guessQueryType import grlc.glogging as glogging import json @@ -23,15 +23,14 @@ def getTextForName(self, query_name): for `query_name='query1'` would return the content of file `query1.rq` from the loader's source (assuming such file exists).""" # The URIs of all candidates - rq_name = query_name + '.rq' - sparql_name = query_name + '.sparql' - tpf_name = query_name + '.tpf' - json_name = query_name + '.json' + candidateNames = [ + query_name + '.rq', + query_name + '.sparql', + query_name + '.tpf', + query_name + '.json' + ] candidates = [ - (rq_name, qType['SPARQL']), - (sparql_name, qType['SPARQL']), - (tpf_name, qType['TPF']), - (json_name, qType['JSON']) + (name, guessQueryType(name)) for name in candidateNames ] for queryFullName, queryType in candidates: @@ -257,7 +256,6 @@ def getRepoDescription(self): return self.api_description - class URLLoader(BaseLoader): """URL specification loader. Retrieves information to construct a grlc specification from a specification YAML file located on a remote server.""" @@ -273,18 +271,33 @@ def __init__(self, spec_url): self.spec = yaml.load(resp.text) self.spec['url'] = spec_url self.spec['files'] = {} - for queryUrl in self.spec['queries']: - queryNameExt = path.basename(queryUrl) - queryName = path.splitext(queryNameExt)[0] # Remove extention + + for query in self.spec['queries']: + queryName, queryUrl = self.extractQueryInfo(query) + item = { 'name': queryName, 'download_url': queryUrl } - self.spec['files'][queryNameExt] = item + self.spec['files'][queryName] = item del self.spec['queries'] else: raise Exception(resp.text) + def extractQueryInfo(self, query): + """Extract query name and URL from specification. These could + either be explicitly declared (values in a dict) or need to be + infered from the URL (which itself could be explicilty declared or + be the only element of query.""" + queryUrl = query['url'] if type(query) is dict else query + + if type(query) is dict and 'name' in query: + queryName = query['name'] + else: + queryNameExt = path.basename(queryUrl) + queryName = path.splitext(queryNameExt)[0] # Remove extention + return queryName, queryUrl + def fetchFiles(self): """Returns a list of file items contained on specification.""" files = [ @@ -300,7 +313,18 @@ def getTextFor(self, fileItem): """Returns the contents of the given file item on the specification.""" # TODO: tiene sentido esto? O es un hack horrible ? nameExt = path.basename(fileItem['download_url']) - return self._getText(nameExt) + return self._getText(fileItem['name']) + + def getTextForName(self, query_name): + """Return the query text and query type for the given query name. + Specific implementation for URLLoader.""" + try: + queryText = self._getText(query_name) + queryType = guessQueryType(self.spec['files'][query_name]['download_url']) + return queryText, queryType + except Exception as e: + # No query found... + return '', None def _getText(self, itemName): """Return the content of the specified item in the specification.""" diff --git a/src/queryTypes.py b/src/queryTypes.py index d52b2af..488810c 100644 --- a/src/queryTypes.py +++ b/src/queryTypes.py @@ -5,3 +5,16 @@ 'TPF': 'tpf', 'JSON': 'json' } + +def guessQueryType(queryUrl): + queryUrl = queryUrl.lower() + if queryUrl.endswith('.rq'): + return qType['SPARQL'] + elif queryUrl.endswith('.sparql'): + return qType['SPARQL'] + elif queryUrl.endswith('.tpf'): + return qType['TPF'] + elif queryUrl.endswith('.json'): + return qType['JSON'] + else: + raise Exception('Unknown query type: ' + queryUrl) From 58722bd77ba2964fd19731108f9e94a609e1fdbd Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Thu, 12 Aug 2021 19:58:37 +0200 Subject: [PATCH 10/47] Update documentation of yaml specification file --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 25651bc..55e1a67 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ A grlc API specification file is a YAML file which includes the necessary inform - `description`: API description - `contact`: Contact details of the API owner. This should include the `name` and `url` properties. - `licence`: A URL pointing to the licence file for the API. - - `queries`: A list of URLs of SPARQL queries (with header decorators). + - `queries`: A list of URLs of SPARQL queries (with header decorators). Alternatively a query can be defined as a dictionary with a `name` and a `url`. For example: ```YAML @@ -97,6 +97,8 @@ queries: - https://www.mywebsite.org/query1.rq - https://www.mywebsite.org/query2.rq - https://www.otherwebsite.org/query3.rq + - name: QueryFour + url: https://www.mywebsite.org/query4.rq ``` ### grlc generated API From cd1f37de2b12503f4666272101f50e321041882d Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Thu, 23 Sep 2021 21:52:31 +0200 Subject: [PATCH 11/47] Add extra debug message --- src/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils.py b/src/utils.py index 15fc21a..4f8ed7d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -208,6 +208,8 @@ def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader, con # Response headers resp = response.text + + glogger.debug('Got HTTP response from to SPARQL endpoint: {}'.format(resp)) headers['Content-Type'] = response.headers['Content-Type'] # If the query is paginated, set link HTTP headers From f1cdb46c04adc3eb4f1353442e7081c760b9d6c2 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 26 Jul 2021 23:26:29 +0200 Subject: [PATCH 12/47] Change warnings into debug messages. See #350 --- src/gquery.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/gquery.py b/src/gquery.py index b4c80bd..1021e33 100644 --- a/src/gquery.py +++ b/src/gquery.py @@ -37,7 +37,7 @@ def guess_endpoint_uri(rq, loader): if has_request_context() and "endpoint" in request.args: endpoint = request.args['endpoint'] - glogger.info("Endpoint provided in request: " + endpoint) + glogger.debug("Endpoint provided in request: " + endpoint) return endpoint, auth # Decorator @@ -45,14 +45,14 @@ def guess_endpoint_uri(rq, loader): decorators = get_yaml_decorators(rq) endpoint = decorators['endpoint'] auth = None - glogger.info("Decorator guessed endpoint: " + endpoint) + glogger.debug("Decorator guessed endpoint: " + endpoint) except (TypeError, KeyError): # File try: endpoint_content = loader.getEndpointText() endpoint = endpoint_content.strip().splitlines()[0] auth = None - glogger.info("File guessed endpoint: " + endpoint) + glogger.debug("File guessed endpoint: " + endpoint) # TODO: except all is really ugly except: # Default @@ -60,7 +60,7 @@ def guess_endpoint_uri(rq, loader): auth = (static.DEFAULT_ENDPOINT_USER, static.DEFAULT_ENDPOINT_PASSWORD) if auth == ('none', 'none'): auth = None - glogger.warning("No endpoint specified, using default ({})".format(endpoint)) + glogger.debug("No endpoint specified, using default ({})".format(endpoint)) return endpoint, auth @@ -175,7 +175,7 @@ def get_parameters(rq, variables, endpoint, query_metadata, auth=None): if vdefault is not None: parameters[vname]['default'] = vdefault - glogger.info('Finished parsing the following parameters: {}'.format(parameters)) + glogger.debug('Finished parsing the following parameters: {}'.format(parameters)) return parameters @@ -213,7 +213,7 @@ def get_enumeration_sparql(rq, v, endpoint, auth=None): """ Returns a list of enumerated values for variable 'v' in query 'rq' """ - glogger.info('Retrieving enumeration for variable {}'.format(v)) + glogger.debug('Retrieving enumeration for variable {}'.format(v)) vcodes = [] # tpattern_matcher = re.compile(".*(FROM\s+)?(?P.*)\s+WHERE.*[\.\{][\n\t\s]*(?P.*\?" + re.escape(v) + ".*?\.).*", flags=re.DOTALL) # tpattern_matcher = re.compile(".*?((FROM\s*)(?P(\<.*\>)+))?\s*WHERE\s*\{(?P.*)\}.*", flags=re.DOTALL) @@ -346,9 +346,9 @@ def get_metadata(rq, endpoint): try: # update query - glogger.info("Trying to parse UPDATE query") + glogger.debug("Trying to parse UPDATE query") parsed_query = UpdateUnit.parseString(rq, parseAll=True) - glogger.info(parsed_query) + glogger.debug(parsed_query) query_metadata['type'] = parsed_query[0]['request'][0].name if query_metadata['type'] == 'InsertData': query_metadata['parameters'] = { @@ -357,7 +357,7 @@ def get_metadata(rq, endpoint): 'data': {'datatype': None, 'enum': [], 'lang': None, 'name': 'data', 'original': '?_data', 'required': True, 'type': 'literal'}} - glogger.info("Update query parsed with {}".format(query_metadata['type'])) + glogger.debug("Update query parsed with {}".format(query_metadata['type'])) # if query_metadata['type'] == 'InsertData': # query_metadata['variables'] = parsed_query.algebra['PV'] except Exception as e: @@ -378,7 +378,7 @@ def paginate_query(query, results_per_page, get_args): split display a maximum of `results_per_page`.""" page = get_args.get('page', 1) - glogger.info("Paginating query for page {}, {} results per page".format(page, results_per_page)) + glogger.debug("Paginating query for page {}, {} results per page".format(page, results_per_page)) # If contains LIMIT or OFFSET, remove them glogger.debug("Original query: " + query) From 26e01b6aa54e5cc8d54915a23fbe61b2183e2aeb Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 4 Aug 2021 19:25:22 +0200 Subject: [PATCH 13/47] Update debug to info message --- src/gquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gquery.py b/src/gquery.py index 1021e33..43e912e 100644 --- a/src/gquery.py +++ b/src/gquery.py @@ -60,7 +60,7 @@ def guess_endpoint_uri(rq, loader): auth = (static.DEFAULT_ENDPOINT_USER, static.DEFAULT_ENDPOINT_PASSWORD) if auth == ('none', 'none'): auth = None - glogger.debug("No endpoint specified, using default ({})".format(endpoint)) + glogger.info("No endpoint specified, using default ({})".format(endpoint)) return endpoint, auth From 4709211d0c248bcd720b3742824dcbab1b62674a Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Thu, 23 Sep 2021 21:52:31 +0200 Subject: [PATCH 14/47] Add extra debug message --- src/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils.py b/src/utils.py index 15fc21a..4f8ed7d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -208,6 +208,8 @@ def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader, con # Response headers resp = response.text + + glogger.debug('Got HTTP response from to SPARQL endpoint: {}'.format(resp)) headers['Content-Type'] = response.headers['Content-Type'] # If the query is paginated, set link HTTP headers From 365e24dbed5a183d99606eac59a753ab7a32f466 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Thu, 23 Sep 2021 22:15:33 +0200 Subject: [PATCH 15/47] Add github action workflows and remove travis config --- .github/workflows/publish_pypi.yml | 25 +++++++++++++++++ .github/workflows/testing.yml | 33 ++++++++++++++++++++++ .travis.yml | 44 ------------------------------ .travis/before_deploy.sh | 9 ------ .travis/before_install.sh | 15 ---------- .travis/install.sh | 24 ---------------- .travis/run.sh | 6 ---- 7 files changed, 58 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/publish_pypi.yml create mode 100644 .github/workflows/testing.yml delete mode 100644 .travis.yml delete mode 100755 .travis/before_deploy.sh delete mode 100755 .travis/before_install.sh delete mode 100755 .travis/install.sh delete mode 100755 .travis/run.sh diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml new file mode 100644 index 0000000..6510392 --- /dev/null +++ b/.github/workflows/publish_pypi.yml @@ -0,0 +1,25 @@ +name: Publish +# Publish to PyPI when a new release is published +on: + release: + types: + - published +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + python setup.py sdist bdist_wheel + - name: Publish package + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..80f818f --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,33 @@ +name: Testing + +on: [push] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install . + pip install -r requirements-test.txt + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest ./tests diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 66e455e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,44 +0,0 @@ -sudo: false -language: python -cache: pip -python: 3.6 - -before_install: - - source .travis/before_install.sh -install: - - source .travis/install.sh -# command to run tests -script: - - source .travis/run.sh - -jobs: - include: - - stage: Tests - os: linux - python: '3.6' - - stage: Tests - os: osx - language: objective-c - env: PYENV_VERSION=3.7.5 - # command to install dependencies - - stage: Tests - os: windows - language: shell - env: - - PATH=/c/Python37:/c/Python37/Scripts:$PATH - - PYENV_VERSION=3.7.5 - - stage: deploy - # deploy automatically to pypi - before_deploy: - - source .travis/before_deploy.sh - script: skip - # deploy automatically to pypi - deploy: - provider: pypi - user: nlesc - distributions: sdist bdist_wheel - password: - secure: IDakMHNli6Ir+GFzCOxcxBc4c/UdP6il9ICzuiAvzxRa5+k2uxUtPwdXVXTq7dOrLSi4lNO3m16ipnJiDxVpM6BbnsLzbQgeGoHFdOUsNMSldt1A9syag+rplNtfyHoqwBb+ogwGDyZ1bjZMllnJfNoUFPJe2TwIQlQ+aHNiVwUEIHWZQBH+AfRNu+eeTArOu3Ln/E8IY8NJazt8UUF+Rwhl45GWGIRWCL6cV0YVdYpikUdAs+Ier+UO6wvRQ+G9/qik26+w2cXeej6HwSk4vl0DKhsxj6POYMwHeGc2wGMlJPn9hCgcBMU3Hgh4+CGXVQ36SiY3XCClUws9MAyrrbIXVGwt1QuuStjqQH77+kABGlGpnz0hfN+J8hLvzdF95316+eRezQ/5kilNqRAPb8+hbANqrev17N8bAQ+gQDBGjqZ2B6cr15YogvJHtunWkiuP44FJeQweTAiOjl/LHmWxc2b6o0Fs3ZYltWUHI4QTK01YIM73zp4dkr6gzfrGBH22Ut0zDpMTqGN+M8gxs/q8pRwvvd7XkuLSaWxrnS07W0rLsRaJHjbaclh1NifuNS3dNEgVxI8vNNKJUYNUELsk9RxCkn+CyBk5AzJa52PrPeCHcHD8YJAV2aT0etndyZvnnSRB7SqulItoZR1sE9JFe1srQ6etl4y+c6IBsbw= - on: - tags: true - branch: master diff --git a/.travis/before_deploy.sh b/.travis/before_deploy.sh deleted file mode 100755 index 91f8a0a..0000000 --- a/.travis/before_deploy.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Deactivate testing virtualenv and reactivate original one -if [[ $TRAVIS_BUILD_STAGE_NAME == 'Deploy' ]]; then - deactivate - echo "Reactivating virtualenv:" - echo $ORIG_ENV - source $ORIG_ENV -fi diff --git a/.travis/before_install.sh b/.travis/before_install.sh deleted file mode 100755 index ad56124..0000000 --- a/.travis/before_install.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Store location of original virtualenv so we can reactivate it on deploy -if [[ $TRAVIS_BUILD_STAGE_NAME == 'Deploy' ]]; then - ORIG_ENV="$(dirname $(which python))/activate" - echo "Original virtualenv" - echo $ORIG_ENV -fi - -if [[ $TRAVIS_OS_NAME == 'osx' ]]; then - brew install openssl pyenv - CONFIGURE_OPTS="--with-openssl=$(brew --prefix openssl)" pyenv install -s $PYENV_VERSION -elif [[ $TRAVIS_OS_NAME == 'windows' ]]; then - choco install python --version $PYENV_VERSION -fi diff --git a/.travis/install.sh b/.travis/install.sh deleted file mode 100755 index d6e7ae9..0000000 --- a/.travis/install.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -if [[ $TRAVIS_OS_NAME == 'osx' ]]; then - pyenv global $PYENV_VERSION - if [[ $PYENV_VERSION == 2* ]]; then - alias pip=pip2 - fi - if [[ $PYENV_VERSION == 3* ]]; then - alias pip=pip3 - fi - export PATH="/Users/travis/.pyenv/shims:${PATH}" -elif [[ $TRAVIS_OS_NAME == 'windows' ]]; then - alias pip="pip --user" -fi - -if [[ $TRAVIS_BUILD_STAGE_NAME == 'Deploy' ]]; then - virtualenv venv -p python$PYENV_VERSION - source venv/bin/activate -fi - -pip install --upgrade pip -pip install docutils==0.17.1 -pip install . -pip install -r requirements-test.txt diff --git a/.travis/run.sh b/.travis/run.sh deleted file mode 100755 index e3a4bf1..0000000 --- a/.travis/run.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -if [[ $TRAVIS_BUILD_STAGE_NAME == 'Deploy' ]]; then - source venv/bin/activate -fi -pytest ./tests From afecfa6d0b8d8dcedfbe6a06fddaad14427147bd Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Thu, 23 Sep 2021 22:26:39 +0200 Subject: [PATCH 16/47] Add flake8 testing --- .github/workflows/testing.yml | 4 ++-- requirements-test.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 80f818f..d27b73d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -25,9 +25,9 @@ jobs: - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | pytest ./tests diff --git a/requirements-test.txt b/requirements-test.txt index 4a10219..773c229 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ mock==2.0.0 pytest==5.2.1 +flake8==3.9.2 From 613375f012b62b29986f3c238739a94704a8effb Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 16 Nov 2021 22:25:41 +0100 Subject: [PATCH 17/47] Test release on test.pypi --- .github/workflows/publish_pypi.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 6510392..b89a6d5 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -19,7 +19,8 @@ jobs: pip install setuptools wheel twine python setup.py sdist bdist_wheel - name: Publish package - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@v1 with: user: __token__ - password: ${{ secrets.PYPI_TOKEN }} + password: ${{ secrets.PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ From 89e6d7d14100014acc21c1e268bb59fa84347bb5 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 16 Nov 2021 22:29:34 +0100 Subject: [PATCH 18/47] Testing on 3.7 and 3.8 --- .github/workflows/testing.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d27b73d..711fc43 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,16 +1,12 @@ -name: Testing - +name: Testing on: [push] - jobs: build: - runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8] - + python-version: [3.7, 3.8] steps: - uses: actions/checkout@master - name: Set up Python ${{ matrix.python-version }} From e3b878cb86826df718f44c2959ae3bc03204934c Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 16 Nov 2021 22:45:21 +0100 Subject: [PATCH 19/47] Disable sphinx --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3033e81..de6190e 100644 --- a/setup.py +++ b/setup.py @@ -48,8 +48,8 @@ # dependency for `python setup.py test` 'pytest-runner', # dependencies for `python setup.py build_sphinx` - 'sphinx', - 'recommonmark' + # 'sphinx', + # 'recommonmark' ], tests_require=tests_require, package_data = { 'grlc': grlc_data }, From 78ab29c343cc05d925480dfc35e4481923422726 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 16 Nov 2021 22:52:22 +0100 Subject: [PATCH 20/47] Use pypi publish v1.4.2 --- .github/workflows/publish_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index b89a6d5..39a452a 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -19,7 +19,7 @@ jobs: pip install setuptools wheel twine python setup.py sdist bdist_wheel - name: Publish package - uses: pypa/gh-action-pypi-publish@v1 + uses: pypa/gh-action-pypi-publish@v1.4.2 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} From c1dbccff2fc3c637d3dc6d043e9fc847f2e7dbdd Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 16 Nov 2021 23:22:12 +0100 Subject: [PATCH 21/47] Do not use test pypi --- .github/workflows/publish_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 39a452a..ec6f923 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -23,4 +23,4 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ + # repository_url: https://test.pypi.org/legacy/ From b7dcbe4c73a3cd78106d418c1cd74290986a02c3 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 23 Jan 2023 17:39:02 +0100 Subject: [PATCH 22/47] Tested with Python 3.7.16 --- requirements.txt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0a3e8e0..ed43caf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,22 +5,26 @@ Flask-Cors==3.0.9 gevent==1.4.0 greenlet==0.4.15 html5lib==1.0.1 -isodate==0.5.4 +isodate==0.5.4; python_version <= '3.6' +isodate==0.6.1; python_version > '3.6' itsdangerous==1.1.0 -keepalive==0.5 +keepalive==0.5; python_version <= '3.6' MarkupSafe==0.23 pyaml==18.11.0 pyparsing==2.0.7 PyYAML==5.4 -rdflib==4.2.2 -rdflib-jsonld==0.4.0 +rdflib==4.2.2; python_version <= '3.6' +rdflib==5.0.0; python_version > '3.6' +rdflib-jsonld==0.4.0; python_version <= '3.6' +rdflib-jsonld==0.6.2; python_version > '3.6' requests==2.20.0 six==1.12.0 simplejson==3.16.0 setuptools>=38.6.0 SPARQLTransformer==2.1.1 SPARQLWrapper==1.8.2 -werkzeug>=0.16.0 -PyGithub==1.43.5 +werkzeug==0.16.0 +PyGithub==1.43.5; python_version <= '3.6' +PyGithub==1.57; python_version > '3.6' gunicorn==19.6.0; sys_platform!="win32" waitress>=1.4.2; sys_platform=="win32" From 2d47aa03c51942da870cb57d88ebd89a14c44703 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 23 Jan 2023 20:03:46 +0100 Subject: [PATCH 23/47] Tested with Python 3.8.16 and 3.9.16 --- requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ed43caf..1ae6132 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,10 @@ docopt==0.6.2 docutils==0.17.1 Flask==1.0.2 Flask-Cors==3.0.9 -gevent==1.4.0 -greenlet==0.4.15 +gevent==1.4.0; python_version <= '3.8' +gevent==22.10.2; python_version > '3.8' +greenlet==0.4.15; python_version <= '3.8' +greenlet==2.0.0; python_version > '3.8' html5lib==1.0.1 isodate==0.5.4; python_version <= '3.6' isodate==0.6.1; python_version > '3.6' From c6fdd6146f240d3f7110a0e500bef5b1ac41be12 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 25 Jan 2023 23:10:58 +0100 Subject: [PATCH 24/47] Add python versions to Github testing action --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 711fc43..b124bf6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Set up Python ${{ matrix.python-version }} From db3009de42bbb23f3de30ad8af052bb78f06d96a Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 31 Jan 2023 22:14:54 +0100 Subject: [PATCH 25/47] Manually incorporatig Gitlab support --- config.default.ini | 5 +- requirements.txt | 3 +- src/fileLoaders.py | 123 ++++++++++++++++++++++++++++++++++++++++++ src/server.py | 72 ++++++++++++++++++++++--- src/static.py | 7 ++- src/swagger.py | 11 ++-- src/utils.py | 21 +++++--- tests/test_grlc.py | 2 +- tests/test_swagger.py | 2 +- 9 files changed, 222 insertions(+), 24 deletions(-) diff --git a/config.default.ini b/config.default.ini index d8df51c..e7c1cd3 100644 --- a/config.default.ini +++ b/config.default.ini @@ -12,4 +12,7 @@ server_name = grlc.io user = none password = none # Logging level -debug = False +debug = True + +[api_gitlab] +gitlab_url=https://gitlab \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0a3e8e0..cd6b73b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,10 +11,11 @@ keepalive==0.5 MarkupSafe==0.23 pyaml==18.11.0 pyparsing==2.0.7 +python-gitlab==2.10.1 PyYAML==5.4 rdflib==4.2.2 rdflib-jsonld==0.4.0 -requests==2.20.0 +requests==2.27.1 six==1.12.0 simplejson==3.16.0 setuptools>=38.6.0 diff --git a/src/fileLoaders.py b/src/fileLoaders.py index f39b3ca..2123e26 100644 --- a/src/fileLoaders.py +++ b/src/fileLoaders.py @@ -1,10 +1,22 @@ +# /* +# * SPDX-License-Identifier: MIT +# * SPDX-FileCopyrightText: Copyright (c) 2022 Orange SA +# * +# * Author: Mihary RANAIVOSON +# * Modifications: Add GitlabLoader +# */ + import grlc.static as static from grlc.queryTypes import qType import grlc.glogging as glogging import json +import gitlab import requests import yaml +import urllib.parse +import base64 +import os from os import path from glob import glob from github import Github @@ -13,8 +25,10 @@ from configparser import ConfigParser from urllib.parse import urljoin +# util variables glogger = glogging.getGrlcLogger(__name__) + class BaseLoader: """Base class for File Loaders""" def getTextForName(self, query_name): @@ -160,6 +174,115 @@ def getRepoDescription(self): return self.gh_repo.description +class GitlabLoader(BaseLoader): + + def __init__(self, user, repo, subdir=None, sha=None, prov=None, branch='main'): + """Create a new GithubLoader. + # TODO: Update to GITLAB ! + + Keyword arguments: + user -- Github user name of the target github repo. + repo -- Repository name of the target github repo. + subdir -- Target subdirectory within the given repo. (default: None). + branch -- Branch + sha -- Github commit identifier hash (default: None). + prov -- grlcPROV object for tracking provenance (default: None).""" + self.user = user + self.repo = repo + self.subdir = (subdir + "/") if subdir else "" + self.branch = branch + self.sha = sha if sha else None + self.prov = prov + gl = gitlab.Gitlab( + url=static.GITLAB_URL, + private_token=static.ACCESS_TOKEN + ) + try: + self.gl_repo = gl.projects.get(user + '/' + repo) + except BadCredentialsException: + raise Exception('BadCredentials: have you set up github_access_token on config.ini ?') + except Exception: + raise Exception('Repo not found: ' + user + '/' + repo) + + def fetchFiles(self): + """Returns a list of file items contained on the github repo.""" + gitlab_files = self.gl_repo.repository_tree(path=self.subdir.strip('/'), ref=self.branch, all=True) + files = [] + for gitlab_file in gitlab_files: + if gitlab_file['type'] == 'blob': + name = gitlab_file['name'] + files.append({ + 'download_url': path.join(self.getRawRepoUri(), self.subdir, name), + 'name': name, + 'decoded_content': str.encode(self._getText(gitlab_file['name'])) + }) + return files + + def getRawRepoUri(self): + """Returns the root url of the github repo.""" + # TODO: replace by gh_repo.html_url ? + return path.join(static.GITLAB_URL, self.user, self.repo, '-', 'raw', self.branch) + + def getTextFor(self, fileItem): + """Returns the contents of the given file item on the github repo.""" + raw_query_uri = fileItem['download_url'] + + # Add query URI as used entity by the logged activity + if self.prov is not None: + self.prov.add_used_entity(raw_query_uri) + return str(fileItem['decoded_content'], 'utf-8') + + def _getText(self, query_name): + """Return the content of the specified file contained in the github repo.""" + try: + file_path = path.join(self.subdir, query_name) + f = self.gl_repo.files.get(file_path=file_path, ref=self.branch) + file_content = base64.b64decode(f.content).decode("utf-8") + return file_content.replace('\\n', '\n').replace('\\t', '\t') + except: + return None + + def getRepoTitle(self): + """Return the title of the github repo.""" + return self.gl_repo.name + + def getContactName(self): + """Return the name of the owner of the gitlab repo.""" + return self.gl_repo.namespace['name'] + + def getContactUrl(self): + """Return the home page of the owner of the gitlab repo.""" + return self.gl_repo.namespace['web_url'] + + def getCommitList(self): + """Return a list of commits on the gitlab repo.""" + return [ c.id for c in self.gl_repo.commits.list() ] + + def getFullName(self): + """Return the full name of the gitlab repo (user/repo).""" + return self.gl_repo.path_with_namespace + + def getRepoURI(self): + """Return the full URI of the gitlab repo.""" + return self.gl_repo.web_url + + def getEndpointText(self): + """Return content of endpoint file (endpoint.txt)""" + return self._getText('endpoint.txt') + + def getLicenceURL(self): + """Returns the URL of the license file in this repository if one exists.""" + for f in self.fetchFiles(): + if f['name'].lower() == 'license' or f['name'].lower() == 'licence': + return f['download_url'] + return None + + def getRepoDescription(self): + """Return the description of the repository""" + return self.gl_repo.description + + + class LocalLoader(BaseLoader): """Local file system file loader. Retrieves information to construct a grlc specification from a local folder.""" diff --git a/src/server.py b/src/server.py index 63347ab..be3e86a 100644 --- a/src/server.py +++ b/src/server.py @@ -27,11 +27,11 @@ def api_docs_template(): """Generate Grlc API page.""" return render_template('api-docs.html', relative_path=relative_path()) -def swagger_spec(user, repo, subdir=None, spec_url=None, sha=None, content=None): +def swagger_spec(user, repo, subdir=None, spec_url=None, sha=None, content=None, git_type=None, branch='main'): """ Generate swagger specification """ - glogger.info("-----> Generating swagger spec for /{}/{}, subdir {}, params {}, on commit {}".format(user, repo, subdir, spec_url, sha)) + glogger.info("-----> Generating swagger spec for /{}/{} ({}), subdir {}, params {}, on commit {}".format(user, repo, git_type, subdir, spec_url, sha)) - swag = utils.build_swagger_spec(user, repo, subdir, spec_url, sha, static.SERVER_NAME) + swag = utils.build_swagger_spec(user, repo, subdir, spec_url, sha, static.SERVER_NAME, git_type, branch) resp_spec = make_response(jsonify(swag)) resp_spec.headers['Content-Type'] = 'application/json' @@ -41,9 +41,9 @@ def swagger_spec(user, repo, subdir=None, spec_url=None, sha=None, content=None) glogger.info("-----> API spec generation for /{}/{}, subdir {}, params {}, on commit {} complete".format(user, repo, subdir, spec_url, sha)) return resp_spec -def query(user, repo, query_name, subdir=None, spec_url=None, sha=None, content=None): +def query(user, repo, query_name, subdir=None, spec_url=None, sha=None, content=None, git_type=None, branch='main'): """Execute SPARQL query for a specific grlc-generated API endpoint""" - glogger.info("-----> Executing call name at /{}/{}/{}/{} on commit {}".format(user, repo, subdir, query_name, sha)) + glogger.info("-----> Executing call name at /{}/{} ({})/{}/{} on commit {}".format(user, repo, git_type, subdir, query_name, sha)) glogger.debug("Request accept header: " + request.headers["Accept"]) requestArgs = request.args @@ -54,7 +54,7 @@ def query(user, repo, query_name, subdir=None, spec_url=None, sha=None, content= query_response, status, headers = utils.dispatch_query(user, repo, query_name, subdir, spec_url, sha=sha, content=content, requestArgs=requestArgs, acceptHeader=acceptHeader, - requestUrl=requestUrl, formData=formData) + requestUrl=requestUrl, formData=formData, git_type=git_type, branch=branch) if isinstance(query_response, list): query_response = jsonify(query_response) @@ -124,6 +124,9 @@ def query_param(query_name, content=None): glogger.debug("Spec URL: {}".format(spec_url)) return query(user=None, repo=None, query_name=query_name, spec_url=spec_url, content=content) + + + ############################## ### Routes for GitHub APIs ### ############################## @@ -161,7 +164,7 @@ def api_docs_git(user, repo, subdir=None, sha=None): @app.route('/api-git////commit//swagger') # backward compatibility route def swagger_spec_git(user, repo, subdir=None, sha=None): """Swagger spec for specifications loaded from a Github repo.""" - return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None) + return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None, git_type="github") # Callname execution @app.route('/api-git///', methods=['GET', 'POST']) @@ -182,7 +185,60 @@ def swagger_spec_git(user, repo, subdir=None, sha=None): @app.route('/api////commit//.', methods=['GET', 'POST']) # backward compatibility route def query_git(user, repo, query_name, subdir=None, sha=None, content=None): """SPARQL query execution for specifications loaded from a Github repo.""" - return query(user, repo, query_name, subdir=subdir, sha=sha, content=content) + return query(user, repo, query_name, subdir=subdir, sha=sha, content=content, git_type="github") + + + + +############################## +### Routes for GitLab APIs ### +############################## + +# Spec generation, front-end +@app.route('/api-gitlab//', strict_slashes=False) +@app.route('/api-gitlab///branch/', strict_slashes=False) +@app.route('/api-gitlab///subdir/', strict_slashes=False) +@app.route('/api-gitlab///branch//subdir/', strict_slashes=False) +@app.route('/api-gitlab///api-docs') +@app.route('/api-gitlab///commit/') +@app.route('/api-gitlab///commit//api-docs') +@app.route('/api-gitlab///subdir//commit/') +@app.route('/api-gitlab///subdir//commit//api-docs') +def api_docs_gitlab(user, repo, subdir=None, sha=None, branch='main'): + """Grlc API page for specifications loaded from a Github repo.""" + glogger.debug("Entry in function: __main__.api_docs_gitlab") + return api_docs_template() + +# Spec generation, JSON +@app.route('/api-gitlab///swagger', methods=['GET']) +@app.route('/api-gitlab///branch//swagger', methods=['GET']) +@app.route('/api-gitlab///subdir//swagger', methods=['GET']) +@app.route('/api-gitlab///branch//subdir//swagger', methods=['GET']) +@app.route('/api-gitlab///commit//swagger') +@app.route('/api-gitlab///subdir//commit//swagger') +@app.route('/api-gitlab////commit//swagger') +def swagger_spec_gitlab(user, repo, subdir=None, sha=None, branch='main'): + """Swagger spec for specifications loaded from a Github repo.""" + glogger.debug("Entry in function: __main__.swagger_spec_gitlab") + return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None, git_type="gitlab", branch=branch) + +# Callname execution +@app.route('/api-gitlab///query/', methods=['GET', 'POST']) +@app.route('/api-gitlab///query/branch//', methods=['GET','POST']) +@app.route('/api-gitlab///query/subdir//', methods=['GET', 'POST']) +@app.route('/api-gitlab///query/branch//subdir//', methods=['GET','POST']) +@app.route('/api-gitlab///query/.', methods=['GET', 'POST']) +@app.route('/api-gitlab///query/subdir//.', methods=['GET', 'POST']) +@app.route('/api-gitlab///query/commit//', methods=['GET', 'POST']) +@app.route('/api-gitlab///query/subdir//commit//', methods=['GET', 'POST']) +@app.route('/api-gitlab///query/commit//.', methods=['GET', 'POST']) +@app.route('/api-gitlab///query/subdir//commit//.', methods=['GET', 'POST']) +def query_gitlab(user, repo, query_name, subdir=None, sha=None, content=None, branch='main'): + """SPARQL query execution for specifications loaded from a Github repo.""" + glogger.debug("Entry in function: __main__.query_gitlab") + return query(user, repo, query_name, subdir=subdir, sha=sha, content=content, git_type="gitlab", branch=branch) + + # Main thread diff --git a/src/static.py b/src/static.py index 1e72ac6..d26e8d4 100644 --- a/src/static.py +++ b/src/static.py @@ -35,12 +35,14 @@ 'password': '', 'server_name': '', 'local_sparql_dir': '', - 'debug': 'False' + 'debug': 'False', + 'gitlab_url': 'https://gitlab' } config = ConfigParser(config_fallbacks) config.add_section('auth') config.add_section('defaults') config.add_section('local') +config.add_section('api_gitlab') config_filename = os.path.join(os.getcwd(), 'config.ini') print('Reading config file: ', config_filename) config.read(config_filename) @@ -54,6 +56,9 @@ # Local folder where queries are loaded from LOCAL_SPARQL_DIR = config.get('local', 'local_sparql_dir') +# api_gitlab +GITLAB_URL = config.get('api_gitlab', 'gitlab_url') + # server name, used by the Flask app and in the swagger spec SERVER_NAME = config.get('defaults', 'server_name') diff --git a/src/swagger.py b/src/swagger.py index 613512b..32e7db7 100644 --- a/src/swagger.py +++ b/src/swagger.py @@ -2,7 +2,7 @@ import grlc.utils import grlc.gquery as gquery import grlc.pagination as pageUtils -from grlc.fileLoaders import GithubLoader, LocalLoader, URLLoader +from grlc.fileLoaders import GithubLoader, LocalLoader, URLLoader, GitlabLoader import traceback import grlc.glogging as glogging @@ -62,6 +62,11 @@ def get_repo_info(loader, sha, prov_g): basePath = '/api-git/' + user_repo + '/' basePath += ('subdir/' + loader.subdir + '/') if loader.subdir else '' basePath += ('commit/' + sha + '/') if sha else '' + if type(loader) is GitlabLoader: + basePath = '/api-gitlab/' + user_repo + '/query/' + basePath += ('branch/' + loader.branch + '/') if loader.branch else '' + basePath += ('subdir/' + loader.subdir.strip('/') + '/') if loader.subdir else '' + basePath += ('commit/' + sha + '/') if sha else '' elif type(loader) is LocalLoader: basePath = '/api-local/' elif type(loader) is URLLoader: @@ -117,9 +122,9 @@ def get_path_for_item(item): return item_path -def build_spec(user, repo, subdir=None, query_url=None, sha=None, prov=None, extraMetadata=[]): +def build_spec(user, repo, subdir=None, query_url=None, sha=None, prov=None, extraMetadata=[], git_type=None, branch='main'): """Build grlc specification for the given github user / repo.""" - loader = grlc.utils.getLoader(user, repo, subdir, query_url, sha=sha, prov=prov) + loader = grlc.utils.getLoader(user, repo, subdir, query_url, sha=sha, prov=prov, git_type=git_type, branch=branch) files = loader.fetchFiles() raw_repo_uri = loader.getRawRepoUri() diff --git a/src/utils.py b/src/utils.py index 4f8ed7d..139eb2a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,7 +3,7 @@ import grlc.pagination as pageUtils import grlc.swagger as swagger from grlc.prov import grlcPROV -from grlc.fileLoaders import GithubLoader, LocalLoader, URLLoader +from grlc.fileLoaders import GithubLoader, LocalLoader, URLLoader, GitlabLoader from grlc.queryTypes import qType from grlc import __version__ as grlc_version @@ -19,14 +19,19 @@ glogger = glogging.getGrlcLogger(__name__) -def getLoader(user, repo, subdir=None, spec_url=None, sha=None, prov=None): +def getLoader(user, repo, subdir=None, spec_url=None, sha=None, prov=None, git_type=None, branch='main'): """Build a fileLoader (LocalLoader, GithubLoader, URLLoader) for the given parameters.""" if user is None and repo is None and not spec_url: loader = LocalLoader() elif spec_url: loader = URLLoader(spec_url) else: - loader = GithubLoader(user, repo, subdir, sha, prov) + if git_type == 'github': + glogger.debug("Building GithubLoader....") + loader = GithubLoader(user, repo, subdir, sha, prov) + else: + glogger.debug("Building GitlabLoader....") + loader = GitlabLoader(user, repo, subdir, sha, prov, branch) return loader @@ -40,7 +45,7 @@ def build_spec(user, repo, subdir=None, sha=None, prov=None, extraMetadata=[]): return items -def build_swagger_spec(user, repo, subdir, spec_url, sha, serverName): +def build_swagger_spec(user, repo, subdir, spec_url, sha, serverName, git_type, branch='main'): """Build grlc specification for the given github user / repo in swagger format.""" if user and repo: # Init provenance recording @@ -52,7 +57,7 @@ def build_swagger_spec(user, repo, subdir, spec_url, sha, serverName): swag['host'] = serverName try: - loader = getLoader(user, repo, subdir, spec_url, sha, prov_g) + loader = getLoader(user, repo, subdir, spec_url, sha, prov_g, git_type, branch) except Exception as e: # If repo does not exits swag['info'] = { @@ -70,7 +75,7 @@ def build_swagger_spec(user, repo, subdir, spec_url, sha, serverName): swag['basePath'] = basePath # TODO: can we pass loader to build_spec ? --> Ideally yes! - spec, warnings = swagger.build_spec(user, repo, subdir, spec_url, sha, prov_g) + spec, warnings = swagger.build_spec(user, repo, subdir, spec_url, sha, prov_g, [], git_type, branch) # Use items to build API paths for item in spec: swag['paths'][item['call_name']] = swagger.get_path_for_item(item) @@ -90,9 +95,9 @@ def build_swagger_spec(user, repo, subdir, spec_url, sha, serverName): def dispatch_query(user, repo, query_name, subdir=None, spec_url=None, sha=None, content=None, requestArgs={}, acceptHeader='application/json', - requestUrl='http://', formData={}): + requestUrl='http://', formData={}, git_type=None, branch='main'): """Executes the specified SPARQL or TPF query.""" - loader = getLoader(user, repo, subdir, spec_url, sha=sha, prov=None) + loader = getLoader(user, repo, subdir, spec_url, sha=sha, prov=None, git_type=git_type, branch=branch) query, q_type = loader.getTextForName(query_name) # Call name implemented with SPARQL query diff --git a/tests/test_grlc.py b/tests/test_grlc.py index 761f6c5..96313bb 100644 --- a/tests/test_grlc.py +++ b/tests/test_grlc.py @@ -23,7 +23,7 @@ def test_build_spec(self, mockQueryText, mockLoaderFiles, mockGithubRepo): user = 'testuser' repo = 'testrepo' - spec, warning = swagger.build_spec(user=user, repo=repo) + spec, warning = swagger.build_spec(user=user, repo=repo, git_type="github") self.assertEqual(len(spec), len(filesInRepo)) diff --git a/tests/test_swagger.py b/tests/test_swagger.py index 5eabeef..2e76329 100644 --- a/tests/test_swagger.py +++ b/tests/test_swagger.py @@ -19,7 +19,7 @@ def test_github(self, mockQueryText, mockLoaderFiles, mockGithubRepo): user = 'testuser' repo = 'testrepo' - spec, warnings = build_spec(user, repo) + spec, warnings = build_spec(user, repo, git_type="github") self.assertEqual(len(spec), len(filesInRepo)) From a179ae31a20ae8c78e5b5b0cec263fdeec3ae1d8 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sat, 4 Feb 2023 11:53:35 +0100 Subject: [PATCH 26/47] Testing Py3.6 on Ubuntu 20.04 --- .github/workflows/testing.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b124bf6..9325d5d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -6,7 +6,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [ 3.7, 3.8, 3.9] + include: + - os: ubuntu-20.04 + python-version: 3.6 steps: - uses: actions/checkout@master - name: Set up Python ${{ matrix.python-version }} From fcb6ae53bc29d08dc6d99bc7fb97877a2b4f38f8 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sat, 4 Feb 2023 12:01:10 +0100 Subject: [PATCH 27/47] Disable Flake8 linting --- .github/workflows/testing.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9325d5d..0df139c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,12 +21,6 @@ jobs: python -m pip install --upgrade pip pip install . pip install -r requirements-test.txt - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | pytest ./tests From 7bbcdcc1982eea8440405cee290e6f0f75e30643 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sat, 11 Feb 2023 17:17:00 +0100 Subject: [PATCH 28/47] Adding (broken) gitlab tests --- tests/mock_data.py | 5 ++++ tests/test_loaders.py | 70 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/tests/mock_data.py b/tests/mock_data.py index 0a8e0d2..8d4f023 100644 --- a/tests/mock_data.py +++ b/tests/mock_data.py @@ -31,6 +31,11 @@ def get_contents(self, filename, ref=None): return None +class MockGitlabRepo: + pass + + + def mock_requestsUrl(url, headers={}, params={}): url = url.replace('http://example.org/', 'tests/repo/') f = open(url, 'r') diff --git a/tests/test_loaders.py b/tests/test_loaders.py index acbf079..717f6d2 100644 --- a/tests/test_loaders.py +++ b/tests/test_loaders.py @@ -3,10 +3,10 @@ from mock import patch from os import path -from grlc.fileLoaders import LocalLoader, GithubLoader, URLLoader +from grlc.fileLoaders import LocalLoader, GithubLoader, GitlabLoader, URLLoader from grlc.queryTypes import qType -from tests.mock_data import MockGithubRepo, mock_requestsUrl +from tests.mock_data import MockGithubRepo, MockGitlabRepo, mock_requestsUrl class TestGithubLoader(unittest.TestCase): @@ -74,6 +74,72 @@ def test_getEndpointText(self): self.assertIsInstance(endpoint, six.string_types, "Should be some text") +class TestGitlabLoader(unittest.TestCase): + @classmethod + # TODO: patch gitlab object? + # @patch('???', return_value=MockGitlabRepo()) + def setUpClass(self, mocked_repo): + self.user = 'fakeuser' + self.repo = 'fakerepo' + self.loader = GitlabLoader(self.user, self.repo, subdir=None, sha=None, prov=None) + + def test_fetchFiles(self): + files = self.loader.fetchFiles() + + # Should return a list of file items + self.assertIsInstance(files, list, "Should return a list of file items") + + # Should have N files (where N=9) + self.assertEqual(len(files), 9, "Should return correct number of files") + + # File items should have a download_url + for fItem in files: + self.assertIn('download_url', fItem, "File items should have a download_url") + + def test_getRawRepoUri(self): + repoUri = self.loader.getRawRepoUri() + + # Should be a string + self.assertIsInstance(repoUri, six.string_types, "Should be a string") + + # For URI shoud contain user / repo + self.assertIn(self.user, repoUri, "Should contain user") + self.assertIn(self.repo, repoUri, "Should contain repo") + + def test_getTextFor(self): + files = self.loader.fetchFiles() + + # the contents of each file + for fItem in files: + text = self.loader.getTextFor(fItem) + + # Should be some text + self.assertIsInstance(text, six.string_types, "Should be some text") + + # Should be non-empty for existing items + self.assertGreater(len(text), 0, "Should be non-empty") + + # Should raise exception for invalid file items + with self.assertRaises(Exception, msg="Should raise exception for invalid file items"): + text = self.loader.getTextFor({}) + + def test_getTextForName(self): + testableNames = [ + ('test-rq', qType['SPARQL']), + ('test-sparql', qType['SPARQL']), + ('test-tpf', qType['TPF']) + ] + for name, expectedType in testableNames: + text, actualType = self.loader.getTextForName(name) + self.assertEqual(expectedType, actualType, "Query type should match %s != %s" % (expectedType, actualType)) + + def test_getEndpointText(self): + endpoint = self.loader.getEndpointText() + + # Should be some text + self.assertIsInstance(endpoint, six.string_types, "Should be some text") + + class TestLocalLoader(unittest.TestCase): @classmethod def setUpClass(self): From 21adf5997b594bcf5ea565e5d32f986e86e5ca76 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sun, 12 Feb 2023 10:55:53 +0100 Subject: [PATCH 29/47] Use a static variable for s --- src/server.py | 8 ++++---- src/static.py | 4 ++++ src/utils.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/server.py b/src/server.py index be3e86a..ece709e 100644 --- a/src/server.py +++ b/src/server.py @@ -164,7 +164,7 @@ def api_docs_git(user, repo, subdir=None, sha=None): @app.route('/api-git////commit//swagger') # backward compatibility route def swagger_spec_git(user, repo, subdir=None, sha=None): """Swagger spec for specifications loaded from a Github repo.""" - return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None, git_type="github") + return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None, git_type=static.TYPE_GITHUB) # Callname execution @app.route('/api-git///', methods=['GET', 'POST']) @@ -185,7 +185,7 @@ def swagger_spec_git(user, repo, subdir=None, sha=None): @app.route('/api////commit//.', methods=['GET', 'POST']) # backward compatibility route def query_git(user, repo, query_name, subdir=None, sha=None, content=None): """SPARQL query execution for specifications loaded from a Github repo.""" - return query(user, repo, query_name, subdir=subdir, sha=sha, content=content, git_type="github") + return query(user, repo, query_name, subdir=subdir, sha=sha, content=content, git_type=static.TYPE_GITHUB) @@ -220,7 +220,7 @@ def api_docs_gitlab(user, repo, subdir=None, sha=None, branch='main'): def swagger_spec_gitlab(user, repo, subdir=None, sha=None, branch='main'): """Swagger spec for specifications loaded from a Github repo.""" glogger.debug("Entry in function: __main__.swagger_spec_gitlab") - return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None, git_type="gitlab", branch=branch) + return swagger_spec(user, repo, subdir=subdir, spec_url=None, sha=sha, content=None, git_type=static.TYPE_GITLAB, branch=branch) # Callname execution @app.route('/api-gitlab///query/', methods=['GET', 'POST']) @@ -236,7 +236,7 @@ def swagger_spec_gitlab(user, repo, subdir=None, sha=None, branch='main'): def query_gitlab(user, repo, query_name, subdir=None, sha=None, content=None, branch='main'): """SPARQL query execution for specifications loaded from a Github repo.""" glogger.debug("Entry in function: __main__.query_gitlab") - return query(user, repo, query_name, subdir=subdir, sha=sha, content=content, git_type="gitlab", branch=branch) + return query(user, repo, query_name, subdir=subdir, sha=sha, content=content, git_type=static.TYPE_GITLAB, branch=branch) diff --git a/src/static.py b/src/static.py index d26e8d4..863f901 100644 --- a/src/static.py +++ b/src/static.py @@ -22,6 +22,10 @@ GITHUB_RAW_BASE_URL = 'https://raw.githubusercontent.com/' GITHUB_API_BASE_URL = 'https://api.github.com/repos/' +# Git types +TYPE_GITHUB = "github" +TYPE_GITLAB = "gitlab" + # Cache control # CACHE_CONTROL_POLICY = 'public, max-age=60' # With the new hash retrieveal and redirect caching becomes obsolete diff --git a/src/utils.py b/src/utils.py index 139eb2a..95aeeb2 100644 --- a/src/utils.py +++ b/src/utils.py @@ -26,7 +26,7 @@ def getLoader(user, repo, subdir=None, spec_url=None, sha=None, prov=None, git_t elif spec_url: loader = URLLoader(spec_url) else: - if git_type == 'github': + if git_type == static.TYPE_GITHUB: glogger.debug("Building GithubLoader....") loader = GithubLoader(user, repo, subdir, sha, prov) else: From 73670f231f081244399729d2d39b783e1a4aadab Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 24 Apr 2023 00:08:09 +0200 Subject: [PATCH 30/47] Update requests required version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1460f62..5bc492f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ rdflib==4.2.2; python_version <= '3.6' rdflib==5.0.0; python_version > '3.6' rdflib-jsonld==0.4.0; python_version <= '3.6' rdflib-jsonld==0.6.2; python_version > '3.6' -requests==2.20.0 +requests>=2.20.0 six==1.12.0 simplejson==3.16.0 setuptools>=38.6.0 From 1b10133f772b4f61737cfd3a75262aadfc1b2c8b Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 24 Apr 2023 08:57:09 +0200 Subject: [PATCH 31/47] Disable GitLab tests --- tests/test_loaders.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_loaders.py b/tests/test_loaders.py index 717f6d2..e01f8cf 100644 --- a/tests/test_loaders.py +++ b/tests/test_loaders.py @@ -77,13 +77,14 @@ def test_getEndpointText(self): class TestGitlabLoader(unittest.TestCase): @classmethod # TODO: patch gitlab object? + # TODO: Enable tests (remove x from 'xtest' names) # @patch('???', return_value=MockGitlabRepo()) def setUpClass(self, mocked_repo): self.user = 'fakeuser' self.repo = 'fakerepo' self.loader = GitlabLoader(self.user, self.repo, subdir=None, sha=None, prov=None) - def test_fetchFiles(self): + def xtest_fetchFiles(self): files = self.loader.fetchFiles() # Should return a list of file items @@ -96,7 +97,7 @@ def test_fetchFiles(self): for fItem in files: self.assertIn('download_url', fItem, "File items should have a download_url") - def test_getRawRepoUri(self): + def xtest_getRawRepoUri(self): repoUri = self.loader.getRawRepoUri() # Should be a string @@ -106,7 +107,7 @@ def test_getRawRepoUri(self): self.assertIn(self.user, repoUri, "Should contain user") self.assertIn(self.repo, repoUri, "Should contain repo") - def test_getTextFor(self): + def xtest_getTextFor(self): files = self.loader.fetchFiles() # the contents of each file @@ -123,7 +124,7 @@ def test_getTextFor(self): with self.assertRaises(Exception, msg="Should raise exception for invalid file items"): text = self.loader.getTextFor({}) - def test_getTextForName(self): + def xtest_getTextForName(self): testableNames = [ ('test-rq', qType['SPARQL']), ('test-sparql', qType['SPARQL']), @@ -133,7 +134,7 @@ def test_getTextForName(self): text, actualType = self.loader.getTextForName(name) self.assertEqual(expectedType, actualType, "Query type should match %s != %s" % (expectedType, actualType)) - def test_getEndpointText(self): + def xtest_getEndpointText(self): endpoint = self.loader.getEndpointText() # Should be some text From 90cf306563ecb9f160cb55189d9027ef75defe29 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 24 Apr 2023 20:57:03 +0200 Subject: [PATCH 32/47] Change README badge to show status of testing GH action --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55e1a67..77fff98 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI version](https://badge.fury.io/py/grlc.svg)](https://badge.fury.io/py/grlc) [![DOI](https://zenodo.org/badge/46131212.svg)](https://zenodo.org/badge/latestdoi/46131212) -[![Build Status](https://travis-ci.org/CLARIAH/grlc.svg?branch=master)](https://travis-ci.org/CLARIAH/grlc) +![Build Status](https://github.com/CLARIAH/grlc/actions/workflows/testing.yml/badge.svg?branch=master) grlc, the git repository linked data API constructor, automatically builds Web APIs using shared SPARQL queries. http://grlc.io/ From db4c9fb766282c34d06f26dd33cf424c79713239 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Mon, 1 May 2023 09:35:04 +0200 Subject: [PATCH 33/47] Update docker image to Python 3.8 --- Dockerfile | 8 ++++---- src/utils.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e768cf9..ae7f438 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6.8 +FROM python:3.8.16 MAINTAINER albert.merono@vu.nl # Default values for env variables @@ -22,14 +22,14 @@ ENV GRLC_INSTALL_DIR="${GRLC_HOME}/grlc" \ GRLC_RUNTIME_DIR="${GRLC_CACHE_DIR}/runtime" RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y nginx git-core logrotate python-pip locales gettext-base sudo build-essential apt-utils \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y nginx git-core logrotate python3-pip locales gettext-base sudo build-essential apt-utils \ && update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \ && locale-gen en_US.UTF-8 \ && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales \ && rm -rf /var/lib/apt/lists/* -RUN curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash - -RUN apt-get update && apt-get install -y nodejs +# RUN curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash - +RUN apt-get update && apt-get install -y nodejs npm COPY ./ ${GRLC_INSTALL_DIR} diff --git a/src/utils.py b/src/utils.py index 95aeeb2..753fbb8 100644 --- a/src/utils.py +++ b/src/utils.py @@ -82,7 +82,7 @@ def build_swagger_spec(user, repo, subdir, spec_url, sha, serverName, git_type, # TODO: Add bootstrap style to top level HTML # Without a better place to display warnings, we can make them part of the description. - if 'description' not in swag['info']: + if 'description' not in swag['info'] or swag['info']['description'] is None: swag['info']['description'] = '' for warn in warnings: swag['info']['description'] += swagger.get_warning_div(warn) From b9f7334d606039992618ee87f31eb59cb55cbcc4 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sun, 21 May 2023 23:32:53 +0200 Subject: [PATCH 34/47] Using POST method for SPARQL updates --- src/server.py | 6 ++++-- src/swagger.py | 3 +++ src/utils.py | 10 +++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/server.py b/src/server.py index ece709e..3e9936b 100644 --- a/src/server.py +++ b/src/server.py @@ -50,11 +50,13 @@ def query(user, repo, query_name, subdir=None, spec_url=None, sha=None, content= acceptHeader = request.headers['Accept'] requestUrl = request.url formData = request.form + method = request.method query_response, status, headers = utils.dispatch_query(user, repo, query_name, subdir, spec_url, sha=sha, content=content, requestArgs=requestArgs, - acceptHeader=acceptHeader, - requestUrl=requestUrl, formData=formData, git_type=git_type, branch=branch) + acceptHeader=acceptHeader, requestUrl=requestUrl, + formData=formData, method=method, git_type=git_type, + branch=branch) if isinstance(query_response, list): query_response = jsonify(query_response) diff --git a/src/swagger.py b/src/swagger.py index 32e7db7..23e98fb 100644 --- a/src/swagger.py +++ b/src/swagger.py @@ -316,6 +316,9 @@ def process_sparql_query_text(query_text, loader, call_name, extraMetadata): elif query_metadata['type'] == 'ConstructQuery': if not method: method = 'get' + elif query_metadata['type'] == 'InsertData': # UPDATE queries should map here + if not method: + method = 'get' elif query_metadata['type'] == 'UNKNOWN': glogger.warning("grlc could not parse this query; assuming a plain, non-parametric SELECT in the API spec") if not method: diff --git a/src/utils.py b/src/utils.py index 95aeeb2..17384b1 100644 --- a/src/utils.py +++ b/src/utils.py @@ -95,7 +95,7 @@ def build_swagger_spec(user, repo, subdir, spec_url, sha, serverName, git_type, def dispatch_query(user, repo, query_name, subdir=None, spec_url=None, sha=None, content=None, requestArgs={}, acceptHeader='application/json', - requestUrl='http://', formData={}, git_type=None, branch='main'): + requestUrl='http://', formData={}, method="POST", git_type=None, branch='main'): """Executes the specified SPARQL or TPF query.""" loader = getLoader(user, repo, subdir, spec_url, sha=sha, prov=None, git_type=git_type, branch=branch) query, q_type = loader.getTextForName(query_name) @@ -103,7 +103,7 @@ def dispatch_query(user, repo, query_name, subdir=None, spec_url=None, sha=None, # Call name implemented with SPARQL query if q_type == qType['SPARQL'] or q_type == qType['JSON']: resp, status, headers = dispatchSPARQLQuery(query, loader, requestArgs, acceptHeader, content, formData, - requestUrl) + requestUrl, method) if acceptHeader == 'application/json': # TODO: transform JSON result if suitable @@ -119,7 +119,7 @@ def dispatch_query(user, repo, query_name, subdir=None, spec_url=None, sha=None, def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader, content, - formData, requestUrl): + formData, requestUrl, method="GET"): """Executes the specified SPARQL query.""" endpoint, auth = gquery.guess_endpoint_uri(raw_sparql_query, loader) if endpoint == '': @@ -179,6 +179,10 @@ def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader, con # Check for INSERT/POST elif query_metadata['type'] == 'InsertData': glogger.debug("Processing INSERT query") + if method != 'POST': + glogger.debug('INSERT queries must use POST method') + return { 'error': 'INSERT queries must use POST method' }, 400, headers + # Rewrite INSERT rewritten_query = rewritten_query.replace("?_g_iri", "{}".format(formData.get('g'))) rewritten_query = rewritten_query.replace("

", formData.get('data')) From 2caec69e31fb84984077680e7e61739d2a8602b0 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sun, 28 May 2023 23:39:33 +0200 Subject: [PATCH 35/47] Add query type 'Modify' --- src/swagger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swagger.py b/src/swagger.py index 23e98fb..630fab5 100644 --- a/src/swagger.py +++ b/src/swagger.py @@ -316,7 +316,7 @@ def process_sparql_query_text(query_text, loader, call_name, extraMetadata): elif query_metadata['type'] == 'ConstructQuery': if not method: method = 'get' - elif query_metadata['type'] == 'InsertData': # UPDATE queries should map here + elif query_metadata['type'] == 'InsertData' or query_metadata['type'] == 'Modify': # UPDATE queries should map here if not method: method = 'get' elif query_metadata['type'] == 'UNKNOWN': @@ -325,6 +325,7 @@ def process_sparql_query_text(query_text, loader, call_name, extraMetadata): method = 'get' else: # TODO: process all other kinds of queries + glogger.debug('Could not parse query {}: Query of type {} is currently unsupported'.format(call_name, query_metadata['type'])) raise Exception('Could not parse query {}: Query of type {} is currently unsupported'.format(call_name, query_metadata['type'])) # Finally: main structure of the callname spec From c40ae0036afdd99c8362a68fbb01020c57982877 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Sun, 25 Jun 2023 22:56:52 +0200 Subject: [PATCH 36/47] Add more recent paper --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 77fff98..d18a650 100644 --- a/README.md +++ b/README.md @@ -390,6 +390,7 @@ Quotes from grlc users: ## Academic publications +- Albert Meroño-Peñuela, Carlos Martinez-Ortiz. “grlc: the git repository linked data API constructor.“ Journal of Open Source Software, 6(67), 2731 (2021), - Albert Meroño-Peñuela, Rinke Hoekstra. “grlc Makes GitHub Taste Like Linked Data APIs”. The Semantic Web – ESWC 2016 Satellite Events, Heraklion, Crete, Greece, May 29 – June 2, 2016, Revised Selected Papers. LNCS 9989, pp. 342-353 (2016). ([PDF](https://link.springer.com/content/pdf/10.1007%2F978-3-319-47602-5_48.pdf)) - Albert Meroño-Peñuela, Rinke Hoekstra. “SPARQL2Git: Transparent SPARQL and Linked Data API Curation via Git”. In: Proceedings of the 14th Extended Semantic Web Conference (ESWC 2017), Poster and Demo Track. Portoroz, Slovenia, May 28th – June 1st, 2017 (2017). ([PDF](https://www.albertmeronyo.org/wp-content/uploads/2017/04/sparql2git-transparent-sparql-4.pdf)) - Albert Meroño-Peñuela, Rinke Hoekstra. “Automatic Query-centric API for Routine Access to Linked Data”. In: The Semantic Web – ISWC 2017, 16th International Semantic Web Conference. Lecture Notes in Computer Science, vol 10587, pp. 334-339 (2017). ([PDF](https://www.albertmeronyo.org/wp-content/uploads/2017/07/ISWC2017_paper_430.pdf)) From e554c2722f800014c30a6bf99f33d8c6a9be1642 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 28 Jun 2023 22:36:52 +0200 Subject: [PATCH 37/47] Default method to post --- src/swagger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swagger.py b/src/swagger.py index 630fab5..b790814 100644 --- a/src/swagger.py +++ b/src/swagger.py @@ -318,7 +318,7 @@ def process_sparql_query_text(query_text, loader, call_name, extraMetadata): method = 'get' elif query_metadata['type'] == 'InsertData' or query_metadata['type'] == 'Modify': # UPDATE queries should map here if not method: - method = 'get' + method = 'post' elif query_metadata['type'] == 'UNKNOWN': glogger.warning("grlc could not parse this query; assuming a plain, non-parametric SELECT in the API spec") if not method: From f2f4996cea784ddbe8fdbad943c0522b9746946b Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 5 Jul 2023 12:27:05 +0200 Subject: [PATCH 38/47] Add grlc book --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d18a650..dc24974 100644 --- a/README.md +++ b/README.md @@ -391,6 +391,7 @@ Quotes from grlc users: ## Academic publications - Albert Meroño-Peñuela, Carlos Martinez-Ortiz. “grlc: the git repository linked data API constructor.“ Journal of Open Source Software, 6(67), 2731 (2021), +- Albert Meroño-Peñuela, Pasquale Lisena, Carlos Martínez-Ortiz. “Web Data APIs for Knowledge Graphs: Easing Access to Semantic Data for Application Developers”. Synthesis Lectures on Data, Semantics, and Knowledge, 12(1), pp.1-118 (2021) (Morgan & Claypool) - Albert Meroño-Peñuela, Rinke Hoekstra. “grlc Makes GitHub Taste Like Linked Data APIs”. The Semantic Web – ESWC 2016 Satellite Events, Heraklion, Crete, Greece, May 29 – June 2, 2016, Revised Selected Papers. LNCS 9989, pp. 342-353 (2016). ([PDF](https://link.springer.com/content/pdf/10.1007%2F978-3-319-47602-5_48.pdf)) - Albert Meroño-Peñuela, Rinke Hoekstra. “SPARQL2Git: Transparent SPARQL and Linked Data API Curation via Git”. In: Proceedings of the 14th Extended Semantic Web Conference (ESWC 2017), Poster and Demo Track. Portoroz, Slovenia, May 28th – June 1st, 2017 (2017). ([PDF](https://www.albertmeronyo.org/wp-content/uploads/2017/04/sparql2git-transparent-sparql-4.pdf)) - Albert Meroño-Peñuela, Rinke Hoekstra. “Automatic Query-centric API for Routine Access to Linked Data”. In: The Semantic Web – ISWC 2017, 16th International Semantic Web Conference. Lecture Notes in Computer Science, vol 10587, pp. 334-339 (2017). ([PDF](https://www.albertmeronyo.org/wp-content/uploads/2017/07/ISWC2017_paper_430.pdf)) From 431460eda1ac4fca70037ea467b2a597d0fa8db9 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 5 Jul 2023 12:28:00 +0200 Subject: [PATCH 39/47] Add doi --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dc24974..d2a6d2f 100644 --- a/README.md +++ b/README.md @@ -391,7 +391,7 @@ Quotes from grlc users: ## Academic publications - Albert Meroño-Peñuela, Carlos Martinez-Ortiz. “grlc: the git repository linked data API constructor.“ Journal of Open Source Software, 6(67), 2731 (2021), -- Albert Meroño-Peñuela, Pasquale Lisena, Carlos Martínez-Ortiz. “Web Data APIs for Knowledge Graphs: Easing Access to Semantic Data for Application Developers”. Synthesis Lectures on Data, Semantics, and Knowledge, 12(1), pp.1-118 (2021) (Morgan & Claypool) +- Albert Meroño-Peñuela, Pasquale Lisena, Carlos Martínez-Ortiz. “Web Data APIs for Knowledge Graphs: Easing Access to Semantic Data for Application Developers”. Synthesis Lectures on Data, Semantics, and Knowledge, 12(1), pp.1-118 (2021) (Morgan & Claypool) - Albert Meroño-Peñuela, Rinke Hoekstra. “grlc Makes GitHub Taste Like Linked Data APIs”. The Semantic Web – ESWC 2016 Satellite Events, Heraklion, Crete, Greece, May 29 – June 2, 2016, Revised Selected Papers. LNCS 9989, pp. 342-353 (2016). ([PDF](https://link.springer.com/content/pdf/10.1007%2F978-3-319-47602-5_48.pdf)) - Albert Meroño-Peñuela, Rinke Hoekstra. “SPARQL2Git: Transparent SPARQL and Linked Data API Curation via Git”. In: Proceedings of the 14th Extended Semantic Web Conference (ESWC 2017), Poster and Demo Track. Portoroz, Slovenia, May 28th – June 1st, 2017 (2017). ([PDF](https://www.albertmeronyo.org/wp-content/uploads/2017/04/sparql2git-transparent-sparql-4.pdf)) - Albert Meroño-Peñuela, Rinke Hoekstra. “Automatic Query-centric API for Routine Access to Linked Data”. In: The Semantic Web – ISWC 2017, 16th International Semantic Web Conference. Lecture Notes in Computer Science, vol 10587, pp. 334-339 (2017). ([PDF](https://www.albertmeronyo.org/wp-content/uploads/2017/07/ISWC2017_paper_430.pdf)) From f8ff5f94b5a991df35a23bfc40f769555c1c3dd3 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Tue, 18 Jul 2023 23:46:30 +0200 Subject: [PATCH 40/47] Fix dependabot config --- .dependabot/config.yml | 10 ---------- .github/dependabot.yml | 12 ++++++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) delete mode 100644 .dependabot/config.yml create mode 100644 .github/dependabot.yml diff --git a/.dependabot/config.yml b/.dependabot/config.yml deleted file mode 100644 index cc1a4ce..0000000 --- a/.dependabot/config.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 1 - -update_configs: - - package_manager: "python" - directory: "/python/helpers" - update_schedule: "monthly" - target_branch: "dev" - default_reviewers: - - "albertmeronyo" - - "c-martinez" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..bc720b6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 + +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + target-branch: "dev" + # Add reviewers + reviewers: + - "albertmeronyo" + - "c-martinez" From 6746665b3aa1ded44bfe4ba232853aafe825b72d Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 19 Jul 2023 22:52:08 +0200 Subject: [PATCH 41/47] Bumping requests version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5bc492f..e640e70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ rdflib==4.2.2; python_version <= '3.6' rdflib==5.0.0; python_version > '3.6' rdflib-jsonld==0.4.0; python_version <= '3.6' rdflib-jsonld==0.6.2; python_version > '3.6' -requests>=2.20.0 +requests>=2.31.0 six==1.12.0 simplejson==3.16.0 setuptools>=38.6.0 From 78a12dcdfa5f5291f2719ace0ef569d193ddd876 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 19 Jul 2023 23:20:10 +0200 Subject: [PATCH 42/47] Keep old version for py3.6 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e640e70..b1bc012 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,8 @@ rdflib==4.2.2; python_version <= '3.6' rdflib==5.0.0; python_version > '3.6' rdflib-jsonld==0.4.0; python_version <= '3.6' rdflib-jsonld==0.6.2; python_version > '3.6' -requests>=2.31.0 +requests==2.20.0; python_version <= '3.6' +requests==2.31.0; python_version > '3.6' six==1.12.0 simplejson==3.16.0 setuptools>=38.6.0 From 268b98e74a37110c740df3fdd32505ad911e6df0 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 19 Jul 2023 23:34:55 +0200 Subject: [PATCH 43/47] Remove testing for py3.6 --- .github/workflows/testing.yml | 5 +---- requirements.txt | 13 ++++--------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 0df139c..26cd1fe 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -6,10 +6,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [ 3.7, 3.8, 3.9] - include: - - os: ubuntu-20.04 - python-version: 3.6 + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Set up Python ${{ matrix.python-version }} diff --git a/requirements.txt b/requirements.txt index 5bc492f..70e657a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,19 +7,15 @@ gevent==22.10.2; python_version > '3.8' greenlet==0.4.15; python_version <= '3.8' greenlet==2.0.0; python_version > '3.8' html5lib==1.0.1 -isodate==0.5.4; python_version <= '3.6' -isodate==0.6.1; python_version > '3.6' +isodate==0.6.1 itsdangerous==1.1.0 -keepalive==0.5; python_version <= '3.6' MarkupSafe==0.23 pyaml==18.11.0 pyparsing==2.0.7 python-gitlab==2.10.1 PyYAML==5.4 -rdflib==4.2.2; python_version <= '3.6' -rdflib==5.0.0; python_version > '3.6' -rdflib-jsonld==0.4.0; python_version <= '3.6' -rdflib-jsonld==0.6.2; python_version > '3.6' +rdflib==5.0.0 +rdflib-jsonld==0.6.2 requests>=2.20.0 six==1.12.0 simplejson==3.16.0 @@ -27,7 +23,6 @@ setuptools>=38.6.0 SPARQLTransformer==2.1.1 SPARQLWrapper==1.8.2 werkzeug==0.16.0 -PyGithub==1.43.5; python_version <= '3.6' -PyGithub==1.57; python_version > '3.6' +PyGithub==1.57 gunicorn==19.6.0; sys_platform!="win32" waitress>=1.4.2; sys_platform=="win32" From 89641869bb1e1c4bc5453f95c1ee8f7dd533c7c7 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 26 Jul 2023 14:57:26 +0200 Subject: [PATCH 44/47] Preparing for 1.3.8 release (#414) * Preparing for 1.3.8 release --- .zenodo.json | 35 ----------------------------------- CITATION.cff | 9 +++++---- CONTRIBUTORS.md | 4 ++++ 3 files changed, 9 insertions(+), 39 deletions(-) delete mode 100644 .zenodo.json diff --git a/.zenodo.json b/.zenodo.json deleted file mode 100644 index 8b38df3..0000000 --- a/.zenodo.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "creators": [ - { - "affiliation": "Vrije Universiteit Amsterdam", - "name": "Meroño-Peñuela, Albert" - }, - { - "name": "Hoekstra, Rinke" - }, - { - "affiliation": "Netherlands eScience Center", - "name": "Martinez, Carlos", - "orcid": "0000-0001-5565-7577" - }, - { - "affiliation": "International Institute of Social History", - "name": "Zijdeman, Richard" - } - ], - "description": "grlc, the git repository linked data API constructor, automatically builds Web APIs using SPARQL queries stored in git repositories.", - "doi": "10.5281/zenodo.1064392", - "keywords": [ - "swagger-ui", - "sparql", - "linked-data", - "semantic-web", - "linked-data-api" - ], - "license": { - "id": "MIT" - }, - "publication_date": "2017-11-22", - "title": "grlc", - "version": "1.3.6" -} diff --git a/CITATION.cff b/CITATION.cff index cb7c2ec..a688988 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -3,9 +3,10 @@ --- authors: - - affiliation: "Vrije Universiteit Amsterdam" + affiliation: "King's College London" family-names: Meroño-Peñuela given-names: Albert + orcid: "https://orcid.org/0000-0003-4646-5842" - family-names: Hoekstra given-names: Rinke @@ -19,8 +20,8 @@ authors: family-names: Zijdeman given-names: Richard cff-version: "1.0.3" -date-released: 2017-11-22 -doi: 10.5281/zenodo.1064392 +date-released: 2023-07-30 +doi: 10.5281/zenodo.1064391 license: MIT message: "If you use this software, please cite it as below." repository-code: "https://github.com/CLARIAH/grlc" @@ -32,4 +33,4 @@ keywords: - "linked-data" - "semantic-web" - "linked-data-api" -version: "1.3.6" +version: "1.3.8" diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 6622b7d..506f75c 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -12,3 +12,7 @@ This is a list of all people who have contributed to grlc. Big thanks to everyon [jspaaks](https://github.com/jspaaks) [ecow](https://github.com/ecow) [rapw3k](https://github.com/rapw3k) +[jaw111](https://github.com/jaw111) +[tkuhn](https://github.com/tkuhn) +[GenEars](https://github.com/GenEars) +[nichtich](https://github.com/nichtich) \ No newline at end of file From 67c08d99f0d68ac8bce0a6b4b27e9faa26ed40e0 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 26 Jul 2023 19:27:38 +0200 Subject: [PATCH 45/47] Resolve version issue during release --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 678a34a..6e5f3fc 100644 --- a/setup.py +++ b/setup.py @@ -60,5 +60,5 @@ package_data = { 'grlc': grlc_data }, include_package_data=True, data_files=[('citation/grlc', ['CITATION.cff'])], - python_requires='>=3.7, <=3.8', + python_requires='>=3.7, <=3.9', ) From bd34749f32dde7991403ea9038fbe957e94e04cf Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 26 Jul 2023 19:37:31 +0200 Subject: [PATCH 46/47] Resolve version issue during release --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 6e5f3fc..1beb748 100644 --- a/setup.py +++ b/setup.py @@ -60,5 +60,4 @@ package_data = { 'grlc': grlc_data }, include_package_data=True, data_files=[('citation/grlc', ['CITATION.cff'])], - python_requires='>=3.7, <=3.9', ) From 9f1828f24c425a053b631e4dc574eee451f4c8f8 Mon Sep 17 00:00:00 2001 From: Carlos Martinez Date: Wed, 26 Jul 2023 22:12:11 +0200 Subject: [PATCH 47/47] Fix itisdangerous version (#417) * Fix itisdangerous version --- CITATION.cff | 5 ++++- requirements.txt | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 46b118a..0f383a0 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -3,10 +3,13 @@ --- authors: - - affiliation: "Vrije Universiteit Amsterdam" + affiliation: "King's College London" family-names: Meroño-Peñuela given-names: Albert orcid: "https://orcid.org/0000-0003-4646-5842" + - + family-names: Hoekstra + given-names: Rinke - affiliation: "Netherlands eScience Center" family-names: Martinez diff --git a/requirements.txt b/requirements.txt index 72e8a56..f30b45f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ greenlet==0.4.15; python_version <= '3.8' greenlet==2.0.0; python_version > '3.8' html5lib==1.0.1 isodate==0.6.1 +itsdangerous==2.0.1 MarkupSafe==0.23 pyaml==18.11.0 pyparsing==2.0.7