From 971d26abbfdee302681b2fa832ef6a069a19a267 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Wed, 6 Apr 2022 08:55:30 +0000 Subject: [PATCH 1/7] Adds single filter. --- aquarius/app/util.py | 10 ++++++++++ tests/test_util.py | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/aquarius/app/util.py b/aquarius/app/util.py index 62c9eb86..5d835c00 100644 --- a/aquarius/app/util.py +++ b/aquarius/app/util.py @@ -11,6 +11,7 @@ from json import JSONDecodeError import logging import os +import requests from web3.main import Web3 from aquarius.app.auth_util import sanitize_addresses @@ -23,6 +24,15 @@ def sanitize_record(data_record): if "_id" in data_record: data_record.pop("_id") + if os.getenv("RBAC_SERVER_URL"): + payload = { + "eventType": "filter_single_result", + "component": "metadatacache", + "ddo": data_record, + } + + return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() + return json.dumps(data_record, default=datetime_converter) diff --git a/tests/test_util.py b/tests/test_util.py index 7911909f..7b9a61fd 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -6,8 +6,9 @@ import json import logging import os +from requests.models import Response from datetime import datetime -from unittest.mock import patch +from unittest.mock import patch, Mock import pytest @@ -112,6 +113,18 @@ def test_sanitize_record(): assert result["other_value"] == "something else" +def test_sanitize_record_through_rbac(monkeypatch): + monkeypatch.setenv("RBAC_SERVER_URL", "test") + + with patch("requests.post") as mock: + response = Mock(spec=Response) + response.json.return_value = {"this_is": "SPARTAAA!"} + mock.return_value = response + + result = sanitize_record({}) + assert result["this_is"] == "SPARTAAA!" + + class BlockProcessingClassChild(BlockProcessingClass): def get_last_processed_block(self): raise Exception("BAD!") From 82e2b96bdd40e2393de5a4b05c6d2684f6cbce1c Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Wed, 6 Apr 2022 09:05:35 +0000 Subject: [PATCH 2/7] Add multiple result (query) filter. --- aquarius/app/assets.py | 9 +++++++-- aquarius/app/util.py | 13 +++++++++++++ tests/test_util.py | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/aquarius/app/assets.py b/aquarius/app/assets.py index cd11fac2..9c21d239 100644 --- a/aquarius/app/assets.py +++ b/aquarius/app/assets.py @@ -10,7 +10,12 @@ import requests from aquarius.app.es_instance import ElasticsearchInstance -from aquarius.app.util import sanitize_record, get_signature_vrs, get_allowed_publishers +from aquarius.app.util import ( + sanitize_record, + sanitize_query_result, + get_signature_vrs, + get_allowed_publishers, +) from aquarius.ddo_checker.shacl_checker import validate_dict from aquarius.events.processors import ( MetadataCreatedProcessor, @@ -283,7 +288,7 @@ def query_ddo(): ) try: - return es_instance.es.search(data) + return sanitize_query_result(es_instance.es.search(data)) except elasticsearch.exceptions.TransportError as e: error = e.error if isinstance(e.error, str) else str(e.error) info = e.info if isinstance(e.info, dict) else "" diff --git a/aquarius/app/util.py b/aquarius/app/util.py index 5d835c00..b6a58f00 100644 --- a/aquarius/app/util.py +++ b/aquarius/app/util.py @@ -36,6 +36,19 @@ def sanitize_record(data_record): return json.dumps(data_record, default=datetime_converter) +def sanitize_query_result(query_result): + if not os.getenv("RBAC_SERVER_URL"): + return query_result + + payload = { + "eventType": "filter_query_result", + "component": "metadatacache", + "query_result": query_result, + } + + return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() + + def get_bool_env_value(envvar_name, default_value=0): assert default_value in (0, 1), "bad default value, must be either 0 or 1" try: diff --git a/tests/test_util.py b/tests/test_util.py index 7b9a61fd..5f2e9101 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -17,6 +17,7 @@ datetime_converter, get_bool_env_value, sanitize_record, + sanitize_query_result, get_aquarius_wallet, AquariusPrivateKeyException, get_signature_vrs, @@ -125,6 +126,21 @@ def test_sanitize_record_through_rbac(monkeypatch): assert result["this_is"] == "SPARTAAA!" +def test_sanitize_query_result(monkeypatch): + result = sanitize_query_result({"this_is": "Athens, for some reason."}) + assert result["this_is"] == "Athens, for some reason." + + monkeypatch.setenv("RBAC_SERVER_URL", "test") + + with patch("requests.post") as mock: + response = Mock(spec=Response) + response.json.return_value = {"this_is": "SPARTAAA!"} + mock.return_value = response + + result = sanitize_query_result({}) + assert result["this_is"] == "SPARTAAA!" + + class BlockProcessingClassChild(BlockProcessingClass): def get_last_processed_block(self): raise Exception("BAD!") From 4b1aa4f8493fa25d1c6bccea6f81b8dbc131d164 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Wed, 6 Apr 2022 09:51:25 +0000 Subject: [PATCH 3/7] Fix RBAC publish test. --- aquarius/app/util.py | 11 +++++++++-- tests/test_util.py | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/aquarius/app/util.py b/aquarius/app/util.py index b6a58f00..e75c0176 100644 --- a/aquarius/app/util.py +++ b/aquarius/app/util.py @@ -31,7 +31,9 @@ def sanitize_record(data_record): "ddo": data_record, } - return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() + response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) + if response.status_code == 200 and response.json(): + return response.json() return json.dumps(data_record, default=datetime_converter) @@ -46,7 +48,12 @@ def sanitize_query_result(query_result): "query_result": query_result, } - return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() + response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) + return ( + response.json() + if response.status_code == 200 and response.json() + else query_result + ) def get_bool_env_value(envvar_name, default_value=0): diff --git a/tests/test_util.py b/tests/test_util.py index 5f2e9101..a4737a90 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -120,6 +120,7 @@ def test_sanitize_record_through_rbac(monkeypatch): with patch("requests.post") as mock: response = Mock(spec=Response) response.json.return_value = {"this_is": "SPARTAAA!"} + response.status_code = 200 mock.return_value = response result = sanitize_record({}) @@ -135,6 +136,7 @@ def test_sanitize_query_result(monkeypatch): with patch("requests.post") as mock: response = Mock(spec=Response) response.json.return_value = {"this_is": "SPARTAAA!"} + response.status_code = 200 mock.return_value = response result = sanitize_query_result({}) From 7c530a851bdb5025faaec7c224d72d1e34a12d0c Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Thu, 7 Apr 2022 07:07:24 +0000 Subject: [PATCH 4/7] Adds logger warning for non-200 status code. --- aquarius/app/util.py | 11 ++++++++++- tests/test_util.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/aquarius/app/util.py b/aquarius/app/util.py index e75c0176..d3fe0dae 100644 --- a/aquarius/app/util.py +++ b/aquarius/app/util.py @@ -32,8 +32,12 @@ def sanitize_record(data_record): } response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) - if response.status_code == 200 and response.json(): + if response.status_code == 200 and response.json() is not False: return response.json() + else: + logger.warning( + f"Expected response code 200 from RBAC server, got {response.status_code}." + ) return json.dumps(data_record, default=datetime_converter) @@ -49,6 +53,11 @@ def sanitize_query_result(query_result): } response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) + + if response.status_code != 200 or response.json() is False: + logger.warning( + f"Expected response code 200 from RBAC server, got {response.status_code}." + ) return ( response.json() if response.status_code == 200 and response.json() diff --git a/tests/test_util.py b/tests/test_util.py index a4737a90..d2334226 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -126,6 +126,14 @@ def test_sanitize_record_through_rbac(monkeypatch): result = sanitize_record({}) assert result["this_is"] == "SPARTAAA!" + with patch("requests.post") as mock: + response = Mock(spec=Response) + response.status_code = 404 + mock.return_value = response + + result = sanitize_record({"this_is": "something else"}) + assert result["this_is"] == "something else" + def test_sanitize_query_result(monkeypatch): result = sanitize_query_result({"this_is": "Athens, for some reason."}) @@ -142,6 +150,14 @@ def test_sanitize_query_result(monkeypatch): result = sanitize_query_result({}) assert result["this_is"] == "SPARTAAA!" + with patch("requests.post") as mock: + response = Mock(spec=Response) + response.status_code = 404 + mock.return_value = response + + result = sanitize_query_result({"this_is": "something else"}) + assert result["this_is"] == "something else" + class BlockProcessingClassChild(BlockProcessingClass): def get_last_processed_block(self): From 92df52e014abc163cb1156a3c6c8008d2958b68a Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Thu, 7 Apr 2022 07:33:22 +0000 Subject: [PATCH 5/7] Intermediary commit. --- aquarius/app/assets.py | 11 ++----- aquarius/app/util.py | 38 +++-------------------- aquarius/rbac.py | 69 ++++++++++++++++++++++++++++++++++++++++++ aquarius/run.py | 8 +++++ 4 files changed, 84 insertions(+), 42 deletions(-) create mode 100644 aquarius/rbac.py diff --git a/aquarius/app/assets.py b/aquarius/app/assets.py index 9c21d239..5de13dd8 100644 --- a/aquarius/app/assets.py +++ b/aquarius/app/assets.py @@ -7,7 +7,6 @@ import json import logging import os -import requests from aquarius.app.es_instance import ElasticsearchInstance from aquarius.app.util import ( @@ -25,6 +24,7 @@ from aquarius.log import setup_logging from aquarius.myapp import app from aquarius.events.purgatory import Purgatory +from aquarius.rbac import RBAC from artifacts import ERC721Template from web3.logs import DISCARD @@ -348,14 +348,7 @@ def validate_remote(): version = data.get("version", None) if os.getenv("RBAC_SERVER_URL"): - payload = { - "eventType": "validateDDO", - "component": "metadatacache", - "ddo": data, - "browserHeaders": {k: v for k, v in request.headers.items()}, - } - - valid = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() + valid = RBAC.validate_ddo_rbac(data) if not valid: return ( diff --git a/aquarius/app/util.py b/aquarius/app/util.py index d3fe0dae..18e3cd71 100644 --- a/aquarius/app/util.py +++ b/aquarius/app/util.py @@ -11,10 +11,10 @@ from json import JSONDecodeError import logging import os -import requests from web3.main import Web3 from aquarius.app.auth_util import sanitize_addresses +from aquarius.rbac import RBAC logger = logging.getLogger("aquarius") keys = KeyAPI(NativeECCBackend) @@ -24,45 +24,17 @@ def sanitize_record(data_record): if "_id" in data_record: data_record.pop("_id") - if os.getenv("RBAC_SERVER_URL"): - payload = { - "eventType": "filter_single_result", - "component": "metadatacache", - "ddo": data_record, - } - - response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) - if response.status_code == 200 and response.json() is not False: - return response.json() - else: - logger.warning( - f"Expected response code 200 from RBAC server, got {response.status_code}." - ) + if not os.getenv("RBAC_SERVER_URL"): + return json.dumps(data_record, default=datetime_converter) - return json.dumps(data_record, default=datetime_converter) + return RBAC.sanitize_record(data_record) def sanitize_query_result(query_result): if not os.getenv("RBAC_SERVER_URL"): return query_result - payload = { - "eventType": "filter_query_result", - "component": "metadatacache", - "query_result": query_result, - } - - response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) - - if response.status_code != 200 or response.json() is False: - logger.warning( - f"Expected response code 200 from RBAC server, got {response.status_code}." - ) - return ( - response.json() - if response.status_code == 200 and response.json() - else query_result - ) + return RBAC.sanitize_query_result(query_result) def get_bool_env_value(envvar_name, default_value=0): diff --git a/aquarius/rbac.py b/aquarius/rbac.py new file mode 100644 index 00000000..8a2ea646 --- /dev/null +++ b/aquarius/rbac.py @@ -0,0 +1,69 @@ +# +# Copyright 2021 Ocean Protocol Foundation +# SPDX-License-Identifier: Apache-2.0 +# +import logging +import requests +import os + +logger = logging.getLogger("aquarius") + + +class RBAC: + @staticmethod + def set_headers(request): + RBAC.headers = {k: v for k, v in request.headers.items()} + + @staticmethod + def sanitize_record(data_record): + payload = { + "eventType": "filter_single_result", + "component": "metadatacache", + "ddo": data_record, + "browserHeaders": getattr(RBAC, "headers", {}) + } + + response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) + if response.status_code != 200: + logger.warning( + f"Expected response code 200 from RBAC server, got {response.status_code}." + ) + + return ( + response.json() + if response.status_code == 200 and response.json() is not False + else data_record + ) + + @staticmethod + def sanitize_query_result(query_result): + payload = { + "eventType": "filter_query_result", + "component": "metadatacache", + "query_result": query_result, + "browserHeaders": getattr(RBAC, "headers", {}) + } + + response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) + + if response.status_code != 200: + logger.warning( + f"Expected response code 200 from RBAC server, got {response.status_code}." + ) + + return ( + response.json() + if response.status_code == 200 and response.json() is not False + else query_result + ) + + @staticmethod + def validate_ddo_rbac(data): + payload = { + "eventType": "validateDDO", + "component": "metadatacache", + "ddo": data, + "browserHeaders": getattr(RBAC, "headers", {}) + } + + return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() diff --git a/aquarius/run.py b/aquarius/run.py index 7e507a66..0a6a31a7 100644 --- a/aquarius/run.py +++ b/aquarius/run.py @@ -11,6 +11,7 @@ from flask import jsonify from flask_swagger import swagger from flask_swagger_ui import get_swaggerui_blueprint +import os from aquarius.app.assets import assets from aquarius.app.chains import chains @@ -20,11 +21,18 @@ from aquarius.events.events_monitor import EventsMonitor from aquarius.events.util import setup_web3 from aquarius.myapp import app +from aquarius.rbac import RBAC config = Config(filename=app.config["AQUARIUS_CONFIG_FILE"]) aquarius_url = config.aquarius_url +@app.before_request +def set_rbac_headers(): + if os.getenv("RBAC_SERVER_URL"): + RBAC.set_rbac_headers(request) + + def get_version(): conf = configparser.ConfigParser() conf.read(".bumpversion.cfg") From ba0bd8a8ec5d0ce317150508849e60646ceb484b Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Thu, 7 Apr 2022 07:45:04 +0000 Subject: [PATCH 6/7] Separate RBAC requests. --- aquarius/rbac.py | 6 +++--- aquarius/run.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aquarius/rbac.py b/aquarius/rbac.py index 8a2ea646..1d4cdf8f 100644 --- a/aquarius/rbac.py +++ b/aquarius/rbac.py @@ -20,7 +20,7 @@ def sanitize_record(data_record): "eventType": "filter_single_result", "component": "metadatacache", "ddo": data_record, - "browserHeaders": getattr(RBAC, "headers", {}) + "browserHeaders": getattr(RBAC, "headers", {}), } response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) @@ -41,7 +41,7 @@ def sanitize_query_result(query_result): "eventType": "filter_query_result", "component": "metadatacache", "query_result": query_result, - "browserHeaders": getattr(RBAC, "headers", {}) + "browserHeaders": getattr(RBAC, "headers", {}), } response = requests.post(os.getenv("RBAC_SERVER_URL"), json=payload) @@ -63,7 +63,7 @@ def validate_ddo_rbac(data): "eventType": "validateDDO", "component": "metadatacache", "ddo": data, - "browserHeaders": getattr(RBAC, "headers", {}) + "browserHeaders": getattr(RBAC, "headers", {}), } return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() diff --git a/aquarius/run.py b/aquarius/run.py index 0a6a31a7..f50baac3 100644 --- a/aquarius/run.py +++ b/aquarius/run.py @@ -8,7 +8,7 @@ import configparser from elasticsearch import Elasticsearch -from flask import jsonify +from flask import jsonify, request from flask_swagger import swagger from flask_swagger_ui import get_swaggerui_blueprint import os @@ -30,7 +30,7 @@ @app.before_request def set_rbac_headers(): if os.getenv("RBAC_SERVER_URL"): - RBAC.set_rbac_headers(request) + RBAC.set_headers(request) def get_version(): From 096a49eec1e8c642d9c0aac5be9b7f99f2592038 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Thu, 7 Apr 2022 07:48:43 +0000 Subject: [PATCH 7/7] Move permission checking from processors to RBAC. --- aquarius/events/processors.py | 13 ++----------- aquarius/rbac.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/aquarius/events/processors.py b/aquarius/events/processors.py index e56dbed2..56d99195 100644 --- a/aquarius/events/processors.py +++ b/aquarius/events/processors.py @@ -9,7 +9,6 @@ from abc import ABC from datetime import datetime -import requests from jsonsempai import magic # noqa: F401 from aquarius.ddo_checker.shacl_checker import validate_dict @@ -21,6 +20,7 @@ from aquarius.events.decryptor import decrypt_ddo from aquarius.events.util import make_did, get_dt_factory from aquarius.graphql import get_number_orders +from aquarius.rbac import RBAC from artifacts import ERC20Template, ERC721Template from web3.logs import DISCARD @@ -61,17 +61,8 @@ def check_permission(self, publisher_address): if self.__class__.__name__ == "MetadataCreatedProcessor" else "update" ) - address = publisher_address - payload = { - "eventType": event_type, - "component": "metadatacache", - "credentials": {"type": "address", "value": address}, - } - try: - return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() - except Exception: - return False + return RBAC.check_permission_rbac(event_type, publisher_address) def add_aqua_data(self, record): """Adds keys that are specific to Aquarius, on top of the DDO structure: diff --git a/aquarius/rbac.py b/aquarius/rbac.py index 1d4cdf8f..f1f7428b 100644 --- a/aquarius/rbac.py +++ b/aquarius/rbac.py @@ -67,3 +67,16 @@ def validate_ddo_rbac(data): } return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() + + @staticmethod + def check_permission_rbac(event_type, address): + payload = { + "eventType": event_type, + "component": "metadatacache", + "credentials": {"type": "address", "value": address}, + } + + try: + return requests.post(os.getenv("RBAC_SERVER_URL"), json=payload).json() + except Exception: + return False