diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c85d6a22..f6d5b622 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,22 +24,31 @@ jobs: repository: "oceanprotocol/barge" path: 'barge' ref: v4 + - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_PASSWORD && env.DOCKERHUB_USERNAME }} + run: | + echo "Login to Docker Hub";echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Run Barge working-directory: ${{ github.workspace }}/barge env: CONTRACTS_VERSION: v1.0.0-alpha.32 run: | bash -x start_ocean.sh --no-dashboard 2>&1 --with-rbac --with-provider2 --with-c2d > start_ocean.log & - for i in $(seq 1 150); do - sleep 5 - [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" -a -f "$HOME/.ocean/ocean/c2d/ready" ] && break - done - ls -la "$HOME/.ocean/ocean-contracts/artifacts/" - name: Install dependencies working-directory: ${{ github.workspace }} run: | python -m pip install --upgrade pip pip install -r requirements_dev.txt + - name: Wait for contracts deployment and C2D cluster to be ready + working-directory: ${{ github.workspace }}/barge + run: | + for i in $(seq 1 250); do + sleep 10 + [ -f "$HOME/.ocean/ocean-contracts/artifacts/ready" -a -f "$HOME/.ocean/ocean-c2d/ready" ] && break + done - name: Test with pytest run: | coverage run --source ocean_provider -m pytest diff --git a/conftest.py b/conftest.py index abd822d4..fc46766d 100644 --- a/conftest.py +++ b/conftest.py @@ -10,6 +10,7 @@ from ocean_provider.run import app from ocean_provider.utils.basics import get_config, get_web3, send_ether +from ocean_provider.utils.provider_fees import get_c2d_environments app = app @@ -92,3 +93,10 @@ def setup_all(provider_address, consumer_address, ganache_wallet): @pytest.fixture def web3(): return get_web3() + + +@pytest.fixture +def free_c2d_env(): + environments = get_c2d_environments() + + return next(env for env in environments if float(env["priceMin"]) == float(0)) diff --git a/ocean_provider/routes/compute.py b/ocean_provider/routes/compute.py index 5e6dd6f6..b581c88f 100644 --- a/ocean_provider/routes/compute.py +++ b/ocean_provider/routes/compute.py @@ -438,9 +438,6 @@ def computeStart(): logger.debug("Sending: %s", workflow) compute_env = data.get("environment") - seconds = ( - datetime.fromtimestamp(validator.valid_until) - datetime.utcnow() - ).seconds nonce, provider_signature = sign_for_compute(provider_wallet, consumer_address) web3 = get_web3() @@ -451,7 +448,7 @@ def computeStart(): "owner": consumer_address, "providerAddress": provider_wallet.address, "environment": compute_env, - "maxDuration": seconds, + "validUntil": validator.valid_until, "nonce": nonce, "chainId": web3.chain_id, } diff --git a/ocean_provider/utils/address.py b/ocean_provider/utils/address.py index 54e0687b..2c438774 100644 --- a/ocean_provider/utils/address.py +++ b/ocean_provider/utils/address.py @@ -3,9 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 # import json +import os from pathlib import Path from typing import Any, Dict, Union from eth_typing.evm import HexAddress +from ocean_provider.utils.basics import get_config + +BLACK_HOLE_ADDRESS = "0x0000000000000000000000000000000000000000" def get_address_json(address_path: Union[str, Path]) -> Dict[str, Any]: @@ -27,3 +31,12 @@ def get_contract_address( for chain_addresses in address_json.values() if chain_addresses["chainId"] == chain_id ) + + +def get_provider_fee_token(chain_id): + fee_token = os.environ.get("PROVIDER_FEE_TOKEN", get_ocean_address(chain_id)) + return fee_token if fee_token else BLACK_HOLE_ADDRESS + + +def get_ocean_address(chain_id): + return get_contract_address(get_config().address_file, "Ocean", chain_id) diff --git a/ocean_provider/utils/compute_environments.py b/ocean_provider/utils/compute_environments.py index 6d6a52b5..d2845894 100644 --- a/ocean_provider/utils/compute_environments.py +++ b/ocean_provider/utils/compute_environments.py @@ -3,6 +3,7 @@ from urllib.parse import urljoin from ocean_provider.requests_session import get_requests_session +from ocean_provider.utils.address import get_provider_fee_token from ocean_provider.utils.basics import get_config, get_web3 @@ -27,9 +28,7 @@ def get_c2d_environments() -> List: # loop envs and add provider token from config envs = response.json() for env in envs: - env["feeToken"] = os.getenv( - "PROVIDER_FEE_TOKEN", "0x0000000000000000000000000000000000000000" - ) + env["feeToken"] = get_provider_fee_token(web3.chain_id) return envs diff --git a/ocean_provider/utils/provider_fees.py b/ocean_provider/utils/provider_fees.py index 64ab0185..8c069374 100644 --- a/ocean_provider/utils/provider_fees.py +++ b/ocean_provider/utils/provider_fees.py @@ -9,6 +9,7 @@ from eth_keys.backends import NativeECCBackend from ocean_provider.requests_session import get_requests_session from ocean_provider.utils.asset import get_asset_from_metadatastore +from ocean_provider.utils.address import get_provider_fee_token from ocean_provider.utils.basics import ( get_provider_wallet, get_metadata_url, @@ -39,9 +40,7 @@ def get_provider_fees( web3 = get_web3() provider_wallet = get_provider_wallet() provider_fee_address = provider_wallet.address - provider_fee_token = os.environ.get( - "PROVIDER_FEE_TOKEN", "0x0000000000000000000000000000000000000000" - ) + provider_fee_token = get_provider_fee_token(web3.chain_id) if compute_env and not force_zero: provider_fee_amount = get_provider_fee_amount( @@ -174,7 +173,7 @@ def get_provider_fee_amount(valid_until, compute_env, web3, provider_fee_token): if provider_fee_token == "0x0000000000000000000000000000000000000000": return 0 - provider_fee_amount = float(seconds * env["priceMin"] / 60) + provider_fee_amount = float(seconds * float(env["priceMin"]) / 60) dt = get_datatoken_contract(web3, provider_fee_token) decimals = dt.caller.decimals() diff --git a/ocean_provider/validation/algo.py b/ocean_provider/validation/algo.py index 680ded4c..269e9992 100644 --- a/ocean_provider/validation/algo.py +++ b/ocean_provider/validation/algo.py @@ -12,6 +12,7 @@ get_asset_from_metadatastore, check_asset_consumable, ) +from ocean_provider.utils.address import get_provider_fee_token from ocean_provider.utils.basics import get_config, get_metadata_url from ocean_provider.utils.datatoken import ( record_consume_request, @@ -112,11 +113,10 @@ def validate_input(self, index=0): valid_until_list.append(self.algo_valid_until) provider_fee_amounts.append(self.algo_fee_amount) - self.valid_until = min(valid_until_list) + self.valid_until = max(valid_until_list) + + provider_fee_token = get_provider_fee_token(self.web3.chain_id) - provider_fee_token = os.environ.get( - "PROVIDER_FEE_TOKEN", "0x0000000000000000000000000000000000000000" - ) required_provider_fee = get_provider_fee_amount( self.valid_until, self.data.get("environment"), diff --git a/ocean_provider/validation/test/test_algo_validation.py b/ocean_provider/validation/test/test_algo_validation.py index e717fedb..80f15340 100644 --- a/ocean_provider/validation/test/test_algo_validation.py +++ b/ocean_provider/validation/test/test_algo_validation.py @@ -7,6 +7,7 @@ import pytest from ocean_provider.utils.asset import Asset +from ocean_provider.utils.basics import get_web3 from ocean_provider.utils.services import Service, ServiceType from ocean_provider.validation.algo import WorkflowValidator from tests.ddo.ddo_sample1_compute import alg_ddo_dict, ddo_dict @@ -31,6 +32,7 @@ ) def test_passes_algo_ddo(provider_wallet, consumer_address, web3): """Tests happy flow of validator with algo ddo.""" + web3 = get_web3() ddo = Asset(ddo_dict) alg_ddo = Asset(alg_ddo_dict) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) @@ -43,6 +45,7 @@ def test_passes_algo_ddo(provider_wallet, consumer_address, web3): "serviceId": sa_compute.id, "transferTxId": "alg_tx_id", }, + "environment": "ocean-compute", } def side_effect(*args, **kwargs): @@ -72,6 +75,7 @@ def side_effect(*args, **kwargs): ) def test_passes_raw(provider_wallet, consumer_address, web3): """Tests happy flow of validator with raw algo.""" + web3 = get_web3() ddo = Asset(ddo_dict) sa = get_first_service_by_type(ddo, ServiceType.COMPUTE) data = { @@ -85,6 +89,7 @@ def test_passes_raw(provider_wallet, consumer_address, web3): "container": {"entrypoint": "node $ALGO", "image": "node", "tag": "10"}, }, }, + "environment": "ocean-compute", } with patch( @@ -229,6 +234,7 @@ def test_fails_meta_issues(provider_wallet, consumer_address, web3): return_value=[{"url": "dummy"}], ) def test_additional_datasets(provider_wallet, consumer_address, web3): + web3 = get_web3() ddo = Asset(ddo_dict) alg_ddo = Asset(alg_ddo_dict) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) @@ -242,6 +248,7 @@ def test_additional_datasets(provider_wallet, consumer_address, web3): "transferTxId": "alg_tx_id", }, "additionalDatasets": "", + "environment": "ocean-compute", } def side_effect(*args, **kwargs): @@ -813,6 +820,7 @@ def test_success_multiple_services_types(provider_wallet, consumer_address, web3 "additionalDatasets": [ {"documentId": ddo.did, "transferTxId": "ddo.did", "serviceId": "access_1"} ], + "environment": "ocean-compute", } def side_effect(*args, **kwargs): @@ -900,6 +908,7 @@ def another_side_effect(*args, **kwargs): ) def test_fee_amount_not_paid(provider_wallet, consumer_address, web3): """Tests happy flow of validator with algo ddo.""" + web3 = get_web3() ddo = Asset(ddo_dict) alg_ddo = Asset(alg_ddo_dict) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) diff --git a/tests/helpers/compute_helpers.py b/tests/helpers/compute_helpers.py index 2cc2e5fb..9af6078f 100644 --- a/tests/helpers/compute_helpers.py +++ b/tests/helpers/compute_helpers.py @@ -26,6 +26,7 @@ def build_and_send_ddo_with_compute_service( do_send=True, short_valid_until=True, timeout=3600, + c2d_environment="ocean-compute", ): web3 = get_web3() algo_metadata = build_metadata_dict_type_algorithm() @@ -82,8 +83,6 @@ def build_and_send_ddo_with_compute_service( datatoken = service.datatoken_address mint_100_datatokens(web3, datatoken, consumer_wallet.address, publisher_wallet) - environments = get_c2d_environments() - if not do_send: return (dataset_ddo_w_compute_service, alg_ddo) @@ -97,7 +96,7 @@ def build_and_send_ddo_with_compute_service( service, consumer_wallet.address, get_future_valid_until(short=short_valid_until), - environments[0]["id"], + c2d_environment, ), consumer_wallet, ) @@ -114,7 +113,7 @@ def build_and_send_ddo_with_compute_service( alg_service, consumer_wallet.address, get_future_valid_until(short=short_valid_until), - environments[0]["id"], + c2d_environment, force_zero=True, ), consumer_wallet, diff --git a/tests/test_compute.py b/tests/test_compute.py index 6f1b47da..ea4b625e 100644 --- a/tests/test_compute.py +++ b/tests/test_compute.py @@ -38,7 +38,13 @@ def test_compute_rejected(client, monkeypatch): @pytest.mark.integration @pytest.mark.parametrize("allow_raw_algos", [True, False]) def test_compute_raw_algo( - client, publisher_wallet, consumer_wallet, consumer_address, web3, allow_raw_algos + client, + publisher_wallet, + consumer_wallet, + consumer_address, + web3, + allow_raw_algos, + free_c2d_env, ): custom_services = "vanilla_compute" if allow_raw_algos else "norawalgo" @@ -57,18 +63,17 @@ def test_compute_raw_algo( "version": "0.1", "container": {"entrypoint": "node $ALGO", "image": "node", "tag": "10"}, } - environments = get_c2d_environments() tx_id, _ = start_order( web3, datatoken, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], sa.index, get_provider_fees( dataset_ddo_w_compute_service.did, sa, consumer_wallet.address, get_future_valid_until(), - environments[0]["id"], + free_c2d_env["id"], ), consumer_wallet, ) @@ -87,7 +92,7 @@ def test_compute_raw_algo( "signature": signature, "nonce": nonce, "consumerAddress": consumer_address, - "environment": environments[0]["id"], + "environment": free_c2d_env["id"], } response = post_to_compute(client, payload) @@ -103,16 +108,16 @@ def test_compute_raw_algo( @pytest.mark.integration def test_compute_specific_algo_dids( - client, publisher_wallet, consumer_wallet, consumer_address + client, publisher_wallet, consumer_wallet, consumer_address, free_c2d_env ): - environments = get_c2d_environments() ddo, tx_id, alg_ddo, _ = build_and_send_ddo_with_compute_service( client, publisher_wallet, consumer_wallet, False, None, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], + free_c2d_env["id"], ) sa = get_first_service_by_type(ddo, ServiceType.COMPUTE) nonce, signature = get_compute_signature(client, consumer_wallet, ddo.did) @@ -133,7 +138,7 @@ def test_compute_specific_algo_dids( "signature": signature, "nonce": nonce, "consumerAddress": consumer_address, - "environment": environments[0]["id"], + "environment": free_c2d_env["id"], } response = post_to_compute(client, payload) @@ -148,15 +153,15 @@ def test_compute_specific_algo_dids( @pytest.mark.integration -def test_compute(client, publisher_wallet, consumer_wallet): - environments = get_c2d_environments() +def test_compute(client, publisher_wallet, consumer_wallet, free_c2d_env): ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( client, publisher_wallet, consumer_wallet, False, None, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], + free_c2d_env["id"], ) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) sa = get_first_service_by_type(ddo, ServiceType.COMPUTE) @@ -173,7 +178,7 @@ def test_compute(client, publisher_wallet, consumer_wallet): "signature": signature, "nonce": nonce, "consumerAddress": consumer_wallet.address, - "environment": environments[0]["id"], + "environment": free_c2d_env["id"], } # Start compute using invalid signature (withOUT nonce), should fail @@ -275,15 +280,15 @@ def test_compute(client, publisher_wallet, consumer_wallet): @pytest.mark.integration -def test_compute_diff_provider(client, publisher_wallet, consumer_wallet): - environments = get_c2d_environments() +def test_compute_diff_provider(client, publisher_wallet, consumer_wallet, free_c2d_env): ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( client, publisher_wallet, consumer_wallet, True, None, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], + free_c2d_env["id"], ) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) sa = get_first_service_by_type(ddo, ServiceType.COMPUTE) @@ -300,7 +305,7 @@ def test_compute_diff_provider(client, publisher_wallet, consumer_wallet): "signature": signature, "nonce": nonce, "consumerAddress": consumer_wallet.address, - "environment": environments[0]["id"], + "environment": free_c2d_env["id"], } response = post_to_compute(client, payload) @@ -308,15 +313,17 @@ def test_compute_diff_provider(client, publisher_wallet, consumer_wallet): @pytest.mark.integration -def test_compute_allow_all_published(client, publisher_wallet, consumer_wallet): - environments = get_c2d_environments() +def test_compute_allow_all_published( + client, publisher_wallet, consumer_wallet, free_c2d_env +): ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( client, publisher_wallet, consumer_wallet, False, "allow_all_published", - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], + free_c2d_env["id"], ) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) sa = get_first_service_by_type(ddo, ServiceType.COMPUTE) @@ -348,23 +355,23 @@ def test_compute_allow_all_published(client, publisher_wallet, consumer_wallet): ) # Start on the correct environment - payload["environment"] = environments[0]["id"] + payload["environment"] = free_c2d_env["id"] response = post_to_compute(client, payload) assert response.status == "200 OK" @pytest.mark.integration def test_compute_additional_input( - client, publisher_wallet, consumer_wallet, monkeypatch + client, publisher_wallet, consumer_wallet, monkeypatch, free_c2d_env ): - environments = get_c2d_environments() ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( client, publisher_wallet, consumer_wallet, False, None, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], + free_c2d_env["id"], ) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) sa = get_first_service_by_type(ddo, ServiceType.COMPUTE) @@ -385,14 +392,14 @@ def test_compute_additional_input( tx_id2, _ = start_order( web3, sa2.datatoken_address, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], sa2.index, get_provider_fees( ddo2.did, sa2, consumer_wallet.address, get_future_valid_until(), - environments[0]["id"], + free_c2d_env["id"], force_zero=True, ), consumer_wallet, @@ -426,7 +433,7 @@ def test_compute_additional_input( "userdata": {"test_key": "test_value"}, } ], - "environment": environments[0]["id"], + "environment": free_c2d_env["id"], } monkeypatch.setenv("RBAC_SERVER_URL", "http://172.15.0.8:3000") @@ -439,16 +446,16 @@ def test_compute_additional_input( @pytest.mark.integration def test_compute_delete_job( - client, publisher_wallet, consumer_wallet, consumer_address + client, publisher_wallet, consumer_wallet, consumer_address, free_c2d_env ): - environments = get_c2d_environments() ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( client, publisher_wallet, consumer_wallet, False, None, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], + free_c2d_env["id"], ) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) sa = get_first_service_by_type(ddo, ServiceType.COMPUTE) @@ -465,7 +472,7 @@ def test_compute_delete_job( "signature": signature, "nonce": nonce, "consumerAddress": consumer_wallet.address, - "environment": environments[0]["id"], + "environment": free_c2d_env["id"], } response = post_to_compute(client, payload) @@ -504,5 +511,6 @@ def test_compute_delete_job( def test_compute_environments(client): compute_envs_endpoint = BaseURLs.SERVICES_URL + "/computeEnvironments" response = client.get(compute_envs_endpoint) - - assert response.json[0]["id"] == "ocean-compute" + for env in response.json: + if env["priceMin"] == 0: + assert env["id"] == "ocean-compute" diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 42f12f20..f993799d 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -25,8 +25,10 @@ from ocean_provider.utils.datatoken import get_datatoken_contract from ocean_provider.utils.did import compute_did_from_data_nft_address_and_chain_id from ocean_provider.utils.encryption import do_encrypt +from ocean_provider.utils.provider_fees import get_c2d_environments from ocean_provider.utils.services import Service, ServiceType from ocean_provider.utils.util import sign_send_and_wait_for_receipt, sign_tx + from tests.helpers.ddo_dict_builders import ( build_credentials_dict, build_ddo_dict, diff --git a/tests/test_initialize.py b/tests/test_initialize.py index 17d25def..bb3e1ccb 100644 --- a/tests/test_initialize.py +++ b/tests/test_initialize.py @@ -223,7 +223,9 @@ def test_initialize_compute_works(client, publisher_wallet, consumer_wallet): @pytest.mark.integration -def test_initialize_compute_order_reused(client, publisher_wallet, consumer_wallet): +def test_initialize_compute_order_reused( + client, publisher_wallet, consumer_wallet, free_c2d_env +): """Call `initializeCompute` when there ARE reusable orders Enumerate all cases: @@ -243,8 +245,6 @@ def test_initialize_compute_order_reused(client, publisher_wallet, consumer_wall Case 4: wrong tx id for dataset order """ - environments = get_c2d_environments() - # Order asset, valid for 60 seconds ddo, tx_id, alg_ddo, alg_tx_id = build_and_send_ddo_with_compute_service( client, @@ -252,9 +252,10 @@ def test_initialize_compute_order_reused(client, publisher_wallet, consumer_wall consumer_wallet, True, None, - environments[0]["consumerAddress"], + free_c2d_env["consumerAddress"], short_valid_until=True, timeout=60, + c2d_environment=free_c2d_env["id"], ) service = get_first_service_by_type(ddo, ServiceType.COMPUTE) sa_compute = get_first_service_by_type(alg_ddo, ServiceType.ACCESS) @@ -275,7 +276,7 @@ def test_initialize_compute_order_reused(client, publisher_wallet, consumer_wall }, "consumerAddress": consumer_wallet.address, "compute": { - "env": environments[0]["id"], + "env": free_c2d_env["id"], "validUntil": get_future_valid_until(short=True), }, }