diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..102feae --- /dev/null +++ b/.env.example @@ -0,0 +1,36 @@ +# DBS RELATED CONFIGS +APP_NAME=dbs +PYTHONUNBUFFERED=1 +DJANGO_LOG_LEVEL=DEBUG + +# FILECOIN RELATED CONFIGS +DBS_URL='http://127.0.0.1:8000' +LOCATION_URL='127.0.0.1' +PRIVATE_KEY='8837837383' +LIGHTHOUSE_API_TOKEN='xyv02u242vfvvdgdv24874nnnnndg' +USER_SQL='root' +PASSWORD_SQL='root' +DATABASE_SQL='filecoin_dbs' +HOST_SQL='127.0.0.1' +DATANAME_SQL='quote' +MUMBAI_RPC='https://polygon-mumbai.blockpi.network/v1/rpc/public' + +# ARWEAVE RELATED CONFIGS +ACCEPTED_PAYMENTS='ethereum,matic' +NODE_RPC_URIS='default,default' +BUNDLR_URI='https://devnet.bundlr.network' +PORT='8081' +# Needs to be provisioned with Matic and GoerliEth +PRIVATE_KEY='0000000000000000000000000000000000000000000000000000000000000000' +SQLITE_DB_PATH='db' +REGISTRATION_INTERVAL=60000 +DBS_URI='http://localhost:8000' +SELF_URI='http://localhost' +IPFS_GATEWAY='https://cloudflare-ipfs.com/ipfs/' +ARWEAVE_GATEWAY='https://arweave.net/' +MAX_UPLOAD_SIZE='1099511627776' +# Needs to be provisioned with Matic/WMatic and GoerliETH/WETH +TEST_PRIVATE_KEY='0000000000000000000000000000000000000000000000000000000000000000' +ENABLE_EXPENSIVE_TESTS='false' +CHAIN_ID='80001' +TOKEN_ADDRESS='0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889' \ No newline at end of file diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 24958fb..1a024f1 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -31,6 +31,9 @@ jobs: run: | python server/manage.py migrate - name: Run Tests + env: + TEST_PRIVATE_KEY: ${{ secrets.TEST_PRIVATE_KEY }} + TOKEN_ADDRESS: ${{ secrets.TOKEN_ADDRESS }} run: | cd server python manage.py test diff --git a/.gitignore b/.gitignore index 8662c1e..d0c7107 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__ venv server/db.sqlite3 +.env diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..639c49d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "services/dbs_filecoin"] + path = services/dbs_filecoin + url = https://github.com/oceanprotocol/dbs_filecoin.git +[submodule "services/dbs_arweave"] + path = services/dbs_arweave + url = https://github.com/oceanprotocol/dbs_arweave.git diff --git a/Dockerfile b/Dockerfile index 7c12eb6..a44101b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,5 @@ COPY . /usr/src/app WORKDIR /usr/src/app/server RUN python manage.py migrate EXPOSE 8000 -CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] \ No newline at end of file + +CMD ["python", "-u", "manage.py", "runserver", "0.0.0.0:8000"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..03b148d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +version: "3.9" +services: + dbs: + build: . + network_mode: host + environment: + - PYTHONUNBUFFERED=1 + - DJANGO_LOG_LEVEL=DEBUG=value + volumes: + - .:/usr/src/app + mariadb: + container_name: ${APP_NAME}_mariadb + image: mariadb + restart: always + environment: + MARIADB_ROOT_PASSWORD: root + MARIADB_USER: test + MARIADB_PASS: pass + MARIADB_DATABASE: filecoin_dbs + MARIADB_HOST: 127.0.0.1 + healthcheck: + test: mariadb --user=root --password=root --silent --execute "use filecoin_dbs;" + interval: 1s + timeout: 3s + retries: 10 + # volumes: + # - ./mounts/mariadb:/var/lib/mariadb + network_mode: host + dbs_filecoin: + container_name: filecoin + build: ./services/dbs_filecoin/ + env_file: .env # Used to provide the filecoin env variables + depends_on: + mariadb: + condition: service_healthy + network_mode: host + dbs_arweave: + container_name: arweave + build: ./services/dbs_arweave/ + env_file: .env + network_mode: host + ipfs: + image: ipfs/kubo + network_mode: host + ipfs-cluster: + image: ipfs/ipfs-cluster + network_mode: host diff --git a/examples/file1.txt b/examples/file1.txt new file mode 100644 index 0000000..e654074 Binary files /dev/null and b/examples/file1.txt differ diff --git a/examples/file2.txt b/examples/file2.txt new file mode 100644 index 0000000..f8c6162 Binary files /dev/null and b/examples/file2.txt differ diff --git a/requirements.txt b/requirements.txt index 7506e2d..8d08c51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,53 +5,72 @@ APScheduler==3.9.1.post1 asgiref==3.5.2 async-timeout==4.0.2 attrs==22.1.0 -backports.zoneinfo;python_version<="3.9" +backports.zoneinfo==0.2.1 base58==2.1.1 +bitarray==2.7.0 certifi==2022.9.24 charset-normalizer==2.1.1 +cytoolz==0.12.1 Django==4.1.2 -django-ipfs-storage==0.0.4 +django-environ==0.9.0 djangorestframework==3.14.0 drf-spectacular==0.24.2 +eth-abi==2.2.0 +eth-account==0.5.9 +eth-hash==0.3.3 +eth-keyfile==0.5.1 +eth-keys==0.3.4 +eth-rlp==0.2.1 +eth-typing==2.3.0 +eth-utils==1.10.0 factory-boy==3.2.1 Faker==15.1.1 flake8==5.0.4 frozenlist==1.3.3 +hexbytes==0.3.0 idna==3.4 importlib-resources==5.10.0 inflection==0.5.1 -ipfsapi==0.4.4 jsonschema==4.16.0 +lru-dict==1.1.8 mccabe==0.7.0 mock==4.0.3 morphys==1.0 multiaddr==0.0.9 multidict==6.0.2 netaddr==0.8.0 -pkgutil-resolve-name==1.3.10 +parsimonious==0.8.1 +pkgutil_resolve_name==1.3.10 +protobuf==3.19.5 psycopg2-binary==2.8.6 py-multibase==1.0.3 py-multicodec==0.2.1 py-multiformats-cid==0.4.4 py-multihash==2.0.1 pycodestyle==2.9.1 +pycryptodome==3.17 pyflakes==2.5.0 pyrsistent==0.18.1 python-baseconv==1.2.2 python-dateutil==2.8.2 +python-dotenv==0.21.1 pytz==2022.5 pytz-deprecation-shim==0.1.0.post0 PyYAML==6.0 requests==2.28.1 responses==0.22.0 +rlp==2.0.1 six==1.16.0 sqlparse==0.4.3 toml==0.10.2 +toolz==0.12.0 types-toml==0.10.8.1 tzdata==2022.6 tzlocal==4.2 uritemplate==4.1.1 urllib3==1.26.12 varint==1.0.2 +web3==5.31.3 +websockets==9.1 yarl==1.8.1 zipp==3.9.0 diff --git a/server/oceandbs/fixtures/storages.json b/server/oceandbs/fixtures/storages.json index a4ba05e..46e1deb 100644 --- a/server/oceandbs/fixtures/storages.json +++ b/server/oceandbs/fixtures/storages.json @@ -2,19 +2,27 @@ { "model": "oceandbs.paymentmethod", "pk": 1, - "fields": { "chainId": "1", "storage": 1 } + "fields": { + "chainId": "80001", + "storage": 1, + "rpcEndpointUrl": "https://rpc-mumbai.maticvigil.com/" + } }, { "model": "oceandbs.paymentmethod", "pk": 2, - "fields": { "chainId": "polygon_chain_id", "storage": 1 } + "fields": { + "chainId": "135", + "storage": 1, + "rpcEndpointUrl": "https://rpc-mumbai.maticvigil.com/" + } }, { "model": "oceandbs.acceptedtoken", "pk": 1, "fields": { "title": "OCEAN", - "value": "0xOCEAN_on_MAINNET", + "value": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889", "paymentMethod": 1 } }, @@ -32,7 +40,7 @@ "pk": 3, "fields": { "title": "OCEAN", - "value": "0xOCEAN_on_POLYGON", + "value": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889", "paymentMethod": 2 } }, @@ -54,6 +62,14 @@ "url": "https://filecoin.org/" } }, + { + "model": "oceandbs.payment", + "pk": 1, + "fields": { + "paymentMethod": 1, + "wallet_address": "0xCC866199C810B216710A3F3714d35920C343a8CD" + } + }, { "model": "oceandbs.quote", "pk": 1, @@ -61,9 +77,10 @@ "quoteId": "123565", "storage": 1, "duration": 2380293823, - "payment": null, - "tokenAddress": "0xOCEAN_on_MAINNET", - "approveAddress": "0x123", + "payment": 1, + "tokenAmount": 500, + "tokenAddress": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889", + "approveAddress": "0xAFcE990754C38Be5E0C341707B2A162C4e67547B", "status": null } } diff --git a/server/oceandbs/migrations/0020_alter_acceptedtoken_paymentmethod_and_more.py b/server/oceandbs/migrations/0020_alter_acceptedtoken_paymentmethod_and_more.py new file mode 100644 index 0000000..f5eb13e --- /dev/null +++ b/server/oceandbs/migrations/0020_alter_acceptedtoken_paymentmethod_and_more.py @@ -0,0 +1,40 @@ +# Generated by Django 4.1.2 on 2023-01-18 16:31 + +import datetime +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('oceandbs', '0019_remove_file_file_file_cid_file_title_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='acceptedtoken', + name='paymentMethod', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='acceptedTokens', to='oceandbs.paymentmethod'), + ), + migrations.AlterField( + model_name='payment', + name='paymentMethod', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='payments', to='oceandbs.paymentmethod'), + ), + migrations.AlterField( + model_name='paymentmethod', + name='storage', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='paymentMethods', to='oceandbs.storage'), + ), + migrations.AlterField( + model_name='quote', + name='expiration', + field=models.DateTimeField(default=datetime.datetime(2023, 1, 18, 17, 1, 41, 275788, tzinfo=datetime.timezone.utc)), + ), + migrations.AlterField( + model_name='quote', + name='nonce', + field=models.DateTimeField(default=datetime.datetime(2023, 1, 11, 16, 31, 41, 275770, tzinfo=datetime.timezone.utc)), + ), + ] diff --git a/server/oceandbs/migrations/0021_alter_paymentmethod_storage_alter_quote_expiration_and_more.py b/server/oceandbs/migrations/0021_alter_paymentmethod_storage_alter_quote_expiration_and_more.py new file mode 100644 index 0000000..1f3785c --- /dev/null +++ b/server/oceandbs/migrations/0021_alter_paymentmethod_storage_alter_quote_expiration_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 4.1.2 on 2023-02-02 17:04 + +import datetime +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('oceandbs', '0020_alter_acceptedtoken_paymentmethod_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='paymentmethod', + name='storage', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='payment', to='oceandbs.storage'), + ), + migrations.AlterField( + model_name='quote', + name='expiration', + field=models.DateTimeField(default=datetime.datetime(2023, 2, 2, 17, 34, 7, 719831, tzinfo=datetime.timezone.utc)), + ), + migrations.AlterField( + model_name='quote', + name='nonce', + field=models.DateTimeField(default=datetime.datetime(2023, 1, 26, 17, 4, 7, 719811, tzinfo=datetime.timezone.utc)), + ), + ] diff --git a/server/oceandbs/migrations/0022_paymentmethod_rpcendpointurl_alter_quote_expiration_and_more.py b/server/oceandbs/migrations/0022_paymentmethod_rpcendpointurl_alter_quote_expiration_and_more.py new file mode 100644 index 0000000..0fe0634 --- /dev/null +++ b/server/oceandbs/migrations/0022_paymentmethod_rpcendpointurl_alter_quote_expiration_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 4.1.2 on 2023-02-13 14:46 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oceandbs', '0021_alter_paymentmethod_storage_alter_quote_expiration_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='paymentmethod', + name='rpcEndpointUrl', + field=models.URLField(default='https://example.com', max_length=2048), + ), + migrations.AlterField( + model_name='quote', + name='expiration', + field=models.DateTimeField(default=datetime.datetime(2023, 2, 13, 15, 16, 25, 188917, tzinfo=datetime.timezone.utc)), + ), + migrations.AlterField( + model_name='quote', + name='nonce', + field=models.DateTimeField(default=datetime.datetime(2023, 2, 6, 14, 46, 25, 188897, tzinfo=datetime.timezone.utc)), + ), + ] diff --git a/server/oceandbs/migrations/0023_alter_quote_expiration_alter_quote_nonce.py b/server/oceandbs/migrations/0023_alter_quote_expiration_alter_quote_nonce.py new file mode 100644 index 0000000..92639aa --- /dev/null +++ b/server/oceandbs/migrations/0023_alter_quote_expiration_alter_quote_nonce.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.2 on 2023-02-13 14:46 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oceandbs', '0022_paymentmethod_rpcendpointurl_alter_quote_expiration_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='quote', + name='expiration', + field=models.DateTimeField(default=datetime.datetime(2023, 2, 13, 15, 16, 28, 315596, tzinfo=datetime.timezone.utc)), + ), + migrations.AlterField( + model_name='quote', + name='nonce', + field=models.DateTimeField(default=datetime.datetime(2023, 2, 6, 14, 46, 28, 315577, tzinfo=datetime.timezone.utc)), + ), + ] diff --git a/server/oceandbs/migrations/0024_alter_quote_expiration_alter_quote_nonce.py b/server/oceandbs/migrations/0024_alter_quote_expiration_alter_quote_nonce.py new file mode 100644 index 0000000..08c7681 --- /dev/null +++ b/server/oceandbs/migrations/0024_alter_quote_expiration_alter_quote_nonce.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.2 on 2023-02-13 14:46 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oceandbs', '0023_alter_quote_expiration_alter_quote_nonce'), + ] + + operations = [ + migrations.AlterField( + model_name='quote', + name='expiration', + field=models.DateTimeField(default=datetime.datetime(2023, 2, 13, 15, 16, 45, 385282, tzinfo=datetime.timezone.utc)), + ), + migrations.AlterField( + model_name='quote', + name='nonce', + field=models.DateTimeField(default=datetime.datetime(2023, 2, 6, 14, 46, 45, 385262, tzinfo=datetime.timezone.utc)), + ), + ] diff --git a/server/oceandbs/models.py b/server/oceandbs/models.py index 74dbe4f..7d9a45c 100644 --- a/server/oceandbs/models.py +++ b/server/oceandbs/models.py @@ -35,7 +35,8 @@ class Meta: class PaymentMethod(models.Model): chainId = models.CharField(max_length=256) - storage = models.ForeignKey(Storage, null=True, on_delete=models.CASCADE, related_name="paymentMethods") + storage = models.ForeignKey(Storage, null=True, on_delete=models.CASCADE, related_name="payment") + rpcEndpointUrl = models.URLField(max_length=2048, default="https://example.com") def __str__(self): return self.chainId + " - " + str(self.storage) diff --git a/server/oceandbs/scripts/approval.py b/server/oceandbs/scripts/approval.py new file mode 100644 index 0000000..06c63c2 --- /dev/null +++ b/server/oceandbs/scripts/approval.py @@ -0,0 +1,56 @@ +import hashlib +import sys, getopt +# from web3.auto import w3 +from web3 import Web3 +from web3.middleware import geth_poa_middleware +import json + +def main(argv): + approvalAddress = '' + tokenAddress = '' + userAddress = '' + + print("Script running") + try: + opts, args = getopt.getopt(argv,"ha:t:u:",["approvalAddress=","tokenAddress=", "userAddress="]) + except getopt.GetoptError: + print ('signature.py -a -t -u ') + sys.exit(2) + + for opt, arg in opts: + if opt == '-h': + print ('signature.py -a -t -u ') + sys.exit() + elif opt in ("-a", "--approvalAddress"): + approvalAddress = arg + elif opt in ("-t", "--tokenAddress"): + tokenAddress = arg + elif opt in ("-u", "--userAddress"): + userAddress = arg + + my_provider = Web3.HTTPProvider("https://rpc-mumbai.maticvigil.com/") + w3 = Web3(my_provider) + w3.middleware_onion.inject(geth_poa_middleware, layer=0) + + # Creating allowance for funds transfer from my user address to the quote paymentAddress + abi = '[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"withdraw","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint8"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transfer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"deposit","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"},{"name":"","type":"address"}],"name":"allowance","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"payable":true,"stateMutability":"payable","type":"fallback"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"dst","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dst","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Deposit","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Withdrawal","type":"event"}]' + abi = json.loads(abi) + contractAddress = w3.toChecksumAddress(tokenAddress) + contract = w3.eth.contract(contractAddress, abi=abi) + print("Contract total supply:", contract.functions.totalSupply().call()) + + userAddress = w3.toChecksumAddress(userAddress) + approvalAddress = w3.toChecksumAddress(approvalAddress) + print("Contract Instanciated:" + str(contract)) + nonce = w3.eth.getTransactionCount(userAddress) + tx_hash = contract.functions.approve(approvalAddress, 123456).buildTransaction({'from': userAddress, 'nonce': nonce}) + signed_tx = w3.eth.account.signTransaction(tx_hash, 'bbb5a2d50f3956e72dd8f38096270d8d951e44da623b1e31422d724e5841c93f') + tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction) + + print("Approval transaction hash:" + str(tx_hash)) + tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) + print("Transaction receipt:" + str(tx_receipt)) + print("Contract allowance:" + str(contract.functions.allowance(userAddress, approvalAddress).call())) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/server/oceandbs/scripts/signature.py b/server/oceandbs/scripts/signature.py new file mode 100644 index 0000000..8033edd --- /dev/null +++ b/server/oceandbs/scripts/signature.py @@ -0,0 +1,32 @@ +import sys, getopt +from ..utils import generate_signature + +def main(argv): + quoteId = '' + nonce = '' + pkey = '' + print("Script running") + try: + opts, args = getopt.getopt(argv,"hq:n:k:",["quoteId=","nonce=", "pkey="]) + except getopt.GetoptError: + print ('signature.py -q -n -k ') + sys.exit(2) + + for opt, arg in opts: + if opt == '-h': + print ('signature.py -q -n -k ') + sys.exit() + elif opt in ("-q", "--quoteId"): + quoteId = arg + elif opt in ("-n", "--nonce"): + nonce = arg + elif opt in ("-k", "--pkey"): + pkey = arg + + signature = generate_signature(quoteId, nonce, pkey) + + print("Signature:" + str(signature.signature)) + print("Nonce:" + str(nonce)) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/server/oceandbs/serializers.py b/server/oceandbs/serializers.py index 75ef427..0ed6153 100644 --- a/server/oceandbs/serializers.py +++ b/server/oceandbs/serializers.py @@ -13,13 +13,13 @@ class Meta: fields=['chainId', 'acceptedTokens'] class CreateStorageSerializer(serializers.ModelSerializer): - paymentMethods = CreatePaymentMethodSerializer(many=True) + payment = CreatePaymentMethodSerializer(many=True) class Meta: model = Storage - fields = ['type', 'description', 'url', 'paymentMethods'] + fields = ['type', 'description', 'url', 'payment'] def create(self, validated_data): - payment_method_data = validated_data.pop('paymentMethods') + payment_method_data = validated_data.pop('payment') storage = Storage.objects.create(**validated_data) for method in payment_method_data: @@ -49,10 +49,10 @@ class Meta: class StorageSerializer(serializers.ModelSerializer): - paymentMethods = PaymentMethodSerializer(many=True) + payment = PaymentMethodSerializer(many=True) class Meta: model = Storage - fields = ['type', 'description', 'paymentMethods'] + fields = ['type', 'description', 'payment'] class PaymentSerializer(serializers.ModelSerializer): diff --git a/server/oceandbs/tests/test_file_upload.py b/server/oceandbs/tests/test_file_upload.py index 29a46c5..a2decd3 100644 --- a/server/oceandbs/tests/test_file_upload.py +++ b/server/oceandbs/tests/test_file_upload.py @@ -2,13 +2,10 @@ from rest_framework.test import APIRequestFactory, APIClient, APITestCase from django.core.files import File from oceandbs.models import File as DBSFile, Quote -from pathlib import Path -import tempfile +from ..utils import generate_signature import mock -import json -import random -from django.utils.encoding import force_str import responses +from django.conf import settings image_mock = mock.MagicMock(spec=File) image_mock.name = 'image.png' @@ -27,6 +24,9 @@ def setUp(self): @responses.activate def test_file_upload(self): + signature = generate_signature(123565, 1768214571, getattr(settings, 'TEST_PRIVATE_KEY', '')) + responses.add_passthru('https://rpc-mumbai.maticvigil.com/') + # Mock call to IPFS for temporary file storage responses.post( url= 'http://127.0.0.1:5001/api/v0/add', @@ -41,7 +41,7 @@ def test_file_upload(self): ) response = self.client.post( - '/upload?quoteId=123565&nonce=1768214571&signature=0ee382b39a39e05500d99233cdca83cd9959be4ff557ce7f3f29c9ce99d3b5de', + '/upload?quoteId=123565&nonce=1768214571&signature=' + str(signature.signature.hex()), {'file1':image_mock, 'file2':image2_mock}, format="multipart" ) diff --git a/server/oceandbs/tests/test_quote_link_endpoint.py b/server/oceandbs/tests/test_quote_link_endpoint.py index 10082c1..015c631 100644 --- a/server/oceandbs/tests/test_quote_link_endpoint.py +++ b/server/oceandbs/tests/test_quote_link_endpoint.py @@ -1,6 +1,9 @@ +from django.conf import settings + from rest_framework import status from rest_framework.test import APIRequestFactory, APIClient, APITestCase, RequestsClient import responses +from ..utils import generate_signature # Create your tests here. # Using the standard RequestFactory API to create a form POST request @@ -13,8 +16,11 @@ def setUp(self): @responses.activate def test_get_link_endpoint(self): + signature = generate_signature(123565, 1768214571, getattr(settings, 'TEST_PRIVATE_KEY', '')) + responses.add_passthru('https://rpc-mumbai.maticvigil.com/') + responses.get( - url= 'https://filecoin.org/getLink?quoteId=123565&nonce=1768214571&signature=0ee382b39a39e05500d99233cdca83cd9959be4ff557ce7f3f29c9ce99d3b5de', + url= 'https://filecoin.org/getLink?quoteId=123565&nonce=1768214571&signature=' + str(signature.signature.hex()), json=[ { "type": "filecoin", @@ -25,7 +31,7 @@ def test_get_link_endpoint(self): ) # For arweave it would be a transaction ID so the tests should be different - response = self.client.get('/getLink?quoteId=123565&nonce=1768214571&signature=0ee382b39a39e05500d99233cdca83cd9959be4ff557ce7f3f29c9ce99d3b5de') + response = self.client.get('/getLink?quoteId=123565&nonce=1768214571&signature=' + str(signature.signature.hex())) # Assert proper HTTP status code self.assertEqual(response.status_code, status.HTTP_200_OK) diff --git a/server/oceandbs/tests/test_quote_status.py b/server/oceandbs/tests/test_quote_status.py index f6211bc..4856d0a 100644 --- a/server/oceandbs/tests/test_quote_status.py +++ b/server/oceandbs/tests/test_quote_status.py @@ -14,7 +14,7 @@ def setUp(self): @responses.activate def test_quote_status_endpoint(self): responses.get( - url= 'https://filecoin.org/quote/123565', + url= 'https://filecoin.org/getStatus?quoteId=123565', json={ 'status': 200 }, diff --git a/server/oceandbs/tests/test_storage_registration.py b/server/oceandbs/tests/test_storage_registration.py index f36c803..87536f2 100644 --- a/server/oceandbs/tests/test_storage_registration.py +++ b/server/oceandbs/tests/test_storage_registration.py @@ -17,7 +17,7 @@ def test_post_storage(self): "type": "filecoin", "description": "File storage on FileCoin", "url": "http://microservice.url", - "paymentMethods":[ + "payment":[ { "chainId": "1", "acceptedTokens": [ @@ -88,7 +88,7 @@ def test_post_storage_already_exists(self): "type": "filecoin", "description": "File storage on FileCoin", "url": "http://microservice.url", - "paymentMethods":[ + "payment":[ { "chainId": "1", "acceptedTokens": [ @@ -147,7 +147,7 @@ def test_post_storage_invalid_notype(self): body = { "description": "File storage on FileCoin", "url": "http://microservice.url", - "paymentMethods":[ + "payment":[ { "chainId": "1", "acceptedTokens": [ diff --git a/server/oceandbs/tests/tests_post_quotes.py b/server/oceandbs/tests/tests_post_quotes.py index ff66fcf..48cfb6d 100644 --- a/server/oceandbs/tests/tests_post_quotes.py +++ b/server/oceandbs/tests/tests_post_quotes.py @@ -22,10 +22,8 @@ def test_quote_creation(self): ], "duration": 4353545453, "payment": { - "payment_method": { - "chainId": 1, - }, - "wallet_address": "0xOCEAN_on_MAINNET" + "chainId": 1, + "tokenAddress": "0xOCEAN_on_MAINNET" }, "userAddress": "0x456" } @@ -70,10 +68,8 @@ def test_quote_creation_no_type(self): ], "duration": 4353545453, "payment": { - "payment_method": { - "chainId": 1, - }, - "wallet_address": "0xOCEAN_on_MAINNET" + "chainId": 1, + "tokenAddress": "0xOCEAN_on_MAINNET" }, "userAddress": "0x456" } @@ -97,10 +93,8 @@ def test_quote_creation_type_mismatch(self): ], "duration": 4353545453, "payment": { - "payment_method": { - "chainId": 1, - }, - "wallet_address": "0xOCEAN_on_MAINNET" + "chainId": 1, + "tokenAddress": "0xOCEAN_on_MAINNET" }, "userAddress": "0x456" } @@ -112,4 +106,4 @@ def test_quote_creation_type_mismatch(self): ) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, 'Chosen storage type does not exist.') \ No newline at end of file + self.assertEqual(response.data, {'error': 'Chosen storage type does not exist.'}) \ No newline at end of file diff --git a/server/oceandbs/utils.py b/server/oceandbs/utils.py index 683e79f..91eaadf 100644 --- a/server/oceandbs/utils.py +++ b/server/oceandbs/utils.py @@ -2,6 +2,16 @@ from django.utils import timezone from datetime import datetime import hashlib +from web3.auto import w3 +from eth_account.messages import encode_defunct + +def generate_signature(quoteId, nonce, pkey): + message = "0x" + hashlib.sha256((str(quoteId) + str(nonce)).encode('utf-8')).hexdigest() + message = encode_defunct(text=message) + print(message) + # Use signMessage from web3 library and etheurem decode_funct to generate the signature + signed_message = w3.eth.account.sign_message(message, private_key=pkey) + return signed_message def check_params_validity(params, quote): @@ -13,15 +23,19 @@ def check_params_validity(params, quote): return Response("Quote already expired, please create a new one.", status=400) # Check nonce - if not str(round(quote.nonce.timestamp())) < params['nonce'][0]: + if str(round(quote.nonce.timestamp())) > params['nonce'][0]: return Response("Nonce value invalid.", status=400) - # Check signature - sha256_hash = hashlib.sha256((str(quote.quoteId) + str(params['nonce'][0])).encode('utf-8')).hexdigest() - if not sha256_hash == params['signature'][0]: - return Response("Invalid signature.", status=400) + message = "0x" + hashlib.sha256((str(quote.quoteId) + str(params['nonce'][0])).encode('utf-8')).hexdigest() + message = encode_defunct(text=message) + + # Use verifyMessage from web3/ethereum API + check_signature = w3.eth.account.recover_message(message, signature=params['signature'][0]) + + if check_signature: + quote.nonce = datetime.fromtimestamp(int(params['nonce'][0]), timezone.utc) + quote.save() - quote.nonce = datetime.fromtimestamp(int(params['nonce'][0]), timezone.utc) - quote.save() + return True - return True + return False diff --git a/server/oceandbs/views.py b/server/oceandbs/views.py index d59362d..46daa16 100644 --- a/server/oceandbs/views.py +++ b/server/oceandbs/views.py @@ -1,23 +1,26 @@ -from django.http import HttpResponse -from django.views.decorators.csrf import csrf_exempt -import requests import json -from rest_framework import serializers +import requests + +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from django.views.decorators.csrf import csrf_exempt -from rest_framework.parsers import JSONParser +from rest_framework import serializers, parsers from rest_framework.views import APIView from rest_framework.response import Response from drf_spectacular.utils import extend_schema, OpenApiResponse, OpenApiExample, inline_serializer, OpenApiParameter -from django.conf import settings + +from web3 import Web3 +from web3.middleware import geth_poa_middleware from .serializers import StorageSerializer, QuoteSerializer, CreateStorageSerializer from .models import Quote, Storage, File, UPLOAD_CODE from .utils import check_params_validity - # Storage service creation class class StorageCreationView(APIView): write_serializer_class = CreateStorageSerializer + parser_classes = (parsers.JSONParser,) @csrf_exempt @extend_schema( @@ -29,8 +32,8 @@ class StorageCreationView(APIView): value={ "type": "filecoin", "description": "File storage on FileCoin", - "url": "http://microservice.url", - "paymentMethods":[ + "url": "http://localhost:3000/", + "payment":[ { "chainId": "1", "acceptedTokens": [ @@ -68,7 +71,7 @@ def post(self, request): """ POST a storage service, handling different error code """ - data = JSONParser().parse(request) + data = request.data # Make sure request data contains type and files if (not 'type' in data): @@ -81,7 +84,8 @@ def post(self, request): if storage: return Response('Chosen storage type already exists.', status=400) except: - for payment_method in data['paymentMethods']: + # print(data['payment']) + for payment_method in data['payment']: transit_table = [] for token in payment_method['acceptedTokens']: accepted_token={} @@ -114,7 +118,7 @@ class StorageListView(APIView): value=[{ "type": "filecoin", "description": "File storage on FileCoin", - "paymentMethods": [ + "payment": [ { "chainId": "1", "acceptedTokens": [ @@ -158,6 +162,7 @@ def get(self, request): # Quote creation endpoint class QuoteCreationView(APIView): + parser_classes = (parsers.JSONParser,) @csrf_exempt @extend_schema( request=[], @@ -166,19 +171,17 @@ class QuoteCreationView(APIView): OpenApiExample( "QuoteCreationRequestExample", value={ - "type": "filecoin", + "type": "arweave", "files": [ - {"length":2343545}, - {"length":2343545} + {"length":234}, + {"length":236} ], - "duration": 4353545453, + "duration": 123, "payment": { - "payment_method": { - "chainId": 1, - }, - "wallet_address": "0xOCEAN_on_MAINNET" + "chainId": 80001, + "tokenAddress": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889" }, - "userAddress": "0x456" + "userAddress": "0xCC866199C810B216710A3F3714d35920C343a8CD" }, request_only=True, # signal that example only applies to requests response_only=False @@ -214,7 +217,7 @@ def post(self, request): """ POST a quote, handle different error code """ - data = JSONParser().parse(request) + data = request.data # Make sure request data contains type and files if (not 'type' in data or not 'files' in data): @@ -222,15 +225,17 @@ def post(self, request): # From type, retrieve associated storage object try: - storage = Storage.objects.get(type=data.pop('type')) + storage = Storage.objects.get(type=data['type']) # If not exists, raise error except: - return Response('Chosen storage type does not exist.', status=400) + return Response({'error': 'Chosen storage type does not exist.'}, status=400) # For the given type of storage, make a call to the associated service API (mock first) to retrieve a cost associated with that + headers = {'User-Agent': 'Mozilla/5.0', 'Content-Type': 'application/json'} response = requests.post( storage.url + 'getQuote/', - data + json.dumps(data), + headers=headers ) # From the response data: @@ -240,7 +245,8 @@ def post(self, request): # Creating the new payment with status still to execute data['storage'] = storage.pk - data['payment']['paymentMethod'] = data['payment'].pop('payment_method') + data['payment']['paymentMethod'] = {'chainId': data['payment']['chainId']} + data['payment']['wallet_adress'] = data['payment']['tokenAddress'] serializer = QuoteSerializer(data=data) if serializer.is_valid(): @@ -253,7 +259,7 @@ def post(self, request): 'tokenAddress': serializer.data['tokenAddress'] }, status=201) return Response(serializer.errors, status=400) - else: return Response('Storage service response badly formatted.', status=400) + else: return Response({'error': 'Storage service response badly formatted.'}, status=400) # Quote detail endpoint displaying the detail of a quote, no update, no deletion for now. @@ -265,7 +271,7 @@ class QuoteStatusView(APIView): OpenApiParameter( name='quoteId', description='Quote ID', - type=int + type=str ) ], examples=[ @@ -300,7 +306,7 @@ def get(self, request): # Request status of quote from micro-service response = requests.get( - quote.storage.url + 'quote/' + str(quoteId) + quote.storage.url + 'getStatus?quoteId=' + str(quoteId) ) if (response.status_code == 200): @@ -311,7 +317,6 @@ def get(self, request): 'status': quote.status }) -# "/upload?quoteId=123565&nonce=1768214571&signature=0ee382b39a39e05500d99233cdca83cd9959be4ff557ce7f3f29c9ce99d3b5de" # Upload file associated with a quote endpoint class UploadFile(APIView): @csrf_exempt @@ -320,7 +325,7 @@ class UploadFile(APIView): OpenApiParameter( name='quoteId', description='Quote ID', - type=int + type=str ), OpenApiParameter( name='nonce', @@ -407,7 +412,38 @@ def post(self, request, format="multipart"): # Forward the files to IPFS, retrieve whatever they provide us (the hash), mocked in the test File.objects.create(quote=quote, **added_file) - files_reference.append(added_file['cid']) + files_reference.append("ipfs://" + str(added_file['cid'])) + + try: + rpcProvider = quote.payment.paymentMethod.rpcEndpointUrl + except ObjectDoesNotExist: + rpcProvider = "https://rpc-mumbai.maticvigil.com/" + + my_provider = Web3.HTTPProvider(rpcProvider) + w3 = Web3(my_provider) + w3.middleware_onion.inject(geth_poa_middleware, layer=0) + + # Creating allowance for funds transfer from my user address to the quote paymentAddress + approvalAddress = quote.approveAddress # '0xAFcE990754C38Be5E0C341707B2A162C4e67547B' + tokenAddress = quote.tokenAddress # '0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889' + userAddress = quote.payment.wallet_address #'0xCC866199C810B216710A3F3714d35920C343a8CD' + + abi = '[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"withdraw","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint8"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transfer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"deposit","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"},{"name":"","type":"address"}],"name":"allowance","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"payable":true,"stateMutability":"payable","type":"fallback"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"dst","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"dst","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Deposit","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Withdrawal","type":"event"}]' + abi = json.loads(abi) + + contractAddress = w3.toChecksumAddress(tokenAddress) + contract = w3.eth.contract(contractAddress, abi=abi) + + userAddress = w3.toChecksumAddress(userAddress) + approvalAddress = w3.toChecksumAddress(approvalAddress) + nonce = w3.eth.getTransactionCount(userAddress) + tx_hash = contract.functions.approve(approvalAddress, quote.tokenAmount).buildTransaction({'from': userAddress, 'nonce': nonce}) + signed_tx = w3.eth.account.signTransaction(tx_hash, 'bbb5a2d50f3956e72dd8f38096270d8d951e44da623b1e31422d724e5841c93f') + tx_hash = w3.eth.sendRawTransaction(signed_tx.rawTransaction) + + tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) + print("Transaction receipt:" + str(tx_receipt)) + print("Contract allowance:" + str(contract.functions.allowance(userAddress, approvalAddress).call())) data = { "quoteId": quote.quoteId, @@ -418,7 +454,7 @@ def post(self, request, format="multipart"): # Upload files to micro-service response = requests.post( - quote.storage.url + 'upload/', + quote.storage.url + 'upload/?quoteId=' + str(quoteId) + '&nonce=' + params['nonce'][0] + '&signature=' + params['signature'][0], data ) @@ -441,7 +477,7 @@ class QuoteLink(APIView): OpenApiParameter( name='quoteId', description='Quote ID', - type=int + type=str ), OpenApiParameter( name='nonce', @@ -500,8 +536,20 @@ def get(self, request): quote.storage.url + 'getLink?quoteId=' + str(quoteId) + '&nonce=' + params['nonce'][0] + '&signature=' + params['signature'][0] ) - #TODO: improve that by managing the different link format from different services. - return Response({ - "type": quote.storage.type, - "CID": json.loads(response.content)[0]['CID'] - }) + print(json.loads(response.content)) + if response.status_code != 200: + return Response(json.loads(response.content), status=400) + + if quote.storage.type == "arweave": + #TODO: improve that by managing the different link format from different services. + responseObj = json.loads(response.content) + # result = [] + # for item in responseObj: + # result.append({''}) + return Response(responseObj, status=200) + elif quote.storage.type == "filecoin": + #TODO: improve that by managing the different link format from different services. + return Response({ + "type": quote.storage.type, + "CID": json.loads(response.content)[0]['CID'] + }) diff --git a/server/server/settings.py b/server/server/settings.py index 904b53f..218ff04 100644 --- a/server/server/settings.py +++ b/server/server/settings.py @@ -12,11 +12,15 @@ import os from django.conf import settings +from dotenv import find_dotenv, load_dotenv + +# Load environment variable files +env_file = find_dotenv("../../.env") +load_dotenv(env_file) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ @@ -25,6 +29,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True +PYTHONUNBUFFERED = True ALLOWED_HOSTS = [] @@ -165,6 +170,12 @@ ######################### # DBS SPECIFIC SETTINGS # ###################"##### -FILECOIN_SERVICE_URL = 'https://storage.filecoin.com' +FILECOIN_SERVICE_URL = 'http://localhost:3000' ARWEAVE_SERVICE_URL = 'https://storage.arweave.com' -DEFAULT_FILE_STORAGE = 'ipfs_storage.InterPlanetaryFileSystemStorage' \ No newline at end of file + +TEST_PRIVATE_KEY = os.environ.get("TEST_PRIVATE_KEY") +TOKEN_ADDRESS = os.environ.get("TOKEN_ADDRESS") + +if os.getenv('GITHUB_WORKFLOW'): + TEST_PRIVATE_KEY = os.getenv("TEST_PRIVATE_KEY") + TOKEN_ADDRESS = os.getenv("TOKEN_ADDRESS") diff --git a/services/dbs_arweave b/services/dbs_arweave new file mode 160000 index 0000000..2b3fc6a --- /dev/null +++ b/services/dbs_arweave @@ -0,0 +1 @@ +Subproject commit 2b3fc6a8c5c9872c1e1995fa86ead57bc790938b diff --git a/services/dbs_filecoin b/services/dbs_filecoin new file mode 160000 index 0000000..44959e2 --- /dev/null +++ b/services/dbs_filecoin @@ -0,0 +1 @@ +Subproject commit 44959e25c2234da91c2a6d7e92de450dd553b84e