Feature/v4 c2d

README.md

@@ -76,6 +76,7 @@ The default versions are always a combination of component versions which are co
 | Graphpgsql          |                    | postgres                          | 172.15.0.7      | 5432 -> 5432  |
 | Dashboard           |                    | portainer/portainer               | 172.15.0.25     | 9100 -> 9000  |
 | Redis               |                    | bitnami/redis:latest              | 172.15.0.18     | 6379 -> 6379  |
+| C2d                 |                    | multiple components               | 172.15.0.12,172.15.0.13     | 31000 -> 31000  |
 
 
 
@@ -101,15 +102,17 @@ export AQUARIUS_VERSION=v2.0.0
 | `--no-aquarius`            | Start up Ocean without the `aquarius` Building Block.                                           |
 | `--no-elasticseach`        | Start up Ocean without the `elasticsearch` Building Block.                                      |
 | `--no-provider`            | Start up Ocean without the `provider` Building Block.                                           |
+| `--no-ipfs`                | Start up Ocean without the `ipfs` Building Block                                                |
 | `--with-provider2`         | Runs a 2nd provider, on port 8031. This is required for ocean.js/ocean.py integration tests. 2nd Provider will use the same image and parameters (log_level, ipfs gateway, compute gateway, etc) as provider1, but has a different private key     |
 | `--no-ganache`             | Start up Ocean without the `ganache` Building Block.                                            |
 | `--no-dashboard`           | Start up Ocean without the `dashboard` Building Block.                                          |
 | `--with-rbac`              | Start up Ocean with RBAC Server                                                                 |
-| `--with-thegraph`          | Start up Ocean with graphnode, ipfs & postgresql                                                |
+| `--with-thegraph`          | Start up Ocean with graphnode,postgresql                                                        |
 | `--skip-deploy`            | Start up Ocean without deploying the contracts. Useful when ethereum node already has contracts.|
 | `--force-pull`             | Force pulling the latest revision of the used Docker images.                                    |
 | `--purge`                  | Removes the Docker containers, volumes, artifact folder and networks used by the script.        |
 | `--exposeip`               | Binds the components to that specific ip. Example: `./start_ocean.sh --exposeip 192.168.0.1`    |
+| `--with-c2d`               | Runs a local C2D Cluster                                                                        |
 
 
 
@@ -197,6 +200,23 @@ The following addresses are preconfigured for testing:  (first 10 addresses from
 
 See the page titled "[Ways to Contribute](https://docs.oceanprotocol.com/concepts/contributing/)" in the Ocean Protocol documentation.
 
+
+## Readiness
+ Several building blocks on barge are going to require some time until they are ready.  Watch for the following files to be created:
+
+| Building block                             |  File                                         |
+| -----------------------------------------  | -------------------------------               |
+| ocean-contracts                            | ${OCEAN_HOME}/ocean-contracts/artifacts/ready |
+| c2d                                        | ${OCEAN_HOME}/ocean-c2d/ready |
+
+
+## Certs
+  Registry certs were created using the following commands:
+  ```bash
+  openssl genrsa 2048 > registry.key
+  openssl req -new -x509 -nodes -sha1 -days 3650 -key registry.key -out registry.crt -addext 'subjectAltName = IP:172.15.0.11'
+  ```
+
 ## License
 
 ```text

certs/registry/registry.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDqDCCApCgAwIBAgIUbDCRZcdoLJNS+X26qn7ZvrKwD2kwDQYJKoZIhvcNAQEF
+BQAwWzELMAkGA1UEBhMCRVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIGA1UEAwwLMTcyLjE1LjAuMTEw
+HhcNMjIwMTEzMTUxMzE0WhcNMzIwMTExMTUxMzE0WjBbMQswCQYDVQQGEwJFVTET
+MBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQ
+dHkgTHRkMRQwEgYDVQQDDAsxNzIuMTUuMC4xMTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAJohuNlU5rRan+n1B9YLKNaee+eJM7yk+uf1OwYPR9rZvJf0
+/TZc728MBTFa0gubYzcXetpo7jn4JHwanQ+VZKQBE7iauUUAY/sKwZBhgMzUanyu
+knmFmBrE+b6yoVrFk2lVSS5VHZ5bLtg9gl/cu7YO4BE8YQ5aEWRqhGt61twAd+ki
+RXXnuc3/NGxs2oKqiIWYZlsvq4yS6/LpKlItaz+v5h/kzpMeewo7fmLRyQziYK0o
+fhmAI38yX9OXO6OXDl505aykHpX/ogG/M7yM3gYpLWKmZ2KexNLrp5YyZugzjSyJ
+/AuRbK6eijw4+fnkEQM7eS5epDYIF+yFByVK4wECAwEAAaNkMGIwHQYDVR0OBBYE
+FPAg1eWFpxiYU3odt00Z0WXtCG2SMB8GA1UdIwQYMBaAFPAg1eWFpxiYU3odt00Z
+0WXtCG2SMA8GA1UdEwEB/wQFMAMBAf8wDwYDVR0RBAgwBocErA8ACzANBgkqhkiG
+9w0BAQUFAAOCAQEAM1xjC6MgOq78zDAeQ4e74IZL1t5bwl3vjvdRS1v5s/WZBy1P
+6Iw/+mz9Gi9zyBlT3OdOcjNpQOtSCASDyYfgP0YJt9OWw9/tJbIUewRJPC977tcA
+kyh5bHjXyHU1b7gT3EX3or32kITOGQzKp3LDxDVqST0brlzmpXzbuMZ9yrmFMYlZ
+A8qDeGRn1s0T/T8wliyWXMa97hK7h8tG8I5kGMByd6FZzey0An5fQv71DWCWPBvU
+sLtdZVjf5C5eY3ngYzELgQfPdI22pMFmzzoOrU6MDbt0sr1vT0Av8Sw6rof7QaeG
+CduCiU9+Rjr66iljywDabfk6QsMH8QkpZWstGQ==
+-----END CERTIFICATE-----

certs/registry/registry.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAmiG42VTmtFqf6fUH1gso1p5754kzvKT65/U7Bg9H2tm8l/T9
+NlzvbwwFMVrSC5tjNxd62mjuOfgkfBqdD5VkpAETuJq5RQBj+wrBkGGAzNRqfK6S
+eYWYGsT5vrKhWsWTaVVJLlUdnlsu2D2CX9y7tg7gETxhDloRZGqEa3rW3AB36SJF
+dee5zf80bGzagqqIhZhmWy+rjJLr8ukqUi1rP6/mH+TOkx57Cjt+YtHJDOJgrSh+
+GYAjfzJf05c7o5cOXnTlrKQelf+iAb8zvIzeBiktYqZnYp7E0uunljJm6DONLIn8
+C5Fsrp6KPDj5+eQRAzt5Ll6kNggX7IUHJUrjAQIDAQABAoIBAAIoGKMJ2hSJ9BTc
+9dgb3d1q6EbsF4o7MLuQJdxG2HWKeGYT6nsZP8xo+aWgJLI+4VuC+5gUwciT3UbT
+RIFUUM0uqqz5lUAO6lc35k9z3c5/5mGKO8hxgxhUyz3M2A8iFpbxt41raXeuxPAH
+pExdFU9N0/rJwn3l6rph9ypd1KXNYlBRuQKs7HI6VjQVaDWiuJCmRlhUrrB1eydE
+pMjNpzHybeQAiFWyA3KWgkpgOoEciJ2lM+G982aSEILXVBzqHpLFZt0R2vOfompY
+XH4l8LEQk6OiGuPUI1o9cNtJxLRd6apNxHfED28SxxUX+lLkfF+Agl7pS+MVj8Oo
+UQwPluUCgYEAzTWrwLiwevs6ZwkePv0OsHemTXBk03KzxFUNm9+r0zOBzZsj3iDV
+VBoSSSwCPz2a3858YvRILK/BM5t0mVrMc97daZRNGKFgxYt9/3iOegg7juCGcsTH
+MEApvcfYcKkZH3P6e1b5sN0ys1+Aka1xkbtOqoVxQGeqasT6Rkkj9iMCgYEAwEex
+VOPChJLmLrOJtcCIhm7c4cBZxjHVNnuTg/4/tcBZsRrUYx9GUgqxwatb/rAAZBCx
+0kwR9P1gHthsoIqcZ3DNd38nR21Wi4IKz58pbXt69mN8xgNz3ZxOlrBOPR5EkxHG
+SLjpeZ3+o89NpcCiuDw7ka7iAAklU3q2YvP7qosCgYBBgThqNIphN4DVExwMj0r5
+GbiET7RfquWnbl8BusxmhrKQp00zwtdWGVxs7xdoiIm/l3ItRD0lxpStnJoDufA5
+rJKQ+T4C9sClTnnR6vHQEoCwgwtn51iOFm08n2Z/z/qtB6CVbpYV1wYwK/QmjG8M
+TfY2iJI2NOE5Y9dgSEiqcQKBgQC3SB9q8afLQ1OSlKTh0OLScf6uWfG5EGtSgbz9
+hm8oSm+0eASXb4pSTcte9n68aPwWZit6b/R1+MeU32u8JCQuHcJc86F3Xmuuk4jG
+m8qbh6aQEgkvnr7lMUdgMDKD48QuBDVjwI8c1FOTxdkX9GWnBBMsLO1UZ2NjDYep
+DzSHPQKBgD4cozTsTPxeVEW3sGi2YBwVuS4zHd+5COJ7LvG7JgRL2k5hQnz+3bwA
+NYBbUeKRy9e99pTsS/jstT1zz3CED/RPzdesDS5drox+vI070l5mi5PaJAevyy1U
+tj6SR6dPrh5Z/obfUN6cnxz1X1jkhzZ7Q8J9VFVg8xV1UclF1B06
+-----END RSA PRIVATE KEY-----

compose-files/c2d.yml

@@ -0,0 +1,39 @@
+version: '3'
+services:
+  kindcluster:
+    image: oceanprotocol/c2d_barge_kind:latest
+    privileged: true
+    ports: 
+      - 8443:8443
+      - 10080:10080
+      - 31000:31000
+    networks:
+      backend:
+        ipv4_address: 172.15.0.13
+    environment:
+      STATIC_IP: 172.15.0.13
+      REGISTRY: "172.15.0.11:5050"
+      REGISTRY_USER: "username"
+      REGISTRY_PASSWORD: '$REGISTRY_TOKEN'
+  computetodata:
+    image: oceanprotocol/c2d_barge_deployer:latest
+    privileged: true
+    environment:
+      KIND_IP: 172.15.0.13
+      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_HTTP_GATEWAY: ${IPFS_HTTP_GATEWAY}
+      REGISTRY: "172.15.0.11:5000/"
+      OPERATOR_SERVICE_IMAGE: oceanprotocol/operator-service:${OPERATOR_SERVICE_VERSION}
+      OPERATOR_ENGINE_IMAGE: oceanprotocol/operator-engine:${OPERATOR_ENGINE_VERSION}
+      POD_CONFIGURATION_IMAGE: oceanprotocol/pod-configuration:${POD_CONFIGURATION_VERSION}
+      POD_PUBLISHING_IMAGE: oceanprotocol/pod-publishing:${POD_PUBLISHING_VERSION}
+    networks:
+      backend:
+        ipv4_address: 172.15.0.12
+    depends_on:
+    - kindcluster
+    volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+    - /usr/bin/docker:/usr/bin/docker
+    - ${OCEAN_C2D_FOLDER}:/ocean/c2d/
+    - ${OCEAN_CERTS_FOLDER}/registry:/certs

compose-files/ipfs.yml

@@ -0,0 +1,15 @@
+version: '3'
+services:
+  ipfs:
+    image: ipfs/go-ipfs:v0.4.23
+    ports:
+      - '5001:5001'
+      - '5005:8080'
+    networks:
+      backend:
+        ipv4_address: 172.15.0.16
+    volumes:
+      - graphipfs:/data/ipfs
+volumes:
+  graphipfs:
+

compose-files/network_volumes.yml

@@ -7,3 +7,4 @@ networks:
       driver: default
       config:
         - subnet: 172.15.0.1/24
+

compose-files/registry.yml

@@ -0,0 +1,21 @@
+version: '3'
+services:
+  registry:
+    restart: always
+    container_name: docker-registry
+    image: registry:2
+    ports:
+      - 5050:5000
+    networks:
+      backend:
+        ipv4_address: 172.15.0.11
+    environment:
+      REGISTRY_HTTP_ADDR: "172.15.0.11:5000"
+      REGISTRY_HTTP_TLS_CERTIFICATE: /certs/registry.crt
+      REGISTRY_HTTP_TLS_KEY: /certs/registry.key
+    volumes:
+      - registryvol:/var/lib/registry
+      - ${OCEAN_CERTS_FOLDER}/registry:/certs
+
+volumes:
+  registryvol:

compose-files/thegraph.yml

@@ -23,15 +23,6 @@ services:
       ipfs: '172.15.0.16:5001'
       ethereum: 'barge:${NETWORK_RPC_URL}'
       RUST_LOG: info
-  ipfs:
-    image: ipfs/go-ipfs:v0.4.23
-    ports:
-      - '5001:5001'
-    networks:
-      backend:
-        ipv4_address: 172.15.0.16
-    volumes:
-      - graphipfs:/data/ipfs
   postgres:
     image: postgres
     ports:
@@ -47,6 +38,5 @@ services:
     volumes:
       - graphpgsql:/var/lib/postgresql/data
 volumes:
-  graphipfs:
   graphpgsql:
 

start_ocean.sh

@@ -32,8 +32,14 @@ export AQUARIUS_VERSION=${AQUARIUS_VERSION:-v4main}
 export PROVIDER_VERSION=${PROVIDER_VERSION:-v4main}
 export CONTRACTS_VERSION=${CONTRACTS_VERSION:-v4main_postaudit}
 export RBAC_VERSION=${RBAC_VERSION:-next}
+
+export OPERATOR_SERVICE_VERSION=${OPERATOR_SERVICE_VERSION:-v4main}
+export OPERATOR_ENGINE_VERSION=${OPERATOR_ENGINE_VERSION:-v4main}
+export POD_CONFIGURATION_VERSION=${POD_CONFIGURATION_VERSION:-v4main}
+export POD_PUBLISHING_VERSION=${POD_PUBLISHING_VERSION:-v4main}
+
 export PROJECT_NAME="ocean"
-export FORCEPULL="true"
+export FORCEPULL="false"
 
 # Export User UID and GID
 export LOCAL_USER_ID=$(id -u)
@@ -57,8 +63,16 @@ export CONTRACTS_OWNER_ROLE_ADDRESS="${CONTRACTS_OWNER_ROLE_ADDRESS}"
 export DEPLOY_CONTRACTS=true
 export OCEAN_ARTIFACTS_FOLDER="${OCEAN_HOME}/ocean-contracts/artifacts"
 mkdir -p ${OCEAN_ARTIFACTS_FOLDER}
+export OCEAN_C2D_FOLDER="${OCEAN_HOME}/ocean-c2d/"
+mkdir -p ${OCEAN_C2D_FOLDER}
 export ADDRESS_FILE="${OCEAN_ARTIFACTS_FOLDER}/address.json"
 echo "export ADDRESS_FILE=${ADDRESS_FILE}"
+
+#certs folder
+export OCEAN_CERTS_FOLDER="${OCEAN_HOME}/ocean-certs/"
+mkdir -p ${OCEAN_CERTS_FOLDER}
+# copy certs
+cp -r ./certs/* ${OCEAN_CERTS_FOLDER}
 # Specify which ethereum client to run or connect to: development
 export CONTRACTS_NETWORK_NAME="development"
 
@@ -75,7 +89,8 @@ export DB_CLIENT_KEY=""
 export DB_CLIENT_CERT=""
 CHECK_ELASTIC_VM_COUNT=true
 
-
+export IPFS_GATEWAY=http://172.15.0.16:5001
+export IPFS_HTTP_GATEWAY=http://172.15.0.16:8080/ipfs/
 #Provider
 export PROVIDER_LOG_LEVEL=${PROVIDER_LOG_LEVEL:-INFO}
 export PROVIDER_WORKERS=10
@@ -90,7 +105,7 @@ else
 fi
 
 #export OPERATOR_SERVICE_URL=http://127.0.0.1:8050
-export OPERATOR_SERVICE_URL=${OPERATOR_SERVICE_URL:-"https://c2d-dev.operator.oceanprotocol.com/"}
+export OPERATOR_SERVICE_URL=${OPERATOR_SERVICE_URL:-"http://172.15.0.13:31000/"}
 
 # Add aquarius to /etc/hosts
 # Workaround mainly for macOS
@@ -161,6 +176,7 @@ COMPOSE_FILES+=" -f ${COMPOSE_DIR}/network_volumes.yml"
 COMPOSE_FILES+=" -f ${COMPOSE_DIR}/dashboard.yml"
 COMPOSE_FILES+=" -f ${COMPOSE_DIR}/aquarius.yml"
 COMPOSE_FILES+=" -f ${COMPOSE_DIR}/elasticsearch.yml"
+COMPOSE_FILES+=" -f ${COMPOSE_DIR}/ipfs.yml"
 COMPOSE_FILES+=" -f ${COMPOSE_DIR}/provider.yml"
 COMPOSE_FILES+=" -f ${COMPOSE_DIR}/redis.yml"
 COMPOSE_FILES+=" -f ${COMPOSE_DIR}/ganache.yml"
@@ -195,12 +211,26 @@ while :; do
 	        COMPOSE_FILES+=" -f ${COMPOSE_DIR}/provider2.yml"
             printf $COLOR_Y'Starting with a 2nd Provider...\n\n'$COLOR_RESET
             ;;
+        --with-registry)
+            COMPOSE_FILES+=" -f ${COMPOSE_DIR}/registry.yml"
+            printf $COLOR_Y'Starting with Registry...\n\n'$COLOR_RESET
+            ;;
+        --with-c2d)
+            COMPOSE_FILES+=" -f ${COMPOSE_DIR}/registry.yml"
+	    COMPOSE_FILES+=" -f ${COMPOSE_DIR}/ipfs.yml"
+            COMPOSE_FILES+=" -f ${COMPOSE_DIR}/c2d.yml"
+            printf $COLOR_Y'Starting with C2D...\n\n'$COLOR_RESET
+            ;;
         --with-rbac)
 	        COMPOSE_FILES+=" -f ${COMPOSE_DIR}/rbac.yml"
             printf $COLOR_Y'Starting with RBAC Server...\n\n'$COLOR_RESET
             ;;
+        --no-ipfs)
+            COMPOSE_FILES="${COMPOSE_FILES/ -f ${COMPOSE_DIR}\/ipfs.yml/}"
+	        printf $COLOR_Y'Starting without IPFS...\n\n'$COLOR_RESET
+            ;;
         --with-thegraph)
-	        COMPOSE_FILES+=" -f ${COMPOSE_DIR}/thegraph.yml"
+            COMPOSE_FILES+=" -f ${COMPOSE_DIR}/thegraph.yml"
             printf $COLOR_Y'Starting with TheGraph...\n\n'$COLOR_RESET
             ;;
         --no-ganache)