Make CI independent of docker registry

.github/workflows/build.yml

@@ -21,7 +21,7 @@ jobs:
         device: [nanos, nanosp, nanox, stax]
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:3.12.0
+      image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -43,32 +43,25 @@ jobs:
           path: app_${{ matrix.device }}_dbg.tgz
           name: app_${{ matrix.device }}_dbg.tgz
 
-  build_docker_integration_tests:
-    uses: ./.github/workflows/docker.yml
-    secrets: inherit
-    with:
-      dockerfile: docker/Dockerfile.integration-tests
-      image_name: integration_tests
 
   build_docker_tezos_ocaml:
+    if: github.repository == 'trilitech/ledger-app-tezos-wallet'
     uses: ./.github/workflows/docker.yml
     secrets: inherit
     with:
       dockerfile: docker/Dockerfile.ocaml
       image_name: tezos_ocaml
 
   integration_tests_basic:
-    needs: [build_app, build_docker_integration_tests]
+    needs: [build_app]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         device: [nanos, nanosp, nanox, stax]
     container:
-      image: ${{ needs.build_docker_integration_tests.outputs.image }}
-      credentials:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+      image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest
+
     steps:
       - name: Set Git Safe Directory
         run: |
@@ -93,9 +86,13 @@ jobs:
             ./tests/integration/run_test_local.sh -F -m ${{ matrix.device }} \
               ./tests/integration/${{ matrix.device }}
           else
+            apk add gmp-dev curl jq libsodium-dev git xxd procps
+            python3 -m venv tezos_test_env --system-site-package
+            source ./tezos_test_env/bin/activate
+            python3 -m pip install -r ./tests/requirements.txt -q
             TMP_DIR=$(mktemp -d /tmp/foo-XXXXXX)
             tar xfz app_${{ matrix.device }}_dbg.tgz -C $TMP_DIR
-            pytest -n 32 tests/integration/nano/ --tb=no                \
+            python3 -m pytest -n 32 tests/integration/nano/ --tb=no                \
                    --device ${{ matrix.device }} --app $TMP_DIR/app.elf \
                    --log-dir integration_tests_log
           fi
@@ -161,18 +158,15 @@ jobs:
           path: tests/samples/operations/nano
 
   integration_tests_samples:
-    needs: [build_app, generate_samples_unit_tests, build_docker_integration_tests]
+    needs: [build_app, generate_samples_unit_tests]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         device: [nanos, nanosp, nanox]
         type: [micheline, operations]
     container:
-      image: ${{ needs.build_docker_integration_tests.outputs.image }}
-      credentials:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+      image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -194,7 +188,12 @@ jobs:
           path: tests
 
       - name: Test
-        run: ./tests/integration/run_test_local.sh -T100 -F -m ${{ matrix.device }} tests
+        run: |
+            apk add gmp-dev curl jq libsodium-dev git xxd procps
+            python3 -m venv tezos_test_env --system-site-package
+            source ./tezos_test_env/bin/activate
+            python3 -m pip install -r ./tests/requirements.txt -q
+            ./tests/integration/run_test_local.sh -T100 -F -m ${{ matrix.device }} tests
 
       - name: Upload results
         uses: actions/upload-artifact@v3

.gitignore

@@ -13,6 +13,7 @@
 /integration_tests_log
 /pattern_registry
 /tests/samples
+/*env/
 
 tags
 __pycache__

Makefile

@@ -25,17 +25,18 @@ docker_speculos:
 	$(DOCKER) image tag $(LEDGERHQ)/speculos speculos
 
 docker_ledger_app_builder:
-	$(DOCKER) pull $(LEDGERHQ)/ledger-app-builder/ledger-app-builder:3.12.0
-	$(DOCKER) image tag $(LEDGERHQ)/ledger-app-builder/ledger-app-builder:3.12.0 \
+	$(DOCKER) pull $(LEDGERHQ)/ledger-app-builder/ledger-app-dev-tools:latest
+	$(DOCKER) image tag $(LEDGERHQ)/ledger-app-builder/ledger-app-dev-tools:latest \
 			ledger-app-builder
 
 docker_ledger_app_ocaml:
 	$(DOCKER) build -t ledger-app-tezos-ocaml \
 			-f docker/Dockerfile.ocaml docker --platform linux/$(CPU)
 
 docker_ledger_app_integration_tests:
-	$(DOCKER) build -t ledger-app-tezos-integration-tests \
-			-f docker/Dockerfile.integration-tests docker --platform linux/$(CPU)
+	$(DOCKER) pull $(LEDGERHQ)/ledger-app-builder/ledger-app-dev-tools:latest
+	$(DOCKER) image tag $(LEDGERHQ)/ledger-app-builder/ledger-app-dev-tools:latest \
+			ledger-app-tezos-integration-tests
 
 docker_images: docker_speculos		\
 	docker_ledger_app_builder	\
@@ -94,17 +95,22 @@ integration_tests_basic_stax:	app_stax.tgz		\
 				tests/integration/stax/*
 	$(RUN_TEST_DOCKER) stax tests/integration/stax
 
-integration_tests_basic_%:	app_%_dbg.tgz			\
+integration_tests_basic_%:	app_%.tgz   \
+				app_%_dbg.tgz			\
 				tests/integration/*		\
 				tests/integration/nano/*	\
 				tests/integration/nano/%/*
-	docker run --user "$$(id -u)":"$$(id -g)" --rm -i -v "$(realpath .):/app" \
-	--entrypoint=/bin/sh ledger-app-tezos-integration-tests -c "              \
-		TMP_DIR=\$$(mktemp -d /tmp/foo-XXXXXX);                           \
-		cd /app;                                                          \
-		tar xfz app_$*_dbg.tgz -C \$$TMP_DIR;                             \
-		pytest -n 32 tests/integration/nano/ --tb=no                      \
-			--device $* --app \$$TMP_DIR/app.elf                      \
+	docker run --rm -i -v "$(realpath .):/app" \
+	--entrypoint=/bin/sh ledger-app-tezos-integration-tests -c "  \
+		TMP_DIR=\$$(mktemp -d /tmp/foo-XXXXXX);                   \
+		cd /app;                                                  \
+		tar xfz app_$*_dbg.tgz -C \$$TMP_DIR;                     \
+		apk add gmp-dev curl jq libsodium-dev git xxd procps;     \
+		python3 -m venv tezos_test_env --system-site-package;     \
+		source ./tezos_test_env/bin/activate;                     \
+		python3 -m pip install -r tests/requirements.txt -q ;  \
+		python3 -m pytest -n 32 tests/integration/nano/ --tb=no   \
+			--device $* --app \$$TMP_DIR/app.elf                  \
 			--log-dir integration_tests_log"
 
 integration_tests_basic:	integration_tests_basic_nanos	\

README.md

@@ -18,8 +18,7 @@ The docker images can be built using the provided Makefile:
 :; make docker-images
 ```
 
-This pulls down two images, `ghcr.io/ledgerhq/speculos` and
-`ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest`.  It
+This pulls down following image, `ghcr.io/ledgerhq/ledger-app-dev-tools`. It
 also builds an image via `docker/Dockerfile.ocaml`.
 
 We do not make the builds of the software conditional on these docker

app/src/app_main.c

@@ -36,6 +36,7 @@
 void
 app_exit(void)
 {
+    PRINTF("[DEBUG] Trying to exit the app. \n");
     os_sched_exit(-1);
 }
 

scripts/test_swap.sh

@@ -105,7 +105,7 @@ _build_app() {
         cd $repo
 
         docker run --rm -ti -v "$(realpath .):/app" --privileged            \
-               ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:3.12.0 \
+               ghcr.io/ledgerhq/ledger-app-builder/ledger-app-dev-tools:latest \
                bash -c "make clean && make -j $params BOLOS_SDK=\$$sdk"
     )
 }
@@ -148,11 +148,11 @@ run_tests() {
     (
         cd $APP_EXCHANGE_REPO
 
-        docker run --privileged  --entrypoint /bin/bash           \
-               --rm -v "$(realpath .):/app"                       \
-               ledger-app-tezos-integration-tests -c              \
-               "cd /app &&                                        \
-                pip install -r test/python/requirements.txt &&    \
+        docker run --privileged  --entrypoint /bin/bash                 \
+               --rm -v "$(realpath .):/app"                             \
+               ledger-app-tezos-integration-tests -c                    \
+               "cd /app &&                                              \
+                pip install -r test/python/requirements.txt -q &&    \
                 pip install protobuf==3.20.3 && pytest test/python $*"
     )
 }

tests/integration/nano/conftest.py

@@ -137,7 +137,6 @@ def backend(app_path: Path,
 
     backend = SpeculosTezosBackend(app_path,
                                    firmware,
-                                   port=port,
                                    args=speculos_args)
 
     with backend as b:

tests/integration/run_test_docker.sh

@@ -19,7 +19,7 @@ set -e
 
 . "`dirname $0`/app_vars.sh"
 
-docker run --user "$(id -u)":"$(id -g)" --rm -i -v "$(realpath .):/app"              	\
+docker run --rm -it -v "$(realpath .):/app"              	\
        --env-file "`dirname $0`/err_codes.sh"           \
        -e VERSION_BYTES=$VERSION_BYTES                  \
        -e COMMIT_BYTES=$COMMIT_BYTES                    \
@@ -29,5 +29,9 @@ docker run --user "$(id -u)":"$(id -g)" --rm -i -v "$(realpath .):/app"
        -e APPVERSION_P=$APPVERSION_P                    \
        -e APPVERSION=$APPVERSION                        \
        -e VERSION_BYTES=$VERSION_BYTES                  \
-    --entrypoint=/bin/sh ledger-app-tezos-integration-tests	\
-    -c "cd /app && ./tests/integration/run_test_local.sh -F -m $*"
+    --entrypoint=/bin/sh ledger-app-tezos-integration-tests	   \
+    -c " apk add gmp-dev curl jq libsodium-dev git xxd procps; \
+		python3 -m venv tezos_test_env --system-site-package;      \
+		source ./tezos_test_env/bin/activate;                      \
+		python3 -m pip install -r tests/requirements.txt -q;    \
+     ./tests/integration/run_test_local.sh -F -m $*"

tests/integration/run_test_local.sh

@@ -37,7 +37,7 @@ start_speculos_runner() {
 
     app_dir="$(mktemp -d $DATA_DIR/appdir-XXXXXX)"
     tar xfz "$tgz" -C $app_dir
-    $SPECULOS --display headless --apdu-port 0 --api-port $PORT          \
+    python3 -m $SPECULOS --display headless --apdu-port 0 --api-port $PORT          \
               --seed "$seed" -m $TARGET $app_dir/app.elf                 \
                 > $SPECULOG 2>&1 < /dev/null &
     speculos_pid=$!
@@ -54,9 +54,12 @@ kill_speculos_runner() {
 
 
 exited() {
+    echo "speculos_pid: $speculos_pid"
     if ps $speculos_pid > /dev/null 2>&1 ; then
+        echo "\n Speculos Still running \n"
         return 1
     else
+        echo "\n Speculos Exited   \n"
         return 0
     fi
 }

tests/integration/stax/utils.py

@@ -168,7 +168,7 @@ def stax_app(prefix) -> TezosAppScreen:
     commit = os.environ["COMMIT_BYTES"]
     version = os.environ["VERSION_BYTES"]
     golden = os.getenv("GOLDEN") != None
-    backend = SpeculosBackend("__unused__", Firmware.STAX, port = port)
+    backend = SpeculosBackend("__unused__", Firmware.STAX,args=["--api-port", port])
     app = TezosAppScreen(backend, Firmware.STAX, commit, version, prefix)
 
     if golden:

tests/integration/test_runtime.sh

@@ -387,7 +387,7 @@ run_a_test() {
                         fi
                         app_dir="$(mktemp -d $DATA_DIR/appdir-XXXXXX)"
                         tar xfz "$tgz" -C $app_dir
-                        pytest $CMD -v \
+                        python3 -m pytest $CMD -v \
                                --device $TARGET \
                                --port $PORT \
                                --app $app_dir/app.elf
@@ -396,6 +396,9 @@ run_a_test() {
                 *.hex)
                     # We skip these...
                     ;;
+                *.txt)
+                    # We skip these...
+                    ;;
                 *)
                     if [ -f "$CMD" ]; then
                         echo Command "$CMD" ends in neither .sh nor .py >&2

tests/requirements.txt

@@ -0,0 +1,8 @@
+wheel
+setuptools
+pkginfo
+cryptography
+base58
+pytezos==3.10.3
+GitPython
+pytest-xdist