Merge branch 'rc-2024.11.1' into stable

.github/workflows/merge.yml

@@ -5,7 +5,7 @@ on:
   # on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors
   # the merge run checks should show on master and enable this clear test/passing history
   merge_group:
-    branches: [main, evm-dev]
+    branches: [main, alpha*, beta*, rc*]
   pull_request:
     branches: ["*"]
 
@@ -16,6 +16,16 @@ env:
   GENESIS_SK: 5ec88891c1098a0fede5b98b07f8abc931d7247b7aa310d21ab430cc957f9f02
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Display Python version
+        run: python --version
+
   cargo-udeps:
     if: "!startsWith(github.event.head_commit.message, 'chore(release):')"
     name: Unused dependency check
@@ -122,7 +132,7 @@ jobs:
         timeout-minutes: 25
         run: cargo test --release --package sn_networking --features="open-metrics, encrypt-records"
 
-        name: Run network tests (without encrypt-records)
+      - name: Run network tests (without encrypt-records)
         timeout-minutes: 25
         run: cargo test --release --package sn_networking --features="open-metrics"
 
@@ -344,6 +354,115 @@ jobs:
           SN_LOG: "v"
         timeout-minutes: 2
 
+      - name: Estimate cost to create a vault
+        if: matrix.os != 'windows-latest'
+        run: |
+          echo "test-file" > upload-test.txt
+          ./target/release/autonomi --log-output-dest=data-dir file upload ./upload-test.txt
+          ./target/release/autonomi --log-output-dest=data-dir register create sample_new_register 1234
+          ./target/release/autonomi --log-output-dest=data-dir vault cost
+          ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 | tee file_list.txt
+          ./target/release/autonomi --log-output-dest=data-dir register list 2>&1 | tee register_list.txt
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 2
+
+      - name: create a vault with existing user data as above
+        if: matrix.os != 'windows-latest'
+        run: ./target/release/autonomi --log-output-dest=data-dir vault create
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 2
+
+      - name: add more files
+        if: matrix.os != 'windows-latest'
+        run: |
+          for i in {1..100}; do
+            dd if=/dev/urandom of=random_file_$i.bin bs=1M count=1 status=none
+            ./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin --public
+            ./target/release/autonomi --log-output-dest=data-dir file upload random_file_$i.bin
+            ./target/release/autonomi --log-output-dest=data-dir register create $i random_file_$i.bin
+          done
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 25
+
+      - name: sync the vault
+        if: matrix.os != 'windows-latest'
+        run: ./target/release/autonomi --log-output-dest=data-dir vault sync
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 2
+
+      - name: vault sync validation
+        if: matrix.os != 'windows-latest'
+        run: |
+          NUM_OF_PUBLIC_FILES=""
+          NUM_OF_PRIVATE_FILES=""
+          NUM_OF_REGISTERS=""
+          NUM_OF_PUBLIC_FILES_IN_VAULT=""
+          NUM_OF_PRIVATE_FILES_IN_VAULT=""
+          NUM_OF_REGISTERS_IN_VAULT=""
+          
+          ./target/release/autonomi --log-output-dest=data-dir file list 2>&1 > file_list.txt
+          
+          # ./target/release/autonomi --log-output-dest=data-dir register list | grep archives > register_list.txt
+          
+          NUM_OF_PUBLIC_FILES=`cat file_list.txt | grep "public" | grep -o '[0-9]\+'`
+          NUM_OF_PRIVATE_FILES=`cat file_list.txt | grep "private" | grep -o '[0-9]\+'`
+          NUM_OF_REGISTERS=`cat register_list.txt | grep "register" | grep -o '[0-9]\+'`
+
+          ./target/release/autonomi --log-output-dest=data-dir vault load 2>&1 > vault_data.txt
+
+          NUM_OF_PUBLIC_FILES_IN_VAULT=`cat vault_data.txt  | grep "public" | grep -o '[0-9]\+'`
+          NUM_OF_PRIVATE_FILES_IN_VAULT=`cat vault_data.txt| grep "private" | grep -o '[0-9]\+'`
+          # NUM_OF_REGISTERS_IN_VAULT=`cat vault_data.txt | grep "register" | grep -o '[0-9]\+'`
+
+          echo "Total Num of local public files is $NUM_OF_PUBLIC_FILES and in vault is $NUM_OF_PUBLIC_FILES_IN_VAULT"
+          echo "Total Num of local private files is $NUM_OF_PRIVATE_FILES and in vault is $NUM_OF_PRIVATE_FILES_IN_VAULT"
+          # echo "Total Num of local registers is $NUM_OF_REGISTERS and in vault is $NUM_OF_REGISTERS_IN_VAULT"
+          
+          rm -rf file_list.txt register_list.txt vault_data.txt
+
+          python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local public Files: {sys.argv[1]} and vault public files: {sys.argv[2]} are Not Equal"' $NUM_OF_PUBLIC_FILES $NUM_OF_PUBLIC_FILES_IN_VAULT
+          python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local private Files: {sys.argv[1]} and vault private files: {sys.argv[2]} are Not Equal"' $NUM_OF_PRIVATE_FILES $NUM_OF_PRIVATE_FILES_IN_VAULT
+          # python3 -c 'import sys; assert sys.argv[1] == sys.argv[2], f"Error: Local registers: {sys.argv[1]} and vault registers: {sys.argv[2]} are Not Equal"' $NUM_OF_REGISTERS $NUM_OF_REGISTERS_IN_VAULT
+          echo "vault synced successfully!"
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 15
+
+      - name: load an existing vault from the network
+        if: matrix.os != 'windows-latest'
+        run: ./target/release/autonomi --log-output-dest=data-dir vault load
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 2
+
+      - name: Time profiling for Different files
+        if: matrix.os != 'windows-latest'
+        run: |
+          # 1 MB
+          python3 -c "with open('random_1MB.bin', 'wb') as f: f.write(bytearray([0xff] * 1 * 1024 * 1024))"
+          # 10 MB
+          python3 -c "with open('random_10MB.bin', 'wb') as f: f.write(bytearray([0xff] * 10 * 1024 * 1024))"
+          # 100 MB
+          python3 -c "with open('random_100MB.bin', 'wb') as f: f.write(bytearray([0xff] * 100 * 1024 * 1024))"
+          # 1 GB
+          python3 -c "with open('random_1GB.bin', 'wb') as f: f.write(bytearray([0xff] * 1000 * 1024 * 1024))"
+
+          ./target/release/autonomi --log-output-dest=data-dir file list
+          time ./target/release/autonomi --log-output-dest=data-dir file upload random_1MB.bin
+          time ./target/release/autonomi --log-output-dest=data-dir file upload random_10MB.bin
+          time ./target/release/autonomi --log-output-dest=data-dir file upload random_100MB.bin
+          time ./target/release/autonomi --log-output-dest=data-dir file upload random_1GB.bin
+          ./target/release/autonomi --log-output-dest=data-dir vault sync
+          rm -rf random*.bin
+          rm -rf ${{ matrix.safe_path }}/autonomi          
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 15
+
       - name: Stop the local network and upload logs
         if: always()
         uses: maidsafe/sn-local-testnet-action@main
@@ -1143,13 +1262,33 @@ jobs:
         shell: bash
         if: always()
 
+      - name: Confirming connection errors
+        shell: bash
+        timeout-minutes: 1
+        env:
+          NODE_DATA_PATH: /home/runner/.local/share/safe/node
+        run: |
+          incoming_connection_errors=$(rg "IncomingConnectionError" $NODE_DATA_PATH -c --stats | \
+            rg "(\d+) matches" | rg "\d+" -o) || { echo "Failed to find IncomingConnectionError error"; exit 0; }
+          if [ -z "$incoming_connection_errors" ]; then
+            echo "Doesn't find any IncomingConnectionError error !"
+          else
+            echo "Found $incoming_connection_errors IncomingConnectionError errors."
+          fi
+          if ! rg "UnexpectedEof" $NODE_DATA_PATH -c --stats; then
+            echo "Doesn't find any UnexpectedEof error !"
+          else
+            echo "Found errors."
+            exit 1
+          fi
+
       - name: Stop the local network and upload logs
         if: always()
         uses: maidsafe/sn-local-testnet-action@main
         with:
           action: stop
           platform: ubuntu-latest
-          log_file_prefix: safe_test_logs_large_file_upload
+          log_file_prefix: safe_test_logs_large_file_upload_no_ws
           build: true
 
   # replication_bench_with_heavy_upload:

.github/workflows/merge_websocket.yml

@@ -0,0 +1,162 @@
+name: Check before merge (websockets)
+
+on:
+  # tests must run for a PR to be valid and pass merge queue muster
+  # on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors
+  # the merge run checks should show on master and enable this clear test/passing history
+  merge_group:
+    branches: [main, alpha*, beta*, rc*]
+  pull_request:
+    branches: ["*"]
+
+env:
+  CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI.
+  WINSW_URL: https://github.com/winsw/winsw/releases/download/v3.0.0-alpha.11/WinSW-x64.exe
+  GENESIS_PK: 9377ab39708a59d02d09bfd3c9bc7548faab9e0c2a2700b9ac7d5c14f0842f0b4bb0df411b6abd3f1a92b9aa1ebf5c3d
+  GENESIS_SK: 5ec88891c1098a0fede5b98b07f8abc931d7247b7aa310d21ab430cc957f9f02
+
+jobs:
+  large_file_upload_test_with_ws:
+    if: "!startsWith(github.event.head_commit.message, 'chore(release):')"
+    name: Large file upload (websockets)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: install ripgrep
+        shell: bash
+        run: sudo apt-get install -y ripgrep
+
+      - name: Check the available space
+        run: |
+          df
+          echo "Home dir:"
+          du -sh /home/runner/
+          echo "Home subdirs:"
+          du -sh /home/runner/*/
+          echo "PWD:"
+          du -sh .
+          echo "PWD subdirs:"
+          du -sh */
+
+      - name: Download material (135MB)
+        shell: bash
+        run: |
+          mkdir test_data_1
+          cd test_data_1
+          wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safe-qiWithListeners-x86_64.tar.gz
+          wget https://sn-node.s3.eu-west-2.amazonaws.com/joshuef/Qi930/safenode-qiWithListeners-x86_64.tar.gz
+          ls -l
+          cd ..
+          tar -cvzf test_data_1.tar.gz test_data_1
+          ls -l
+
+      - name: Build binaries
+        run: cargo build --release --features local,websockets --bin safenode --bin autonomi
+        timeout-minutes: 30
+
+      - name: Start a local network
+        uses: maidsafe/sn-local-testnet-action@main
+        with:
+          action: start
+          enable-evm-testnet: true
+          node-path: target/release/safenode
+          platform: ubuntu-latest
+          build: true
+          sn-log: ""
+
+      - name: Check if SAFE_PEERS and EVM_NETWORK are set
+        shell: bash
+        run: |
+          if [[ -z "$SAFE_PEERS" ]]; then
+            echo "The SAFE_PEERS variable has not been set"
+            exit 1
+          elif [[ -z "$EVM_NETWORK" ]]; then
+            echo "The EVM_NETWORK variable has not been set"
+            exit 1
+          else
+            echo "SAFE_PEERS has been set to $SAFE_PEERS"
+            echo "EVM_NETWORK has been set to $EVM_NETWORK"
+          fi
+
+      - name: Check the available space post download
+        run: |
+          df
+          echo "Home dir:"
+          du -sh /home/runner/
+          echo "Home subdirs:"
+          du -sh /home/runner/*/
+          echo "PWD:"
+          du -sh .
+          echo "PWD subdirs:"
+          du -sh */
+
+      - name: export default secret key
+        run: echo "SECRET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" >> $GITHUB_ENV
+        shell: bash
+
+      - name: File upload
+        run: ./target/release/autonomi --log-output-dest=data-dir file upload "./test_data_1.tar.gz" > ./upload_output 2>&1
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 5
+
+      - name: showing the upload terminal output
+        run: cat upload_output
+        shell: bash
+        if: always()
+
+      - name: parse address
+        run: |
+          UPLOAD_ADDRESS=$(rg "At address: ([0-9a-f]*)" -o -r '$1' ./upload_output)
+          echo "UPLOAD_ADDRESS=$UPLOAD_ADDRESS" >> $GITHUB_ENV
+        shell: bash
+
+      - name: File Download
+        run: ./target/release/autonomi --log-output-dest=data-dir file download ${{ env.UPLOAD_ADDRESS }} ./downloaded_resources > ./download_output 2>&1
+        env:
+          SN_LOG: "v"
+        timeout-minutes: 5
+
+      - name: showing the download terminal output
+        run: |
+          cat download_output
+          ls -l
+          cd downloaded_resources
+          ls -l
+        shell: bash
+        if: always()
+
+      - name: Confirming connection errors
+        shell: bash
+        timeout-minutes: 1
+        env:
+          NODE_DATA_PATH: /home/runner/.local/share/safe/node
+        run: |
+          incoming_connection_errors=$(rg "IncomingConnectionError" $NODE_DATA_PATH -c --stats | \
+            rg "(\d+) matches" | rg "\d+" -o) || { echo "Failed to find IncomingConnectionError error"; exit 0; }
+          if [ -z "$incoming_connection_errors" ]; then
+            echo "Doesn't find any IncomingConnectionError error !"
+          else
+            echo "Found $incoming_connection_errors IncomingConnectionError errors."
+          fi
+          unexpected_eof_errors=$(rg "UnexpectedEof" $NODE_DATA_PATH -c --stats | \
+            rg "(\d+) matches" | rg "\d+" -o) || { echo "Failed to find UnexpectedEof error"; exit 0; }
+          if [ -z "$unexpected_eof_errors" ]; then
+            echo "Doesn't find any UnexpectedEof error !"
+          else
+            echo "Found $unexpected_eof_errors UnexpectedEof errors."
+          fi
+
+      - name: Stop the local network and upload logs
+        if: always()
+        uses: maidsafe/sn-local-testnet-action@main
+        with:
+          action: stop
+          platform: ubuntu-latest
+          log_file_prefix: safe_test_logs_large_file_upload_with_ws
+          build: true

.github/workflows/nightly_wan.yml

@@ -1,8 +1,9 @@
 name: Nightly -- Full WAN Network Tests
 
 on:
-  schedule:
-    - cron: "0 0 * * *"
+  # To do: this is broken, need to fix and enable later
+  # schedule:
+  #   - cron: "0 0 * * *"
   # enable as below for testing purpose.
   # pull_request:
   #   branches: ["*"]