Post 1.1.0 updates

.github/workflows/wasm-package.yml

@@ -27,7 +27,7 @@ jobs:
       - run: sudo contrib/bullseye_deps.sh -j -n -w
 
       # Build NPM package into a tgz file (pack internally triggers the build/prepare script)
-      - run: cd src/wasm_package && npm install && npm pack --foreground-scripts
+      - run: cd src/wasm_package && npm ci && npm pack --foreground-scripts
         name: Build & Pack
 
       # Report the SHA256 digest of the final package. This should be deterministic (including generated WASM),

.gitlab-ci.yml

@@ -2,7 +2,7 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
 
 build_wally_release_files:
-  image: greenaddress/wallycore@sha256:21112c3d05f0f541a43636bb5c7e1826292f9c66fca0b68546b963adac4fc4c3
+  image: greenaddress/wallycore@sha256:f159b338f3d25e13958d79fbd709238e3864de5468b9d743625466a0a681e52b
   artifacts:
     expire_in: 7 days
     name: wallycore-bindings
@@ -36,7 +36,7 @@ build_wally_release_files:
     - rmdir wally_dist
 
 build_mingw_static:
-  image: greenaddress/wallycore@sha256:21112c3d05f0f541a43636bb5c7e1826292f9c66fca0b68546b963adac4fc4c3
+  image: greenaddress/wallycore@sha256:f159b338f3d25e13958d79fbd709238e3864de5468b9d743625466a0a681e52b
   tags:
     - ga
   script:
@@ -45,7 +45,7 @@ build_mingw_static:
     - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
 
 run_tests:
-  image: greenaddress/wallycore@sha256:21112c3d05f0f541a43636bb5c7e1826292f9c66fca0b68546b963adac4fc4c3
+  image: greenaddress/wallycore@sha256:f159b338f3d25e13958d79fbd709238e3864de5468b9d743625466a0a681e52b
   tags:
     - ga
   artifacts:
@@ -59,3 +59,24 @@ run_tests:
     - for t in $(ls src/.libs/test_* | egrep -v '_clear|xml|json' | tr '\n' ' '); do LD_LIBRARY_PATH=./src/.libs/ valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose --xml=yes --xml-file=$t.xml $t; done
     - for t in $(ls src/.libs/test_* | egrep -v '_clear|xml|json' | tr '\n' ' '); do valgrind-codequality --input-file $t.xml --output-file $t.json; done
     - jq '[.[]|.[]]' -s ./src/.libs/test_*.json > valgrind.json
+
+run_asan_ubsan_gcc:
+  image: greenaddress/wallycore@sha256:f159b338f3d25e13958d79fbd709238e3864de5468b9d743625466a0a681e52b
+  tags:
+    - ga
+  script:
+    - ./tools/cleanup.sh && ./tools/autogen.sh
+    - CC=gcc CFLAGS="-O1 -fsanitize=address -fsanitize=undefined -fsanitize=alignment -fsanitize-address-use-after-scope -fno-sanitize-recover=all" ./configure --enable-export-all --enable-swig-python --enable-swig-java --enable-shared --disable-static --disable-clear-tests --disable-asm
+    - sed -i 's/^PYTHON = /PYTHON = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/10\/libasan.so /g' src/Makefile
+    - sed -i 's/^JAVA = /JAVA = LD_PRELOAD=\/usr\/lib\/gcc\/x86_64-linux-gnu\/10\/libasan.so /g' src/Makefile
+    - make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))
+    - ASAN_OPTIONS=abort_on_error=1:fast_unwind_on_malloc=0:detect_leaks=0 UBSAN_OPTIONS=print_stacktrace=1 make check V=1
+
+run_scan_build_clang:
+  image: greenaddress/wallycore@sha256:f159b338f3d25e13958d79fbd709238e3864de5468b9d743625466a0a681e52b
+  tags:
+    - ga
+  script:
+    - ./tools/cleanup.sh && ./tools/autogen.sh
+    - CC=clang CFLAGS="-O0" scan-build-11 ./configure --enable-export-all --enable-swig-python --enable-swig-java --disable-clear-tests --disable-asm
+    - scan-build-11 --keep-cc --exclude src/secp256k1/ make -j $(($(grep ^processor /proc/cpuinfo | wc -l) / 2))

CHANGES.md

@@ -1,5 +1,16 @@
 # Changes
 
+## Unreleased
+
+### Added
+
+### Changed
+
+- Javascript: The npm build now uses nodejs 20, as nodejs 16 is end-of-life.
+
+### Fixed
+
+
 ## Version 1.1.0
 
 ### Added

README.md

@@ -158,7 +158,7 @@ the Android NDK. The script `tools/android_helpers.sh` can be sourced from
 the shell or scripts to make it easier to produce builds:
 
 ```
-$ export ANDROID_NDK=/opt/android-ndk-r23b # r22 is the minimum supported version
+$ export ANDROID_NDK=/opt/android-ndk-r26b # r22 is the minimum supported version
 $ . ./tools/android_helpers.sh
 
 $ android_get_arch_list

contrib/Dockerfile_bullseye

@@ -3,10 +3,10 @@
 # build from this directory with e.g:
 #     DOCKER_BUILDKIT=1 docker build . -t greenaddress/wallycore -f Dockerfile_bullseye
 #
-FROM debian:bullseye@sha256:d0bf7d85ad1e61ab3520b1d22d1e2a136799defd4e0d1e3f998d3b9045f38551
+FROM debian:bullseye@sha256:71cb300d5448af821aedfe63afd55ba05f45a6a79f00dcd131b96b780bb99fe4
 WORKDIR /root
 COPY bullseye_deps.sh ./deps.sh
 COPY requirements.txt ./contrib/requirements.txt
 RUN ./deps.sh && rm ./deps.sh
 ENV JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64
-ENV ANDROID_NDK=/opt/android-ndk-r23b
+ENV ANDROID_NDK=/opt/android-ndk-r26b

contrib/bullseye_deps.sh

@@ -45,17 +45,24 @@ pip3 install valgrind-codequality -r contrib/requirements.txt
 pushd /opt
 
 if [ -z "$skip_ndk" ]; then
-    curl -L -o ndk.zip https://dl.google.com/android/repository/android-ndk-r23b-linux.zip
-    echo "c6e97f9c8cfe5b7be0a9e6c15af8e7a179475b7ded23e2d1c1fa0945d6fb4382 ndk.zip" | sha256sum --check
+    curl -L -o ndk.zip https://dl.google.com/android/repository/android-ndk-r26b-linux.zip
+    echo "ad73c0370f0b0a87d1671ed2fd5a9ac9acfd1eb5c43a7fbfbd330f85d19dd632  ndk.zip" | shasum -a 256 -c
     unzip ndk.zip
     rm ndk.zip
 fi
 
 if [ -z "$skip_emsdk" ]; then
+    # Install node 20
+    curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
+    apt install nodejs -yqq
+    # Install emsdk
     git clone https://github.com/emscripten-core/emsdk
     cd emsdk
-    ./emsdk install 3.1.27
-    ./emsdk activate 3.1.27
+    ./emsdk install 3.1.52
+    ./emsdk activate 3.1.52
+    # Force emsdk to use the installed node version instead of its own
+    sed -i "s/^NODE_JS = .*$/NODE_JS = 'node'/g" /opt/emsdk/.emscripten
+    # Make emsdk commands available
     source ./emsdk_env.sh
 fi
 

contrib/requirements.txt

@@ -1,7 +1,7 @@
 build
-sphinx<2
-sphinx-rtd-theme<0.5
-readthedocs-sphinx-ext<2.2
-docutils<0.18
-mkdocs==1.2.3
-jinja2==3.0.0
+sphinx
+sphinx-rtd-theme
+readthedocs-sphinx-ext
+docutils
+mkdocs
+jinja2

include/wally_descriptor.h

@@ -360,7 +360,7 @@ WALLY_CORE_API int wally_descriptor_to_script(
  * :param multi_index: The multi-path item to generate. See `wally_descriptor_get_num_paths`.
  * :param child_num: The BIP32 child number to derive, or zero for static descriptors.
  * :param flags: For future use. Must be 0.
- * :param output: Destination for the resulting addresss.
+ * :param output: Destination for the resulting address.
  *|    The string returned should be freed using `wally_free_string`.
  */
 WALLY_CORE_API int wally_descriptor_to_address(

src/aes.c

@@ -75,7 +75,7 @@ int wally_aes_len(const unsigned char *key, size_t key_len,
                   const unsigned char *bytes, size_t bytes_len,
                   uint32_t flags, size_t *written)
 {
-    if (!written)
+    if (written)
         *written = 0;
     if (!are_valid_args(key, key_len, bytes, bytes_len, flags) ||
         !bytes_len || bytes_len % AES_BLOCK_LEN || !written)

src/bip32.c

@@ -149,7 +149,7 @@ static int path_from_str_n(const char *str, size_t str_len,
             if (multi < multi_index && i == start + 1)
                 goto fail; /* No number found */
         }
-        while (str[i] >= '0' && str[i] <= '9' && i < str_len) {
+        while (i < str_len && str[i] >= '0' && str[i] <= '9') {
             v = v * 10 + (str[i] - '0');
             if (v >= BIP32_INITIAL_HARDENED_CHILD)
                 goto fail; /* Derivation index too large */
@@ -185,7 +185,7 @@ static int path_from_str_n(const char *str, size_t str_len,
             ++i;
             v = child_num; /* Use the given child number for the wildcard value */
         }
-        if (is_hardened_indicator(str[i], allow_upper, features)) {
+        if (i < str_len && is_hardened_indicator(str[i], allow_upper, features)) {
             v |= BIP32_INITIAL_HARDENED_CHILD;
             ++i;
         }

src/internal.c

@@ -434,7 +434,9 @@ char *wally_strdup_n(const char *str, size_t str_len)
 {
     char *new_str = (char *)wally_malloc(str_len + 1);
     if (new_str) {
-        memcpy(new_str, str, str_len);
+        if (str_len) {
+            memcpy(new_str, str, str_len);
+        }
         new_str[str_len] = '\0';
     }
     return new_str;

src/psbt.c

@@ -3470,7 +3470,7 @@ static int merge_value_commitment(struct wally_map *dst_fields, uint64_t *dst_am
         if (ret != WALLY_OK)
             return ret;
         if (!for_clone) {
-            /* Not cloning: clear the amount when we have a committment to it */
+            /* Not cloning: clear the amount when we have a commitment to it */
             *dst_amount = 0;
         }
     }

src/test/test_transaction.py

@@ -394,7 +394,7 @@ def test_hash_prevouts(self):
         cases = [
             (None,     len(txhashes), indices, len(indices), out,  out_len), # NULL txhashes
             (txhashes, 0,             indices, len(indices), out,  out_len), # Zero hash len
-            (txhashes, 16,            indices, len(indices), out,  out_len), # Incorect hash len
+            (txhashes, 16,            indices, len(indices), out,  out_len), # Incorrect hash len
             (txhashes, len(txhashes), None,    len(indices), out,  out_len), # NULL indices
             (txhashes, len(txhashes), indices, 0,            out,  out_len), # Zero num indices
             (txhashes, len(txhashes), indices, 1,            out,  out_len), # Num indices != num hashes

src/wasm_package/build.sh

@@ -1,12 +1,6 @@
 #! /usr/bin/env bash
 set -xeo pipefail
 
-# Update function list in wasm_exports.sh and run codegen for the public API in functions.js
-(cd ../.. && ./tools/build_wrappers.py)
-
-# Update WASM package constants and version to match libwally
-(cd ../.. && ./tools/update_wasm_package.sh)
-
 # Build WASM (Note Elements is always enabled)
 (cd ../.. && ./tools/build_wasm.sh)
 mkdir -p libwally_wasm && cp ../../dist/wallycore.{js,wasm} libwally_wasm/

tools/build_android_libraries.sh

@@ -30,11 +30,8 @@ if [ -n "$1" ]; then
 fi
 
 for arch in $ARCH_LIST; do
-    # Use API level 19 for non-64 bit targets for better device coverage
-    api="19"
-    if [[ $arch == *"64"* ]]; then
-        api="21"
-    fi
+    # Use API level 23
+    api="23"
 
     # Location of the NDK tools to build with
     toolsdir=$(android_get_build_tools_dir)
@@ -52,7 +49,7 @@ for arch in $ARCH_LIST; do
 done
 
 # Copy headers and Java wrapper
-# The andoid release files can be used from Java or in native code
+# The android release files can be used from Java or in native code
 mkdir -p $PWD/release/include $PWD/release/src/swig_java/src/com/blockstream/libwally
 cp $PWD/include/*.h $PWD/release/include
 cp $PWD/src/swig_java/src/com/blockstream/libwally/Wally.java $PWD/release/src/swig_java/src/com/blockstream/libwally

tools/cleanup.sh

@@ -13,6 +13,7 @@ find . -name "__pycache__" -exec rm -rf {} 2>/dev/null \;
 
 rm -f */*~
 rm -f *~
+rm -f a.*
 rm -f aclocal.m4
 rm -rf build/
 rm -f config.h.in
@@ -34,6 +35,9 @@ rm -f src/swig_python/wallycore/__init__.py*
 rm -f src/swig_python/swig_python_wrap.c
 rm -rf src/.libs
 rm -f src/secp256k1/build-aux/ltmain.sh-e
+rm -f src/secp256k1/a.*
+rm -rf src/wasm_package/browser-dist/
+rm -f src/wasm_package/wallycore*.tgz
 rm -f tools/build-aux/ar-lib
 rm -f tools/build-aux/compile
 rm -f tools/build-aux/config.guess

tools/update_wasm_package.sh

@@ -16,4 +16,4 @@ current_version=$(grep -oP 'AC_INIT\(\[libwallycore\],\[\K[^\]]+' configure.ac)
 
 
 # Calling build_wrappers.py is also necessary to update the wasm functions.js file.
-# This is handled by update_generated.sh and by the wasm package build script.
+# This is handled by update_generated.sh which calls this script.