Update install scripts

contributors/runbtc.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 5, 2024.
+
+Signed: runbtc

production/install

@@ -375,20 +375,30 @@ LIQUIDTESTNET_ASSET_REGISTRY_DB_NAME=asset_registry_testnet_db
 ##### OS packages #####
 #######################
 
+# packages needed to run this script
+DEBIAN_PREREQ_PKG=()
+DEBIAN_PREREQ_PKG+=(dialog)
+
 # packages needed for mempool ecosystem
 DEBIAN_PKG=()
-DEBIAN_PKG+=(zsh vim curl screen openssl python3 dialog cron)
+DEBIAN_PKG+=(zsh vim curl screen openssl python3 cron acl)
 DEBIAN_PKG+=(build-essential git git-lfs clang cmake jq)
 DEBIAN_PKG+=(autotools-dev autoconf automake pkg-config bsdmainutils)
 DEBIAN_PKG+=(libevent-dev libdb-dev libssl-dev libtool autotools-dev)
 DEBIAN_PKG+=(libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev)
 DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python3-certbot-nginx rsync ufw)
-DEBIAN_PKG+=(geoipupdate)
+DEBIAN_PKG+=(geoipupdate redis)
+DEBIAN_PKG+=(cargo)
+DEBIAN_PKG+=(openjdk-11-jdk)
 
 DEBIAN_UNFURL_PKG=()
 DEBIAN_UNFURL_PKG+=(cups chromium-bsu libatk1.0 libatk-bridge2.0 libxkbcommon-dev libxcomposite-dev)
 DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev libasound-dev)
 
+# packages needed to run this script
+FREEBSD_PREREQ_PKG=()
+FREEBSD_PREREQ_PKG+=(dialog)
+
 # packages needed for mempool ecosystem
 FREEBSD_PKG=()
 FREEBSD_PKG+=(zsh sudo git git-lfs screen curl wget calc neovim)
@@ -470,6 +480,18 @@ osPackageInstallAll()
     esac
 }
 
+osPackageInstallPreReq()
+{
+    case $OS in
+        FreeBSD)
+            osPackageInstall ${FREEBSD_PREREQ_PKG[@]}
+        ;;
+        Debian)
+            osPackageInstall ${DEBIAN_PREREQ_PKG[@]}
+        ;;
+    esac
+}
+
 # osUserCreate username home_directory main_groupname [additional_group]
 osUserCreate()
 {
@@ -633,6 +655,14 @@ zfsCreateFilesystems()
 ext4CreateDir()
 {
     mkdir -p "/backup" "${ELEMENTS_HOME}" "${BITCOIN_HOME}" "${MINFEE_HOME}" "${ELECTRS_HOME}" "${MEMPOOL_HOME}" "${MYSQL_HOME}" "${BITCOIN_ELECTRS_HOME}" "${ELEMENTS_HOME}/liquidv1" "${ELEMENTS_ELECTRS_HOME}"
+    mkdir -p /var/cache/nginx/services /var/cache/nginx/api
+    chown "${NGINX_USER}:${NGINX_GROUP}" /var/cache/nginx/services /var/cache/nginx/api
+
+    # create /bitcoin/socket with custom ACL for electrs unix sockets
+    mkdir -p "${BITCOIN_HOME}/socket"
+
+    # create /elements/socket with custom ACL for electrs unix sockets
+    mkdir -p "${ELEMENTS_HOME}/socket"
 
     # Bitcoin Mainnet
     if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
@@ -714,6 +744,11 @@ ext4CreateDir()
     fi
 }
 
+#################################
+# Install prerequisite packages #
+#################################
+
+osPackageInstallPreReq
 
 ##### Perform sanity checks before trying anything
 
@@ -1038,6 +1073,9 @@ case $OS in
         echo "[*] Installing syslog configuration"
         osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/linux/rsyslog.conf" /etc/rsyslog.d/10-mempool.conf
         osSudo "${ROOT_USER}" sed -i.orig -e 's/^\*\.\*;auth,authpriv\.none/*\.*;auth,authpriv\.none,local7\.none/' /etc/rsyslog.d/50-default.conf
+
+        echo "[*] Installing redis configuration"
+        osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/linux/redis.conf" /etc/redis/redis.conf
     ;;
 esac
 
@@ -1379,9 +1417,6 @@ if [ "${BISQ_INSTALL}" = ON ];then
             echo "[*] Cloning Bisq top-level repo"
             osSudo "${BISQ_USER}" git clone --branch "${BISQ_REPO_BRANCH}" "${BISQ_REPO_URL}" "${BISQ_HOME}/${BISQ_REPO_NAME}"
 
-            echo "[*] Installing OpenJDK from Bisq install_java_linux.sh script"
-            osSudo "${ROOT_USER}" "${BISQ_HOME}/${BISQ_REPO_NAME}/scripts/install_java_linux.sh"
-
             echo "[*] Checking out Bisq ${BISQ_LATEST_RELEASE}"
             osSudo "${BISQ_USER}" sh -c "cd ${BISQ_HOME}/${BISQ_REPO_NAME} && git checkout ${BISQ_LATEST_RELEASE}"
 
@@ -1402,28 +1437,38 @@ if [ "${BISQ_INSTALL}" = ON ];then
         ;;
 
         Debian)
+            echo "[*] Installing Bisq service file"
             osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/bisq.service" "${DEBIAN_SERVICE_HOME}/bisq.service"
-            osSudo "${ROOT_USER}" sed -i.orig "s/#Requires=bitcoin.service/Requires=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service"
-            osSudo "${ROOT_USER}" sed -i.orig "s/#BindsTo=bitcoin.service/BindsTo=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service"
+            if [ "${BITCOIN_INSTALL}" = ON ];then
+                osSudo "${ROOT_USER}" sed -i.orig "s/After=network.target/After=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service"
+                osSudo "${ROOT_USER}" sed -i.orig "s/#Requires=bitcoin.service/Requires=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service"
+                osSudo "${ROOT_USER}" sed -i.orig "s/#BindsTo=bitcoin.service/BindsTo=bitcoin.service/" "${DEBIAN_SERVICE_HOME}/bisq.service"
+            fi
             osSudo "${ROOT_USER}" sed -i.orig "s/__BISQ_REPO_NAME__/${BISQ_REPO_NAME}/" "${DEBIAN_SERVICE_HOME}/bisq.service"
             osSudo "${ROOT_USER}" sed -i.orig "s!__BISQ_HOME__!${BISQ_HOME}!" "${DEBIAN_SERVICE_HOME}/bisq.service"
 
             echo "[*] Installing Bisq environment file"
             osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/bisq.env" "${DEBIAN_ENV_HOME}/bisq.env"
-            osSudo "${ROOT_USER}" sed -i.orig "s!__BISQ_APP_NAME__!${BISQ_APP_NAME}!" "${DEBIAN_ENV_HOME}/bisq.env"
             osSudo "${ROOT_USER}" sed -i.orig "s!__BISQ_HOME__!${BISQ_HOME}!" "${DEBIAN_ENV_HOME}/bisq.env"
-
-            echo "[*] Configuring Bisq environment file with Bitcoin RPC credentials"
             osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_P2P_HOST__/${BITCOIN_MAINNET_P2P_HOST}/" "${DEBIAN_ENV_HOME}/bisq.env"
             osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_P2P_PORT__/${BITCOIN_MAINNET_P2P_PORT}/" "${DEBIAN_ENV_HOME}/bisq.env"
             osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_HOST__/${BITCOIN_MAINNET_RPC_HOST}/" "${DEBIAN_ENV_HOME}/bisq.env"
             osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PORT__/${BITCOIN_MAINNET_RPC_PORT}/" "${DEBIAN_ENV_HOME}/bisq.env"
             osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${DEBIAN_ENV_HOME}/bisq.env"
             osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${DEBIAN_ENV_HOME}/bisq.env"
-
-            #echo "[*] Updating Bitcoin configuration for Bisq"
-            #osSudo "${ROOT_USER}" sed -i.orig "s/#blocknotify/blocknotify/" "${BITCOIN_HOME}/bitcoin.conf"
-            #osSudo "${BITCOIN_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/blocknotify.sh" "${BITCOIN_HOME}/blocknotify.sh"
+            osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_BLOCKNOTIFY_HOST__/${BISQ_BLOCKNOTIFY_HOST}/" "${DEBIAN_ENV_HOME}/bisq.env"
+            osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_BLOCKNOTIFY_PORT__/${BISQ_BLOCKNOTIFY_PORT}/" "${DEBIAN_ENV_HOME}/bisq.env"
+            osSudo "${ROOT_USER}" sed -i.orig "s!BISQ_APP_NAME=bisq-seednode!BISQ_APP_NAME=bisq-statsnode!" "${DEBIAN_ENV_HOME}/bisq.env"
+            osSudo "${ROOT_USER}" sed -i.orig "s!BISQ_ENTRYPOINT=seednode/build/app/bin/bisq-seednode!BISQ_ENTRYPOINT=statsnode/build/app/bin/bisq-statsnode!" "${DEBIAN_ENV_HOME}/bisq.env"
+            osSudo "${ROOT_USER}" sed -i.orig "s/BISQ_DUMP_BLOCKCHAIN=false/BISQ_DUMP_BLOCKCHAIN=true/" "${DEBIAN_ENV_HOME}/bisq.env"
+            osSudo "${ROOT_USER}" sed -i.orig "s/BISQ_DUMP_STATISTICS=false/BISQ_DUMP_STATISTICS=true/" "${DEBIAN_ENV_HOME}/bisq.env"
+            osSudo "${ROOT_USER}" sed -i.orig "s!-Xmx4096M!-Xmx8192M!" "${DEBIAN_SERVICE_HOME}/bisq.env"
+
+            if [ "${BITCOIN_INSTALL}" = ON ];then
+                echo "[*] Updating Bitcoin configuration for Bisq"
+                osSudo "${ROOT_USER}" sed -i.orig "1iblocknotify=${BITCOIN_HOME}/blocknotify.sh %s" "${BITCOIN_HOME}/bitcoin.conf"
+                osSudo "${BITCOIN_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${BISQ_HOME}/${BISQ_REPO_NAME}/seednode/blocknotify.sh" "${BITCOIN_HOME}/blocknotify.sh"
+            fi
         ;;
 
     esac
@@ -1435,7 +1480,7 @@ fi
 
 if [ "${UNFURL_INSTALL}" = ON ];then
 
-    echo "[*] Insalling Unfurl source"
+    echo "[*] Installing Unfurl source"
     case $OS in
 
         FreeBSD)
@@ -1520,7 +1565,7 @@ fi
 # Bitcoin instance for Mainnet Minfee #
 #######################################
 
-if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
+if [ "${BITCOIN_MAINNET_MINFEE_ENABLE}" = ON ];then
     echo "[*] Installing Bitcoin Minfee service"
     case $OS in
 
@@ -1588,8 +1633,8 @@ fi
 # Bitcoin instance for Liquid Testnet #
 #######################################
 
-if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
-    echo "[*] Installing Bitcoin Liquid service"
+if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
+    echo "[*] Installing Bitcoin Liquid Testnet service"
     case $OS in
 
         FreeBSD)
@@ -1880,13 +1925,25 @@ echo "[*] Setting permissions for electrs sockets"
 case $OS in
 
     FreeBSD)
-        setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
-        chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
-        setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
-        chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"
+        if [ "${BITCOIN_INSTALL}" = ON ];then
+            setfacl -m "user:bitcoin:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${BITCOIN_HOME}/socket"
+            chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
+        fi
+        if [ "${ELEMENTS_INSTALL}" = ON ];then
+            setfacl -m "user:elements:full_set:f:allow,user:mempool:full_set:f:allow,user:www:full_set:f:allow,everyone@::f:allow" "${ELEMENTS_HOME}/socket"
+            chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"
+        fi
     ;;
 
     Debian)
+        if [ "${BITCOIN_INSTALL}" = ON ];then
+            setfacl -Rdm "u:bitcoin:rwx,u:mempool:rwx,u:www-data:rwx,o::rw-" "${BITCOIN_HOME}/socket"
+            chown "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_HOME}/socket"
+        fi
+        if [ "${ELEMENTS_INSTALL}" = ON ];then
+            setfacl -Rdm "u:elements:rwx,u:mempool:rwx,u:www-data:rwx,o::rw-" "${ELEMENTS_HOME}/socket"
+            chown "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}/socket"
+        fi
     ;;
 esac
 

production/linux/redis.conf

@@ -0,0 +1,79 @@
+save 3600 1 300 100 60 10000
+unixsocket /var/run/redis/redis.sock
+unixsocketperm 666
+
+bind 127.0.0.1 -::1
+protected-mode yes
+port 6379
+tcp-backlog 511
+timeout 0
+tcp-keepalive 300
+daemonize yes
+pidfile /var/run/redis/redis.pid
+loglevel notice
+logfile /var/log/redis/redis.log
+databases 16
+always-show-logo no
+set-proc-title yes
+proc-title-template "{title} {listen-addr} {server-mode}"
+#locale-collate ""
+stop-writes-on-bgsave-error yes
+rdbcompression yes
+rdbchecksum yes
+dbfilename dump.rdb
+rdb-del-sync-files no
+dir /var/lib/redis/
+replica-serve-stale-data yes
+replica-read-only yes
+repl-diskless-sync yes
+repl-diskless-sync-delay 5
+repl-diskless-sync-max-replicas 0
+repl-diskless-load disabled
+repl-disable-tcp-nodelay no
+replica-priority 100
+acllog-max-len 128
+lazyfree-lazy-eviction no
+lazyfree-lazy-expire no
+lazyfree-lazy-server-del no
+replica-lazy-flush no
+lazyfree-lazy-user-del no
+lazyfree-lazy-user-flush no
+oom-score-adj no
+oom-score-adj-values 0 200 800
+disable-thp yes
+appendonly no
+appendfilename "appendonly.aof"
+appenddirname "appendonlydir"
+appendfsync everysec
+no-appendfsync-on-rewrite no
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb
+aof-load-truncated yes
+aof-use-rdb-preamble yes
+aof-timestamp-enabled no
+
+slowlog-log-slower-than 10000
+slowlog-max-len 128
+latency-monitor-threshold 0
+notify-keyspace-events ""
+hash-max-listpack-entries 512
+hash-max-listpack-value 64
+list-max-listpack-size -2
+list-compress-depth 0
+set-max-intset-entries 512
+#set-max-listpack-entries 128
+#set-max-listpack-value 64
+zset-max-listpack-entries 128
+zset-max-listpack-value 64
+hll-sparse-max-bytes 3000
+stream-node-max-bytes 4096
+stream-node-max-entries 100
+activerehashing yes
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit replica 256mb 64mb 60
+client-output-buffer-limit pubsub 32mb 8mb 60
+hz 10
+dynamic-hz yes
+aof-rewrite-incremental-fsync yes
+rdb-save-incremental-fsync yes
+jemalloc-bg-thread yes

production/mempool-build-all

@@ -1,4 +1,19 @@
 #!/usr/bin/env zsh
+
+case `uname -s` in
+
+    FreeBSD)
+        DB_SOCKET=/var/run/mysql/mysql.sock
+        REDIS_SOCKET=/tmp/redis.sock
+    ;;
+
+    Linux)
+        DB_SOCKET=/run/mysqld/mysqld.sock
+        REDIS_SOCKET=/var/run/redis/redis.sock
+    ;;
+
+esac
+
 PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
 HOSTNAME=$(hostname)
 LOCATION=$(hostname|cut -d . -f2)
@@ -110,6 +125,8 @@ build_backend()
         -e "s!__MEMPOOL_LIQUIDTESTNET_PASS__!${MEMPOOL_LIQUIDTESTNET_PASS}!" \
         -e "s!__MEMPOOL_BISQ_USER__!${MEMPOOL_BISQ_USER}!" \
         -e "s!__MEMPOOL_BISQ_PASS__!${MEMPOOL_BISQ_PASS}!" \
+        -e "s!__DB_SOCKET__!${DB_SOCKET}!" \
+        -e "s!__REDIS_SOCKET__!${REDIS_SOCKET}!" \
 	"mempool-config.json"
     fi
     npm install --omit=dev --omit=optional || exit 1
@@ -170,8 +187,4 @@ for target in mainnet liquid bisq;do
     ship_frontend "${target}"
 done
 
-# notify everyone
-echo "${HOSTNAME} updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
-echo "${HOSTNAME} updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
-
 exit 0

production/mempool-config.bisq.json

@@ -1,6 +1,6 @@
 {
   "MEMPOOL": {
-    "OFFICIAL": true,
+    "OFFICIAL": false,
     "NETWORK": "bisq",
     "BACKEND": "esplora",
     "HTTP_PORT": 8996,
@@ -20,9 +20,9 @@
     "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
   },
   "DATABASE": {
-    "ENABLED": false,
+    "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "SOCKET": "__DB_SOCKET__",
     "USERNAME": "__MEMPOOL_BISQ_USER__",
     "PASSWORD": "__MEMPOOL_BISQ_PASS__",
     "DATABASE": "mempool_bisq"
@@ -33,6 +33,6 @@
   },
   "BISQ": {
     "ENABLED": true,
-    "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
+    "DATA_PATH": "/bisq/bisq-statsnode/btc_mainnet/db"
   }
 }