chore: easy setup fleets for lpt

Makefile

@@ -342,6 +342,33 @@ docker-image:
 docker-push:
 	docker push $(DOCKER_IMAGE_NAME)
 
+####################################
+## Container lite-protocol-tester ##
+####################################
+# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
+# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
+DOCKER_LPT_NIMFLAGS ?= -d:chronicles_colors:none -d:insecure
+
+# build a docker image for the fleet
+docker-liteprotocoltester: DOCKER_LPT_TAG ?= latest
+docker-liteprotocoltester: DOCKER_LPT_NAME ?= wakuorg/liteprotocoltester:$(DOCKER_LPT_TAG)
+docker-liteprotocoltester:
+	docker build \
+	  --no-cache \
+		--build-arg="MAKE_TARGET=liteprotocoltester" \
+		--build-arg="NIMFLAGS=$(DOCKER_LPT_NIMFLAGS)" \
+		--build-arg="NIM_COMMIT=$(DOCKER_NIM_COMMIT)" \
+		--build-arg="LOG_LEVEL=TRACE" \
+		--label="commit=$(shell git rev-parse HEAD)" \
+		--label="version=$(GIT_VERSION)" \
+		--target $(TARGET) \
+		--tag $(DOCKER_LPT_NAME) \
+		--file apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile \
+		.
+
+docker-liteprotocoltester-push:
+	docker push $(DOCKER_LPT_NAME)
+
 
 ################
 ## C Bindings ##

apps/liteprotocoltester/.env

@@ -17,11 +17,11 @@ MAX_MESSAGE_SIZE=145Kb
 #CLUSTER_ID=66
 
 ## for status.prod
-#PUBSUB=/waku/2/rs/16/32
-#CONTENT_TOPIC=/tester/2/light-pubsub-test/fleet
-#CLUSTER_ID=16
+PUBSUB=/waku/2/rs/16/32
+CONTENT_TOPIC=/tester/2/light-pubsub-test/fleet
+CLUSTER_ID=16
 
 ## for TWN
-PUBSUB=/waku/2/rs/1/4
-CONTENT_TOPIC=/tester/2/light-pubsub-test/twn
-CLUSTER_ID=1
+#PUBSUB=/waku/2/rs/1/4
+#CONTENT_TOPIC=/tester/2/light-pubsub-test/twn
+#CLUSTER_ID=1

apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile

@@ -1,58 +1,57 @@
 # BUILD NIM APP ----------------------------------------------------------------
-  FROM rust:1.77.1-alpine3.18  AS nim-build
+FROM rust:1.77.1-alpine3.18  AS nim-build
 
-  ARG NIMFLAGS
-  ARG MAKE_TARGET=liteprotocoltester
-  ARG NIM_COMMIT
-  ARG LOG_LEVEL=DEBUG
+ARG NIMFLAGS
+ARG MAKE_TARGET=liteprotocoltester
+ARG NIM_COMMIT
+ARG LOG_LEVEL=TRACE
 
-  # Get build tools and required header files
-  RUN apk add --no-cache bash git build-base pcre-dev linux-headers curl jq
+# Get build tools and required header files
+RUN apk add --no-cache bash git build-base openssl-dev pcre-dev linux-headers curl jq
 
-  WORKDIR /app
-  COPY . .
+WORKDIR /app
+COPY . .
 
-  # workaround for alpine issue: https://github.com/alpinelinux/docker-alpine/issues/383
-  RUN apk update && apk upgrade
+# workaround for alpine issue: https://github.com/alpinelinux/docker-alpine/issues/383
+RUN apk update && apk upgrade
 
-  # Ran separately from 'make' to avoid re-doing
-  RUN git submodule update --init --recursive
+# Ran separately from 'make' to avoid re-doing
+RUN git submodule update --init --recursive
 
-  # Slowest build step for the sake of caching layers
-  RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT}
+# Slowest build step for the sake of caching layers
+RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT}
 
-  # Build the final node binary
-  RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET LOG_LEVEL=${LOG_LEVEL} NIMFLAGS="${NIMFLAGS}"
+# Build the final node binary
+RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET LOG_LEVEL=${LOG_LEVEL} NIMFLAGS="${NIMFLAGS}"
 
 
-  # PRODUCTION IMAGE -------------------------------------------------------------
+# PRODUCTION IMAGE -------------------------------------------------------------
 
-  FROM alpine:3.18 AS prod
+FROM alpine:3.18 AS prod
 
-  ARG MAKE_TARGET=liteprotocoltester
+ARG MAKE_TARGET=liteprotocoltester
 
-  LABEL maintainer="jakub@status.im"
-  LABEL source="https://github.com/waku-org/nwaku"
-  LABEL description="Lite Protocol Tester: Waku light-client"
-  LABEL commit="unknown"
-  LABEL version="unknown"
+LABEL maintainer="zoltan@status.im"
+LABEL source="https://github.com/waku-org/nwaku"
+LABEL description="Lite Protocol Tester: Waku light-client"
+LABEL commit="unknown"
+LABEL version="unknown"
 
-  # DevP2P, LibP2P, and JSON RPC ports
-  EXPOSE 30303 60000 8545
+# DevP2P, LibP2P, and JSON RPC ports
+EXPOSE 30303 60000 8545
 
-  # Referenced in the binary
-  RUN apk add --no-cache libgcc pcre-dev libpq-dev
+# Referenced in the binary
+RUN apk add --no-cache libgcc pcre-dev libpq-dev \
+  wget \
+  iproute2
 
-  # Fix for 'Error loading shared library libpcre.so.3: No such file or directory'
-  RUN ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3
+# Fix for 'Error loading shared library libpcre.so.3: No such file or directory'
+RUN ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3
 
-  # Copy to separate location to accomodate different MAKE_TARGET values
-  COPY --from=nim-build /app/build/$MAKE_TARGET /usr/bin/
+COPY --from=nim-build /app/build/liteprotocoltester /usr/bin/
+COPY --from=nim-build /app/apps/liteprotocoltester/run_tester_node.sh /usr/bin/
 
-  # Copy migration scripts for DB upgrades
-  COPY --from=nim-build /app/migrations/ /app/migrations/
+ENTRYPOINT ["/usr/bin/run_tester_node.sh", "/usr/bin/liteprotocoltester"]
 
-  ENTRYPOINT ["/usr/bin/liteprotocoltester"]
-
-  # By default just show help if called without arguments
-  CMD ["--help"]
+# # By default just show help if called without arguments
+CMD ["--help"]

apps/liteprotocoltester/README.md

@@ -17,43 +17,26 @@ and multiple receivers.
 
 Publishers are fill all message payloads with information about the test message and sender, helping the receiver side to calculate results.
 
-## Phases of development
-
-### Phase 1
-
-At the first phase we aims to demonstrate the concept of the testing all boundled into a docker-compose environment where we run
-one service (full)node and a publisher and a receiver node.
-At this stage we can only configure number of messages and fixed frequency of the message pump. We do not expect message losses and any significant latency hence the test setup is very simple.
-
-### Further plans
-
-- Add more configurability (randomized message sizes, usage of more content topics and support for static sharding).
-- Extend collected metrics and polish reporting.
-  - Add test metrics to graphana dashboard.
-- Support for static sharding and auto sharding for being able to test under different conditions.
-- ...
-
 ## Usage
 
-### Phase 1
-
-> NOTICE: This part is obsolate due integration with waku-simulator.
-> It needs some rework to make it work again standalone.
+### Using lpt-runner
 
-Lite Protocol Tester application is built under name `liteprotocoltester` in apps/liteprotocoltester folder.
+For ease of use, you can clone lpt-runner repository. That will utilize previously pushed liteprotocoltester docker image.
+It is recommended to use this method for fleet testing.
 
-Starting from nwaku repository root:
 ```bash
-make liteprotocoltester
-cd apps/liteprotocoltester
-docker compose build
+git clone https://github.com/waku-org/lpt-runner.git
+cd lpt-runner
+
+# check Reame.md for more information
+# edit .env file to your needs
+
 docker compose up -d
-docker compose logs -f receivernode
-```
 
-### Phase 2
+# navigate localhost:3033 to see the lite-protocol-tester dashboard
+```
 
-> Integration with waku-simulator!
+### Integration with waku-simulator!
 
 - For convenience, integration is done in cooperation with waku-simulator repository, but nothing is tightly coupled.
 - waku-simulator must be started separately with its own configuration.
@@ -100,9 +83,7 @@ docker compose -f docker-compose-on-simularor.yml logs -f receivernode
 
 Navigate to http://localhost:3033 to see the lite-protocol-tester dashboard.
 
-### Phase 3
-
-> Run independently on a chosen waku fleet
+### Run independently on a chosen waku fleet
 
 This option is simple as is just to run the built liteprotocoltester binary with run_tester_node.sh script.
 
@@ -136,7 +117,7 @@ Run a SENDER role liteprotocoltester and a RECEIVER role one on different termin
 
 > RECEIVER side will periodically print statistics to standard output.
 
-## Configure
+## Configuration
 
 ### Environment variables for docker compose runs
 
@@ -158,6 +139,7 @@ Run a SENDER role liteprotocoltester and a RECEIVER role one on different termin
 | :--- | :--- | :--- |
 | --test_func | separation of PUBLISHER or RECEIVER mode | RECEIVER |
 | --service-node| Address of the service node to use for lightpush and/or filter service | - |
+| --bootstrap-node| Address of the fleet's bootstrap node to use to determine service peer randomly choosen from the network. `--service-node` switch has precedence over this | - |
 | --num-messages | Number of message to publish | 120 |
 | --delay-messages | Frequency of messages in milliseconds | 1000 |
 | --min-message-size | Minimum message size in bytes | 1KiB |
@@ -174,21 +156,28 @@ Run a SENDER role liteprotocoltester and a RECEIVER role one on different termin
 | --log-level | Log level for the application | DEBUG |
 | --log-format | Logging output format (TEXT or JSON) | TEXT |
 
+### Specifying peer addresses
+
+Service node or bootstrap addresses can be specified in multiadress or ENR form.
 
+### Using bootstrap nodes
+
+There are multiple benefits of using bootstrap nodes. By using them liteprotocoltester will use Peer Exchange protocol to get possible peers from the network that are capable to serve as service peers for testing. Additionally it will test dial them to verify their connectivity - this will be reported in the logs and on dashboard metrics.
+Also by using bootstrap node and peer exchange discovery, litprotocoltester will be able to simulate service peer switch in case of failures. There are built in tresholds for service peer failures during test and service peer can be switched during the test. Also these service peer failures are reported, thus extening network reliability measures.
 
 ### Docker image notice
 
 #### Building for docker compose runs on simulator or standalone
 Please note that currently to ease testing and development tester application docker image is based on ubuntu and uses the externally pre-built binary of 'liteprotocoltester'.
 This speeds up image creation. Another dokcer build file is provided for proper build of boundle image.
 
-> `Dockerfile.liteprotocoltester.copy` will create an image with the binary copied from the build directory.
+> `Dockerfile.liteprotocoltester` will create an ubuntu based image with the binary copied from the build directory.
 
-> `Dockerfile.liteprotocoltester.compile` will create an image completely compiled from source. This can be quite slow.
+> `Dockerfile.liteprotocoltester.compile` will create an ubuntu based image completely compiled from source. This can be slow.
 
 #### Creating standalone runner docker image
 
-To ease the work with lite-proto-tester, a docker image is possible to build.
+To ease the work with lite-protocol-tester, a docker image is possible to build.
 With that image it is easy to run the application in a container.
 
 > `Dockerfile.liteprotocoltester` will create an ubuntu image with the binary copied from the build directory. You need to pre-build the application.
@@ -205,7 +194,17 @@ docker build -t liteprotocoltester:latest -f Dockerfile.liteprotocoltester ../..
 
 # edit and adjust .env file to your needs and for the network configuration
 
-docker run --env-file .env liteprotocoltester:latest RECEIVER <service-node-ip4-peer-address>
+docker run --env-file .env liteprotocoltester:latest RECEIVER <service-node-peer-address>
 
-docker run --env-file .env liteprotocoltester:latest SENDER <service-node-ip4-peer-address>
+docker run --env-file .env liteprotocoltester:latest SENDER <service-node-peer-address>
 ```
+
+#### Run test with auto service peer selection from a fleet using bootstrap node
+
+```bash
+
+docker run --env-file .env liteprotocoltester:latest RECEIVER <bootstrap-node-peer-address> BOOTSTRAP
+
+docker run --env-file .env liteprotocoltester:latest SENDER <bootstrap-node-peer-address> BOOTSTRAP
+```
+