Skip to content

Commit

Permalink
chore: easy setup fleets for lpt (#3125)
Browse files Browse the repository at this point in the history 
* Added bootstrap peer exchange discovery option for easy setup ltp
* Extended with PX discovery, auto-dial of PX cap peers, added switching service peers if failed with original
* Added peer-exchange, found capable peers test, metrics on peer stability and availability, dashboard adjustments
* Updated and actualized README.md for liteprotocoltester
* Created jenkinsfile for liteprotocoltester deployment
* Fixed dial exception during lightpublish
* Add configuration for requesting and testing peer exchange peers
* Extended examples added to Readme
* Added metrics port configurability
---------

Co-authored-by: gabrielmer <[email protected]>
  • Loading branch information
@NagyZoltanPeter @gabrielmer
NagyZoltanPeter and gabrielmer authored Oct 25, 2024
1 parent 02c8f35 commit 268e7e6
Show file tree
Hide file tree
Showing 19 changed files with 1,832 additions and 337 deletions.
27 changes: 27 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,33 @@ docker-image:
docker-push:
docker push $(DOCKER_IMAGE_NAME)

####################################
## Container lite-protocol-tester ##
####################################
# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
DOCKER_LPT_NIMFLAGS ?= -d:chronicles_colors:none -d:insecure

# build a docker image for the fleet
docker-liteprotocoltester: DOCKER_LPT_TAG ?= latest
docker-liteprotocoltester: DOCKER_LPT_NAME ?= wakuorg/liteprotocoltester:$(DOCKER_LPT_TAG)
docker-liteprotocoltester:
docker build \
--no-cache \
--build-arg="MAKE_TARGET=liteprotocoltester" \
--build-arg="NIMFLAGS=$(DOCKER_LPT_NIMFLAGS)" \
--build-arg="NIM_COMMIT=$(DOCKER_NIM_COMMIT)" \
--build-arg="LOG_LEVEL=TRACE" \
--label="commit=$(shell git rev-parse HEAD)" \
--label="version=$(GIT_VERSION)" \
--target $(TARGET) \
--tag $(DOCKER_LPT_NAME) \
--file apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile \
.

docker-liteprotocoltester-push:
docker push $(DOCKER_LPT_NAME)


################
## C Bindings ##
Expand Down
12 changes: 6 additions & 6 deletions apps/liteprotocoltester/.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ MAX_MESSAGE_SIZE=145Kb
#CLUSTER_ID=66

## for status.prod
#PUBSUB=/waku/2/rs/16/32
#CONTENT_TOPIC=/tester/2/light-pubsub-test/fleet
#CLUSTER_ID=16
PUBSUB=/waku/2/rs/16/32
CONTENT_TOPIC=/tester/2/light-pubsub-test/fleet
CLUSTER_ID=16

## for TWN
PUBSUB=/waku/2/rs/1/4
CONTENT_TOPIC=/tester/2/light-pubsub-test/twn
CLUSTER_ID=1
#PUBSUB=/waku/2/rs/1/4
#CONTENT_TOPIC=/tester/2/light-pubsub-test/twn
#CLUSTER_ID=1
77 changes: 38 additions & 39 deletions apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile
View file
Original file line number Diff line number Diff line change
@@ -1,58 +1,57 @@
# BUILD NIM APP ----------------------------------------------------------------
FROM rust:1.77.1-alpine3.18 AS nim-build
FROM rust:1.77.1-alpine3.18 AS nim-build

ARG NIMFLAGS
ARG MAKE_TARGET=liteprotocoltester
ARG NIM_COMMIT
ARG LOG_LEVEL=DEBUG
ARG NIMFLAGS
ARG MAKE_TARGET=liteprotocoltester
ARG NIM_COMMIT
ARG LOG_LEVEL=TRACE

# Get build tools and required header files
RUN apk add --no-cache bash git build-base pcre-dev linux-headers curl jq
# Get build tools and required header files
RUN apk add --no-cache bash git build-base openssl-dev pcre-dev linux-headers curl jq

WORKDIR /app
COPY . .
WORKDIR /app
COPY . .

# workaround for alpine issue: https://github.com/alpinelinux/docker-alpine/issues/383
RUN apk update && apk upgrade
# workaround for alpine issue: https://github.com/alpinelinux/docker-alpine/issues/383
RUN apk update && apk upgrade

# Ran separately from 'make' to avoid re-doing
RUN git submodule update --init --recursive
# Ran separately from 'make' to avoid re-doing
RUN git submodule update --init --recursive

# Slowest build step for the sake of caching layers
RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT}
# Slowest build step for the sake of caching layers
RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT}

# Build the final node binary
RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET LOG_LEVEL=${LOG_LEVEL} NIMFLAGS="${NIMFLAGS}"
# Build the final node binary
RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET LOG_LEVEL=${LOG_LEVEL} NIMFLAGS="${NIMFLAGS}"


# PRODUCTION IMAGE -------------------------------------------------------------
# PRODUCTION IMAGE -------------------------------------------------------------

FROM alpine:3.18 AS prod
FROM alpine:3.18 AS prod

ARG MAKE_TARGET=liteprotocoltester
ARG MAKE_TARGET=liteprotocoltester

LABEL maintainer="jakub@status.im"
LABEL source="https://github.com/waku-org/nwaku"
LABEL description="Lite Protocol Tester: Waku light-client"
LABEL commit="unknown"
LABEL version="unknown"
LABEL maintainer="zoltan@status.im"
LABEL source="https://github.com/waku-org/nwaku"
LABEL description="Lite Protocol Tester: Waku light-client"
LABEL commit="unknown"
LABEL version="unknown"

# DevP2P, LibP2P, and JSON RPC ports
EXPOSE 30303 60000 8545
# DevP2P, LibP2P, and JSON RPC ports
EXPOSE 30303 60000 8545

# Referenced in the binary
RUN apk add --no-cache libgcc pcre-dev libpq-dev
# Referenced in the binary
RUN apk add --no-cache libgcc pcre-dev libpq-dev \
wget \
iproute2

# Fix for 'Error loading shared library libpcre.so.3: No such file or directory'
RUN ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3
# Fix for 'Error loading shared library libpcre.so.3: No such file or directory'
RUN ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3

# Copy to separate location to accomodate different MAKE_TARGET values
COPY --from=nim-build /app/build/$MAKE_TARGET /usr/bin/
COPY --from=nim-build /app/build/liteprotocoltester /usr/bin/
COPY --from=nim-build /app/apps/liteprotocoltester/run_tester_node.sh /usr/bin/

# Copy migration scripts for DB upgrades
COPY --from=nim-build /app/migrations/ /app/migrations/
ENTRYPOINT ["/usr/bin/run_tester_node.sh", "/usr/bin/liteprotocoltester"]

ENTRYPOINT ["/usr/bin/liteprotocoltester"]

# By default just show help if called without arguments
CMD ["--help"]
# # By default just show help if called without arguments
CMD ["--help"]
164 changes: 127 additions & 37 deletions apps/liteprotocoltester/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,43 +17,28 @@ and multiple receivers.

Publishers are fill all message payloads with information about the test message and sender, helping the receiver side to calculate results.

## Phases of development

### Phase 1

At the first phase we aims to demonstrate the concept of the testing all boundled into a docker-compose environment where we run
one service (full)node and a publisher and a receiver node.
At this stage we can only configure number of messages and fixed frequency of the message pump. We do not expect message losses and any significant latency hence the test setup is very simple.

### Further plans

- Add more configurability (randomized message sizes, usage of more content topics and support for static sharding).
- Extend collected metrics and polish reporting.
- Add test metrics to graphana dashboard.
- Support for static sharding and auto sharding for being able to test under different conditions.
- ...

## Usage

### Phase 1

> NOTICE: This part is obsolate due integration with waku-simulator.
> It needs some rework to make it work again standalone.
### Using lpt-runner

Lite Protocol Tester application is built under name `liteprotocoltester` in apps/liteprotocoltester folder.
For ease of use, you can clone lpt-runner repository. That will utilize previously pushed liteprotocoltester docker image.
It is recommended to use this method for fleet testing.

Starting from nwaku repository root:
```bash
make liteprotocoltester
cd apps/liteprotocoltester
docker compose build
git clone https://github.com/waku-org/lpt-runner.git
cd lpt-runner

# check Reame.md for more information
# edit .env file to your needs

docker compose up -d
docker compose logs -f receivernode

# navigate localhost:3033 to see the lite-protocol-tester dashboard
```

### Phase 2
> See more detailed examples below.
> Integration with waku-simulator!
### Integration with waku-simulator!

- For convenience, integration is done in cooperation with waku-simulator repository, but nothing is tightly coupled.
- waku-simulator must be started separately with its own configuration.
Expand Down Expand Up @@ -100,9 +85,7 @@ docker compose -f docker-compose-on-simularor.yml logs -f receivernode

Navigate to http://localhost:3033 to see the lite-protocol-tester dashboard.

### Phase 3

> Run independently on a chosen waku fleet
### Run independently on a chosen waku fleet

This option is simple as is just to run the built liteprotocoltester binary with run_tester_node.sh script.

Expand Down Expand Up @@ -136,7 +119,7 @@ Run a SENDER role liteprotocoltester and a RECEIVER role one on different termin

> RECEIVER side will periodically print statistics to standard output.
## Configure
## Configuration

### Environment variables for docker compose runs

Expand All @@ -158,6 +141,7 @@ Run a SENDER role liteprotocoltester and a RECEIVER role one on different termin
| :--- | :--- | :--- |
| --test_func | separation of PUBLISHER or RECEIVER mode | RECEIVER |
| --service-node| Address of the service node to use for lightpush and/or filter service | - |
| --bootstrap-node| Address of the fleet's bootstrap node to use to determine service peer randomly choosen from the network. `--service-node` switch has precedence over this | - |
| --num-messages | Number of message to publish | 120 |
| --delay-messages | Frequency of messages in milliseconds | 1000 |
| --min-message-size | Minimum message size in bytes | 1KiB |
Expand All @@ -173,22 +157,31 @@ Run a SENDER role liteprotocoltester and a RECEIVER role one on different termin
| --rest-allow-origin | For convenience rest configuration can be done here | * |
| --log-level | Log level for the application | DEBUG |
| --log-format | Logging output format (TEXT or JSON) | TEXT |
| --metrics-port | Metrics scarpe port | 8003 |

### Specifying peer addresses

Service node or bootstrap addresses can be specified in multiadress or ENR form.

### Using bootstrap nodes

There are multiple benefits of using bootstrap nodes. By using them liteprotocoltester will use Peer Exchange protocol to get possible peers from the network that are capable to serve as service peers for testing. Additionally it will test dial them to verify their connectivity - this will be reported in the logs and on dashboard metrics.
Also by using bootstrap node and peer exchange discovery, litprotocoltester will be able to simulate service peer switch in case of failures. There are built in tresholds count for service peer failures (3) after service peer will be switched during the test. Also there will be max 10 trials of switching peer before test declared failed and quit.
These service peer failures are reported, thus extending network reliability measures.

### Docker image notice

#### Building for docker compose runs on simulator or standalone
Please note that currently to ease testing and development tester application docker image is based on ubuntu and uses the externally pre-built binary of 'liteprotocoltester'.
This speeds up image creation. Another dokcer build file is provided for proper build of boundle image.

> `Dockerfile.liteprotocoltester.copy` will create an image with the binary copied from the build directory.
> `Dockerfile.liteprotocoltester` will create an ubuntu based image with the binary copied from the build directory.
> `Dockerfile.liteprotocoltester.compile` will create an image completely compiled from source. This can be quite slow.
> `Dockerfile.liteprotocoltester.compile` will create an ubuntu based image completely compiled from source. This can be slow.
#### Creating standalone runner docker image

To ease the work with lite-proto-tester, a docker image is possible to build.
To ease the work with lite-protocol-tester, a docker image is possible to build.
With that image it is easy to run the application in a container.

> `Dockerfile.liteprotocoltester` will create an ubuntu image with the binary copied from the build directory. You need to pre-build the application.
Expand All @@ -205,7 +198,104 @@ docker build -t liteprotocoltester:latest -f Dockerfile.liteprotocoltester ../..

# edit and adjust .env file to your needs and for the network configuration

docker run --env-file .env liteprotocoltester:latest RECEIVER <service-node-ip4-peer-address>
docker run --env-file .env liteprotocoltester:latest RECEIVER <service-node-peer-address>

docker run --env-file .env liteprotocoltester:latest SENDER <service-node-peer-address>
```

#### Run test with auto service peer selection from a fleet using bootstrap node

```bash

docker run --env-file .env liteprotocoltester:latest RECEIVER <bootstrap-node-peer-address> BOOTSTRAP

docker run --env-file .env liteprotocoltester:latest SENDER <bootstrap-node-peer-address> BOOTSTRAP
```

> Notice that official image is also available at harbor.status.im/wakuorg/liteprotocoltester:latest
## Examples

### Bootstrap or Service node selection

The easiest way to get the proper bootstrap nodes for the tests from https://fleets.status.im page.
Adjust on which fleets you would like to run the tests.

> Please note that not all of them configured to support Peer Exchange protocol, those ones cannot be for bootstrap nodes for `liteprotocoltester`.
### Environment variables
You need not necessary to use .env file, although it can be more convenient.
Anytime you can override all or part of the environment variables defined in the .env file.

### Run standalone

Example of running the liteprotocoltester in standalone mode on status.stagin network.
Testing includes using bootstrap nodes to gather service peers from the network via Peer Exchange protocol.
Both parties will test-dial all the peers retrieved with the corresponding protocol.
Sender will start publishing messages after 60 seconds, sending 200 messages with 1 second delay between them.
Message size will be between 15KiB and 145KiB.
Cluster id and Pubsub-topic must be accurately set according to the network configuration.

The example shows that either multiaddress or ENR form accepted.

```bash
export START_PUBLISHING_AFTER=60
export NUM_MESSAGES=200
export DELAY_MESSAGES=1000
export MIN_MESSAGE_SIZE=15Kb
export MAX_MESSAGE_SIZE=145Kb
export PUBSUB=/waku/2/rs/16/32
export CONTENT_TOPIC=/tester/2/light-pubsub-test/fleet
export CLUSTER_ID=16

docker run harbor.status.im/wakuorg/liteprotocoltester:latest RECEIVER /dns4/boot-01.do-ams3.status.staging.status.im/tcp/30303/p2p/16Uiu2HAmQE7FXQc6iZHdBzYfw3qCSDa9dLc1wsBJKoP4aZvztq2d BOOTSTRAP

# in different terminal session, repeat the exports and run the other party of the test.
docker run harbor.status.im/wakuorg/liteprotocoltester:latest SENDER enr:-QEiuECJPv2vL00Jp5sTEMAFyW7qXkK2cFgphlU_G8-FJuJqoW_D5aWIy3ylGdv2K8DkiG7PWgng4Ql_VI7Qc2RhBdwfAYJpZIJ2NIJpcIQvTKi6im11bHRpYWRkcnO4cgA2NjFib290LTAxLmFjLWNuLWhvbmdrb25nLWMuc3RhdHVzLnN0YWdpbmcuc3RhdHVzLmltBnZfADg2MWJvb3QtMDEuYWMtY24taG9uZ2tvbmctYy5zdGF0dXMuc3RhZ2luZy5zdGF0dXMuaW0GAbveA4Jyc40AEAUAAQAgAEAAgAEAiXNlY3AyNTZrMaEDkbgV7oqPNmFtX5FzSPi9WH8kkmrPB1R3n9xRXge91M-DdGNwgnZfg3VkcIIjKIV3YWt1Mg0 BOOTSTRAP

docker run --env-file .env liteprotocoltester:latest SENDER <service-node-ip4-peer-address>
```

### Use of lpt-runner

Another method is to use [lpt-runner repository](https://github.com/waku-org/lpt-runner/tree/master).
This extends testing with grafana dashboard and ease the test setup.
Please read the corresponding [README](https://github.com/waku-org/lpt-runner/blob/master/README.md) there as well.

In this example we will run similar test as above but there will be 3 instances of publisher nodes and 1 receiver node.
This test uses waku.sandbox fleet which is connected to TWN. This implies lower message rates due to the RLN rate limation.
Also leave a gap of 120 seconds before starting to publish messages to let receiver side fully finish peer test-dialing.
For TWN network it is always wise to use bootstrap nodes with Peer Exchange support.

> Theoritically we can use the same bootstrap nodes for both parties, but it is recommended to use different ones to simulate different network edges, thus getting more meaningful results.
```bash
git clone https://github.com/waku-org/lpt-runner.git
cd lpt-runner

export NUM_PUBLISHER_NODES=3
export NUM_RECEIVER_NODES=1
export START_PUBLISHING_AFTER=120
export NUM_MESSAGES=300
export DELAY_MESSAGES=7000
export MIN_MESSAGE_SIZE=15Kb
export MAX_MESSAGE_SIZE=145Kb
export PUBSUB=/waku/2/rs/1/4
export CONTENT_TOPIC=/tester/2/light-pubsub-test/twn
export CLUSTER_ID=1

export FILTER_BOOTSTRAP=/dns4/node-01.ac-cn-hongkong-c.waku.sandbox.status.im/tcp/30303/p2p/16Uiu2HAmQYiojgZ8APsh9wqbWNyCstVhnp9gbeNrxSEQnLJchC92
export LIGHTPUSH_BOOTSTRAP=/dns4/node-01.do-ams3.waku.sandbox.status.im/tcp/30303/p2p/16Uiu2HAmNaeL4p3WEYzC9mgXBmBWSgWjPHRvatZTXnp8Jgv3iKsb

docker compose up -d

# we can check logs from one or all SENDER
docker compose logs -f --index 1 publishernode

# for checking receiver side performance
docker compose logs -f receivernode

# when test completed
docker compose down
```

For dashboard navigate to http://localhost:3033
Loading

0 comments on commit 268e7e6

Please sign in to comment.