From b04039a33a301729568adb0a7862a5ef92fa0f1c Mon Sep 17 00:00:00 2001 From: yshashix Date: Thu, 19 Sep 2024 17:16:01 +0200 Subject: [PATCH] added support for ubuntu 24.04 and 22.04 and adjusted github workflow - docker v27.2 - miniconda for python 3.10 - sqlite3-pcre2 --- .github/workflows/build.yaml | 25 ++++++++++++++++--- .github/workflows/k8s-tests.yaml | 2 ++ helm/README.md | 2 +- semantic-model/shacl2flink/Makefile | 4 +-- semantic-model/shacl2flink/README.md | 25 +++++++++++-------- semantic-model/shacl2flink/pyenv_setup.sh | 8 ++++++ .../shacl2flink/udf/sqlite3_insert.py | 2 +- .../shacl2flink/udf/sqlite3_select.py | 2 +- ...prepare-platform-for-self-hosted-runner.sh | 10 ++++++++ test/prepare-platform.sh | 20 +++++++++++---- 10 files changed, 77 insertions(+), 23 deletions(-) create mode 100755 semantic-model/shacl2flink/pyenv_setup.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 00fce577..c0d9fb95 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -9,26 +9,45 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 - name: Build the Docker images run: docker compose build + # Use conda-incubator to install Miniconda + - name: Setup Miniconda with Python 3.10 + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: "3.10" # Specify Python 3.10 + auto-activate-base: false # Do not auto-activate the base environment + use-mamba: true # Optional: use mamba for faster environment creation + - name: Create a new Conda environment + run: conda create --name py310 python=3.10 -y - name: Build shacl2flink run: | - sudo apt install sqlite3 sqlite3-pcre + sudo apt install -y sqlite3 libsqlite3-dev libpcre2-dev + sudo mkdir -p /usr/lib/sqlite3 + git clone https://github.com/christian-proust/sqlite3-pcre2.git + cd sqlite3-pcre2 && make && make install + cd ../ && rm -rf sqlite3-pcre2 + source $CONDA/bin/activate + conda activate py310 cd semantic-model/shacl2flink && make setup && make lint test build test-kms - name: Build Dataservice run: | + source $CONDA/bin/activate + conda activate py310 cd semantic-model/dataservice && make setup && make lint test - name: Build datamodel tools run: | cd semantic-model/datamodel/tools && make setup && make lint test - name: Build opcua tools run: | + source $CONDA/bin/activate + conda activate py310 pip3 install asyncua cd semantic-model/opcua && make setup && make lint test - name: Build iff-agent run: | (cd NgsildAgent/config && cp config.json.template config.json) - cd NgsildAgent && npm install && npm run lint && npm run test && npm audit --production --audit-level high \ No newline at end of file + cd NgsildAgent && npm install && npm run lint && npm run test && npm audit --production --audit-level high diff --git a/.github/workflows/k8s-tests.yaml b/.github/workflows/k8s-tests.yaml index abff4434..c03a4b9c 100644 --- a/.github/workflows/k8s-tests.yaml +++ b/.github/workflows/k8s-tests.yaml @@ -47,6 +47,8 @@ jobs: killall kubefwd - name: Build and test shacl2flink run: | + source ./miniconda3/etc/profile.d/conda.sh + conda activate py310 ( cd semantic-model/shacl2flink && make setup && make lint test test-kms ) ( cd semantic-model/shacl2flink && make test-full-flink-deployment ) diff --git a/helm/README.md b/helm/README.md index e762a116..92adeed3 100644 --- a/helm/README.md +++ b/helm/README.md @@ -17,7 +17,7 @@ The Services consist of # Building and Installation of Platform Locally -The following setup procedure is executed in the Kubernetes end to end [`test`](../.github/workflows/k8s-tests.yaml) as part of the Continous Integration procedure. It is tested and regularly executed on `Ubuntu 20.04` and `Ubuntu 22.04`. +The following setup procedure is executed in the Kubernetes end to end [`test`](../.github/workflows/k8s-tests.yaml) as part of the Continous Integration procedure. It is tested and regularly executed on `Ubuntu 22.04` and `Ubuntu 24.04`. --- diff --git a/semantic-model/shacl2flink/Makefile b/semantic-model/shacl2flink/Makefile index 66227857..70443ca1 100644 --- a/semantic-model/shacl2flink/Makefile +++ b/semantic-model/shacl2flink/Makefile @@ -48,7 +48,7 @@ setup: requirements.txt setup-dev $(PIP) install -r requirements.txt echo ".headers on" > ~/.sqliterc echo ".mode column" >> ~/.sqliterc - echo ".load /usr/lib/sqlite3/pcre.so" >> ~/.sqliterc + echo ".load /usr/lib/sqlite3/pcre2.so" >> ~/.sqliterc update-rdf-cm: delete-rdf-cm kubectl -n $(NAMESPACE) create -f output/rdf-maps.yaml @@ -175,4 +175,4 @@ ontology2kms: clean: @rm -rf $(OUTPUTDIR) -.PHONY: clean \ No newline at end of file +.PHONY: clean diff --git a/semantic-model/shacl2flink/README.md b/semantic-model/shacl2flink/README.md index 0fc963aa..83bd4288 100644 --- a/semantic-model/shacl2flink/README.md +++ b/semantic-model/shacl2flink/README.md @@ -23,25 +23,29 @@ A first [overview](../datamodel/README.md) and [tutorial](../datamodel/Tutorial. ## Requirements -- Python > 3.8 -- Virtualenv installed - -In addition, install `sqlite3` and `sqlite3-pcre` +- You need to have Python > 3.8 +- Virtualenv needs to be installed +- `sqlite3` and `sqlite3-pcre` need to be installed ```bash - sudo apt install sqlite3 sqlite3-pcre + sudo apt install sqlite3 libsqlite3-dev libpcre2-dev ``` ## Installation +If miniconda installed with python3.10 environment (using prepare-platform.sh), move to step 2 else use below script to install and create python env +### Step 1 : ```bash -python -m venv venv +bash pyenv_setup.sh +source ./miniconda3/bin/activate +conda create -n py310 python=3.10 -y ``` - -Everytime you are starting a new shell you need to enable the Virtual Environment: +### Step 2 : +Everytime you are starting a new shell you need to enable the miniconda Virtual Environment which runs python 3.10 sourcing miniconda installation path: ```bash -source venv/bin/activate +source ./miniconda3/bin/activate +conda activate py310 make setup ``` @@ -56,7 +60,8 @@ Press `Ctrl + Shift + p` and type `Python: Select Interpreter` and select the vi Install the development dependencies: ```bash -source venv/bin/activate +source ./miniconda3/bin/activate +conda activate py310 pip install -r requirements-dev.txt ``` diff --git a/semantic-model/shacl2flink/pyenv_setup.sh b/semantic-model/shacl2flink/pyenv_setup.sh new file mode 100755 index 00000000..341bc82a --- /dev/null +++ b/semantic-model/shacl2flink/pyenv_setup.sh @@ -0,0 +1,8 @@ +#! /bin/bash + +#Install miniconda with python 3.10 version +mkdir -p ./miniconda3 +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ./miniconda3/miniconda.sh +bash ./miniconda3/miniconda.sh -b -u -p ./miniconda3 +source ./miniconda3/bin/conda init +rm ./miniconda3/miniconda.sh \ No newline at end of file diff --git a/semantic-model/shacl2flink/udf/sqlite3_insert.py b/semantic-model/shacl2flink/udf/sqlite3_insert.py index ef7ebd55..34651c0d 100644 --- a/semantic-model/shacl2flink/udf/sqlite3_insert.py +++ b/semantic-model/shacl2flink/udf/sqlite3_insert.py @@ -24,7 +24,7 @@ def main(databasefile, sqlscript): with sqlite3.connect(databasefile) as conn: with open(sqlscript, 'r') as file: conn.enable_load_extension(True) - conn.load_extension('/usr/lib/sqlite3/pcre.so') + conn.load_extension('/usr/lib/sqlite3/pcre2.so') conn.create_aggregate('statetime', 2, sqlite_udf_statetime.Statetime) conn.create_function('hash', 1, sqlite_udf_hash.eval) cur = conn.cursor() diff --git a/semantic-model/shacl2flink/udf/sqlite3_select.py b/semantic-model/shacl2flink/udf/sqlite3_select.py index c9a91618..1673db38 100644 --- a/semantic-model/shacl2flink/udf/sqlite3_select.py +++ b/semantic-model/shacl2flink/udf/sqlite3_select.py @@ -23,7 +23,7 @@ def main(databasefile, sqlscript): with sqlite3.connect(databasefile) as conn: with open(sqlscript, 'r') as file: conn.enable_load_extension(True) - conn.load_extension('/usr/lib/sqlite3/pcre.so') + conn.load_extension('/usr/lib/sqlite3/pcre2.so') conn.create_aggregate('statetime', 2, sqlite_udf_statetime.Statetime) conn.create_function('hash', 1, sqlite_udf_hash.eval) cur = conn.cursor() diff --git a/test/prepare-platform-for-self-hosted-runner.sh b/test/prepare-platform-for-self-hosted-runner.sh index baa05219..1da49516 100644 --- a/test/prepare-platform-for-self-hosted-runner.sh +++ b/test/prepare-platform-for-self-hosted-runner.sh @@ -36,3 +36,13 @@ cd ../helm || exit 1 wget https://github.com/helmfile/helmfile/releases/download/v0.149.0/helmfile_0.149.0_linux_amd64.tar.gz tar -zxvf helmfile_0.149.0_linux_amd64.tar.gz chmod u+x helmfile + +echo Install miniconda with python 3.10 env py310 +echo --------------------------------- +mkdir ../miniconda3 +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ../miniconda3/miniconda.sh +bash ../miniconda3/miniconda.sh -b -u -p ../miniconda3 +source ../miniconda3/bin/conda init +source ../miniconda3/bin/activate +conda create -n py310 python=3.10 -y +rm ../miniconda3/miniconda.sh diff --git a/test/prepare-platform.sh b/test/prepare-platform.sh index eb8cc791..335c0d37 100644 --- a/test/prepare-platform.sh +++ b/test/prepare-platform.sh @@ -23,10 +23,15 @@ printf "\033[1mInstalling docker\n" printf -- "-----------------\033[0m\n" sudo apt -qq update sudo apt-get -qq install apt-transport-https ca-certificates curl gnupg-agent software-properties-common mosquitto-clients postgresql-client-common postgresql-client -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc +# Add the docker repository to Apt sources: +echo \ + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt -qq update -sudo apt-get install -y docker-ce=5:23.0.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) docker-ce-cli=5:23.0.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) containerd.io +sudo apt-get install -y docker-ce=5:27.2.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) docker-ce-cli=5:27.2.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) containerd.io printf "\033[1mSuccessfully installed %s\033[0m\n" "$(docker --version)" printf "\n" @@ -148,7 +153,13 @@ if [ -z "$BUILDONLY" ];then echo Install sqlite and pcre component echo --------------------------------- - sudo apt install -yq sqlite3 sqlite3-pcre + sudo apt install -yq sqlite3 libsqlite3-dev libpcre2-dev + sudo mkdir -p /usr/lib/sqlite3 + # Install sqlite3-pcre2 loadable library on location /usr/lib/sqlite3/pcre2.so + # Using Christian proust repos which is forked from original sqlite3-pcre code return from Alexey Tourbin + git clone https://github.com/christian-proust/sqlite3-pcre2.git + cd sqlite3-pcre2 && make && make install + cd ../ && rm -rf sqlite3-pcre2 echo Install python3-pip echo ------------------- @@ -162,7 +173,6 @@ if [ -z "$BUILDONLY" ];then sudo bash /tmp/nodesource_setup.sh sudo apt install -y nodejs - echo Install Bats echo ------------------- sudo apt install -yq bats