added support for ubuntu 24.04 and 22.04 with updated docker v27.2

.github/workflows/build.yaml

@@ -9,26 +9,45 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v2
     - name: Build the Docker images
       run: docker compose build
+    # Use conda-incubator to install Miniconda
+    - name: Setup Miniconda with Python 3.10
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: "3.10"  # Specify Python 3.10
+        auto-activate-base: false  # Do not auto-activate the base environment
+        use-mamba: true  # Optional: use mamba for faster environment creation
+    - name: Create a new Conda environment
+      run: conda create --name py310 python=3.10 -y
     - name: Build shacl2flink
       run: |
-        sudo  apt install sqlite3 sqlite3-pcre
+        sudo  apt install -y sqlite3 libsqlite3-dev libpcre2-dev
+        sudo mkdir -p /usr/lib/sqlite3
+        git clone https://github.com/christian-proust/sqlite3-pcre2.git
+        cd sqlite3-pcre2 && make && make install
+        cd ../ && rm -rf sqlite3-pcre2
+        source $CONDA/bin/activate
+        conda activate py310
         cd semantic-model/shacl2flink && make setup && make lint test build test-kms
     - name: Build Dataservice
       run: |
+        source $CONDA/bin/activate
+        conda activate py310
         cd semantic-model/dataservice && make setup && make lint test
     - name: Build datamodel tools
       run: |
         cd semantic-model/datamodel/tools && make setup && make lint test
     - name: Build opcua tools
       run: |
+        source $CONDA/bin/activate
+        conda activate py310
         pip3 install asyncua
         cd semantic-model/opcua && make setup && make lint test
     - name: Build iff-agent
       run: |
         (cd NgsildAgent/config && cp config.json.template config.json)  
-        cd NgsildAgent && npm install && npm run lint && npm run test && npm audit --production --audit-level high
+        cd NgsildAgent && npm install && npm run lint && npm run test && npm audit --production --audit-level high

.github/workflows/k8s-tests.yaml

@@ -47,6 +47,8 @@ jobs:
         killall kubefwd
     - name: Build and test shacl2flink
       run: |
+        source ./miniconda3/etc/profile.d/conda.sh
+        conda activate py310
         ( cd semantic-model/shacl2flink && make setup && make lint test test-kms )
         ( cd semantic-model/shacl2flink && make test-full-flink-deployment )
 

helm/README.md

@@ -17,7 +17,7 @@ The Services consist of
 
 # Building and Installation of Platform Locally
 
-The following setup procedure is executed in the Kubernetes end to end [`test`](../.github/workflows/k8s-tests.yaml) as part of the Continous Integration procedure. It is tested and regularly executed on `Ubuntu 20.04` and `Ubuntu 22.04`.
+The following setup procedure is executed in the Kubernetes end to end [`test`](../.github/workflows/k8s-tests.yaml) as part of the Continous Integration procedure. It is tested and regularly executed on `Ubuntu 22.04` and `Ubuntu 24.04`.
 
 ---
 

semantic-model/shacl2flink/Makefile

@@ -48,7 +48,7 @@ setup: requirements.txt setup-dev
 	$(PIP) install -r requirements.txt
 	echo ".headers on" > ~/.sqliterc
 	echo ".mode column" >> ~/.sqliterc
-	echo ".load /usr/lib/sqlite3/pcre.so" >> ~/.sqliterc
+	echo ".load /usr/lib/sqlite3/pcre2.so" >> ~/.sqliterc
 
 update-rdf-cm: delete-rdf-cm
 	kubectl -n $(NAMESPACE) create -f output/rdf-maps.yaml
@@ -175,4 +175,4 @@ ontology2kms:
 clean:
 	@rm -rf $(OUTPUTDIR)
 
-.PHONY: clean
+.PHONY: clean

semantic-model/shacl2flink/README.md

@@ -23,25 +23,29 @@ A first [overview](../datamodel/README.md) and [tutorial](../datamodel/Tutorial.
 
 ## Requirements
 
-- Python > 3.8
-- Virtualenv installed
-
-In addition, install `sqlite3` and `sqlite3-pcre`
+- You need to have Python > 3.8
+- Virtualenv needs to be installed
+- `sqlite3` and `sqlite3-pcre` need to be installed
 
   ```bash
-  sudo  apt install sqlite3 sqlite3-pcre
+  sudo  apt install sqlite3 libsqlite3-dev libpcre2-dev
   ```
 
 ## Installation
 
+If miniconda installed with python3.10 environment (using prepare-platform.sh), move to step 2 else use below script to install and create python env
+### Step 1 :
 ```bash
-python -m venv venv
+bash pyenv_setup.sh
+source ./miniconda3/bin/activate
+conda create -n py310 python=3.10 -y
 ```
-
-Everytime you are starting a new shell you need to enable the Virtual Environment:
+### Step 2 :
+Everytime you are starting a new shell you need to enable the miniconda Virtual Environment which runs python 3.10 sourcing miniconda installation path:
 
 ```bash
-source venv/bin/activate
+source ./miniconda3/bin/activate
+conda activate py310
 make setup
 ```
 
@@ -56,7 +60,8 @@ Press `Ctrl + Shift + p` and type `Python: Select Interpreter` and select the vi
 Install the development dependencies:
 
 ```bash
-source venv/bin/activate
+source ./miniconda3/bin/activate
+conda activate py310
 pip install -r requirements-dev.txt
 ```
 

semantic-model/shacl2flink/pyenv_setup.sh

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+#Install miniconda with python 3.10 version
+mkdir -p ./miniconda3
+wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ./miniconda3/miniconda.sh
+bash ./miniconda3/miniconda.sh -b -u -p ./miniconda3
+source ./miniconda3/bin/conda init
+rm ./miniconda3/miniconda.sh

semantic-model/shacl2flink/udf/sqlite3_insert.py

@@ -24,7 +24,7 @@ def main(databasefile, sqlscript):
     with sqlite3.connect(databasefile) as conn:
         with open(sqlscript, 'r') as file:
             conn.enable_load_extension(True)
-            conn.load_extension('/usr/lib/sqlite3/pcre.so')
+            conn.load_extension('/usr/lib/sqlite3/pcre2.so')
             conn.create_aggregate('statetime', 2, sqlite_udf_statetime.Statetime)
             conn.create_function('hash', 1, sqlite_udf_hash.eval)
             cur = conn.cursor()

semantic-model/shacl2flink/udf/sqlite3_select.py

@@ -23,7 +23,7 @@ def main(databasefile, sqlscript):
     with sqlite3.connect(databasefile) as conn:
         with open(sqlscript, 'r') as file:
             conn.enable_load_extension(True)
-            conn.load_extension('/usr/lib/sqlite3/pcre.so')
+            conn.load_extension('/usr/lib/sqlite3/pcre2.so')
             conn.create_aggregate('statetime', 2, sqlite_udf_statetime.Statetime)
             conn.create_function('hash', 1, sqlite_udf_hash.eval)
             cur = conn.cursor()

test/prepare-platform-for-self-hosted-runner.sh

@@ -36,3 +36,13 @@ cd ../helm || exit 1
 wget https://github.com/helmfile/helmfile/releases/download/v0.149.0/helmfile_0.149.0_linux_amd64.tar.gz
 tar -zxvf helmfile_0.149.0_linux_amd64.tar.gz
 chmod u+x helmfile
+
+echo Install miniconda with python 3.10 env py310
+echo ---------------------------------
+mkdir ../miniconda3
+wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ../miniconda3/miniconda.sh
+bash ../miniconda3/miniconda.sh -b -u -p ../miniconda3
+source ../miniconda3/bin/conda init
+source ../miniconda3/bin/activate
+conda create -n py310 python=3.10 -y
+rm ../miniconda3/miniconda.sh

test/prepare-platform.sh

@@ -23,10 +23,15 @@ printf "\033[1mInstalling docker\n"
 printf -- "-----------------\033[0m\n"
 sudo apt -qq update
 sudo apt-get -qq install apt-transport-https ca-certificates curl gnupg-agent software-properties-common mosquitto-clients postgresql-client-common postgresql-client
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+# Add the docker repository to Apt sources:
+echo \
+  "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 sudo apt -qq update
-sudo apt-get install -y docker-ce=5:23.0.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) docker-ce-cli=5:23.0.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) containerd.io
+sudo apt-get install -y docker-ce=5:27.2.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) docker-ce-cli=5:27.2.1-1~ubuntu.$(lsb_release -sr)~$(lsb_release -cs) containerd.io
 printf "\033[1mSuccessfully installed %s\033[0m\n" "$(docker --version)"
 printf "\n"
 
@@ -148,7 +153,13 @@ if [ -z "$BUILDONLY" ];then
 
     echo Install sqlite and pcre component
     echo ---------------------------------
-    sudo  apt install -yq sqlite3 sqlite3-pcre
+    sudo  apt install -yq sqlite3 libsqlite3-dev libpcre2-dev
+    sudo mkdir -p /usr/lib/sqlite3
+    # Install sqlite3-pcre2 loadable library on location /usr/lib/sqlite3/pcre2.so 
+    # Using Christian proust repos which is forked from original sqlite3-pcre code return from Alexey Tourbin <[email protected]> 
+    git clone https://github.com/christian-proust/sqlite3-pcre2.git
+    cd sqlite3-pcre2 && make && make install
+    cd ../ && rm -rf sqlite3-pcre2
 
     echo Install python3-pip
     echo -------------------
@@ -162,7 +173,6 @@ if [ -z "$BUILDONLY" ];then
     sudo bash /tmp/nodesource_setup.sh
     sudo apt install -y nodejs
 
-
     echo Install Bats
     echo -------------------
     sudo apt install -yq bats