Merge develop

config.template/apache/.htpasswd

@@ -0,0 +1 @@
+<<Please generate valid htpasswd for web dashboard and rest api file with common utils>>

config.template/apache/apache-route.yaml

@@ -0,0 +1,19 @@
+apiVersion: route.openshift.io/v1
+kind: Route
+metadata:
+  annotations:
+    haproxy.router.openshift.io/ip_whitelist: <ip_of_client_1> <ip_of_client_2>
+  labels:
+    app: apache-dashboard
+  name: apache-dashboard
+spec:
+  host: flink-dashboard.apps.dcw1.paas.psnc.pl
+  port:
+    targetPort: 8080-tcp
+  tls:
+    termination: edge
+  to:
+    kind: Service
+    name: apache-dashboard
+    weight: 100
+  wildcardPolicy: None

docker/build-docker-images.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+echo building flink-poc docker image that can be deployed on the openshift cluster.
+
+#images
+cd flink_java21
+echo building :: flink java 21 base $(pwd)
+docker build --no-cache -t flink:1.18.1-java21_poc .
+cd ..
+cd flink-node
+echo building :: flink-node $(pwd)
+docker build --no-cache -t flink-node-poc .
+cd ..
+cd ..
+echo building :: flink-rich-node $(pwd)
+docker build --no-cache -t flink-rich-node-poc -f ./docker/flink-node-with-application/Dockerfile .
+

docker/docker-compose.yml

@@ -0,0 +1,122 @@
+version: '3.8'
+
+services:
+  cassandra:
+    image: cassandra:3.11.16
+    container_name: flink-cassandra
+    restart: always
+    ports:
+      - 9042:9042
+      - 9160:9160
+    volumes:
+      - /data/cassandra:/var/lib/cassandra
+
+  zookeeper:
+    image: zookeeper:3.9.2
+    container_name: flink-zookeeper
+    restart: always
+    ports:
+      - 2181:2181
+
+  solr:
+    image: solr:7.7.3
+    container_name: flink-solr
+    restart: always
+    ports:
+      - 8983:8983
+    volumes:
+      - /data/solr:/var/solr
+
+  flink-jobmanager:
+    image: registry.paas.psnc.pl/ecloud-poc/flink-rich-node-poc:latest
+    container_name: flink-rich-poc-jm
+    command: jobmanager
+    environment:
+       FLINK_PROPERTIES: |+ 
+          jobmanager.rpc.address: flink-jobmanager 
+          taskmanager.numberOfTaskSlots: 4 
+          blob.server.port: 6124 
+          jobmanager.rpc.port: 6123 
+          taskmanager.rpc.port: 6122 
+          jobmanager.memory.process.size: 2728m 
+          taskmanager.memory.process.size: 2728m 
+          taskmanager.memory.jvm-metaspace.size: 512MB 
+          jobmanager.memory.jvm-metaspace.size: 512MB 
+          parallelism.default: 2 
+          env.java.opts.all: "-XX:MaxDirectMemorySize=1170210816 --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1099 -Dcom.sun.management.jmxremote.rmi.port=1099 -Djava.rmi.server.hostname=127.0.0.1 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dumps --add-opens java.base/jdk.internal.misc=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true" 
+          web.upload.dir: /web-upload 
+          jobstore.expiration-time: 172800 
+          jobstore.type: File 
+          process.jobmanager.working-dir: /working-directory 
+          jobmanager.resource-id: poc1
+          state.backend.type: filesystem
+          state.checkpoints.dir: file:///data/flink/checkpoints                        
+          execution.runtime-mode: STREAMING
+          execution.checkpointing.interval: 30000
+          execution.checkpointing.min-pause: 30000
+          execution.checkpointing.tolerable-failed-checkpoints: 5
+          execution.checkpointing.mode: AT_LEAST_ONCE
+          taskmanager.network.memory.buffer-debloat.enabled: true
+          jobmanager.scheduler: adaptive
+          slot.idle.timeout: 5000
+          slot.request.timeout: 20000
+    volumes:
+      - /data/flink/deployments:/deployments:rw
+      - /data/flink/jobs-config:/jobs-config
+      - /data/flink/working-directory:/working-directory:rw
+      - /data/flink/web-upload:/web-upload:rw
+      - /data/flink/log:/opt/flink/log:rw
+      - flink-checkpoints:/data/flink/checkpoints
+    ports:
+      - '6123:6123'
+      - '6124:6124'
+      - '8081:8081'
+    networks:
+      - flink-network
+  flink-taskmanager:
+    image: registry.paas.psnc.pl/ecloud-poc/flink-rich-node-poc
+    container_name: flink-rich-poc-tm
+    user: 1000:1000
+    command: taskmanager
+    environment:
+      FLINK_PROPERTIES: |+
+        jobmanager.rpc.address: jobmanager 
+        taskmanager.numberOfTaskSlots: 4 
+        blob.server.port: 6124 
+        jobmanager.rpc.port: 6123 
+        taskmanager.rpc.port: 6122 
+        jobmanager.memory.process.size: 2728m 
+        taskmanager.memory.process.size: 2728m 
+        taskmanager.memory.jvm-metaspace.size: 512MB 
+        jobmanager.memory.jvm-metaspace.size: 512MB 
+        parallelism.default: 2 
+        env.java.opts.all: "-XX:MaxDirectMemorySize=1170210816 --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1099 -Dcom.sun.management.jmxremote.rmi.port=1099 -Djava.rmi.server.hostname=127.0.0.1 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dumps" 
+        web.upload.dir: /web-upload 
+        jobstore.expiration-time: 172800 
+        jobstore.type: File 
+        process.jobmanager.working-dir: /working-directory 
+        state.backend.type: filesystem 
+        state.checkpoints.dir: file:///data/flink/checkpoints                        
+        execution.checkpointing.interval: 30000
+        execution.checkpointing.min-pause: 30000
+        execution.checkpointing.tolerable-failed-checkpoints: 5
+        execution.checkpointing.mode: AT_LEAST_ONCE
+        jobmanager.resource-id: poc1
+    ports:
+      - '6122:6122'
+    volumes:
+      - /data/flink/log:/opt/flink/log:rw
+    networks:
+      - flink-network
+    depends_on:
+      - flink-jobmanager
+networks:
+  flink-network:
+
+volumes:
+  flink-checkpoints:
+    name: flink-checkpoints
+    driver: local
+  flink-log:
+    name: flink-log
+    driver: local

docker/flink-node-with-application/Dockerfile

@@ -0,0 +1,6 @@
+FROM flink-node-poc:latest
+
+MAINTAINER PSNC
+COPY ./docker/flink-node-with-application/flink-connector-cassandra_2.12-3.1.0-1.17.jar /opt/flink/lib/
+COPY ./docker/flink-node-with-application/metrics-core-3.0.2.jar /opt/flink/lib/
+COPY ./flinkPoCPG/target/flink-poc-postgres.jar /opt/flink/lib/

docker/flink-node-with-application/release.metadata

@@ -0,0 +1,2 @@
+Tags: flink-rich-node-poc:latest
+Architectures: amd64,arm64v8

docker/flink-node/Dockerfile

@@ -0,0 +1,44 @@
+FROM flink:1.18.1-java21_poc
+
+MAINTAINER PSNC
+
+RUN apt-get update && apt-get install -y \
+    xz-utils \
+    telnet  \
+    iputils-ping    \
+    procps \
+    traceroute \
+    curl \
+    wget \
+    net-tools \
+    dnsutils \
+    vim \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY install.sh /
+
+RUN  chmod 555 /install.sh
+RUN  /install.sh
+
+RUN apt-get update && apt-get install -y \
+    zip \
+    unzip \
+    lsof \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY jattach /usr/bin/
+RUN chmod 755 /usr/bin/jattach
+
+#Kubernetes client - kubectl - for better diagnostic in case of problems
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+    && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256" \
+    && echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check \
+    && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+#openshift client - oc - for better diagnostic in case of problems
+RUN mkdir /oc-temp \
+    && cd /oc-temp \
+    && curl -LO https://github.com/okd-project/okd/releases/download/4.15.0-0.okd-2024-03-10-010116/openshift-client-linux-4.15.0-0.okd-2024-03-10-010116.tar.gz \
+    && tar -xzf openshift-client-linux-4.15.0-0.okd-2024-03-10-010116.tar.gz \
+    && mv oc /usr/bin \
+    && rm -rf /oc-temp \

docker/flink-node/install.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -e
+set -u
+
+apt-get update
+apt-get install -y build-essential
+#
+###########
+wget http://www.ijg.org/files/jpegsrc.v8c.tar.gz
+tar -xf jpegsrc.v8c.tar.gz
+rm  jpegsrc.v8c.tar.gz
+cd jpeg-8c
+./configure
+make
+make install
+cd ..
+rm -rf jpeg-8c
+
+#ghostscript
+wget https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs923/ghostscript-9.23.tar.xz
+tar -xf ghostscript-9.23.tar.xz
+rm ghostscript-9.23.tar.xz
+cd ghostscript-9.23
+./configure
+make
+make install
+cd ..
+rm -rf ghostscript-9.23
+
+#libtiff
+wget http://download.osgeo.org/libtiff/tiff-4.0.6.tar.gz
+tar -xf tiff-4.0.6.tar.gz
+rm  tiff-4.0.6.tar.gz
+cd tiff-4.0.6
+./configure --disable-static
+make
+make install
+cd ..
+rm -rf tiff-4.0.6
+cd /
+
+#image magic
+apt-get install -y imagemagick
+
+#Installing ffprobe
+wget https://www.johnvansickle.com/ffmpeg/old-releases/ffmpeg-4.2.2-amd64-static.tar.xz
+tar xvf ffmpeg-4.2.2-amd64-static.tar.xz
+mv ffmpeg-4.2.2-amd64-static/ffprobe /usr/local/bin/
+rm -rf ffmpeg-4.2.2-amd64-static
+rm -rf ffmpeg-4.2.2-amd64-static.tar.xz
+
+
+ranlib /usr/local/lib/libjpeg.a
+#to have it configured after restart: create /etc/ld.so.conf.d/imagemagick.conf file (with root) and put /usr/local/lib into it. Then run ldconfig and reboot
+ldconfig /usr/local/lib
+
+#clearing apt-get cache and other
+apt-get remove -y build-essential
+apt autoremove -y
+rm -rf /var/lib/apt/lists/*

docker/flink-node/release.metadata

@@ -0,0 +1,2 @@
+Tags: flink-node-poc:latest
+Architectures: amd64,arm64v8

docker/flink_java21/Dockerfile

@@ -0,0 +1,95 @@
+###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM eclipse-temurin:21-jre-jammy
+
+# Install dependencies
+RUN set -ex; \
+  apt-get update; \
+  apt-get -y install gpg libsnappy1v5 gettext-base libjemalloc-dev; \
+  rm -rf /var/lib/apt/lists/*
+
+# Grab gosu for easy step-down from root
+ENV GOSU_VERSION 1.11
+RUN set -ex; \
+  wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \
+  wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \
+  export GNUPGHOME="$(mktemp -d)"; \
+  for server in ha.pool.sks-keyservers.net $(shuf -e \
+                          hkp://p80.pool.sks-keyservers.net:80 \
+                          keyserver.ubuntu.com \
+                          hkp://keyserver.ubuntu.com:80 \
+                          pgp.mit.edu) ; do \
+      gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \
+  done && \
+  gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
+  gpgconf --kill all; \
+  rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
+  chmod +x /usr/local/bin/gosu; \
+  gosu nobody true
+
+# Configure Flink version
+ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz \
+    FLINK_ASC_URL=https://downloads.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz.asc \
+    GPG_KEY=96AE0E32CBE6E0753CE6DF6CB078D1D3253A8D82 \
+    CHECK_GPG=true
+
+# Prepare environment
+ENV FLINK_HOME=/opt/flink
+ENV PATH=$FLINK_HOME/bin:$PATH
+RUN groupadd --system --gid=9999 flink && \
+    useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink
+WORKDIR $FLINK_HOME
+
+# Install Flink
+RUN set -ex; \
+  wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
+  \
+  if [ "$CHECK_GPG" = "true" ]; then \
+    wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+    done && \
+    gpg --batch --verify flink.tgz.asc flink.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  fi; \
+  \
+  tar -xf flink.tgz --strip-components=1; \
+  rm flink.tgz; \
+  \
+  chown -R flink:flink .; \
+  \
+  # Replace default REST/RPC endpoint bind address to use the container's network interface \
+  sed -i 's/rest.address: localhost/rest.address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i 's/rest.bind-address: localhost/rest.bind-address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i 's/jobmanager.bind-host: localhost/jobmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i 's/taskmanager.bind-host: localhost/taskmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i '/taskmanager.host: localhost/d' $FLINK_HOME/conf/flink-conf.yaml;
+
+# Configure container
+COPY docker-entrypoint.sh /
+RUN chmod 755 /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]
+EXPOSE 6123 8081
+CMD ["help"]