Merge pull request #639 from mendix/develop

Release 2023-06-06
mendix · Jun 6, 2023 · 8be13f0 · 8be13f0
2 parents c4ba778 + 9e4cb20
commit 8be13f0
Show file tree

Hide file tree

Showing 19 changed files with 154 additions and 72 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -42,13 +42,13 @@ jobs:
   build:
     name: Build
     needs: get-version
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.6.15
+          python-version: 3.10.6
           cache: pip
       - name: Install Python requirements
         run: make install_requirements

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -11,7 +11,7 @@ on:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: false
 
 jobs:
   pre:
@@ -49,7 +49,7 @@ jobs:
   integration-test:
     needs: [build, setup-integration-tests]
     name: Run integration tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 30
     strategy:
       matrix: ${{fromJson(needs.setup-integration-tests.outputs.matrix)}}
@@ -64,9 +64,9 @@ jobs:
         name: dist
         path: dist
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.6.15
+        python-version: 3.10.6
         cache: pip
     - name: Install Python requirements
       run: make install_requirements
@@ -77,7 +77,7 @@ jobs:
         echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
         sudo apt-get update && sudo apt-get install -y cf-cli
         sudo curl -s https://download.mendix.com/Mendix-CA-G2.crt -o /usr/local/share/ca-certificates/ca.crt && sudo update-ca-certificates
-        cf install-plugin cflocal -f
+        cf install-plugin https://cdn.mendix.com/mx-buildpack/cflocal/cflocal-0.20.0-rc1-linux -f
     - name: Export hostname for use in tests
       run: |
         TEST_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,14 +19,7 @@ repos:
     rev: v0.12.0
     hooks:
       - id: markdownlint_docker
-  - repo: https://github.com/psf/black
-    rev: 22.8.0
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.0.261'
     hooks:
-      - id: black
-        language_version: python3.6
-        files: >
-          (?x)^(
-              buildpack/.*|
-              lib/m2ee/.*|
-              tests/.*
-          )$
+      - id: ruff
diff --git a/DEVELOPING.md b/DEVELOPING.md
@@ -13,7 +13,7 @@ The buildpack project features the following directories:
 * `lib` : A forked version of M2EE Tools suited for the cloud lives here, used for working with the Mendix Runtime
 * `tests` : All test code lives here
 
-The following directories are not included in the repositoty, but are used in the development process:
+The following directories are not included in the repository, but are used in the development process:
 
 * `build` : Working directory for building the buildpack
 * `dist` : The final build artifact lives here
@@ -32,7 +32,7 @@ For developing the buildpack, you must set up the following:
 
 ### Installing `pyenv`
 
-[`pyenv`](https://github.com/pyenv/pyenv) in combination with [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) can be used to create a local Python virtual environment to develop in. Note that you'll have to create an environment with the latest version of Python 3.6 - the default in the Cloud Foundry root filesystem ( `cflinuxfs3` ) we use to deploy applications on.
+[`pyenv`](https://github.com/pyenv/pyenv) in combination with [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) can be used to create a local Python virtual environment to develop in. Note that you'll have to create an environment with the latest version of Python 3.10 - the default in the Cloud Foundry root filesystem ( `cflinuxfs4` ) we use to deploy applications on.
 
 ### Developing in Docker
 

diff --git a/Makefile b/Makefile
@@ -1,16 +1,14 @@
 PROJECT_NAME ?= cf-mendix-buildpack
 PREFIX = $(shell p='$(TEST_PREFIX)'; echo "$${p:-test}")
-TEST_PROCESSES ?= 2
 TEST_FILES ?= tests/integration/test_*.py
-MAX_LINE_LENGTH = $(shell cat .pylintrc | grep max-line-length | cut -d '=' -f 2 | xargs)
 
 VERSION ?= $(shell git tag --list --sort=-version:refname "v*" | head -n 1)
 COMMIT ?= $(shell git rev-parse --short HEAD)
 
-PIP_TOOLS_VERSION ?= 6.4.0
-PIP_VERSION ?= 21.3.1
+PIP_TOOLS_VERSION ?= 6.13.0
+PIP_VERSION ?= 23.1
 PYTHON_PLATFORM ?= manylinux2014_x86_64
-PYTHON_VERSION ?= 36
+PYTHON_VERSION ?= 310
 
 .PHONY: vendor
 vendor: create_build_dirs copy_vendored_dependencies download_wheels
@@ -60,8 +58,8 @@ install_requirements: install_piptools requirements
 
 .PHONY: requirements
 requirements: install_piptools
-	pip-compile requirements*.in -o requirements-all.txt
-	pip-compile requirements.in
+	pip-compile --resolver=backtracking requirements*.in -o requirements-all.txt
+	pip-compile --resolver=backtracking requirements.in
 
 .PHONY: write_version
 write_version:
@@ -102,9 +100,9 @@ test: test_unit test_integration
 
 .PHONY: format
 format:
-	black --line-length=${MAX_LINE_LENGTH} buildpack lib/m2ee/* tests/*/
+	ruff check buildpack lib/m2ee/* tests/*/ --fix-only --show-fixes
 
 .PHONY: lint
 lint:
-	black --line-length=${MAX_LINE_LENGTH} --check --diff buildpack lib/m2ee/* tests/*/
+	ruff check buildpack lib/m2ee/* tests/*/ --show-source
 	pylint --disable=W,R,C buildpack lib/m2ee/* tests/*/
diff --git a/README.md b/README.md
@@ -55,7 +55,7 @@ Release notes are available for the [buildpack](https://github.com/mendix/cf-men
 
 ## Requirements
 
-The buildpack requires a cluster which supports the `cflinuxfs3` stack.
+The buildpack requires a cluster which supports the `cflinuxfs4` stack.
 
 Additionally, we recommend a base level knowledge of Cloud Foundry. You should at least be familiar with the Cloud Foundry CLI.
 
@@ -74,6 +74,15 @@ The following table shows which specific buildpack release introduced or removed
 
 We recommend using a [maintained (LTS / MTS)](https://docs.mendix.com/releasenotes/studio-pro/lts-mts) Mendix version. Additionally, we recommend to always [use / "pin to" a specific buildpack release](#buildpack-releases-and-version-pinning).
 
+## Supported CFLinuxFS Versions
+
+| Buildpack | Supported    | End-of-Support |
+|-----------|--------------|----------------|
+| `4.x`     | `cflinuxfs3` | 2023 Q3        |
+| `5.x`     | `cflinuxfs4` | -              |
+
+We recommend switching to version 5.x with cflinuxfs4 since cflinuxfs3 is no longer maintained.
+
 ## Buildpack Releases and Version Pinning
 
 We recommend "pinning to" - using a specific [release](https://github.com/mendix/cf-mendix-buildpack/releases) of - the buildpack. This will prevent you from being affected by bugs that are inadvertently introduced, but you will need to set up a procedure to regularly move to new versions of the buildpack.
@@ -584,7 +593,7 @@ To enable New Relic, simply bind a New Relic service to this app and settings wi
 
 #### Set up Splunk integration
 
-To collect Mendix Runtime logs to [Splunk Cloud Platform](https://www.splunk.com/en_us/products/splunk-cloud-platform.html), 
+To collect Mendix Runtime logs to [Splunk Cloud Platform](https://www.splunk.com/en_us/products/splunk-cloud-platform.html),
 [Fluent Bit](https://docs.fluentbit.io/manual/) is used.
 
 To enable Splunk integration for a Mendix application, following environment variables should be configured.
@@ -873,7 +882,7 @@ purposes, you can do the following:
 
 ```shell
 cf ssh <YOUR_APP>
-export PYTHONPATH=/home/vcap/app/.local/lib/python3.6/site-packages/:/home/vcap/app/lib/
+export PYTHONPATH=/home/vcap/app/.local/lib/python3.10/site-packages/:/home/vcap/app/lib/
 python3
 ```
 

diff --git a/bin/bootstrap-python b/bin/bootstrap-python
@@ -6,8 +6,8 @@ BUILDPACK_PATH=$(dirname "$0")/..
 BUILD_PATH=$1
 CACHE_PATH=$2
 
-# This is set as a (temporary) solution for pip 21.2 and https://github.com/pypa/pip/issues/10151
-export _PIP_LOCATIONS_NO_WARN_ON_MISMATCH=1
+# Silence warning about outdated pip version
+export PIP_DISABLE_PIP_VERSION_CHECK=1
 
 # Bootstrap and install the Python requirements from wheels.
 # Minimal packaging for the buildpack includes pip and setuptools wheels.
@@ -39,7 +39,12 @@ then
 fi
 
 echo " ---> Bootstrapping pip..."
-PIP_CMD="pip"
+if [ "$CF_STACK" == "cflinuxfs4" ]; then
+  PIP_CMD="pip3"
+else
+  PIP_CMD="pip"
+fi
+
 if ! [[ -x "$(command -v ${PIP_CMD})" ]]
 then
     # pip is not included on the root FS
@@ -49,12 +54,15 @@ then
         python3 ${PIP_WHL_PATH}/pip install $PIP_VERBOSITY_FLAGS --user --no-warn-script-location --no-index --find-links=$PIP_WHEELDIR pip setuptools wheel
     else
         # Bootstrap wheels are not bundled in the buildpack
-        curl https://bootstrap.pypa.io/pip/3.6/get-pip.py -o get-pip.py
+        curl -sS https://bootstrap.pypa.io/pip/get-pip.py -o get-pip.py
         python3 get-pip.py install $PIP_VERBOSITY_FLAGS --user --no-warn-script-location
     fi
     PIP_CMD="/home/vcap/.local/bin/pip"
 else
     # pip is bundled on the root FS
+    if [ "$CF_STACK" == "cflinuxfs4" ]; then
+      PIP_CMD="python3 $(dirname "$(command -v python3)")/pip3"
+    fi
     $PIP_CMD install --upgrade --user --no-warn-script-location pip setuptools wheel
 fi
 

diff --git a/bin/finalize b/bin/finalize
@@ -3,9 +3,9 @@
 # Finalize script is executed from /home/vcap and buildpack is downloaded in /tmp/buildpacks/<<something>>/
 # Determine the buildpack folder based on this file
 BUILDPACK_PATH=$(dirname "$0")/..
-BUILD_PATH=$1
+BUILD_PATH=$1  #/tmp/app -> /home/vcap/app
 CACHE_PATH=$2
-DEPS_PATH=$3/$4
+DEPS_PATH=$3/$4  #tmp/deps/0/ -> /home/vcap/deps/0/
 PROFILE_DIR=${5:-}
 
 VERSION_PATH="${BUILDPACK_PATH}/VERSION"
@@ -23,6 +23,14 @@ name: mendix
 version: ${VERSION}
 EOF
 
+# Install python if stack is cflinuxfs4
+if [ "$CF_STACK" == "cflinuxfs4" ]; then
+  echo " ---> Detected cflinuxfs4 without built-in python-dev..."
+  PYTHON_DIR="$DEPS_PATH/python"
+  mkdir -p $PYTHON_DIR
+  source "$BUILDPACK_PATH/bin/install-python" "$PYTHON_DIR" "$BUILDPACK_PATH"
+fi
+
 # Bootstrap the Python dependencies needed for the buildpack
 source ${BUILDPACK_PATH}/bin/bootstrap-python
 

diff --git a/bin/install-python b/bin/install-python
@@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+shopt -s expand_aliases
+
+# This file is a copy of https://github.com/cloudfoundry/php-buildpack/commit/5074091c420bcb569b98cf5af151b6e35d79d59f
+# please refer to that location for future changes.
+
+function main() {
+  install_dir="$1"
+  buildpack_dir="$2"
+
+  if [ ! -d "/tmp/python/bin" ]; then
+    python_dep_name=$(get_python_from_manifest "$buildpack_dir")
+    echo " -------> Providing Python $(echo $python_dep_name | cut -d'_' -f 2) (required by the buildpack to run in cflinuxfs4)"
+
+    if [ ! -d "$buildpack_dir/dependencies" ]; then
+      setup_online "$python_dep_name" "$install_dir" "$buildpack_dir"
+    else
+      setup_offline "$python_dep_name" "$install_dir" "$buildpack_dir"
+    fi
+  elif [ $install_dir != "/tmp/python" ]; then
+    cp -r "/tmp/python/." "$install_dir"
+  fi
+
+  export LD_LIBRARY_PATH="$install_dir/lib:${LD_LIBRARY_PATH:-}"
+  export PATH="$install_dir/bin:${PATH:-}"
+}
+
+function setup_offline() {
+  python_dep_name="$1"
+  install_dir="$2"
+  buildpack_dir="$3"
+
+  tar -xzf "$buildpack_dir/dependencies/https___buildpacks.cloudfoundry.org_dependencies_python_$python_dep_name" -C "$install_dir"
+}
+
+function setup_online(){
+  python_dep_name="$1"
+  install_dir="$2"
+  buildpack_dir="$3"
+
+  curl -Ls "https://buildpacks.cloudfoundry.org/dependencies/python/$python_dep_name" | tar -xzf - -C "$install_dir"
+}
+
+function get_python_from_manifest() {
+  buildpack_dir="$1"
+  python_dep_info=$(cat "$buildpack_dir/dependencies-stage.yml" | grep -A 3 "name: python" | grep "uri:" | awk '{print $2}' | cut -d'/' -f 6)
+  echo "$python_dep_info"
+}
+
+main "${@:-}"
diff --git a/bin/release b/bin/release
@@ -2,5 +2,5 @@
 
 cat << EOF
 default_process_types:
-  web: PYTHONPATH=$PYTHONPATH:buildpack:lib python3 buildpack/start.py
+  web: PATH="/home/vcap/deps/0/python/bin:$PATH" PYTHONPATH=$PYTHONPATH:buildpack:lib python3 buildpack/start.py
 EOF
diff --git a/buildpack/stage.py b/buildpack/stage.py
@@ -35,7 +35,7 @@
     PROFILE_DIR = os.path.abspath(os.path.join(BUILD_DIR, ".profile.d"))
 
 SUPPORTED_STACKS = [
-    "cflinuxfs3",
+    "cflinuxfs4",
     None,
 ]  # None is allowed, but not supported
 

diff --git a/buildpack/start.py b/buildpack/start.py
@@ -65,7 +65,15 @@ def _kill_process_group_with_signal(signum):
             )
         except OSError as error:
             logging.debug(
-                "Failed to send [%s] to process group [%s]: %s",
+                "Failed to send [%s] to process group [%s]: (OSError) %s",
+                signum.name,
+                process_group,
+                error,
+            )
+        except SystemExit as error:
+            # Workaround for UPV4-2859 - https://github.com/python/cpython/issues/103512#issuecomment-1541021187
+            logging.debug(
+                "Failed to send [%s] to process group [%s]: (SystemExit) %s",
                 signum.name,
                 process_group,
                 error,

diff --git a/dependencies-stage.yml b/dependencies-stage.yml
@@ -0,0 +1,8 @@
+---
+- name: python
+  version: 3.10.6
+  uri: https://buildpacks.cloudfoundry.org/dependencies/python/python_3.10.6_linux_x64_cflinuxfs4_62e69f06.tgz
+  sha256: 62e69f0664e0bbb05a6351d4d96dc9ea6b959c4c2e4ccdb124f932d0bdac0e41
+  cf_stacks:
+  - cflinuxfs4
+  source_sha256: 848cb06a5caa85da5c45bd7a9221bb821e33fc2bdcba088c127c58fad44e6343
diff --git a/dependencies.yml b/dependencies.yml
@@ -29,7 +29,7 @@ dependencies:
       artifact: "{{ url }}/e/{{ environment }}/api/v1/deployment/installer/agent/unix/paas/latest?include=java&bitness=64&Api-Token={{ token }}"
       managed: false
   fluentbit:
-    artifact: fluentbit/fluent-bit-{{ version }}.tar.gz
+    artifact: fluentbit/fluent-bit-{{ version }}-cflinuxfs4.tar.gz
     version: 1.9.2
   java:
     keyutil:
@@ -76,21 +76,22 @@ dependencies:
       bom_type: framework
       cpe: cpe:2.3:a:mono-project:mono:{{ version }}:*:*:*:*:*:*:*
       type:
-        - bionic
+        - jammy
       version:
-        - "3": 3.10.0
         - "4": 4.6.2.16
         - "5": 5.20.1.27
   newrelic:
     agent:
       artifact: newrelic/newrelic-java-{{ version }}.zip
       version: 6.5.4
   nginx:
-    artifact: nginx_{{ version }}_linux_x64_cflinuxfs3_{{ commit }}.tgz
-    commit: f0918d6b
+    artifact: nginx_{{ version }}_linux_x64_{{ fs }}_{{ commit }}.tgz
+    commit: 909b06a9
+    fs: cflinuxfs4
     cpe: cpe:2.3:a:f5:nginx:{{ version }}:*:*:*:*:*:*:*
-    version: 1.21.1
+    version: 1.22.1
   telegraf:
     agent:
       artifact: telegraf/telegraf-{{ version }}_linux_amd64.tar.gz
       version: 1.24.4
+# https://github.com/cloudfoundry/public-buildpacks-ci-robots/tree/main/binary-builds-new
diff --git a/dev/Dockerfile b/dev/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.6
+FROM --platform=linux/amd64 python:3.10
 
 WORKDIR /app
 

diff --git a/etc/fluentbit/metadata.lua b/etc/fluentbit/metadata.lua
@@ -22,4 +22,4 @@ function add_metadata(tag, timestamp, record)
     end
 
     return 2, timestamp, record
-end
+end