Skip to content

Commit

Permalink
Merge pull request #707 from mendix/develop
Browse files Browse the repository at this point in the history
Release 2023-12-21
  • Loading branch information
mbyregowMx authored Dec 21, 2023
2 parents 9b8f652 + f9ef92c commit 20f07d8
Show file tree
Hide file tree
Showing 66 changed files with 6,275 additions and 121 deletions.
1 change: 1 addition & 0 deletions .github/.mdl_style.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
all
exclude_rule 'MD007'
exclude_rule 'MD013'
exclude_rule 'MD024'
exclude_rule 'MD029'
exclude_rule 'MD034'
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
fail_fast: false
repos:
- repo: https://github.com/adrienverge/yamllint.git
rev: v1.30.0
rev: v1.33.0
hooks:
- id: yamllint
args: ["--format", "parsable", "--strict"]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v4.5.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
Expand All @@ -20,6 +20,6 @@ repos:
hooks:
- id: markdownlint_docker
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: 'v0.0.261'
rev: 'v0.1.6'
hooks:
- id: ruff
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ TEST_FILES ?= tests/integration/test_*.py
VERSION ?= $(shell git tag --list --sort=-version:refname "v*" | head -n 1)
COMMIT ?= $(shell git rev-parse --short HEAD)

PIP_TOOLS_VERSION ?= 6.13.0
PIP_VERSION ?= 23.1
PIP_TOOLS_VERSION ?= 7.3.0
PIP_VERSION ?= 23.3.1
PYTHON_PLATFORM ?= manylinux2014_x86_64
PYTHON_VERSION ?= 310

Expand Down
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ cf set-env <YOUR_APP> DATABASE_CONNECTION_PARAMS '{"tcpKeepAlive": "true", "conn

*:warning: If you set `DATABASE_URL` as JDBC connection string (prefixed with `jdbc:` and including parameters, `DATABASE_CONNECTION_PARAMS` is not required.*

To allow connection to an AWS RDS database the buildpack selects the regional CA certificate stored in [`rds-certificates`](etc/rds-certificates). If the region's certificate doesn't exist, the buildpack will fail with an error `Could not find database CA certificate in map`.

#### Supported VCAP Schemas

Cloud Foundry database services are detected from Cloud Foundry service bindings ([VCAP](https://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html#VCAP-SERVICES)) and translated into Mendix Runtime configuration. In case no database service is bound, the fallback is the environment variable `DATABASE_URL`.
Expand Down Expand Up @@ -895,7 +897,7 @@ For the third-party integrations explained above, in addition to the metrics col
This configuration also has a filtering mechanism that allows users to specify metrics they allow or deny for the vendor they are using.
To filter the ingestion of custom runtime metrics to third party APMs, users should provide a list of prefixes of the metrics they want to allow/deny using the environment variables listed below.

Note: Custom database metrics cannot be filtered by name, to turn them off, the `APPMETRICS_INCLUDE_DB` environment variable should be set to false.
Note: Custom database metrics cannot be filtered by name, to turn them off, the `APPMETRICS_INCLUDE_DB` environment variable should be set to false.

#### APM_METRICS_FILTER_ALLOW

Expand All @@ -909,7 +911,7 @@ cf set-env <YOUR_APP> APM_METRICS_FILTER_ALLOW 'mx.runtime.stats.sessions'

#### APM_METRICS_FILTER_DENY

Comma-separated list of prefixes for the metrics to be denied.
Comma-separated list of prefixes for the metrics to be denied.

For example, to deny all metrics starting with jetty or mx.runtime, the environment variable should be set to `jetty,mx.runtime`:

Expand All @@ -925,7 +927,6 @@ If this environment variable is set to `true`, all metrics will be denied regard
cf set-env <YOUR_APP> APM_METRICS_FILTER_DENY_ALL true
```


## Using the Buildpack without an Internet Connection

If you are running Cloud Foundry without a connection to the Internet, you should specify an on-premises web server that hosts Mendix Runtime files and other buildpack dependencies. You can set the endpoint with the following environment variable:
Expand Down
2 changes: 1 addition & 1 deletion bin/install-python
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,4 @@ function get_python_from_manifest() {
echo "$python_dep_info"
}

main "${@:-}"
main "${@:-}"
2 changes: 1 addition & 1 deletion buildpack/core/java.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def stage(buildpack_path, cache_path, local_path, java_major_version):


def _compose_jvm_target_dir(dependency):
return f"usr/lib/jvm/{dependency['vendor']}-{dependency['type']}-{dependency['version']}-{dependency['vendor']}-x64" # noqa: line-too-long
return f"usr/lib/jvm/{dependency['vendor']}-{dependency['type']}-{dependency['version']}-{dependency['vendor']}-x64" # noqa: C0301


def _get_java_dependency(
Expand Down
12 changes: 6 additions & 6 deletions buildpack/core/nginx.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@
from lib.m2ee.util import strtobool

ALLOWED_HEADERS = {
"X-Frame-Options": r"(?i)(^allow-from https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$|^deny$|^sameorigin$)", # noqa: line-too-long
"Referrer-Policy": r"(?i)(^no-referrer$|^no-referrer-when-downgrade$|^origin|origin-when-cross-origin$|^same-origin|strict-origin$|^strict-origin-when-cross-origin$|^unsafe-url$)", # noqa: line-too-long
"Access-Control-Allow-Origin": r"(?i)(^\*$|^null$|^https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)", # noqa: line-too-long
"X-Frame-Options": r"(?i)(^allow-from https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$|^deny$|^sameorigin$)", # noqa: C0301
"Referrer-Policy": r"(?i)(^no-referrer$|^no-referrer-when-downgrade$|^origin|origin-when-cross-origin$|^same-origin|strict-origin$|^strict-origin-when-cross-origin$|^unsafe-url$)", # noqa: C0301
"Access-Control-Allow-Origin": r"(?i)(^\*$|^null$|^https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)", # noqa: C0301
"X-Content-Type-Options": r"(?i)(^nosniff$)",
"Content-Security-Policy": r"[a-zA-Z0-9:;/''\"\*_\- \.\n?=%&+]+",
"Strict-Transport-Security": r"(?i)(^max-age=[0-9]*$|^max-age=[0-9]*; includeSubDomains$|^max-age=[0-9]*; preload$)", # noqa: line-too-long
"X-Permitted-Cross-Domain-Policies": r"(?i)(^all$|^none$|^master-only$|^by-content-type$|^by-ftp-filename$)", # noqa: line-too-long
"Strict-Transport-Security": r"(?i)(^max-age=[0-9]*$|^max-age=[0-9]*; includeSubDomains$|^max-age=[0-9]*; preload$)", # noqa: C0301
"X-Permitted-Cross-Domain-Policies": r"(?i)(^all$|^none$|^master-only$|^by-content-type$|^by-ftp-filename$)", # noqa: C0301
"Origin-Trial": r"[a-zA-Z0-9:;/''\"\*_\- \.\n?=%&+]+",
"X-XSS-Protection": r"(?i)(^0$|^1$|^1; mode=block$|^1; report=https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)", # noqa: line-too-long
"X-XSS-Protection": r"(?i)(^0$|^1$|^1; mode=block$|^1; report=https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)", # noqa: C0301
}

CONFIG_FILE = "nginx/conf/nginx.conf"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import ( # noqa: line-too-long
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import ( # noqa: C0301
DebeziumInterface,
)
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_default import ( # noqa: line-too-long
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_default import ( # noqa: C0301
DebeziumDefault,
)

# Do not remove this import, it allows automatic class load
from buildpack.databroker.config_generator.scripts.generators.debezium_configs import * # noqa: line-too-long,F403
from buildpack.databroker.config_generator.scripts.generators.debezium_configs import * # noqa: C0301, F403


def generate_config(config):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import ( # noqa: line-too-long
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import ( # noqa: C0301
DebeziumInterface,
)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import ( # noqa: line-too-long
from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import ( # noqa: C0301
DebeziumInterface,
)
from buildpack.databroker.config_generator.scripts.constants import (
Expand Down
12 changes: 6 additions & 6 deletions buildpack/databroker/config_generator/templates/jmx/consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
},
{
"include": {
"bean_regex": "kafka\.consumer:type=consumer-coordinator-metrics,client-id=.*", # noqa: line-too-long
"bean_regex": "kafka\.consumer:type=consumer-coordinator-metrics,client-id=.*", # noqa: C0301
"attribute": {
"commit-latency-avg": {
"alias": "kafka.consumer.coordinator.commit-latency-avg"
Expand Down Expand Up @@ -82,10 +82,10 @@
"alias": "kafka.consumer.coordinator.partitions-revoked-latency-max"
},
"partitions-assigned-latency-avg": {
"alias": "kafka.consumer.coordinator.partitions-assigned-latency-avg" # noqa: line-too-long
"alias": "kafka.consumer.coordinator.partitions-assigned-latency-avg" # noqa: C0301
},
"partitions-assigned-latency-max": {
"alias": "kafka.consumer.coordinator.partitions-assigned-latency-max" # noqa: line-too-long
"alias": "kafka.consumer.coordinator.partitions-assigned-latency-max" # noqa: C0301
},
"partitions-lost-latency-avg": {
"alias": "kafka.consumer.coordinator.partitions-lost-latency-avg"
Expand All @@ -98,7 +98,7 @@
},
{
"include": {
"bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*", # noqa: line-too-long
"bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*", # noqa: C0301
"attribute": {
"bytes-consumed-rate": {
"alias": "kafka.consumer.fetch.manager.bytes-consumed-rate"
Expand Down Expand Up @@ -146,7 +146,7 @@
},
{
"include": {
"bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*,topic=.*", # noqa: line-too-long
"bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*,topic=.*", # noqa: C0301
"attribute": {
"bytes-consumed-rate": {
"alias": "kafka.consumer.fetch.manager.bytes-consumed-rate"
Expand Down Expand Up @@ -174,7 +174,7 @@
},
{
"include": {
"bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,partition=.*,topic=.*,client-id=.*", # noqa: line-too-long
"bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,partition=.*,topic=.*,client-id=.*", # noqa: C0301
"attribute": {
"preferred-read-replica": {
"alias": "kafka.consumer.fetch.manager.preferred-read-replica"
Expand Down
105 changes: 86 additions & 19 deletions buildpack/infrastructure/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from buildpack import util




def get_config():
# the following options are validated to get database credentials
# 1) existence of custom runtime settings Database.... values
Expand Down Expand Up @@ -251,9 +253,39 @@ def __init__(self, url, env_vars=None):
self.m2ee_config = None

def init(self):
region_pem_map = {
"us-east-1": "us-east-1-bundle.pem",
"us-east-2": "us-east-2-bundle.pem",
"us-west-1": "us-west-1-bundle.pem",
"us-west-2": "us-west-2-bundle.pem",
"af-south-1": "af-south-1-bundle.pem",
"ap-east-1": "ap-east-1-bundle.pem",
"ap-south-2": "ap-south-2-bundle.pem",
"ap-southeast-3": "ap-southeast-3-bundle.pem",
"ap-southeast-4": "ap-southeast-4-bundle.pem",
"ap-south-1": "ap-south-1-bundle.pem",
"ap-northeast-3": "ap-northeast-3-bundle.pem",
"ap-northeast-1": "ap-northeast-1-bundle.pem",
"ap-northeast-2": "ap-northeast-2-bundle.pem",
"ap-southeast-1": "ap-southeast-1-bundle.pem",
"ap-southeast-2": "ap-southeast-2-bundle.pem",
"ca-central-1": "ca-central-1-bundle.pem",
"eu-central-1": "eu-central-1-bundle.pem",
"eu-west-1": "eu-west-1-bundle.pem",
"eu-west-2": "eu-west-2-bundle.pem",
"eu-south-1": "eu-south-1-bundle.pem",
"eu-west-3": "eu-west-3-bundle.pem",
"eu-south-2": "eu-south-2-bundle.pem",
"eu-north-1": "eu-north-1-bundle.pem",
"eu-central-2": "eu-central-2-bundle.pem",
"il-central-1": "il-central-1-bundle.pem",
"me-south-1": "me-south-1-bundle.pem",
"me-central-1": "me-central-1-bundle.pem",
"sa-east-1": "sa-east-1-bundle.pem"
}
patterns = [
r"(?P<type>[a-zA-Z0-9]+)://(?P<user>[^:]+):(?P<password>[^@]+)@(?P<host>[^/]+)/(?P<dbname>[^?]*)(?P<extra>\?.*)?", # noqa: line-too-long
r"jdbc:(?P<type>[a-zA-Z0-9]+)://(?P<host>[^;]+);database=(?P<dbname>[^;]*);user=(?P<user>[^;]+);password=(?P<password>.*)$", # noqa: line-too-long
r"(?P<type>[a-zA-Z0-9]+)://(?P<user>[^:]+):(?P<password>[^@]+)@(?P<host>[^/]+)/(?P<dbname>[^?]*)(?P<extra>\?.*)?", # noqa: C0301
r"jdbc:(?P<type>[a-zA-Z0-9]+)://(?P<host>[^;]+);database=(?P<dbname>[^;]*);user=(?P<user>[^;]+);password=(?P<password>.*)$", # noqa: C0301
]

supported_databases = {
Expand Down Expand Up @@ -300,18 +332,22 @@ def init(self):
if database_type == "PostgreSQL":
jdbc_params.update({"tcpKeepAlive": "true"})

if database_type == "PostgreSQL" and config["DatabaseHost"].split(":")[
0
].endswith(".rds.amazonaws.com"):
jdbc_params.update(
{
"sslrootcert": os.path.expandvars(
"$HOME/.postgresql/amazon-rds-ca.pem"
)
}
)
jdbc_params.update({"sslmode": "verify-full"})

try:
if database_type == "PostgreSQL" and config["DatabaseHost"].split(":")[
0
].endswith(".rds.amazonaws.com"):
database_region = config["DatabaseHost"].split('.')[2]
jdbc_params.update(
{
"sslrootcert": os.path.expandvars(
"$HOME/.postgresql/"+region_pem_map[database_region]
)
}
)
jdbc_params.update({"sslmode": "verify-full"})
except Exception:
raise Exception("Could not find database CA certificate in map")

if database_type == "PostgreSQL" and not self.url.startswith("jdbc:"):
self.extract_inline_cert(jdbc_params, self.SSLCERT, "postgresql.crt")
self.extract_inline_cert(jdbc_params, self.SSLKEY, "postgresql.pk8")
Expand Down Expand Up @@ -465,7 +501,7 @@ def get_database_password(self):
def get_database_jdbc_url(self):
"""Return the database jdbc url for the M2EE configuration"""
url = self.credentials.get("url", "")
pattern = r"jdbc:sap://(?P<host>[^:]+):(?P<port>[0-9]+)/?(?P<q>\?(?P<params>.*))?$" # noqa:line-too-long
pattern = r"jdbc:sap://(?P<host>[^:]+):(?P<port>[0-9]+)/?(?P<q>\?(?P<params>.*))?$" # noqa:C0301
match = re.search(pattern, url)
if match is None:
logging.error("Unable to parse Hana JDBC url string for parameters")
Expand Down Expand Up @@ -495,11 +531,42 @@ def get_additional_m2ee_config(self):

def stage(buildpack_dir, build_dir):
logging.debug("Staging database...")
region_pem_map = {
"us-east-1": "us-east-1-bundle.pem",
"us-east-2": "us-east-2-bundle.pem",
"us-west-1": "us-west-1-bundle.pem",
"us-west-2": "us-west-2-bundle.pem",
"af-south-1": "af-south-1-bundle.pem",
"ap-east-1": "ap-east-1-bundle.pem",
"ap-south-2": "ap-south-2-bundle.pem",
"ap-southeast-3": "ap-southeast-3-bundle.pem",
"ap-southeast-4": "ap-southeast-4-bundle.pem",
"ap-south-1": "ap-south-1-bundle.pem",
"ap-northeast-3": "ap-northeast-3-bundle.pem",
"ap-northeast-1": "ap-northeast-1-bundle.pem",
"ap-northeast-2": "ap-northeast-2-bundle.pem",
"ap-southeast-1": "ap-southeast-1-bundle.pem",
"ap-southeast-2": "ap-southeast-2-bundle.pem",
"ca-central-1": "ca-central-1-bundle.pem",
"eu-central-1": "eu-central-1-bundle.pem",
"eu-west-1": "eu-west-1-bundle.pem",
"eu-west-2": "eu-west-2-bundle.pem",
"eu-south-1": "eu-south-1-bundle.pem",
"eu-west-3": "eu-west-3-bundle.pem",
"eu-south-2": "eu-south-2-bundle.pem",
"eu-north-1": "eu-north-1-bundle.pem",
"eu-central-2": "eu-central-2-bundle.pem",
"il-central-1": "il-central-1-bundle.pem",
"me-south-1": "me-south-1-bundle.pem",
"me-central-1": "me-central-1-bundle.pem",
"sa-east-1": "sa-east-1-bundle.pem"
}
util.mkdir_p(os.path.join(build_dir, ".postgresql"))
shutil.copy(
os.path.join(buildpack_dir, "etc", "amazon-rds-ca.pem"),
os.path.join(build_dir, ".postgresql", "amazon-rds-ca.pem"),
)
for key, value in region_pem_map.items():
shutil.copy(
os.path.join(buildpack_dir, "etc/rds-certificates", value),
os.path.join(build_dir, ".postgresql", value ),
)


def update_config(m2ee):
Expand Down
4 changes: 2 additions & 2 deletions buildpack/telemetry/appdynamics.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"APPDYNAMICS_AGENT_APPLICATION_NAME",
default=util.get_app_from_domain(),
),
"APPDYNAMICS_AGENT_NODE_NAME": f"{os.getenv('APPDYNAMICS_AGENT_NODE_NAME', default='node')}-{CF_APPLICATION_INDEX}", # noqa: line-too-long
"APPDYNAMICS_AGENT_NODE_NAME": f"{os.getenv('APPDYNAMICS_AGENT_NODE_NAME', default='node')}-{CF_APPLICATION_INDEX}", # noqa: C0301
"APPDYNAMICS_AGENT_TIER_NAME": os.getenv(
"APPDYNAMICS_AGENT_TIER_NAME", default=CF_APPLICATION_NAME
),
Expand All @@ -33,7 +33,7 @@
"APPDYNAMICS_CONTROLLER_SSL_ENABLED": os.getenv(
"APPDYNAMICS_CONTROLLER_SSL_ENABLED", default="true"
),
"APPDYNAMICS_AGENT_UNIQUE_HOST_ID": f"{os.getenv('APPDYNAMICS_AGENT_UNIQUE_HOST_ID', default=CF_APPLICATION_NAME),}-{CF_APPLICATION_INDEX}", # noqa: line-too-long
"APPDYNAMICS_AGENT_UNIQUE_HOST_ID": f"{os.getenv('APPDYNAMICS_AGENT_UNIQUE_HOST_ID', default=CF_APPLICATION_NAME),}-{CF_APPLICATION_INDEX}", # noqa: C0301
}


Expand Down
2 changes: 1 addition & 1 deletion buildpack/telemetry/appdynamics_telegraf_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def convert_and_push_payload():
AppDynamics Machine Agent HTTP listener.
"""
# AppDynamics Docs: https://docs.appdynamics.com/22.2/en/infrastructure-visibility/machine-agent/extensions-and-custom-metrics/machine-agent-http-listener # noqa: line-too-long
# AppDynamics Docs: https://docs.appdynamics.com/22.2/en/infrastructure-visibility/machine-agent/extensions-and-custom-metrics/machine-agent-http-listener # noqa: C0301

metrics_str = input()
metrics_dict = json.loads(metrics_str)
Expand Down
Loading

0 comments on commit 20f07d8

Please sign in to comment.