diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt index 2f2b64723f0edd..fa07b4184a6bc0 100644 --- a/docker/datahub-ingestion-base/base-requirements.txt +++ b/docker/datahub-ingestion-base/base-requirements.txt @@ -1,19 +1,20 @@ # Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate. -acryl-datahub-classify==0.0.10 +acryl-datahub-classify==0.0.11 acryl-PyHive==0.6.16 -acryl-sqlglot==24.0.1.dev7 +acryl-sqlglot==25.3.1.dev3 aenum==3.1.15 -aiohttp==3.9.5 +aiohappyeyeballs==2.3.2 +aiohttp==3.10.0 aiosignal==1.3.1 -alembic==1.13.1 +alembic==1.13.2 altair==4.2.0 anyio==4.4.0 -apache-airflow==2.9.2 +apache-airflow==2.9.3 apache-airflow-providers-common-io==1.3.2 -apache-airflow-providers-common-sql==1.14.0 -apache-airflow-providers-fab==1.1.1 -apache-airflow-providers-ftp==3.9.1 -apache-airflow-providers-http==4.11.1 +apache-airflow-providers-common-sql==1.14.2 +apache-airflow-providers-fab==1.2.2 +apache-airflow-providers-ftp==3.10.0 +apache-airflow-providers-http==4.12.0 apache-airflow-providers-imap==3.6.1 apache-airflow-providers-smtp==1.7.1 apache-airflow-providers-sqlite==3.8.1 @@ -26,25 +27,30 @@ asgiref==3.8.1 asn1crypto==1.5.1 asttokens==2.4.1 async-timeout==4.0.3 -asynch==0.2.3 +asynch==0.2.4 attrs==23.2.0 avro==1.11.3 avro-gen3==0.7.13 +azure-common==1.1.28 +azure-core==1.29.4 +azure-identity==1.14.1 +azure-storage-blob==12.21.0 +azure-storage-file-datalake==12.16.0 Babel==2.15.0 backoff==2.2.1 beautifulsoup4==4.12.3 bleach==6.1.0 blinker==1.8.2 blis==0.7.11 -boto3==1.34.129 -botocore==1.34.129 +boto3==1.34.151 +botocore==1.34.151 bracex==2.4 cached-property==1.5.2 cachelib==0.9.0 -cachetools==5.3.3 +cachetools==5.4.0 catalogue==2.0.10 cattrs==23.2.3 -certifi==2024.6.2 +certifi==2024.7.4 cffi==1.16.0 chardet==5.2.0 charset-normalizer==3.3.2 @@ -55,25 +61,26 @@ click-spinner==0.1.10 clickclick==20.10.2 clickhouse-driver==0.2.8 clickhouse-sqlalchemy==0.2.4 +cloudpathlib==0.18.1 cloudpickle==3.0.0 colorama==0.4.6 colorlog==4.8.0 comm==0.2.2 confection==0.1.5 ConfigUpdater==3.2 -confluent-kafka==2.4.0 -connexion==2.14.1 +confluent-kafka==2.5.0 +connexion==2.14.2 cron-descriptor==1.4.3 -croniter==2.0.5 +croniter==3.0.3 cryptography==42.0.8 cx_Oracle==8.3.0 cymem==2.0.8 databricks-dbapi==0.6.0 -databricks-sdk==0.28.0 +databricks-sdk==0.29.0 databricks-sql-connector==2.9.6 dataflows-tabulator==1.54.3 db-dtypes==1.2.0 -debugpy==1.8.1 +debugpy==1.8.2 decorator==5.1.1 defusedxml==0.7.1 deltalake==0.17.4 @@ -84,42 +91,42 @@ docker==7.1.0 docutils==0.21.2 ecdsa==0.19.0 elasticsearch==7.13.4 -email_validator==2.1.2 +email_validator==2.2.0 entrypoints==0.4 et-xmlfile==1.1.0 -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 executing==2.0.1 expandvars==0.12.0 -fastavro==1.9.4 +fastavro==1.9.5 fastjsonschema==2.20.0 -filelock==3.15.1 +filelock==3.15.4 Flask==2.2.5 flatdict==4.0.1 frozenlist==1.4.1 fsspec==2023.12.2 future==1.0.0 -GeoAlchemy2==0.15.1 +GeoAlchemy2==0.15.2 gitdb==4.0.11 GitPython==3.1.43 -google-api-core==2.19.0 -google-auth==2.30.0 -google-cloud-appengine-logging==1.4.3 +google-api-core==2.19.1 +google-auth==2.32.0 +google-cloud-appengine-logging==1.4.5 google-cloud-audit-log==0.2.5 -google-cloud-bigquery==3.24.0 +google-cloud-bigquery==3.25.0 google-cloud-core==2.4.1 -google-cloud-datacatalog==3.19.0 +google-cloud-datacatalog==3.20.0 google-cloud-datacatalog-lineage==0.2.2 google-cloud-logging==3.5.0 google-crc32c==1.5.0 -google-re2==1.1.20240601 +google-re2==1.1.20240702 google-resumable-media==2.7.1 -googleapis-common-protos==1.63.1 +googleapis-common-protos==1.63.2 gql==3.5.0 graphql-core==3.2.3 great-expectations==0.15.50 greenlet==3.0.3 -grpc-google-iam-v1==0.13.0 -grpcio==1.64.1 +grpc-google-iam-v1==0.13.1 +grpcio==1.65.2 grpcio-status==1.62.2 grpcio-tools==1.62.2 gssapi==1.8.3 @@ -130,7 +137,7 @@ httpx==0.27.0 humanfriendly==10.0 idna==3.7 ijson==3.3.0 -importlib_metadata==7.1.0 +importlib_metadata==7.2.1 importlib_resources==6.4.0 inflection==0.5.1 ipaddress==1.0.23 @@ -148,7 +155,7 @@ jsonlines==4.0.0 jsonpatch==1.33 jsonpointer==3.0.0 jsonref==1.1.0 -jsonschema==4.22.0 +jsonschema==4.23.0 jsonschema-specifications==2023.12.1 jupyter-server==1.16.0 jupyter_client==7.4.9 @@ -159,16 +166,16 @@ langcodes==3.4.0 language_data==1.2.0 lark==1.1.4 lazy-object-proxy==1.10.0 -leb128==1.0.7 -limits==3.12.0 +leb128==1.0.8 +limits==3.13.0 linear-tsv==1.1.0 linkify-it-py==2.0.3 -lkml==1.3.4 +lkml==1.3.5 lockfile==0.12.2 looker-sdk==23.0.0 lxml==5.2.2 lz4==4.3.3 -makefun==1.15.2 +makefun==1.15.4 Mako==1.3.5 marisa-trie==1.2.0 markdown-it-py==3.0.0 @@ -182,11 +189,12 @@ mdurl==0.1.2 methodtools==0.4.7 mistune==3.0.2 mixpanel==4.10.1 -mlflow-skinny==2.14.0 +mlflow-skinny==2.15.0 mmhash3==3.0.1 more-itertools==10.3.0 moto==4.2.14 msal==1.22.0 +msal-extensions==1.1.0 multidict==6.0.5 murmurhash==1.0.10 mypy-extensions==1.0.0 @@ -201,47 +209,46 @@ notebook_shim==0.2.4 numpy==1.26.4 oauthlib==3.2.2 okta==1.7.0 -openlineage-airflow==1.12.0 -openlineage-integration-common==1.12.0 -openlineage-python==1.12.0 -openlineage_sql==1.12.0 -openpyxl==3.1.4 -opentelemetry-api==1.25.0 -opentelemetry-exporter-otlp==1.25.0 -opentelemetry-exporter-otlp-proto-common==1.25.0 -opentelemetry-exporter-otlp-proto-grpc==1.25.0 -opentelemetry-exporter-otlp-proto-http==1.25.0 -opentelemetry-proto==1.25.0 -opentelemetry-sdk==1.25.0 -opentelemetry-semantic-conventions==0.46b0 +openlineage-airflow==1.18.0 +openlineage-integration-common==1.18.0 +openlineage-python==1.18.0 +openlineage_sql==1.18.0 +openpyxl==3.1.5 +opentelemetry-api==1.26.0 +opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-proto==1.26.0 +opentelemetry-sdk==1.26.0 +opentelemetry-semantic-conventions==0.47b0 ordered-set==4.1.0 packaging==24.1 pandas==2.1.4 pandocfilters==1.5.1 parse==1.20.2 parso==0.8.4 -pathlib_abc==0.1.1 pathspec==0.12.1 -pathy==0.11.0 pendulum==3.0.0 pexpect==4.9.0 phonenumbers==8.13.0 platformdirs==4.2.2 pluggy==1.5.0 +portalocker==2.10.1 preshed==3.0.9 prison==0.2.1 progressbar2==4.4.2 prometheus_client==0.20.0 prompt_toolkit==3.0.47 -proto-plus==1.23.0 -protobuf==4.25.3 +proto-plus==1.24.0 +protobuf==4.25.4 psutil==6.0.0 psycopg2-binary==2.9.9 ptyprocess==0.7.0 -pure-eval==0.2.2 pure-sasl==0.6.2 +pure_eval==0.2.3 py-partiql-parser==0.5.0 -pyarrow==16.1.0 +pyarrow==17.0.0 pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 @@ -249,16 +256,16 @@ pyathena==2.25.2 pycountry==24.6.1 pycparser==2.22 pycryptodome==3.20.0 -pydantic==1.10.16 -pydash==8.0.1 +pydantic==1.10.17 +pydash==8.0.3 pydruid==0.6.9 Pygments==2.18.0 pyiceberg==0.4.0 -pymongo==4.7.3 +pymongo==4.8.0 PyMySQL==1.1.1 -pyOpenSSL==24.1.0 +pyOpenSSL==24.2.1 pyparsing==3.0.9 -pyspnego==0.11.0 +pyspnego==0.11.1 python-daemon==3.0.1 python-dateutil==2.9.0.post0 python-jose==3.3.0 @@ -273,9 +280,9 @@ pytz==2024.1 PyYAML==6.0.1 pyzmq==26.0.3 redash-toolbelt==0.1.9 -redshift-connector==2.1.1 +redshift-connector==2.1.2 referencing==0.35.1 -regex==2024.5.15 +regex==2024.7.24 requests==2.32.3 requests-file==2.1.0 requests-gssapi==1.3.0 @@ -286,31 +293,32 @@ rfc3339-validator==0.1.4 rfc3986==2.0.0 rich==13.7.1 rich-argparse==1.5.2 -rpds-py==0.18.1 +rpds-py==0.19.1 rsa==4.9 rstr==3.2.2 ruamel.yaml==0.17.17 -s3transfer==0.10.1 +s3transfer==0.10.2 schwifty==2024.6.1 -scipy==1.13.1 +scipy==1.14.0 scramp==1.4.5 Send2Trash==1.8.3 -sentry-sdk==2.5.1 +sentry-sdk==2.12.0 setproctitle==1.3.3 +shellingham==1.5.4 simple-salesforce==1.12.6 six==1.16.0 slack-sdk==3.18.1 -smart-open==6.4.0 +smart-open==7.0.4 smmap==5.0.1 sniffio==1.3.1 -snowflake-connector-python==3.11.0 -snowflake-sqlalchemy==1.5.3 +snowflake-connector-python==3.12.0 +snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 soupsieve==2.5 -spacy==3.5.0 +spacy==3.7.5 spacy-legacy==3.0.12 spacy-loggers==1.0.5 -sql-metadata==2.2.2 +sql_metadata==2.12.0 SQLAlchemy==1.4.44 sqlalchemy-bigquery==1.11.0 sqlalchemy-cockroachdb==1.4.4 @@ -318,7 +326,7 @@ SQLAlchemy-JSONField==1.0.2 sqlalchemy-pytds==0.3.5 sqlalchemy-redshift==0.8.14 SQLAlchemy-Utils==0.41.2 -sqlglotrs==0.2.5 +sqlglotrs==0.2.7 sqllineage==1.3.8 sqlparse==0.4.4 srsly==2.4.8 @@ -327,25 +335,25 @@ strictyaml==1.7.3 tableauserverclient==0.25 tableschema==1.20.11 tabulate==0.9.0 -tenacity==8.4.1 -teradatasql==20.0.0.12 +tenacity==9.0.0 +teradatasql==20.0.0.14 teradatasqlalchemy==20.0.0.1 termcolor==2.4.0 terminado==0.18.1 text-unidecode==1.3 -thinc==8.1.12 +thinc==8.2.5 thrift==0.16.0 thrift-sasl==0.4.3 -time-machine==2.14.1 +time-machine==2.14.2 tinycss2==1.3.0 toml==0.10.2 -tomlkit==0.12.5 +tomlkit==0.13.0 toolz==0.12.1 tornado==6.4.1 tqdm==4.66.4 traitlets==5.2.1.post0 -trino==0.328.0 -typer==0.7.0 +trino==0.329.0 +typer==0.12.3 typing-inspect==0.9.0 typing_extensions==4.12.2 tzdata==2024.1 @@ -355,15 +363,16 @@ ujson==5.10.0 unicodecsv==0.14.1 universal_pathlib==0.2.2 urllib3==1.26.19 -vertica-python==1.3.8 +vertica-python==1.4.0 vertica-sqlalchemy-dialect==0.0.8.2 vininfo==1.8.0 wasabi==1.1.3 wcmatch==8.5.2 wcwidth==0.2.13 +weasel==0.4.1 webencodings==0.5.1 websocket-client==1.8.0 -Werkzeug==2.3.8 +Werkzeug==2.2.3 widgetsnbextension==4.0.11 wirerope==0.4.7 wrapt==1.16.0 diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle index faa0589cfbfbbf..5652fedcd93b3b 100644 --- a/docker/datahub-ingestion-base/build.gradle +++ b/docker/datahub-ingestion-base/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 3 // increment to trigger rebuild + revision = 4 // increment to trigger rebuild } docker { diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle index b9ab546674a031..6757be7cd6f221 100644 --- a/docker/datahub-ingestion/build.gradle +++ b/docker/datahub-ingestion/build.gradle @@ -12,7 +12,7 @@ ext { docker_target = project.getProperties().getOrDefault("dockerTarget", "slim") docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}" - revision = 3 // increment to trigger rebuild + revision = 4 // increment to trigger rebuild } dependencies { diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index d722b07b9a7af4..de766f76cb626e 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -144,7 +144,7 @@ services: - neo4jdata:/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -159,7 +159,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - 29092:29092 - 9092:9092 @@ -177,7 +177,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - 2181:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index eae36fb849fd5c..748a2cc9e04167 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -123,7 +123,7 @@ services: - esdata:/usr/share/elasticsearch/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -138,7 +138,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -155,7 +155,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 96f37496859a46..ae55861580becd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -143,7 +143,7 @@ services: - neo4jdata:/data schema-registry: hostname: schema-registry - image: confluentinc/cp-schema-registry:7.4.0 + image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 env_file: schema-registry/env/docker.env @@ -158,7 +158,7 @@ services: condition: service_healthy broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 env_file: broker/env/docker.env @@ -175,7 +175,7 @@ services: - broker:/var/lib/kafka/data/ zookeeper: hostname: zookeeper - image: confluentinc/cp-zookeeper:7.4.0 + image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} ports: - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 env_file: zookeeper/env/docker.env diff --git a/docker/profiles/docker-compose.prerequisites.yml b/docker/profiles/docker-compose.prerequisites.yml index 08ebc8b65d8c9c..7cd9c9039539cc 100644 --- a/docker/profiles/docker-compose.prerequisites.yml +++ b/docker/profiles/docker-compose.prerequisites.yml @@ -210,7 +210,7 @@ services: - neo4jdata:/data kafka-broker: hostname: broker - image: confluentinc/cp-kafka:7.4.0 + image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0} command: - /bin/bash - -c