Upgrade the Agent build to embed Python 3.8.1

.gitlab-ci.yml

@@ -68,10 +68,10 @@ variables:
   S3_DSD6_URI: s3://dsd6-staging/linux
   RELEASE_VERSION_6: nightly
   RELEASE_VERSION_7: nightly
-  DATADOG_AGENT_BUILDIMAGES: v2028839-bd64f94
-  DATADOG_AGENT_BUILDERS: v2093589-8bb07dc
-  DATADOG_AGENT_WINBUILDIMAGES: v2068298-dc0e899
-  DATADOG_AGENT_ARMBUILDIMAGES: v2028839-bd64f94
+  DATADOG_AGENT_BUILDIMAGES: v2123669-fbee190
+  DATADOG_AGENT_BUILDERS: v2123666-a884483
+  DATADOG_AGENT_WINBUILDIMAGES: v2123666-a884483
+  DATADOG_AGENT_ARMBUILDIMAGES: v2123669-fbee190
 
 
 # Default before_script for all the jobs. If you create a new job and don't want this to execute
@@ -276,6 +276,7 @@ run_tests_ebpf:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/ebpf:go1.10.1
   before_script:
     - cd $SRC_PATH
+    - pip3 install distro -c requirements.txt # needed for some of our invoke tasks until the builders image contain it
     - inv -e deps --verbose --dep-vendor-only
   tags: [ "runner:main", "size:large" ]
   script:
@@ -352,7 +353,7 @@ build_dogstatsd-deb_x64:
 # build dogstatsd static for deb-arm
 build_dogstatsd_static-deb_arm64:
   stage: binary_build
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_ARMBUILDIMAGES
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_BUILDIMAGES
   tags: ["runner:docker-arm", "platform:arm64"]
   needs: [ "fail_on_non_triggered_tag", "run_tests_deb-x64-py3" ]
   variables:
@@ -371,7 +372,7 @@ build_dogstatsd_static-deb_arm64:
 # build dogstatsd linked for deb-arm
 build_dogstatsd-deb_arm64:
   stage: binary_build
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_ARMBUILDIMAGES
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_BUILDIMAGES
   tags: ["runner:docker-arm", "platform:arm64"]
   needs: [ "fail_on_non_triggered_tag", "run_tests_deb-x64-py3" ]
   variables:
@@ -403,7 +404,7 @@ build_puppy_agent-deb_x64:
 # build puppy agent for ARM, to make sure the build is not broken because of build targets
 build_puppy_agent-deb_arm64:
   stage: binary_build
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_ARMBUILDIMAGES
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_BUILDIMAGES
   tags: ["runner:docker-arm", "platform:arm64"]
   needs: [ "fail_on_non_triggered_tag", "run_tests_deb-x64-py3" ]
   variables:
@@ -440,7 +441,7 @@ cluster_agent-build_amd64:
 
 cluster_agent-build_arm64:
   <<: *cluster_agent-build_common
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_ARMBUILDIMAGES
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64:$DATADOG_AGENT_BUILDIMAGES
   tags: ["runner:docker-arm", "platform:arm64"]
   variables:
     ARCH: arm64

docs/dev/agent_dev_env.md

@@ -7,8 +7,8 @@ development files to be available in the dev env. The Agent can embed Python2
 and/or Python3, you will need development files for all versions you want to
 support.
 
-If you're on OSX/macOS, installing Python 2.7 and/or 3.7 with [Homebrew](https://brew.sh) will
-bring along all the development files needed:
+If you're on OSX/macOS, installing Python 2.7 and/or 3.8 with [Homebrew](https://brew.sh)
+brings along all the development files needed:
 ```
 brew install python@2
 brew install python@3
@@ -21,7 +21,7 @@ sudo apt-get install python2.7-dev
 sudo apt-get install python2.3-dev
 ```
 
-On Windows, install Python 2.7 and/or 3.7 via the [official installer](https://www.python.org/downloads/).
+On Windows, install Python 2.7 and/or 3.8 via the [official installer](https://www.python.org/downloads/).
 
 ### Additional Windows Tools
 You will also need the Visual Studio for [Visual Studio for Python installer](http://aka.ms/vcpython27)

omnibus/config/software/datadog-agent-integrations-py3.rb

@@ -31,8 +31,8 @@
 end
 
 relative_path 'integrations-core'
-whitelist_file "embedded/lib/python3.7/site-packages/psycopg2"
-whitelist_file "embedded/lib/python3.7/site-packages/pymqi"
+whitelist_file "embedded/lib/python3.8/site-packages/psycopg2"
+whitelist_file "embedded/lib/python3.8/site-packages/pymqi"
 
 source git: 'https://github.com/DataDog/integrations-core.git'
 
@@ -100,8 +100,8 @@
     # Prepare the build env, these dependencies are only needed to build and
     # install the core integrations.
     #
-    command "#{pip} install wheel==0.30.0"
-    command "#{pip} install pip-tools==2.0.2"
+    command "#{pip} install wheel==0.34.1"
+    command "#{pip} install pip-tools==4.2.0"
     uninstall_buildtime_deps = ['rtloader', 'click', 'first', 'pip-tools']
     nix_build_env = {
       "CFLAGS" => "-I#{install_dir}/embedded/include -I/opt/mqm/inc",
@@ -259,7 +259,7 @@
     if windows?
       patch :source => "jpype_0_7.patch", :target => "#{python_3_embedded}/Lib/site-packages/jaydebeapi/__init__.py"
     else
-      patch :source => "jpype_0_7.patch", :target => "#{install_dir}/embedded/lib/python3.7/site-packages/jaydebeapi/__init__.py"
+      patch :source => "jpype_0_7.patch", :target => "#{install_dir}/embedded/lib/python3.8/site-packages/jaydebeapi/__init__.py"
     end
 
   end

omnibus/config/software/python3.rb

@@ -1,6 +1,6 @@
 name "python3"
 
-default_version "3.7.6"
+default_version "3.8.1"
 
 if ohai["platform"] != "windows"
   dependency "libffi"
@@ -13,7 +13,7 @@
   dependency "libyaml"
 
   source :url => "https://python.org/ftp/python/#{version}/Python-#{version}.tgz",
-         :sha256 => "aeee681c235ad336af116f08ab6563361a0c81c537072c1b309d6e4050aa2114"
+         :sha256 => "c7cfa39a43b994621b245e029769e9126caa2a93571cee2e743b213cceac35fb"
 
   relative_path "Python-#{version}"
 
@@ -48,7 +48,7 @@
     command python_configure.join(" "), :env => env
     command "make -j #{workers}", :env => env
     command "make install", :env => env
-    delete "#{install_dir}/embedded/lib/python3.7/test"
+    delete "#{install_dir}/embedded/lib/python3.8/test"
 
     # There exists no configure flag to tell Python to not compile readline support :(
     major, minor, bugfix = version.split(".")
@@ -65,12 +65,12 @@
     dependency "vc_ucrt_redist"
 
     source :url => "http://s3.amazonaws.com/dd-agent-omnibus/python-windows-#{version}-x86.zip",
-            :sha256 => "946335d0f15e0c4c2820b5ce1ecd4b6470b8f6f28b774d4572279f2d0478c8ab"
+            :sha256 => "44f4a665392912fe172223c42c62dd193670392fee5010bc9f89c5cd2722964c"
   else
 
     # note that startring with 3.7.3 on Windows, the zip should be created without the built-in pip
     source :url => "https://s3.amazonaws.com/dd-agent-omnibus/python-windows-#{version}-amd64.zip",
-         :sha256 => "618325664cadaa80766a3c89f480904bbf23fcbb61467ec36c5d87be026c00cf"
+         :sha256 => "58563ca60891025923572107e02b8f07439928eb5222dd10466cc92089072c2a"
 
   end
   build do

omnibus/package-scripts/agent/preinst

@@ -96,9 +96,14 @@ if [ "$DISTRIBUTION" != "Darwin" ]; then
             rm -rf $DOWNLOADER_DATA
         fi
 
-        DOWNLOADER_DATA3="$INSTALL_DIR/embedded/lib/python3.7/site-packages/datadog_checks/downloader/data"
-        if [ -d $DOWNLOADER_DATA3 ]; then
-            rm -rf $DOWNLOADER_DATA3
+        DOWNLOADER_DATA37="$INSTALL_DIR/embedded/lib/python3.7/site-packages/datadog_checks/downloader/data"
+        if [ -d $DOWNLOADER_DATA37 ]; then
+            rm -rf $DOWNLOADER_DATA37
+        fi
+
+        DOWNLOADER_DATA38="$INSTALL_DIR/embedded/lib/python3.8/site-packages/datadog_checks/downloader/data"
+        if [ -d $DOWNLOADER_DATA38 ]; then
+            rm -rf $DOWNLOADER_DATA38
         fi
     else
         echo "[ FAILED ]\tYour system is currently not supported by this script.";

omnibus/package-scripts/agent/prerm

@@ -109,9 +109,9 @@ remove_downloader_data()
         rm -rf $DOWNLOADER_DATA
     fi
 
-    DOWNLOADER_DATA3="$INSTALL_DIR/embedded/lib/python3.7/site-packages/datadog_checks/downloader/data"
-    if [ -d $DOWNLOADER_DATA3 ]; then
-        rm -rf $DOWNLOADER_DATA3
+    DOWNLOADER_DATA38="$INSTALL_DIR/embedded/lib/python3.8/site-packages/datadog_checks/downloader/data"
+    if [ -d $DOWNLOADER_DATA38 ]; then
+        rm -rf $DOWNLOADER_DATA38
     fi
 }
 

releasenotes/notes/python-3-8-c66cde69f972eb1e.yaml

@@ -0,0 +1,14 @@
+# Each section from every release note is combined when the
+# CHANGELOG.rst is rendered. Therefore, the text should be worded in a way
+# that does not depend on any information available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Upgrade embedded Python 3 to 3.8.1
+    Link to Python 3.8 changelog: https://docs.python.org/3/whatsnew/3.8.html
+    Agent v6 will continue to ship both Python 2 & 3, respectively version 2.7.17 (unchanged) and 3.8.1
+    Agent v7 will ship Python 3.8.1

requirements.txt

@@ -5,3 +5,4 @@ docker==3.0.1
 requests==2.20.1
 PyYAML==5.1
 toml==0.9.4
+distro==1.4.0

rtloader/README.md

@@ -34,7 +34,7 @@ Most of the code used to extend the embedded interpreter is there.
 
 * C/C++ compiler
 * Python 2.7.x development packages
-* Python 3.7.x development packages
+* Python 3.8.x development packages
 * Cmake version 3.12 or above
 * Go compiler with `cgo` capabilities to run the tests
 

rtloader/appveyor.yml

@@ -12,7 +12,7 @@ environment:
   GOPATH: C:\gopath
   # Give hints to CMake to find Pythons
   Python2_ROOT_DIR: C:\Python27-x64
-  Python3_ROOT_DIR: C:\Python37-x64
+  Python3_ROOT_DIR: C:\Python38-x64
   PIP2: pip2 -q
   PIP3: pip3 -q
   MSYS_ROOT: C:\msys64

tasks/agent.py

@@ -7,7 +7,7 @@
 import os
 import shutil
 import sys
-import platform
+import distro
 from distutils.dir_util import copy_tree
 
 import invoke
@@ -103,7 +103,7 @@ def build(ctx, rebuild=False, race=False, build_include=None, build_exclude=None
                 build_exclude.append(ex)
 
     # remove all tags that are only available on debian distributions
-    distname = platform.linux_distribution()[0].lower()
+    distname = distro.id().lower()
     if distname not in REDHAT_AND_DEBIAN_DIST:
         for ex in REDHAT_AND_DEBIAN_ONLY_TAGS:
             if ex not in build_exclude:

tasks/android.py

@@ -7,7 +7,7 @@
 import os
 import shutil
 import sys
-import platform
+import distro
 from distutils.dir_util import copy_tree
 
 import invoke
@@ -63,7 +63,7 @@ def build(ctx, rebuild=False, race=False, build_include=None, build_exclude=None
                 build_exclude.append(ex)
 
     # remove all tags that are only available on debian distributions
-    distname = platform.linux_distribution()[0].lower()
+    distname = distro.id().lower()
     if distname not in REDHAT_AND_DEBIAN_DIST:
         for ex in REDHAT_AND_DEBIAN_ONLY_TAGS:
             if ex not in build_exclude:

tasks/build_tags.py

@@ -2,7 +2,7 @@
 Utilities to manage build tags
 """
 import sys
-import platform
+import distro
 from invoke import task
 
 # ALL_TAGS lists any available build tag
@@ -55,7 +55,7 @@
 REDHAT_AND_DEBIAN_DIST = [
     'centos',
     'debian',
-    'redhat',
+    'rhel',
     'ubuntu',
 ]
 
@@ -77,7 +77,7 @@ def get_default_build_tags(puppy=False, process=False):
         exclude = exclude + PROCESS_ONLY_TAGS
 
     # remove all tags that are only available on debian distributions
-    distname = platform.linux_distribution()[0].lower()
+    distname = distro.id().lower()
     if distname not in REDHAT_AND_DEBIAN_DIST:
         exclude = exclude + REDHAT_AND_DEBIAN_ONLY_TAGS
 

tasks/release.py

@@ -156,7 +156,7 @@ def _stringify_version(version_dict):
         .format(version_dict["major"],
                 version_dict["minor"],
                 version_dict["patch"])
-    if version_dict["rc"] is not 0:
+    if version_dict["rc"] != 0:
         version = "{}-rc.{}".format(version, version_dict["rc"])
     return version
 
@@ -296,15 +296,15 @@ def create_new_version(
 
     if not integration_version:
         integration_version = _get_highest_repo_version(auth, "integrations-core", highest_version, version_re)
-        if integration_version["rc"] is not 0:
+        if integration_version["rc"] != 0:
             print("ERROR: Integration-Core tag is still and RC tag. That's probably NOT what you want in the final artifact. Aborting.")
             #return Exit(code=1)
         integration_version = _stringify_version(integration_version)
     print("Integration-Core's tag is {}".format(integration_version))
 
     if not omnibus_software_version:
         omnibus_software_version = _get_highest_repo_version(auth, "omnibus-software", highest_version, version_re)
-        if omnibus_software_version["rc"] is not 0:
+        if omnibus_software_version["rc"] != 0:
             print("ERROR: Omnibus-Software tag is still and RC tag. That's probably NOT what you want in the final artifact. Aborting.")
             #return Exit(code=1)
         omnibus_software_version = _stringify_version(omnibus_software_version)
@@ -403,4 +403,4 @@ def create_rc(
         integration_version,
         omnibus_software_version,
         omnibus_ruby_version,
-        jmxfetch_version)
+        jmxfetch_version)