Rework static requirements

AUTHORS

@@ -115,9 +115,9 @@ Growing Community
 --------------------------------
 
 Salt is a rapidly growing project with a large community, to view all
-contributors please check ohloh, this file can sometimes be out of date:
+contributors please check Github, this file can sometimes be out of date:
 
-https://www.ohloh.net/p/salt/contributors
+https://github.com/saltstack/salt/graphs/contributors
 
 To see current statistics check out the GitHub Pulse page:
 

LICENSE

@@ -342,7 +342,7 @@ License: Apache-2.0
  .
  Files in this directory were created in-house.
 
-Files: tests/utils/cptestcase.py
+Files: tests/support/cptestcase.py
 Copyright: (c) 2014 Adam Hajari
 The MIT License (MIT)
 

MANIFEST.in

@@ -17,7 +17,6 @@ recursive-include scripts *
 recursive-include conf *
 recursive-include pkg *
 recursive-include salt *.jinja
-recursive-include salt *.flo
 recursive-include templates *
 include salt/templates/git/*
 include salt/templates/lxc/*

noxfile.py

@@ -185,69 +185,77 @@ def _get_pip_requirements_file(session, transport, crypto=None):
     if IS_WINDOWS:
         if crypto is None:
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "{}-windows.txt".format(transport)
+                "requirements",
+                "static",
+                "ci",
+                pydir,
+                "{}-windows.txt".format(transport),
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "windows.txt"
+                "requirements", "static", "ci", pydir, "windows.txt"
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
         _requirements_file = os.path.join(
-            "requirements", "static", pydir, "windows-crypto.txt"
+            "requirements", "static", "ci", pydir, "windows-crypto.txt"
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
     elif IS_DARWIN:
         if crypto is None:
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "{}-darwin.txt".format(transport)
+                "requirements", "static", "ci", pydir, "{}-darwin.txt".format(transport)
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "darwin.txt"
+                "requirements", "static", "ci", pydir, "darwin.txt"
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
         _requirements_file = os.path.join(
-            "requirements", "static", pydir, "darwin-crypto.txt"
+            "requirements", "static", "ci", pydir, "darwin-crypto.txt"
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
     elif IS_FREEBSD:
         if crypto is None:
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "{}-freebsd.txt".format(transport)
+                "requirements",
+                "static",
+                "ci",
+                pydir,
+                "{}-freebsd.txt".format(transport),
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "freebsd.txt"
+                "requirements", "static", "ci", pydir, "freebsd.txt"
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
         _requirements_file = os.path.join(
-            "requirements", "static", pydir, "freebsd-crypto.txt"
+            "requirements", "static", "ci", pydir, "freebsd-crypto.txt"
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
     else:
         _install_system_packages(session)
         if crypto is None:
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "{}-linux.txt".format(transport)
+                "requirements", "static", "ci", pydir, "{}-linux.txt".format(transport)
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
             _requirements_file = os.path.join(
-                "requirements", "static", pydir, "linux.txt"
+                "requirements", "static", "ci", pydir, "linux.txt"
             )
             if os.path.exists(_requirements_file):
                 return _requirements_file
         _requirements_file = os.path.join(
-            "requirements", "static", pydir, "linux-crypto.txt"
+            "requirements", "static", "ci", pydir, "linux-crypto.txt"
         )
         if os.path.exists(_requirements_file):
             return _requirements_file
@@ -632,7 +640,7 @@ def runtests_cloud(session, coverage):
     _install_requirements(session, "zeromq", "unittest-xml-reporting==2.2.1")
 
     requirements_file = os.path.join(
-        "requirements", "static", _get_pydir(session), "cloud.txt"
+        "requirements", "static", "ci", _get_pydir(session), "cloud.txt"
     )
 
     install_command = ["--progress-bar=off", "-r", requirements_file]
@@ -865,7 +873,7 @@ def pytest_cloud(session, coverage):
     # Install requirements
     _install_requirements(session, "zeromq")
     requirements_file = os.path.join(
-        "requirements", "static", _get_pydir(session), "cloud.txt"
+        "requirements", "static", "ci", _get_pydir(session), "cloud.txt"
     )
 
     install_command = ["--progress-bar=off", "-r", requirements_file]
@@ -998,7 +1006,7 @@ def fileno(self):
 def _lint(session, rcfile, flags, paths, tee_output=True):
     _install_requirements(session, "zeromq")
     requirements_file = os.path.join(
-        "requirements", "static", _get_pydir(session), "lint.txt"
+        "requirements", "static", "ci", _get_pydir(session), "lint.txt"
     )
     install_command = ["--progress-bar=off", "-r", requirements_file]
     session.install(*install_command, silent=PIP_INSTALL_SILENT)
@@ -1160,7 +1168,7 @@ def docs_html(session, compress):
     """
     pydir = _get_pydir(session)
     requirements_file = os.path.join(
-        "requirements", "static", _get_pydir(session), "docs.txt"
+        "requirements", "static", "ci", _get_pydir(session), "docs.txt"
     )
     install_command = ["--progress-bar=off", "-r", requirements_file]
     session.install(*install_command, silent=PIP_INSTALL_SILENT)
@@ -1181,7 +1189,7 @@ def docs_man(session, compress, update):
     """
     pydir = _get_pydir(session)
     requirements_file = os.path.join(
-        "requirements", "static", _get_pydir(session), "docs.txt"
+        "requirements", "static", "ci", _get_pydir(session), "docs.txt"
     )
     install_command = ["--progress-bar=off", "-r", requirements_file]
     session.install(*install_command, silent=PIP_INSTALL_SILENT)
@@ -1201,7 +1209,7 @@ def _invoke(session):
     Run invoke tasks
     """
     requirements_file = os.path.join(
-        "requirements", "static", _get_pydir(session), "invoke.txt"
+        "requirements", "static", "ci", _get_pydir(session), "invoke.txt"
     )
     install_command = ["--progress-bar=off", "-r", requirements_file]
     session.install(*install_command, silent=PIP_INSTALL_SILENT)
@@ -1277,7 +1285,7 @@ def changelog(session, draft):
     Generate salt's changelog
     """
     requirements_file = os.path.join(
-        "requirements", "static", _get_pydir(session), "changelog.txt"
+        "requirements", "static", "ci", _get_pydir(session), "changelog.txt"
     )
     install_command = ["--progress-bar=off", "-r", requirements_file]
     session.install(*install_command, silent=PIP_INSTALL_SILENT)

pkg/osx/build_env.sh

@@ -61,13 +61,14 @@ sysctl -w kern.maxfilesperproc=$SET_ULIMIT
 launchctl limit maxfiles $SET_ULIMIT $SET_ULIMIT
 ulimit -n $SET_ULIMIT
 
+PY_VERSION=3.7
 SRCDIR=`git rev-parse --show-toplevel`
 SCRIPTDIR=`pwd`
 SHADIR=$SCRIPTDIR/shasums
 INSTALL_DIR=/opt/salt
 PKG_CONFIG=$INSTALL_DIR/bin/pkg-config
 PKG_CONFIG_PATH=$INSTALL_DIR/lib/pkgconfig
-PYDIR=$INSTALL_DIR/lib/python3.7
+PYDIR=$INSTALL_DIR/lib/python$PY_VERSION
 PYTHON=$INSTALL_DIR/bin/python3
 PIP=$INSTALL_DIR/bin/pip3
 
@@ -284,7 +285,7 @@ cd $BUILDDIR
 echo "################################################################################"
 echo "Installing Salt Dependencies with pip (normal)"
 echo "################################################################################"
-$PIP install -r $SRCDIR/pkg/osx/req.txt -r $SRCDIR/pkg/osx/req_pyobjc.txt \
+$PIP install -r $SRCDIR/requirements/static/pkg/py$PY_VERSION/darwin.txt \
              --target=$PYDIR/site-packages \
              --ignore-installed \
              --no-cache-dir

pkg/windows/build.bat

@@ -125,6 +125,8 @@ if Defined x (
 @echo ---------------------------------------------------------------------
 if %Python%==3 (
     Set "PyDir=C:\Python37"
+    Set "PyVerMajor=3"
+    Set "PyVerMinor=7"
 ) else (
     :: Placeholder for future version
     :: Set "PyDir=C:\Python4"

pkg/windows/build_env.ps1

@@ -303,25 +303,17 @@ If (!($Path.ToLower().Contains("$($ini['Settings']['Scripts3Dir'])".ToLower())))
 Write-Output " ----------------------------------------------------------------"
 Write-Output " - $script_name :: Updating PIP and SetupTools . . ."
 Write-Output " ----------------------------------------------------------------"
-Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_pip.txt" "python pip"
+Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -U pip `"setuptools<50.0.0`"" "python pip"
 
 
-#==============================================================================
-# Install windows specific pypi resources using pip
-#==============================================================================
-Write-Output " ----------------------------------------------------------------"
-Write-Output " - $script_name :: Installing windows specific pypi resources using pip . . ."
-Write-Output " ----------------------------------------------------------------"
-Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_win.txt" "pip install"
-
 #==============================================================================
 # Install pypi resources using pip
 #==============================================================================
 If ($NoPipDependencies -eq $false) {
   Write-Output " ----------------------------------------------------------------"
   Write-Output " - $script_name :: Installing pypi resources using pip . . ."
   Write-Output " ----------------------------------------------------------------"
-  Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install"
+  Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($ini['Settings']['SrcDir'])\requirements\static\pkg\py$($ini['Settings']['PyVerMajor']).$($ini['Settings']['PyVerMinor'])\windows.txt" "pip install"
 }
 
 If (Test-Path "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32" -PathType Container )

pkg/windows/modules/get-settings.psm1

@@ -14,12 +14,15 @@ Function Get-Settings {
 
         # Location where the files are kept
         $Settings = @{
-            "SaltRepo"    = "https://repo.saltstack.com/windows/dependencies"
-            "SaltDir"     = "C:\salt"
-            "Python3Dir"   = "C:\Python37"
-            "Scripts3Dir"  = "C:\Python37\Scripts"
-            "SitePkgs3Dir" = "C:\Python37\Lib\site-packages"
-            "DownloadDir" = "$env:Temp\DevSalt"
+            "SrcDir"       = "$env:SrcDir"
+            "SaltRepo"     = "https://repo.saltstack.com/windows/dependencies"
+            "SaltDir"      = "C:\salt"
+            "PyVerMajor"   = "$env:PyVerMajor"
+            "PyVerMinor"   = "$env:PyVerMinor"
+            "Python3Dir"   = "$env:PyDir"
+            "Scripts3Dir"  = "$env:PyDir\Scripts"
+            "SitePkgs3Dir" = "$env:PyDir\Lib\site-packages"
+            "DownloadDir"  = "$env:Temp\DevSalt"
             }
 
         $ini.Add("Settings", $Settings)

requirements/base.txt

@@ -3,4 +3,4 @@ msgpack>=0.5,!=0.5.5
 PyYAML
 MarkupSafe
 requests>=1.0.0
-distro
+distro>=1.5

requirements/darwin.txt

@@ -0,0 +1,22 @@
+# Darwin source distribution requirements
+
+-r zeromq.txt
+
+apache-libcloud>=2.4.0
+backports.ssl_match_hostname>=3.7.0.1; python_version < '3.7'
+cherrypy>=17.4.1
+cryptography>=2.6.1
+gitpython>=2.1.15
+idna>=2.8
+ipaddress>=1.0.22
+linode-python>=1.1.1
+mako>=1.0.7
+psutil>=5.6.6
+pyasn1>=0.4.8
+pycparser>=2.19
+pyopenssl>=19.0.0
+python-dateutil>=2.8.0
+python-gnupg>=0.4.4
+setproctitle>=1.1.10
+timelib>=0.2.5
+vultr>=1.0.1

requirements/static/darwin.in → requirements/static/ci/darwin.in

@@ -12,7 +12,6 @@ jxmlease
 keyring==5.7.1
 kubernetes<4.0
 mock>=3.0.5; python_version < '3.6'
-more-itertools==5.0.0
 moto
 napalm; python_version >= '3.6'
 # XXX: Temporarily do not install pylxd.
@@ -22,7 +21,6 @@ pyopenssl
 python-etcd>0.4.2
 pyvmomi
 rfc3987
-salttesting==2017.6.1
 strict_rfc3339
 toml
 virtualenv>=20.0.20