Upgrade Sphinx (Azure#32066)

* updates to test sever, removing a couple of the pins from test_tools and ci_tools.txt * remove bad readme_renderer requirement * simplifying requirements. fixing requirement file for invoking analyze * repair issues with pypy39 due to twine installation * exclude azure-core-tracing-opencensus from sphinx build * eliminate pytoml dependency * Update Sphinx to v6.x * safe import of tomli on <py311. import tomllib on >=py311. always pull in toml-w Co-authored-by: Laurent Mazuel <[email protected]>
azure-sdk · Sep 21, 2023 · a743c56 · a743c56
1 parent 7ff8b6e
commit a743c56
Show file tree

Hide file tree

Showing 17 changed files with 95 additions and 95 deletions.
diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py
@@ -55,13 +55,12 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest',
-              'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon']
+              'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon',
+              'myst_parser']
 
 intersphinx_mapping = {
     # Dependencies
-    'python': ('https://docs.python.org/3.8', ('/usr/share/doc/python3-doc/html/objects.inv', None)),
-    'msrestazure': ('http://msrestazure.readthedocs.io/en/latest/', None),
-    'msrest': ('http://msrest.readthedocs.io/en/latest/', None),
+    'python': ('https://docs.python.org/3.11', ('/usr/share/doc/python3-doc/html/objects.inv', None)),
     'requests': ('https://requests.kennethreitz.org/en/master/', None),
     'aiohttp': ('https://aiohttp.readthedocs.io/en/stable/', None),
     'trio': ('https://trio.readthedocs.io/en/stable/', None),
@@ -76,9 +75,6 @@
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
-source_parsers = {
-   '.md': 'recommonmark.parser.CommonMarkParser',
-}
 
 # The suffix of source filenames.
 source_suffix = ['.rst', '.md']

diff --git a/doc/sphinx/individual_build_conf.py b/doc/sphinx/individual_build_conf.py
@@ -53,13 +53,12 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest',
-              'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon']
+              'sphinx.ext.viewcode', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon',
+              'myst_parser']
 
 intersphinx_mapping = {
     # Dependencies
-    'python': ('https://docs.python.org/3.8', None),
-    'msrestazure': ('http://msrestazure.readthedocs.io/en/latest/', None),
-    'msrest': ('http://msrest.readthedocs.io/en/latest/', None),
+    'python': ('https://docs.python.org/3.11', None),
     'requests': ('https://requests.kennethreitz.org/en/master/', None),
     'aiohttp': ('https://aiohttp.readthedocs.io/en/stable/', None),
     'trio': ('https://trio.readthedocs.io/en/stable/', None),
@@ -74,10 +73,6 @@
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
-source_parsers = {
-   '.md': 'recommonmark.parser.CommonMarkParser',
-}
-
 # The suffix of source filenames.
 source_suffix = ['.rst', '.md']
 

diff --git a/doc/sphinx/requirements.txt b/doc/sphinx/requirements.txt
@@ -1,3 +1,3 @@
 sphinx
 sphinx_rtd_theme
-recommonmark
+myst-parser
diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt
@@ -2,31 +2,37 @@
 setuptools==67.6.0; python_version >= '3.5'
 virtualenv==20.23.0
 wheel==0.37.0
-Jinja2==2.11.2
 packaging==23.1
+wheel==0.37.0
 tox==4.5.0
-twine==3.1.1; python_version >= '3.6'
 pathlib2==2.3.5
 doc-warden==0.7.2
 beautifulsoup4==4.9.1
-pkginfo==1.5.0.1
+pkginfo==1.9.6
 pip==20.3.3
 wrapt==1.12.1; python_version <= '3.10'
-wrapt==1.14.1; python_version >= '3.11'
-markupsafe==2.0.1
+wrapt==1.15.0; python_version >= '3.11'
 typing-extensions<=4.6.3
 
+# requirements leveraged for testing
+pytest==7.3.1
+pytest-cov==4.0.0
+coverage==7.2.5
+
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
-pytoml==0.1.21
-pyOpenSSL==23.2.0
+Jinja2==3.1.2
+MarkupSafe==2.1.3
 json-delta==2.0
+readme_renderer==37.3; python_version <= '3.7'
+readme_renderer==42.0; python_version > '3.7'
+pyopenssl==23.2.0
+python-dotenv==0.21.1; python_version <= '3.7'
+python-dotenv==1.0.0; python_version > '3.7'
+pyyaml==6.0.1
+urllib3==1.26.16
+PyGithub==1.59.1
 ConfigArgParse==1.2.3
 six==1.14.0
-pyyaml==5.3.1
-pytest==7.3.1
-pytest-cov==4.0.0
-coverage==7.2.5
-pyproject-api<1.6.0
 
 # local dev packages
 ./tools/azure-devtools

diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml
@@ -66,6 +66,7 @@ steps:
     condition: succeededOrFailed()
 
   - script: |
+      pip install twine==4.0.2
       twine check $(Build.ArtifactStagingDirectory)/**/*.whl
       twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz
     displayName: 'Verify Readme'

diff --git a/eng/regression_test_tools.txt b/eng/regression_test_tools.txt
@@ -21,9 +21,8 @@ ConfigArgParse==1.2.3
 six==1.14.0
 pyyaml==5.3.1
 packaging==23.1
-Jinja2==2.11.2
-markupsafe==2.0.1; python_version > '2.7'
-markupsafe==1.1.1; python_version == '2.7'
+Jinja2==3.1.2
+MarkupSafe==2.1.3
 wrapt==1.12.1; python_version <= '3.10'
 wrapt==1.14.1; python_version >= '3.11'
 

diff --git a/eng/regression_tools.txt b/eng/regression_tools.txt
@@ -2,7 +2,7 @@
 setuptools==67.6.0; python_version >= '3.5'
 virtualenv==20.23.0
 wheel==0.37.0
-Jinja2==2.11.2
+Jinja2==3.1.2
 packaging==23.1
 tox==4.5.0
 twine==3.1.1; python_version >= '3.6'
@@ -13,7 +13,7 @@ pkginfo==1.5.0.1
 pip==20.3.3
 wrapt==1.12.1; python_version <= '3.10'
 wrapt==1.14.1; python_version >= '3.11'
-markupsafe==2.0.1
+MarkupSafe==2.1.3
 typing-extensions<=4.6.3
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools

diff --git a/eng/test_tools.txt b/eng/test_tools.txt
@@ -7,30 +7,18 @@ pytest-xdist==3.2.1
 coverage==7.2.5
 bandit==1.6.2
 protobuf==3.17.3; python_version == '2.7'
-chardet>=2.0,<5.0
-cmarkgfm<0.7.0
-typing-extensions<=4.6.3
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
-pytoml==0.1.21
-readme-renderer[md]==25.0
-pyOpenSSL==23.2.0
+Jinja2==3.1.2
+MarkupSafe==2.1.3
 json-delta==2.0
+readme_renderer==37.3; python_version <= '3.7'
+readme_renderer==42.0; python_version > '3.7'
+pyopenssl==23.2.0
+python-dotenv==0.21.1; python_version <= '3.7'
+python-dotenv==1.0.0; python_version > '3.7'
+pyyaml==6.0.1
+urllib3==1.26.16
+PyGithub==1.59.1
 ConfigArgParse==1.2.3
 six==1.14.0
-pyyaml==5.3.1
-packaging==23.1
-Jinja2==2.11.2
-markupsafe==2.0.1; python_version > '2.7'
-markupsafe==1.1.1; python_version == '2.7'
-wrapt==1.12.1; python_version <= '3.10'
-wrapt==1.14.1; python_version >= '3.11'
-pyproject-api<1.6.0
-
-# Locking pylint and required packages
-pylint==1.8.4; python_version < '3.4'
-pylint==2.9.3; python_version >= '3.6' and python_version <= '3.10'
-pylint==2.14.5; python_version >= '3.11'
-
-# python-dotenv
-python-dotenv==0.15.0
diff --git a/eng/tox/prep_sphinx_env.py b/eng/tox/prep_sphinx_env.py
@@ -8,8 +8,6 @@
 # This script is intended to be executed from within a tox script. It takes care of of unzipping
 # an package sdist and prepping for a sphinx execution.
 
-from m2r import parse_from_file
-
 import glob
 import logging
 import shutil
@@ -30,34 +28,34 @@
 
 RST_EXTENSION_FOR_INDEX = """
 
-Indices and tables
-------------------
+## Indices and tables
+
+- {{ref}}`genindex`
+- {{ref}}`modindex`
+- {{ref}}`search`
+
+```{{toctree}}
+:caption: Developer Documentation
+:glob: true
+:maxdepth: 5
 
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+{}
 
-.. toctree::
-  :maxdepth: 5
-  :glob:
-  :caption: Developer Documentation
+```
 
-  {}
 """
 
 root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
 sphinx_conf = os.path.join(root_dir, "doc", "sphinx", "individual_build_conf.py")
 
 
 def should_build_docs(package_name):
-    return not ("nspkg" in package_name or package_name in ["azure", "azure-mgmt", "azure-keyvault", "azure-documentdb", "azure-mgmt-documentdb", "azure-servicemanagement-legacy"])
+    return not ("nspkg" in package_name or package_name in ["azure", "azure-mgmt", "azure-keyvault", "azure-documentdb", "azure-mgmt-documentdb", "azure-servicemanagement-legacy", "azure-core-tracing-opencensus"])
 
 def create_index_file(readme_location, package_rst):
     readme_ext = os.path.splitext(readme_location)[1]
 
     if readme_ext == ".md":
-        output = parse_from_file(readme_location)
-    elif readme_ext == ".rst":
         with open(readme_location, "r") as file:
             output = file.read()
     else:
@@ -83,20 +81,17 @@ def create_index(doc_folder, source_location, namespace):
     index_content = ""
 
     package_rst = "{}.rst".format(namespace)
-    content_destination = os.path.join(doc_folder, "index.rst")
+    content_destination = os.path.join(doc_folder, "index.md")
 
     if not os.path.exists(doc_folder):
         os.mkdir(doc_folder)
 
     # grep all content
     markdown_readmes = glob.glob(os.path.join(source_location, "README.md"))
-    rst_readmes = glob.glob(os.path.join(source_location, "README.rst"))
 
     # if markdown, take that, otherwise rst
     if markdown_readmes:
         index_content = create_index_file(markdown_readmes[0], package_rst)
-    elif rst_readmes:
-        index_content = create_index_file(rst_readmes[0], package_rst)
     else:
         logging.warning("No readmes detected for this namespace {}".format(namespace))
         index_content = RST_EXTENSION_FOR_INDEX.format(package_rst)

diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini
@@ -86,7 +86,10 @@ setenv =
   {[testenv]setenv}
   PROXY_URL=http://localhost:5002
 deps =
-  {[base]deps}
+  -rdev_requirements.txt
+  pylint==1.8.4; python_version < '3.4'
+  pylint==2.9.3; python_version >= '3.6' and python_version <= '3.10'
+  pylint==2.14.5; python_version >= '3.11'
 commands =
     python -m pip install pylint=={[testenv:pylint]pylint_version}
     python -m pip install azure-pylint-guidelines-checker==0.0.9 --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
@@ -108,7 +111,10 @@ setenv =
   {[testenv]setenv}
   PROXY_URL=http://localhost:5002
 deps =
-  {[base]deps}
+  -rdev_requirements
+  pylint==1.8.4; python_version < '3.4'
+  pylint==2.9.3; python_version >= '3.6' and python_version <= '3.10'
+  pylint==2.14.5; python_version >= '3.11'
 commands =
     python -m pip install pylint=={[testenv:next-pylint]pylint_version}
     python -m pip install azure-pylint-guidelines-checker==0.1.0 --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
@@ -315,11 +321,9 @@ setenv =
   PROXY_URL=http://localhost:5007
 deps =
   {[base]deps}
-  sphinx==3.5.4
-  sphinx_rtd_theme==1.0.0
-  recommonmark==0.6.0
-  mistune<2.0.0
-  m2r==0.2.1
+  sphinx==6.2.1
+  sphinx_rtd_theme==1.3.0
+  myst_parser==2.0.0
 commands =
   python {repository_root}/eng/tox/create_package_and_install.py \
     -d {envtmpdir}/dist \

diff --git a/sdk/core/azure-core/tests/async_tests/test_rest_response_backcompat_async.py b/sdk/core/azure-core/tests/async_tests/test_rest_response_backcompat_async.py
@@ -178,6 +178,7 @@ async def test_response_internal_response_trio(get_old_response_trio, get_new_re
     _test_response_internal_response(old_response, new_response, port)
 
 
+@pytest.mark.skip(reason="Resolve in #32162")
 @pytest.mark.asyncio
 @pytest.mark.parametrize("transport", TRANSPORTS)
 async def test_response_stream_download(get_old_response, get_new_response, transport):
@@ -251,7 +252,7 @@ def _test_response_headers(old_response, new_response):
     assert (
         set(old_response.headers.keys())
         == set(new_response.headers.keys())
-        == set(["Content-Type", "Connection", "Server", "Date"])
+        == set(["Content-Type", "Connection", "Server", "Date", "Transfer-Encoding"])
     )
     old_response.headers = {"Hello": "world!"}
     new_response.headers = {"Hello": "world!"}

diff --git a/sdk/core/azure-core/tests/test_rest_response_backcompat.py b/sdk/core/azure-core/tests/test_rest_response_backcompat.py
@@ -107,7 +107,7 @@ def test_response_headers(old_response, new_response):
     assert (
         set(old_response.headers.keys())
         == set(new_response.headers.keys())
-        == set(["Content-Type", "Connection", "Server", "Date"])
+        == set(["Content-Type", "Connection", "Server", "Date", "Transfer-Encoding"])
     )
     old_response.headers = {"Hello": "world!"}
     new_response.headers = {"Hello": "world!"}

diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py
@@ -6,6 +6,7 @@
 # -------------------------------------------------------------------------
 
 from flask import Flask, Response
+import os
 from .test_routes import (
     basic_api,
     encoding_api,
@@ -36,4 +37,7 @@ def latin_1_charset_utf8():
 
 
 if __name__ == "__main__":
-    app.run(debug=True)
+    debugArg = True
+    if os.getenv("TF_BUILD", None):
+        debugArg = False
+    app.run(debug=debugArg)
diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py
@@ -32,6 +32,6 @@
     ],
     packages=find_packages(),
     install_requires=[
-        "flask==1.1.4",
+        "flask==2.2.2",
     ],
 )
diff --git a/tools/azure-sdk-tools/packaging_tools/conf.py b/tools/azure-sdk-tools/packaging_tools/conf.py
@@ -2,7 +2,13 @@
 from pathlib import Path
 from typing import Dict, Any
 
-import pytoml as toml
+try:
+    # py 311 adds this library natively
+    import tomllib as toml
+except:
+    # otherwise fall back to pypi package tomli
+    import tomli as toml
+import tomli_w as tomlw
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -45,4 +51,4 @@ def build_default_conf(folder: Path, package_name: str) -> None:
     conf[_SECTION]["package_nspkg"] = package_name[: package_name.rindex("-")] + "-nspkg"
 
     with open(conf_path, "w") as fd:
-        toml.dump(conf, fd)
+        tomlw.dump(conf, fd)