Add environment.yml to unify conda versioning

doc/dev/conda-builds.md

@@ -16,6 +16,20 @@ A Conda Artifact defines:
 
 ## How to Build an Azure SDK Conda Package Locally
 
+### Set up your conda environment
+
+You will notice that all the azure-sdk conda distributions have the **same** version number and requirement set. This is due to the fact that the azure-sdk team pushes our conda packages out in waves. To support this, all versions are set via a common environment variable `AZURESDK_CONDA_VERSION`.
+
+We keep this environment variable set properly across all our builds by using a common `environment.yml` when creating our build environment. This environment definition ensures that 
+
+1. Our channel `https://azuresdkconda.blob.core.windows.net/channel1/` is added to the set to download packages
+2. The environment variable `AZURESDK_CONDA_VERSION` will be set exactly once.
+
+
+Reference the `environment.yml` in your local build by pass `-f <path to environment.yml>` when you create your conda environment.
+```
+conda env create --yes --quiet --name ${{ artifact.name }} -f $(Build.SourcesDirectory)/eng/environment.yml
+```
 
 ### Create Your Build Directory
 Given how Conda packages are comprised of multiple source distributions _combined_, the buildable source does not exist directly within the azure-sdk-for-python repo. Currently, there is _some_ manual work that needs to be done.
@@ -84,6 +98,7 @@ python `build_conda_artifacts.py`
     -r "azure/storage"
     -n "azure-storage"
     -s "storage"
+    -e "<resolvable path to repo root>/eng/environment.yml"
 ```
 
 ### Generate the Conda Package

eng/environment.yml

@@ -0,0 +1,6 @@
+channels:
+  - https://azuresdkconda.blob.core.windows.net/channel1/
+  - defaults
+
+variables:
+  AZURESDK_CONDA_VERSION: '2021.05.01'

eng/pipelines/templates/steps/build-conda-artifacts.yml

@@ -53,18 +53,29 @@ steps:
       displayName: 'Build Source Distribution for ${{ artifact.name }}'
       inputs:
         scriptPath: 'scripts/devops_tasks/build_conda_artifacts.py'
-        arguments: '-d "$(conda.output)" -b "$(conda.build)" -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}" -r "${{ artifact.common_root }}" -n "${{ artifact.name }}" -s "${{ parameters.ServiceDirectory }}" -o "${{ upper(parameters.ServiceDirectory) }}_SOURCE_DISTRIBUTION"'
+        arguments: >-
+          -d "$(conda.output)"
+          -b "$(conda.build)"
+          -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}"
+          -r "${{ artifact.common_root }}"
+          -n "${{ artifact.name }}"
+          -s "${{ parameters.ServiceDirectory }}"
+          -o "${{ upper(parameters.ServiceDirectory) }}_SOURCE_DISTRIBUTION"
+          -e "$(Build.SourcesDirectory)/eng/environment.yml"
 
     - bash: |
         echo "##vso[task.prependpath]$CONDA/bin"
       displayName: 'Prepend PATH with Conda and INIT'
 
     - bash: |
-        conda create --yes --quiet --name ${{ artifact.name }}
-        source activate ${{ artifact.name }}
+        conda env create --name ${{ artifact.name }} --file $(Build.SourcesDirectory)/eng/environment.yml
         conda install --yes --quiet --name ${{ artifact.name }} conda-build
       displayName: 'Prepare Conda Environment for building ${{ artifact.name }}'
 
+    - pwsh: |
+
+      displayName: 'Set Conda Version Variable'
+
     - bash: |
         source activate ${{ artifact.name }}
         conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c https://azuresdkconda.blob.core.windows.net/channel1/

eng/pipelines/templates/steps/get-tagged-code.yml

@@ -15,9 +15,7 @@ parameters:
 steps:
   - pwsh: |
       $targetPath = "$(Agent.TempDirectory)/${{ parameters.Package }}"
-      if (!(Test-Path $targetPath)) {
-        mkdir $targetPath
-      }
+      mkdir -p $targetPath
 
       Write-Host "##vso[task.setvariable variable=Package.Clone]$targetPath"
     displayName: 'Prep for Sparse Checkout'

scripts/devops_tasks/build_conda_artifacts.py

@@ -30,7 +30,7 @@
 from subprocess import check_call
 from distutils.dir_util import copy_tree
 
-VERSION_REGEX = re.compile(r"\{\%\s*set\s*version\s*=\s*\"(.*)\"\s*\%\}")
+VERSION_REGEX = re.compile(r"\s*AZURESDK_CONDA_VERSION\s*:\s*[\'](.*)[\']\s*")
 
 NAMESPACE_EXTENSION_TEMPLATE = """__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: str
 """
@@ -142,17 +142,15 @@ def create_sdist_skeleton(build_directory, artifact_name, common_root):
                 dest = os.path.join(ns_dir, directory)
                 shutil.copytree(src, dest)
 
-
-def get_version_from_meta(meta_yaml_location):
-    with open(os.path.abspath((meta_yaml_location)), "r") as f:
+def get_version_from_config(environment_config):
+    with open(os.path.abspath((environment_config)), "r") as f:
         lines = f.readlines()
     for line in lines:
         result = VERSION_REGEX.match(line)
         if result:
             return result.group(1)
     return "0.0.0"
 
-
 def get_manifest_includes(common_root):
     levels = common_root.split("/")
     breadcrumbs = []
@@ -165,15 +163,15 @@ def get_manifest_includes(common_root):
     return breadcrumbs
 
 
-def create_setup_files(build_directory, common_root, artifact_name, service, meta_yaml):
+def create_setup_files(build_directory, common_root, artifact_name, service, meta_yaml, environment_config):
     sdist_directory = os.path.join(build_directory, artifact_name)
     setup_location = os.path.join(sdist_directory, "setup.py")
     manifest_location = os.path.join(sdist_directory, "MANIFEST.in")
     cfg_location = os.path.join(sdist_directory, "setup.cfg")
 
     setup_template = CONDA_PKG_SETUP_TEMPLATE.format(
         conda_package_name=artifact_name,
-        version=get_version_from_meta(meta_yaml),
+        version=get_version_from_config(environment_config),
         service=service,
         package_excludes="'azure', 'tests', '{}'".format(common_root.replace("/", ".")),
     )
@@ -193,7 +191,7 @@ def create_setup_files(build_directory, common_root, artifact_name, service, met
 
 
 def create_combined_sdist(
-    output_directory, build_directory, artifact_name, common_root, service, meta_yaml
+    output_directory, build_directory, artifact_name, common_root, service, meta_yaml, environment_config
 ):
     singular_dependency = (
         len(get_pkgs_from_build_directory(build_directory, artifact_name)) == 0
@@ -202,7 +200,7 @@ def create_combined_sdist(
     if not singular_dependency:
         create_sdist_skeleton(build_directory, artifact_name, common_root)
         create_setup_files(
-            build_directory, common_root, artifact_name, service, meta_yaml
+            build_directory, common_root, artifact_name, service, meta_yaml, environment_config
         )
 
     sdist_location = os.path.join(build_directory, artifact_name)
@@ -283,6 +281,15 @@ def create_combined_sdist(
         required=False,
     )
 
+    parser.add_argument(
+        "-e",
+        "--environment_config",
+        dest="environment_config",
+        help="The location of the environment.yml used to create the conda environments. This file has necessary common configuration information within.",
+        required=False,
+    )
+
+
     args = parser.parse_args()
     output_source_location = create_combined_sdist(
         args.distribution_directory,
@@ -291,6 +298,7 @@ def create_combined_sdist(
         args.common_root,
         args.service,
         args.meta_yml,
+        args.environment_config
     )
 
     if args.output_var:

sdk/core/meta.yaml

@@ -1,9 +1,8 @@
 {% set name = "azure-core" %}
-{% set version = "2021.05.01" %}
 
 package:
   name: "{{ name|lower }}"
-  version: "{{ version }}"
+  version: {{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
 
 source:
   url: {{ environ.get('CORE_SOURCE_DISTRIBUTION', '') }}
@@ -39,7 +38,7 @@ about:
   license: MIT
   license_family: MIT
   license_file: 
-  summary: "Microsoft Azure Core Library for Python"
+  summary: {{ environ.get('AZURESDK_CONDA_SUMMARY', 'Microsoft Azure Core Library for Python') }}
   doc_url: 
   dev_url: 
 

sdk/storage/ci.yml

@@ -39,6 +39,7 @@ extends:
     MatrixConfigs:
       - Name: storage_ci_matrix
         Path: eng/pipelines/templates/stages/platform-matrix-cryptography-dependency.json
+        GenerateDocs: False
         Selection: sparse
         GenerateVMJobs: true
     Artifacts:

sdk/storage/meta.yaml

@@ -1,9 +1,8 @@
 {% set name = "azure-storage" %}
-{% set version = "2021.05.01" %}
 
 package:
   name: "{{ name|lower }}"
-  version: "{{ version }}"
+  version: {{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
 
 source:
   url: {{ environ.get('STORAGE_SOURCE_DISTRIBUTION', '') }}
@@ -17,14 +16,14 @@ build:
 
 requirements:
   host:
-    - azure-core >=2021.05.01
+    - azure-core >={{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
     - cryptography >=2.1.4
     - msrest >=2021.05.01
     - pip
     - python
     - aiohttp
   run:
-    - azure-core >=2021.05.01
+    - azure-core >={{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
     - cryptography >=2.1.4
     - msrest >=2021.05.01
     - python
@@ -42,8 +41,8 @@ about:
   home: "https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/storage"
   license: MIT
   license_family: MIT
-  license_file: 
-  summary: "Microsoft Azure Storage Client Library for Python"
+  license_file:
+  summary: {{ environ.get('AZURESDK_CONDA_SUMMARY', 'Microsoft Azure Storage Client Library for Python') }}
   doc_url: 
   dev_url: