Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Find list of installed packages using pkg_resources #12591

Merged
merged 2 commits into from
Jul 18, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions eng/tox/install_dev_build_dependency.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,11 @@
from os import path
from subprocess import check_call

from pip._internal.operations import freeze

# import common_task module
root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", ".."))
common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks"))
sys.path.append(common_task_path)
from common_tasks import process_glob_string
from common_tasks import process_glob_string, get_installed_packages
from tox_helper_tasks import get_package_details

EXCLUDED_PKGS = [
Expand All @@ -34,10 +32,10 @@
# This script verifies installed package version and ensure all installed pacakges are dev build version


def get_installed_packages(pkg_name_to_exclude):
def get_installed_azure_packages(pkg_name_to_exclude):
# This method returns a list of installed azure sdk packages
installed_pkgs = [
p.split("==")[0] for p in freeze.freeze() if p.startswith("azure-")
p.split("==")[0] for p in get_installed_packages() if p.startswith("azure-")
]

# Get valid list of Azure SDK packages in repo
Expand Down Expand Up @@ -100,7 +98,7 @@ def install_packages(packages):

def install_dev_build_packages(pkg_name_to_exclude):
# Uninstall GA version and reinstall dev build version of dependent packages
azure_pkgs = get_installed_packages(pkg_name_to_exclude)
azure_pkgs = get_installed_azure_packages(pkg_name_to_exclude)
uninstall_packages(azure_pkgs)
install_packages(azure_pkgs)

Expand Down
9 changes: 7 additions & 2 deletions eng/tox/verify_installed_packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,13 @@
import os
import sys
import logging
from pip._internal.operations import freeze
from os import path

# import common_task module
root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", ".."))
common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks"))
sys.path.append(common_task_path)
from common_tasks import get_installed_packages

def verify_packages(package_file_path):
# this method verifies packages installed on machine is matching the expected package version
Expand All @@ -29,7 +34,7 @@ def verify_packages(package_file_path):
sys.exit(1)

# find installed and expected packages
installed = dict(p.split('==') for p in freeze.freeze() if p.startswith('azure') and "==" in p)
installed = dict(p.split('==') for p in get_installed_packages() if p.startswith('azure') and "==" in p)
expected = dict(p.split('==') for p in packages)

missing_packages = [pkg for pkg in expected.keys() if installed.get(pkg) != expected.get(pkg)]
Expand Down
15 changes: 13 additions & 2 deletions scripts/devops_tasks/common_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import pdb

# Assumes the presence of setuptools
from pkg_resources import parse_version, parse_requirements, Requirement
from pkg_resources import parse_version, parse_requirements, Requirement, WorkingSet, working_set

# this assumes the presence of "packaging"
from packaging.specifiers import SpecifierSet
Expand Down Expand Up @@ -420,4 +420,15 @@ def find_tools_packages(root_path):
glob_string = os.path.join(root_path, "tools", "*", "setup.py")
pkgs = [os.path.basename(os.path.dirname(p)) for p in glob.glob(glob_string)]
logging.info("Packages in tools: {}".format(pkgs))
return pkgs
return pkgs


def get_installed_packages(paths = None):
"""Find packages in default or given lib paths
"""
# WorkingSet returns installed packages in given path
# working_set returns installed packages in default path
# if paths is set then find installed packages from given paths
ws = WorkingSet(paths) if paths else working_set
return ["{0}=={1}".format(p.project_name, p.version) for p in ws]

8 changes: 4 additions & 4 deletions scripts/devops_tasks/test_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@
filter_dev_requirements,
find_packages_missing_on_pypi,
find_whl,
find_tools_packages
find_tools_packages,
get_installed_packages
)
from git_helper import get_release_tag, git_checkout_tag, git_checkout_branch, clone_repo
from pip._internal.operations import freeze

AZURE_GLOB_STRING = "azure*"

Expand Down Expand Up @@ -222,7 +222,7 @@ def _install_packages(self, dependent_pkg_path, pkg_to_exclude):
temp_dir = self.context.temp_path

list_to_exclude = [pkg_to_exclude,]
installed_pkgs = [p.split('==')[0] for p in list(freeze.freeze(paths=self.context.venv.lib_paths)) if p.startswith('azure-')]
installed_pkgs = [p.split('==')[0] for p in get_installed_packages(self.context.venv.lib_paths) if p.startswith('azure-')]
logging.info("Installed azure sdk packages:{}".format(installed_pkgs))

# Do not exclude list of packages in tools directory and so these tools packages will be reinstalled from repo branch we are testing
Expand Down Expand Up @@ -257,7 +257,7 @@ def _is_package_installed(self, package, version):
venv_root = self.context.venv.path
site_packages = self.context.venv.lib_paths
logging.info("Searching for packages in :{}".format(site_packages))
installed_pkgs = list(freeze.freeze(paths=site_packages))
installed_pkgs = get_installed_packages(site_packages)
logging.info("Installed packages: {}".format(installed_pkgs))
# Verify installed package version
# Search for exact version or dev build version of current version.
Expand Down