From ecc5dd108481464f5630c701ad0133230b036f0b Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Thu, 18 Feb 2021 21:12:40 -0800 Subject: [PATCH 01/25] Initial add of modelsrepository removed partial expanded fallback logic Black formatting Removed unnecessary comment New structure, added nspkg wip removed nspkg Refactored (again) black formatting Implementation --- sdk/iot/azure-iot-modelsrepository/.flake8 | 7 + sdk/iot/azure-iot-modelsrepository/README.md | 17 ++ .../azure/__init__.py | 1 + .../azure/iot/__init__.py | 1 + .../azure/iot/modelsrepository/__init__.py | 18 ++ .../modelsrepository/chainable_exception.py | 24 +++ .../azure/iot/modelsrepository/client.py | 173 ++++++++++++++++++ .../iot/modelsrepository/pseudo_parser.py | 58 ++++++ .../azure/iot/modelsrepository/resolver.py | 148 +++++++++++++++ sdk/iot/azure-iot-modelsrepository/pytest.ini | 5 + .../requirements.txt | 9 + .../samples/README.md | 10 + .../samples/client_configuration_sample.py | 42 +++++ .../samples/get_models_sample.py | 55 ++++++ sdk/iot/azure-iot-modelsrepository/setup.py | 78 ++++++++ .../tests/__init__.py | 0 .../tests/conftest.py | 14 ++ sdk/iot/azure-iot-modelsrepository/tox.ini | 9 + 18 files changed, 669 insertions(+) create mode 100644 sdk/iot/azure-iot-modelsrepository/.flake8 create mode 100644 sdk/iot/azure-iot-modelsrepository/README.md create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/__init__.py create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/__init__.py create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/chainable_exception.py create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py create mode 100644 sdk/iot/azure-iot-modelsrepository/pytest.ini create mode 100644 sdk/iot/azure-iot-modelsrepository/requirements.txt create mode 100644 sdk/iot/azure-iot-modelsrepository/samples/README.md create mode 100644 sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py create mode 100644 sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py create mode 100644 sdk/iot/azure-iot-modelsrepository/setup.py create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/__init__.py create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/conftest.py create mode 100644 sdk/iot/azure-iot-modelsrepository/tox.ini diff --git a/sdk/iot/azure-iot-modelsrepository/.flake8 b/sdk/iot/azure-iot-modelsrepository/.flake8 new file mode 100644 index 000000000000..942c4cef4914 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/.flake8 @@ -0,0 +1,7 @@ +[flake8] +# E501: line length (black formatting will handle) +# W503, E203: Not PEP8 compliant (incompatible with black formatting) +ignore = E501,W503,E203 +exclude = + .git, + __pycache__, \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/README.md b/sdk/iot/azure-iot-modelsrepository/README.md new file mode 100644 index 000000000000..fd181dae0f8e --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/README.md @@ -0,0 +1,17 @@ +# Azure IoT Models Repository Library + +The Azure IoT Models Repository Library for Python provides functionality for working with the Azure IoT Models Repository + +## Installation + +This package is not yet available on pip. Please install locally from source: + +```Shell +python -m pip install -e +``` + +## Features + +* ### Resolver + * Allows retrieval of model DTDLs from remote URLs or local filesystems + * This feature is provisionally included in this package pending review diff --git a/sdk/iot/azure-iot-modelsrepository/azure/__init__.py b/sdk/iot/azure-iot-modelsrepository/azure/__init__.py new file mode 100644 index 000000000000..8db66d3d0f0f --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/__init__.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/__init__.py new file mode 100644 index 000000000000..8db66d3d0f0f --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py new file mode 100644 index 000000000000..fc4b5ce647bb --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +# Main Client +from .client import ModelsRepositoryClient + +# Constants +from .client import ( + DEPENDENCY_MODE_DISABLED, + DEPENDENCY_MODE_ENABLED, + DEPENDENCY_MODE_TRY_FROM_EXPANDED, +) + +# Error handling +from .resolver import ResolverError diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/chainable_exception.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/chainable_exception.py new file mode 100644 index 000000000000..bad9523e3a45 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/chainable_exception.py @@ -0,0 +1,24 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + + +class ChainableException(Exception): + """This exception stores a reference to a previous exception which has caused + the current one""" + + def __init__(self, message=None, cause=None): + # By using .__cause__, this will allow typical stack trace behavior in Python 3, + # while still being able to operate in Python 2. + self.__cause__ = cause + super(ChainableException, self).__init__(message) + + def __str__(self): + if self.__cause__: + return "{} caused by {}".format( + super(ChainableException, self).__repr__(), self.__cause__.__repr__() + ) + else: + return super(ChainableException, self).__repr__() diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py new file mode 100644 index 000000000000..fc7afe77b114 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -0,0 +1,173 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import six.moves.urllib as urllib +import re +from azure.core import PipelineClient +from azure.core.pipeline.transport import RequestsTransport +from azure.core.configuration import Configuration +from azure.core.pipeline.policies import ( + UserAgentPolicy, + HeadersPolicy, + RetryPolicy, + RedirectPolicy, + BearerTokenCredentialPolicy, + ContentDecodePolicy, + NetworkTraceLoggingPolicy, + ProxyPolicy, +) +from . import resolver as resolver +from . import pseudo_parser +from .chainable_exception import ChainableException + + +# Public constants exposed to consumers +DEPENDENCY_MODE_TRY_FROM_EXPANDED = "tryFromExpanded" +DEPENDENCY_MODE_DISABLED = "disabled" +DEPENDENCY_MODE_ENABLED = "enabled" + + +# Convention-private constants +_DEFAULT_LOCATION = "https://devicemodels.azure.com" +_DEFAULT_API_VERSION = "2021-02-11" +_REMOTE_PROTOCOLS = ["http", "https"] + + +class ModelsRepositoryClient(object): + """Client providing APIs for Models Repository operations""" + + def __init__(self, repository_location=None, api_version=None, **kwargs): + """ + :param str repository_location: Location of the Models Repository you wish to access. + This location can be a remote HTTP/HTTPS URL, or a local filesystem path. + If omitted, will default to using "https://devicemodels.azure.com". + + :raises: ValueError if repository_location is invalid + """ + repository_location = ( + _DEFAULT_LOCATION if repository_location is None else repository_location + ) + # api_version = _DEFAULT_API_VERSION if api_version is None else api_version + + kwargs.setdefault("api_verison", api_version) + + # NOTE: depending on how this class develops over time, may need to adjust relationship + # between some of these objects + self.fetcher = _create_fetcher( + location=repository_location, api_version=api_version, **kwargs + ) + self.resolver = resolver.DtmiResolver(self.fetcher) + self.pseudo_parser = pseudo_parser.PseudoParser(self.resolver) + + def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): + """Retrieve a model from the Models Repository. + + :param list[str]: The DTMIs for the models you wish to retrieve + :param str dependency_resolution : Dependency resolution mode. Possible values: + - "disabled": Do not resolve model dependencies + - "enabled": Resolve model dependencies from the repository + - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded DTDL + document in the repository. If this is not successful, will fall back on + manually resolving dependencies in the repository + + :raises: ValueError if given an invalid dependency resolution mode + :raises: ResolverError if there is an error retreiving a model + + :returns: Dictionary mapping DTMIs to models + :rtype: dict + """ + if dependency_resolution == DEPENDENCY_MODE_DISABLED: + model_map = self.resolver.resolve(dtmis) + elif dependency_resolution == DEPENDENCY_MODE_ENABLED: + # Manually resolve dependencies using pseudo-parser + base_model_map = model_map = self.resolver.resolve(dtmis) + base_model_list = [model for model in base_model_map.values()] + model_map = self.pseudo_parser.expand(base_model_list) + elif dependency_resolution == DEPENDENCY_MODE_TRY_FROM_EXPANDED: + # Try to use an expanded DTDL to resolve dependencies + try: + model_map = self.resolver.resolve(dtmis, expanded_model=True) + except resolver.ResolverError: + # Fallback to manual dependency resolution + base_model_map = model_map = self.resolver.resolve(dtmis) + base_model_list = [model for model in base_model_map.items()] + model_map = self.pseudo_parser.expand(base_model_list) + else: + raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) + return model_map + + +class ModelsRepositoryClientConfiguration(Configuration): + """ModelsRepositoryClient-specific variant of the Azure Core Configuration for Pipelines""" + + def __init__(self, **kwargs): + super(ModelsRepositoryClientConfiguration, self).__init__(**kwargs) + # NOTE: There might be some further organization to do here as it's kind of weird that + # the generic config (which could be used for any remote repository) always will have + # the default repository's api version stored. Keep this in mind when expanding the + # scope of the client in the future - perhaps there may need to eventually be unique + # configs for default repository vs. custom repository endpoints + self._api_version = kwargs.get("api_version", _DEFAULT_API_VERSION) + + +def _create_fetcher(location, **kwargs): + """Return a Fetcher based upon the type of location""" + scheme = urllib.parse.urlparse(location).scheme + if scheme in _REMOTE_PROTOCOLS: + # HTTP/HTTPS URL + client = _create_pipeline_client(base_url=location, **kwargs) + fetcher = resolver.HttpFetcher(client) + elif scheme == "file": + # Filesystem URI + location = location[len("file://") :] + fetcher = resolver.FilesystemFetcher(location) + elif scheme == "" and location.startswith("/"): + # POSIX filesystem path + fetcher = resolver.FilesystemFetcher(location) + elif scheme == "" and re.search(r"\.[a-zA-z]{2,63}$", location[: location.find("/")]): + # Web URL with protocol unspecified - default to HTTPS + location = "https://" + location + client = _create_pipeline_client(base_url=location, **kwargs) + fetcher = resolver.HttpFetcher(client) + elif scheme != "" and len(scheme) == 1 and scheme.isalpha(): + # Filesystem path using drive letters (e.g. "C:", "D:", etc.) + fetcher = resolver.FilesystemFetcher(location) + else: + raise ValueError("Unable to identify location: {}".format(location)) + return fetcher + + +def _create_pipeline_client(base_url, **kwargs): + """Creates and returns a PipelineClient configured for the provided base_url and kwargs""" + transport = kwargs.get("transport", RequestsTransport(**kwargs)) + config = _create_config(**kwargs) + policies = [ + config.user_agent_policy, + config.headers_policy, + config.authentication_policy, + ContentDecodePolicy(), + config.proxy_policy, + config.redirect_policy, + config.retry_policy, + config.logging_policy, + ] + return PipelineClient(base_url=base_url, config=config, policies=policies, transport=transport) + + +def _create_config(**kwargs): + """Creates and returns a ModelsRepositoryConfiguration object""" + config = ModelsRepositoryClientConfiguration(**kwargs) + config.headers_policy = kwargs.get( + "headers_policy", HeadersPolicy({"CustomHeader": "Value"}, **kwargs) + ) + config.user_agent_policy = kwargs.get( + "user_agent_policy", UserAgentPolicy("ServiceUserAgentValue", **kwargs) + ) + config.authentication_policy = kwargs.get("authentication_policy") + config.retry_policy = kwargs.get("retry_policy", RetryPolicy(**kwargs)) + config.redirect_policy = kwargs.get("redirect_policy", RedirectPolicy(**kwargs)) + config.logging_policy = kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)) + config.proxy_policy = kwargs.get("proxy_policy", ProxyPolicy(**kwargs)) + return config diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py new file mode 100644 index 000000000000..d76911c6eb75 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py @@ -0,0 +1,58 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""This module contains a partial parsing implementation that is strictly +scoped for parsing dependencies. Ideally, this would be made obsolete by +a full Python parser implementation, as it is supposed to be a single source +of truth for the DTDL model specifications. + +Note that this implementation is not representative of what an eventual full +parser implementation would necessarily look like from an API perspective +""" + + +class PseudoParser(object): + def __init__(self, resolver): + self.resolver = resolver + + def expand(self, models): + """""" + expanded_map = {} + for model in models: + expanded_map[model["@id"]] = model + self._expand(model, expanded_map) + return expanded_map + + def _expand(self, model, model_map): + dependencies = get_dependency_list(model) + dependencies_to_resolve = [ + dependency for dependency in dependencies if dependency not in model_map + ] + + if dependencies_to_resolve: + resolved_dependency_map = self.resolver.resolve(dependencies_to_resolve) + model_map.update(resolved_dependency_map) + for dependency_model in resolved_dependency_map.items(): + self._expand(dependency_model, model_map) + + +def get_dependency_list(model): + """Return a list of DTMIs for model dependencies""" + if "contents" in model: + components = [item["schema"] for item in model["contents"] if item["@type"] == "Component"] + else: + components = [] + + if "extends" in model: + # Models defined in a DTDL can implement extensions of up to two interfaces + if isinstance(model["extends"], list): + interfaces = model["extends"] + else: + interfaces = [model["extends"]] + else: + interfaces = [] + + dependencies = components + interfaces + return dependencies diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py new file mode 100644 index 000000000000..bc81d16325d5 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py @@ -0,0 +1,148 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +import six +import json +import abc +import re +import os +from .chainable_exception import ChainableException + +logger = logging.getLogger(__name__) + + +class ResolverError(ChainableException): + pass + + +class DtmiResolver(object): + def __init__(self, fetcher): + """ + :param fetcher: A Fetcher configured to an endpoint to resolve DTMIs from + :type fetcher: :class:`azure.iot.modelsrepository.resolver` + """ + self.fetcher = fetcher + + def resolve(self, dtmis, expanded_model=False): + """Resolve a DTMI from the configured endpoint and return the resulting JSON model. + + :param list[str] dtmis: DTMIs to resolve + :param bool expanded_model: Indicates whether to resolve a regular or expanded model + + :raises: ValueError if the DTMI is invalid + :raises: ResolverError if the DTMI cannot be resolved to a model + + :returns: A dictionary mapping DTMIs to models + :rtype: dict + """ + model_map = {} + for dtmi in dtmis: + dtdl_path = _convert_dtmi_to_path(dtmi) + if expanded_model: + dtdl_path = dtdl_path.replace(".json", ".expanded.json") + + try: + dtdl = self.fetcher.fetch(dtdl_path) + except FetcherError as e: + raise ResolverError("Failed to resolve dtmi: {}".format(dtmi), cause=e) + + if expanded_model: + for model in dtdl: + model_map[model["@id"]] = model + else: + model_map[dtmi] = dtdl + return model_map + + +class FetcherError(ChainableException): + pass + + +@six.add_metaclass(abc.ABCMeta) +class Fetcher(object): + """Interface for fetching from a generic location""" + + @abc.abstractmethod + def fetch(self, path): + pass + + +class HttpFetcher(Fetcher): + """Fetches JSON data from a web endpoint""" + + def __init__(self, http_client): + """ + :param http_client: PipelineClient that has been configured for an endpoint + :type http_client: :class:`azure.core.PipelineClient` + """ + self.client = http_client + + def fetch(self, path): + """Fetch and return the contents of a JSON file at a given web path. + The path can be relative to the path configured in the Fetcher's HttpClient, + or it can be an absolute path. + + :raises: FetcherError if data cannot be fetched + + :returns: JSON data at the path + :rtype: JSON object + """ + request = self.client.get(url=path) + response = self.client._pipeline.run(request).http_response + if response.status_code != 200: + raise FetcherError("Failed to fetch from remote endpoint") + json_response = json.loads(response.text()) + return json_response + + +class FilesystemFetcher(Fetcher): + """Fetches JSON data from a local filesystem endpoint""" + + def __init__(self, base_path): + """ + :param str base_path: The base filepath for fetching from + """ + self.base_path = base_path + + def fetch(self, path): + """Fetch and return the contents of a JSON file at a given filesystem path. + The path can be relative to the Fetcher's base_path, or it can be an absolute path. + + :raises: FetcherError if data cannot be fetched + + :returns: JSON data at the path + :rtype: JSON object + """ + # Format path + path = os.path.join(self.base_path, path) + path = os.path.normcase(path) + path = os.path.normpath(path) + + # TODO: Ensure support for relative and absolute paths + # TODO: Need robust suite of testing for different types of paths + + # Fetch + try: + with open(path) as f: + file_str = f.read() + except Exception as e: + raise FetcherError("Failed to fetch from Filesystem", e) + return json.loads(file_str) + + +def _convert_dtmi_to_path(dtmi): + """Converts a DTMI into a DTMI path + + E.g: + dtmi:com:example:Thermostat;1 -> dtmi/com/example/thermostat-1.json + """ + pattern = re.compile( + "^dtmi:[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?(?::[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?)*;[1-9][0-9]{0,8}$" + ) + if not pattern.match(dtmi): + raise ValueError("Invalid DTMI") + else: + return dtmi.lower().replace(":", "/").replace(";", "-") + ".json" diff --git a/sdk/iot/azure-iot-modelsrepository/pytest.ini b/sdk/iot/azure-iot-modelsrepository/pytest.ini new file mode 100644 index 000000000000..cff99159612d --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = --testdox +testdox_format = plaintext +norecursedirs=__pycache__, *.egg-info +filterwarnings = ignore::DeprecationWarning diff --git a/sdk/iot/azure-iot-modelsrepository/requirements.txt b/sdk/iot/azure-iot-modelsrepository/requirements.txt new file mode 100644 index 000000000000..2a3ce200e4fb --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/requirements.txt @@ -0,0 +1,9 @@ +pytest +pytest-mock +pytest-testdox +flake8 + +azure-core +urllib3>1.21.1,<1.26 +requests>=2.22.0 +six diff --git a/sdk/iot/azure-iot-modelsrepository/samples/README.md b/sdk/iot/azure-iot-modelsrepository/samples/README.md new file mode 100644 index 000000000000..38e3759fba2f --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/samples/README.md @@ -0,0 +1,10 @@ +# Azure IoT Models Repository Library Samples + +This directory contains samples showing how to use the features of the Azure IoT Models Repository Library. + +The pre-configured endpoints and DTMIs within the sampmles refer to example DTDL documents that can be found on [devicemodels.azure.com](https://devicemodels.azure.com/). These values can be replaced to reflect the locations of your own DTDLs, wherever they may be. + +## Resolver Samples +* [get_models_sample.py](get_models_sample.py) - Retrieve a model/models (and possibly dependencies) from a Model Repository, given a DTMI or DTMIs + +* [client_configuration_sample.py](client_configuration_sample.py) - Configure the client to work with local or remote repositories, as well as custom policies and transports \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py new file mode 100644 index 000000000000..734f582866a8 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py @@ -0,0 +1,42 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from azure.iot.modelsrepository import ModelsRepositoryClient +from azure.core.pipeline.transport import RequestsTransport +from azure.core.pipeline.policies import UserAgentPolicy, RetryPolicy + + +def default_client(): + # By default, this client will be configured for the Azure Device Models Repository + # i.e. https://devicemodels.azure.com/ + client = ModelsRepositoryClient() + + +def use_remote_repository(): + # You can specify a custom remote endpoint where your Models Repository is located + client = ModelsRepositoryClient(repository_location="https://fake.myrepository.com/") + + +def use_local_repository(): + # You can also specify a custom local filesystem path where your Models Repository is located. + # Paths can be specified as relative or absolute, as well as in URI format + client = ModelsRepositoryClient(repository_location="file://home/fake/myrepository") + + +def custom_policies(): + # You can customize policies for remote operations in the client by passing through kwargs. + # Please refer to documentation for the azure.core libraries for reference on all the possible + # policy options. + user_agent_policy = UserAgentPolicy("myuseragent") + retry_policy = RetryPolicy(retry_total=10) + client = ModelsRepositoryClient(user_agent_policy=user_agent_policy, retry_policy=retry_policy) + + +def custom_transport(): + # You can supply your own transport for remote operations in the client by passing through + # kwargs. Please refer to documentation for the azure.core libraries for reference on the + # possible transport options. + transport = RequestsTransport(use_env_settings=False) + client = ModelsRepositoryClient(transport=transport) diff --git a/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py new file mode 100644 index 000000000000..4408c1ff98ff --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py @@ -0,0 +1,55 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from azure.iot.modelsrepository import ( + ModelsRepositoryClient, + DEPENDENCY_MODE_TRY_FROM_EXPANDED, + DEPENDENCY_MODE_ENABLED, +) +import pprint + +dtmi = "dtmi:com:example:TemperatureController;1" +dtmi2 = "dtmi:com:example:TemperatureController;2" + +# By default this client will use the Azure Device Models Repository endpoint +# i.e. https://devicemodels.azure.com/ +# See client_configuration_sample.py for examples of alternate configurations +client = ModelsRepositoryClient() + + +def get_model(): + # This API call will return a dictionary mapping DTMI to its corresponding model from + # a DTDL document at the specified endpoint + # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.json + model_map = client.get_models([dtmi]) + pprint.pprint(model_map) + + +def get_models(): + # This API call will return a dictionary mapping DTMIs to corresponding models for from the + # DTDL documents at the specified endpoint + # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.json + # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-2.json + model_map = client.get_models([dtmi, dtmi2]) + pprint.pprint(model_map) + + +def get_model_expanded_dtdl(): + # This API call will return a dictionary mapping DTMIs to corresponding models for all elements + # of an expanded DTDL document at the specified endpoint + # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.expanded.json + model_map = client.get_models( + dtmis=[dtmi], dependency_resolution=DEPENDENCY_MODE_TRY_FROM_EXPANDED + ) + pprint.pprint(model_map) + + +def get_model_and_dependencies(): + # This API call will return a dictionary mapping the specified DTMI to it's corresponding model, + # from a DTDL document at the specified endpoint, as well as the DTMIs and models for all + # dependencies on components and interfaces + # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.json + model_map = client.get_models(dtmis=[dtmi], dependency_resolution=DEPENDENCY_MODE_ENABLED) + pprint.pprint(model_map) diff --git a/sdk/iot/azure-iot-modelsrepository/setup.py b/sdk/iot/azure-iot-modelsrepository/setup.py new file mode 100644 index 000000000000..f612b0032110 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/setup.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from setuptools import setup, find_packages + +# azure v0.x is not compatible with this package +# azure v0.x used to have a __version__ attribute (newer versions don't) +try: + import azure + + try: + ver = azure.__version__ + raise Exception( + "This package is incompatible with azure=={}. ".format(ver) + + 'Uninstall it with "pip uninstall azure".' + ) + except AttributeError: + pass +except ImportError: + pass + +with open("README.md", "r") as fh: + _long_description = fh.read() + +setup( + name="azure-iot-modelsrepository", + version="0.0.0-preview", + description="Microsoft Azure IoT Models Repository Library", + license="MIT License", + author="Microsoft Corporation", + author_email="azpysdkhelp@microsoft.com", + url="https://github.com/Azure/azure-sdk-for-python", + long_description=_long_description, + long_description_content_type="text/markdown", + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], + install_requires=[ + "azure-core" + # Define sub-dependencies due to pip dependency resolution bug + # https://github.com/pypa/pip/issues/988 + # ---requests dependencies--- + # requests 2.22+ does not support urllib3 1.25.0 or 1.25.1 (https://github.com/psf/requests/pull/5092) + "urllib3>1.21.1,<1.26,!=1.25.0,!=1.25.1;python_version!='3.4'", + # Actual project dependencies + "requests>=2.22.0", + "six", + ], + extras_require={":python_version<'3.0'": ["azure-iot-nspkg>=1.0.1"]}, + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3*, !=3.4.*", + packages=find_packages( + exclude=[ + "tests", + "tests.*", + "samples", + "samples.*", + # Exclude packages that will be covered by PEP420 or nspkg + "azure", + "azure.iot", + ] + ), + zip_safe=False, +) diff --git a/sdk/iot/azure-iot-modelsrepository/tests/__init__.py b/sdk/iot/azure-iot-modelsrepository/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/iot/azure-iot-modelsrepository/tests/conftest.py b/sdk/iot/azure-iot-modelsrepository/tests/conftest.py new file mode 100644 index 000000000000..3e49705aa942 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/conftest.py @@ -0,0 +1,14 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest + + +@pytest.fixture +def arbitrary_exception(): + class ArbitraryException(Exception): + pass + + return ArbitraryException("This exception is completely arbitrary") diff --git a/sdk/iot/azure-iot-modelsrepository/tox.ini b/sdk/iot/azure-iot-modelsrepository/tox.ini new file mode 100644 index 000000000000..bd1501900709 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py27, py35, py36, py37, py38, py39 + +[testenv] +deps = + pytest + pytest-mock + pytest-testdox +commands = pytest \ No newline at end of file From 69c9345602ec8725f3bcee1c0e2950c493750565 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Mon, 8 Mar 2021 20:30:58 -0800 Subject: [PATCH 02/25] Updated README + removed unnecessary dependencies Updated README + removed unnecessary dependencies --- sdk/iot/azure-iot-modelsrepository/README.md | 3 +-- sdk/iot/azure-iot-modelsrepository/requirements.txt | 2 -- sdk/iot/azure-iot-modelsrepository/setup.py | 7 ------- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/README.md b/sdk/iot/azure-iot-modelsrepository/README.md index fd181dae0f8e..28e32b7ce465 100644 --- a/sdk/iot/azure-iot-modelsrepository/README.md +++ b/sdk/iot/azure-iot-modelsrepository/README.md @@ -12,6 +12,5 @@ python -m pip install -e ## Features -* ### Resolver +* ### ModelsRepositoryClient * Allows retrieval of model DTDLs from remote URLs or local filesystems - * This feature is provisionally included in this package pending review diff --git a/sdk/iot/azure-iot-modelsrepository/requirements.txt b/sdk/iot/azure-iot-modelsrepository/requirements.txt index 2a3ce200e4fb..7c35e7d07437 100644 --- a/sdk/iot/azure-iot-modelsrepository/requirements.txt +++ b/sdk/iot/azure-iot-modelsrepository/requirements.txt @@ -4,6 +4,4 @@ pytest-testdox flake8 azure-core -urllib3>1.21.1,<1.26 -requests>=2.22.0 six diff --git a/sdk/iot/azure-iot-modelsrepository/setup.py b/sdk/iot/azure-iot-modelsrepository/setup.py index f612b0032110..5969dee9b52f 100644 --- a/sdk/iot/azure-iot-modelsrepository/setup.py +++ b/sdk/iot/azure-iot-modelsrepository/setup.py @@ -52,13 +52,6 @@ ], install_requires=[ "azure-core" - # Define sub-dependencies due to pip dependency resolution bug - # https://github.com/pypa/pip/issues/988 - # ---requests dependencies--- - # requests 2.22+ does not support urllib3 1.25.0 or 1.25.1 (https://github.com/psf/requests/pull/5092) - "urllib3>1.21.1,<1.26,!=1.25.0,!=1.25.1;python_version!='3.4'", - # Actual project dependencies - "requests>=2.22.0", "six", ], extras_require={":python_version<'3.0'": ["azure-iot-nspkg>=1.0.1"]}, From bc04e91827e293bb1088bc85e6c6c137b6f12092 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Wed, 10 Mar 2021 15:24:44 -0800 Subject: [PATCH 03/25] DTMI utils --- sdk/iot/azure-iot-modelsrepository/.flake8 | 2 +- .../azure/iot/modelsrepository/client.py | 6 +- .../azure/iot/modelsrepository/dtmi_utils.py | 64 ++ .../azure/iot/modelsrepository/resolver.py | 18 +- .../tests/test_client.py | 43 ++ .../tests/test_dtmi_utils.py | 70 +++ .../tests/test_resolver.py | 558 ++++++++++++++++++ sdk/iot/azure-iot-modelsrepository/tox.ini | 2 +- 8 files changed, 743 insertions(+), 20 deletions(-) create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_client.py create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_utils.py create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py diff --git a/sdk/iot/azure-iot-modelsrepository/.flake8 b/sdk/iot/azure-iot-modelsrepository/.flake8 index 942c4cef4914..dbf3f4fb8f1e 100644 --- a/sdk/iot/azure-iot-modelsrepository/.flake8 +++ b/sdk/iot/azure-iot-modelsrepository/.flake8 @@ -4,4 +4,4 @@ ignore = E501,W503,E203 exclude = .git, - __pycache__, \ No newline at end of file + __pycache__, diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index fc7afe77b114..faeb14e2b9e7 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -18,7 +18,7 @@ NetworkTraceLoggingPolicy, ProxyPolicy, ) -from . import resolver as resolver +from . import resolver from . import pseudo_parser from .chainable_exception import ChainableException @@ -38,6 +38,7 @@ class ModelsRepositoryClient(object): """Client providing APIs for Models Repository operations""" + # TODO: Should api_version be a kwarg? def __init__(self, repository_location=None, api_version=None, **kwargs): """ :param str repository_location: Location of the Models Repository you wish to access. @@ -73,12 +74,13 @@ def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): manually resolving dependencies in the repository :raises: ValueError if given an invalid dependency resolution mode - :raises: ResolverError if there is an error retreiving a model + :raises: ResolverError if there is an error retrieving a model :returns: Dictionary mapping DTMIs to models :rtype: dict """ if dependency_resolution == DEPENDENCY_MODE_DISABLED: + # Simply retrieve the model(s) model_map = self.resolver.resolve(dtmis) elif dependency_resolution == DEPENDENCY_MODE_ENABLED: # Manually resolve dependencies using pseudo-parser diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py new file mode 100644 index 000000000000..7dafe2f6f2ea --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py @@ -0,0 +1,64 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import re +import pathlib + + +def is_valid_dtmi(dtmi): + """Checks validity of a DTMI + + :param dtmi str: DTMI + + :returns: Boolean indicating if DTMI is valid + :rtype: bool + """ + pattern = re.compile( + "^dtmi:[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?(?::[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?)*;[1-9][0-9]{0,8}$" + ) + if not pattern.match(dtmi): + return False + else: + return True + + +def get_model_uri(dtmi, repository_uri, expanded=False): + """Get the URI representing the absolute location of a model in a Models Repository + + :param dtmi str: DTMI for a model + :param repository_uri str: URI for a Models Repository + :param expanded bool: Indicates if the URI should be for an expanded model (Default: False) + + :raises: ValueError if given an invalid DTMI + + :returns: The URI for the model in the Models Repository + :rtype: str + """ + if not repository_uri.endswith("/"): + repository_uri += "/" + model_uri = repository_uri + _convert_dtmi_to_path(dtmi) + if expanded: + model_uri.replace(".json", ".expanded.json") + return model_uri + + +def _convert_dtmi_to_path(dtmi): + """Returns the DTMI path for a DTMI + E.g: + dtmi:com:example:Thermostat;1 -> dtmi/com/example/thermostat-1.json + + :param dtmi str: DTMI + + :raises ValueError if DTMI is invalid + + :returns: Relative path + """ + pattern = re.compile( + "^dtmi:[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?(?::[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?)*;[1-9][0-9]{0,8}$" + ) + if not pattern.match(dtmi): + raise ValueError("Invalid DTMI") + else: + return dtmi.lower().replace(":", "/").replace(";", "-") + ".json" diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py index bc81d16325d5..7ef99a7658bf 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py @@ -9,6 +9,7 @@ import abc import re import os +from . import dtmi_utils from .chainable_exception import ChainableException logger = logging.getLogger(__name__) @@ -40,7 +41,7 @@ def resolve(self, dtmis, expanded_model=False): """ model_map = {} for dtmi in dtmis: - dtdl_path = _convert_dtmi_to_path(dtmi) + dtdl_path = dtmi_utils._convert_dtmi_to_path(dtmi) if expanded_model: dtdl_path = dtdl_path.replace(".json", ".expanded.json") @@ -131,18 +132,3 @@ def fetch(self, path): except Exception as e: raise FetcherError("Failed to fetch from Filesystem", e) return json.loads(file_str) - - -def _convert_dtmi_to_path(dtmi): - """Converts a DTMI into a DTMI path - - E.g: - dtmi:com:example:Thermostat;1 -> dtmi/com/example/thermostat-1.json - """ - pattern = re.compile( - "^dtmi:[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?(?::[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?)*;[1-9][0-9]{0,8}$" - ) - if not pattern.match(dtmi): - raise ValueError("Invalid DTMI") - else: - return dtmi.lower().replace(":", "/").replace(";", "-") + ".json" diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_client.py b/sdk/iot/azure-iot-modelsrepository/tests/test_client.py new file mode 100644 index 000000000000..2d08448c0983 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_client.py @@ -0,0 +1,43 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +import logging +from azure.iot.modelsrepository import ModelsRepositoryClient, DEPENDENCY_MODE_ENABLED, DEPENDENCY_MODE_TRY_FROM_EXPANDED +from azure.iot.modelsrepository import resolver + +logging.basicConfig(level=logging.DEBUG) + + +@pytest.mark.describe("ModelsRepositoryClient -- Instantiation (Remote Repository)") +class TestModelsRepositoryClientInstantiation(object): + @pytest.fixture + def location(self): + return "https://my.fake.respository.com" + + @pytest.mark.it("Instantiates a PipelineClient that uses the provided repository location") + @pytest.mark.parametrize("location, expected_pl_base_url", [ + pytest.param("http://repository.mydomain.com/", "http://repository.mydomain.com/", id="HTTP URL"), + pytest.param("https://repository.mydomain.com/", "https://repository.mydomain.com/", id="HTTPS URL"), + pytest.param("repository.mydomain.com", "https://repository.mydomain.com", id="No protocol specified on URL (defaults to HTTPS)"), + pytest.param("https://repository.mydomain.com", "https://repository.mydomain.com", id="No trailing '/' on URL") + ]) + def test_pipeline_client(self, location, expected_pl_base_url): + client = ModelsRepositoryClient(repository_location=location) + assert client.fetcher.client._base_url == expected_pl_base_url + + @pytest.mark.it("Configures the PipelineClient ") + + # @pytest.mark.it("Instantiates a DtmiResolver from an HttpFetcher if the client is being created from a remote repository location") + # def test_resolver_fetcher(self): + # location = "https://my.fake.respository.com" + # client = ModelsRepositoryClient(repository_location=location) + + # assert isinstance(client.resolver, resolver.DtmiResolver) + # assert isinstance(client.fetcher, resolver.HttpFetcher) + # assert client.resolver.fetcher is client.fetcher + # assert client.fetcher.client._base_url == location + + # @pytest.mark.it("Instantiates a PseudoParser from the DtmiResolver") \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_utils.py b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_utils.py new file mode 100644 index 000000000000..34629cc61ba8 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_utils.py @@ -0,0 +1,70 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +import logging +import azure.iot.modelsrepository.dtmi_utils as dtmi_utils + +logging.basicConfig(level=logging.DEBUG) + +@pytest.mark.describe(".is_valid_dtmi()") +class TestIsValidDTMI(object): + @pytest.mark.it("Returns True if given a valid DTMI") + @pytest.mark.parametrize("dtmi", [ + pytest.param("dtmi:FooDTDL;1", id="Short DTMI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", id="Long DTMI") + ]) + def test_valid_dtmi(self, dtmi): + assert dtmi_utils.is_valid_dtmi(dtmi) + + @pytest.mark.it("Returns False if given an invalid DTMI") + @pytest.mark.parametrize("dtmi", [ + pytest.param("", id="Empty string"), + pytest.param("not a dtmi", id="Not a DTMI"), + pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), + pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), + pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), + ]) + def test_invalid_dtmi(self, dtmi): + assert not dtmi_utils.is_valid_dtmi(dtmi) + + +@pytest.mark.describe(".get_model_uri()") +class TestGetModelURI(object): + @pytest.mark.it("Returns the URI for a specified model at a specified repository") + @pytest.mark.parametrize("dtmi, repository_uri, expected_model_uri", [ + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "https://myrepository/", "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTPS repository URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository/", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTP repository URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file://localhost/myrepository/", "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI w/ host"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Repository URI without trailing '/'") + ]) + def test_uri(self, dtmi, repository_uri, expected_model_uri): + model_uri = dtmi_utils.get_model_uri(dtmi, repository_uri) + assert model_uri == expected_model_uri + + @pytest.mark.it("Returns the URI for a specified expanded model at a specified repository") + @pytest.mark.parametrize("dtmi, repository_uri, expected_model_uri", [ + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "https://myrepository/", "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTPS repository URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository/", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTP repository URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file://localhost/myrepository/", "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI w/ host"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Repository URI without trailing '/'") + ]) + def test_uri(self, dtmi, repository_uri, expected_model_uri): + model_uri = dtmi_utils.get_model_uri(dtmi, repository_uri, expanded=True) + assert model_uri == expected_model_uri + + @pytest.mark.it("Raises ValueError if given an invalid DTMI") + @pytest.mark.parametrize("dtmi", [ + pytest.param("", id="Empty string"), + pytest.param("not a dtmi", id="Not a DTMI"), + pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), + pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), + pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), + ]) + def test_invalid_dtmi(self, dtmi): + with pytest.raises(ValueError): + dtmi_utils.get_model_uri(dtmi, "https://myrepository/") \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py b/sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py new file mode 100644 index 000000000000..62d7299d2e08 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py @@ -0,0 +1,558 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +import logging +import json +import os +from azure.iot.modelsrepository import resolver + +logging.basicConfig(level=logging.DEBUG) + + +@pytest.fixture +def dtmi(): + return "dtmi:com:somedomain:example:FooDTDL;1" + + +@pytest.fixture +def dtdl(): + return { + "@context": "dtmi:dtdl:context;1", + "@id": "dtmi:com:somedomain:example:FooDTDL;1", + "@type": "Interface", + "displayName": "Foo", + "contents": [ + { + "@type": "Property", + "name": "fooproperty", + "displayName": "Foo Property", + "schema": "string", + "description": "A string representing some value. This isn't real", + }, + ], + } + + +# NOTE: maybe move this to a fetcher specific class +@pytest.fixture +def path(): + return "some/path/to/a/dtdl.json" + + +################################################################# + + +@pytest.fixture +def foo_dtmi(): + return "dtmi:com:somedomain:example:FooDTDL;1" + + +# @pytest.fixture +# def foo_dtdl_json(): +# # Testing Notes: +# # - Contains a single property +# # - Contains multiple components +# # - Contains an extension of an interface +# # - Contains two different components with the same model +# return { +# "@context": "dtmi:dtdl:context;1", +# "@id": "dtmi:com:somedomain:example:FooDTDL;1", +# "@type": "Interface", +# "displayName": "Foo", +# "extends": "dtmi:com:somedomain:example:BazDTDL;1", +# "contents": [ +# { +# "@type": "Property", +# "name": "fooproperty", +# "displayName": "Foo Property", +# "schema": "string", +# "description": "A string representing some value. This isn't real", +# }, +# { +# "@type": "Component", +# "name": "bar", +# "displayName": "Bar 1", +# "schema": "dtmi:com:somedomain:example:BarDTDL;1", +# "description": "Bar component 1", +# }, +# { +# "@type": "Component", +# "name": "bar", +# "displayName": "Bar 2", +# "schema": "dtmi:com:somedomain:example:BarDTDL;1", +# "description": "Bar component 2", +# }, +# { +# "@type": "Component", +# "name": "buzz", +# "displayName": "Buzz", +# "schema": "dtmi:com:somedomain:example:BuzzDTDL;1", +# }, +# ], +# } + + +# @pytest.fixture +# def bar_dtdl_json(): +# # Testing Notes: +# # - Contains a telemetry +# return { +# "@context": "dtmi:dtdl:context;1", +# "@id": "dtmi:com:somedomain:example:BarDTDL;1", +# "@type": "Interface", +# "displayName": "Bar", +# "contents": [ +# { +# "@type": "Property", +# "name": "barproperty", +# "displayName": "Bar Property", +# "schema": "string", +# "description": "A string representing some value. This isn't real", +# }, +# {"@type": "Telemetry", "name": "bartelemetry", "schema": "double"}, +# ], +# } + + +# @pytest.fixture +# def buzz_dtdl_json(): +# # Testing Notes: +# # - Contains two extensions of interfaces (maximum value) +# # - Contains a single property +# return { +# "@context": "dtmi:dtdl:context;1", +# "@id": "dtmi:com:somedomain:example:BuzzDTDL;1", +# "@type": "Interface", +# "displayName": "Buzz", +# "extends": [ +# "dtmi:com:somedomain:example:QuxDTDL;1", +# "dtmi:com:somedomain:example:QuzDTDL;1", +# ], +# "contents": [ +# { +# "@type": "Property", +# "name": "buzzproperty", +# "displayName": "Buzz Property", +# "schema": "string", +# "description": "A string representing some value. This isn't real", +# }, +# ], +# } + + +# @pytest.fixture +# def baz_dtdl_json(): +# # Testing Notes: +# # - Contains multiple properties +# return { +# "@context": "dtmi:dtdl:context;1", +# "@id": "dtmi:com:somedomain:example:BazDTDL;1", +# "@type": "Interface", +# "displayName": "Baz", +# "contents": [ +# { +# "@type": "Property", +# "name": "bazproperty1", +# "displayName": "Baz Property 1", +# "schema": "string", +# "description": "A string representing some value. This isn't real", +# }, +# { +# "@type": "Property", +# "name": "bazproperty2", +# "displayName": "Baz Property 2", +# "schema": "string", +# "description": "A string representing some value. This isn't real", +# }, +# ], +# } + + +# @pytest.fixture +# def qux_dtdl_json(): +# # Testing Notes: +# # - Contains a Command +# return { +# "@context": "dtmi:dtdl:context;1", +# "@id": "dtmi:com:somedomain:example:QuxDTDL;1", +# "@type": "Interface", +# "displayName": "Qux", +# "contents": [ +# { +# "@type": "Command", +# "name": "quxcommand", +# "request": { +# "name": "quxcommandtime", +# "displayName": "Qux Command Time", +# "description": "It's a command. For Qux.", +# "schema": "dateTime", +# }, +# "response": {"name": "quxresponsetime", "schema": "dateTime"}, +# } +# ], +# } + + +# @pytest.fixture +# def quz_dtdl_json(): +# # Testing Notes: +# # - Contains no contents (doesn't make much sense, but an edge case to test nontheless) +# return { +# "@context": "dtmi:dtdl:context;1", +# "@id": "dtmi:com:somedomain:example:QuzDTDL;1", +# "@type": "Interface", +# "displayName": "Quz", +# } + + +# @pytest.fixture +# def foo_dtdl_expanded_json( +# foo_dtdl_json, bar_dtdl_json, buzz_dtdl_json, qux_dtdl_json, quz_dtdl_json, baz_dtdl_json +# ): +# return [ +# foo_dtdl_json, +# bar_dtdl_json, +# buzz_dtdl_json, +# qux_dtdl_json, +# quz_dtdl_json, +# baz_dtdl_json, +# ] + + +# @pytest.fixture +# def dtmi_to_path_mappings(): +# # NOTE: Does not include .exapnded.json paths. +# # Manually replace .json with .expanded.json if necessary +# path_map = {} +# path_map["dtmi:com:somedomain:example:FooDTDL;1"] = "dtmi/com/somedomain/example/foodtdl-1.json" +# path_map["dtmi:com:somedomain:example:BarDTDL;1"] = "dtmi/com/somedomain/example/bardtdl-1.json" +# path_map[ +# "dtmi:com:somedomain:example:BuzzDTDL;1" +# ] = "dtmi/com/somedomain/example/buzzdtdl-1.json" +# path_map["dtmi:com:somedomain:example:QuxDTDL;1"] = "dtmi/com/somedomain/example/quxdtdl-1.json" +# path_map["dtmi:com:somedomain:example:QuzDTDL;1"] = "dtmi/com/somedomain/example/quzdtdl-1.json" +# path_map["dtmi:com:somedomain:example:BazDTDL;1"] = "dtmi/com/somedomain/example/bazdtdl-1.json" +# return path_map + + +# @pytest.fixture +# def path_to_dtdl_mappings( +# foo_dtdl_json, +# bar_dtdl_json, +# buzz_dtdl_json, +# qux_dtdl_json, +# quz_dtdl_json, +# baz_dtdl_json, +# foo_dtdl_expanded_json, +# dtmi_to_path_mappings, +# ): +# # NOTE: Keep this fixture updated with any new models added for testing +# dtdl_map = {} +# dtdl_list = [ +# # (Regular DTDL, Expanded DTDL) +# (foo_dtdl_json, foo_dtdl_expanded_json), +# (bar_dtdl_json, None), +# (buzz_dtdl_json, None), +# (qux_dtdl_json, None), +# (quz_dtdl_json, None), +# (baz_dtdl_json, None), +# ] +# for dtdl_tuple in dtdl_list: +# dtdl = dtdl_tuple[0] +# expanded_dtdl = dtdl_tuple[1] +# path = dtmi_to_path_mappings[dtdl["@id"]] +# dtdl_map[path] = dtdl +# if expanded_dtdl: +# expanded_path = path.replace(".json", ".expanded.json") +# dtdl_map[expanded_path] = expanded_dtdl +# return dtdl_map + + +# class DtmiResolverResolveSharedTests(object): +# @pytest.fixture +# def fetcher(self, mocker, path_to_dtdl_mappings): +# fetcher_mock = mocker.MagicMock() +# fetcher_mock.cached_mock_fetch_returns = [] +# fetcher_mock.fail_expanded = False + +# def mocked_fetch(path): +# if path.endswith(".expanded.json") and fetcher_mock.fail_expanded: +# raise resolver.FetcherError() +# try: +# dtdl = path_to_dtdl_mappings[path] +# except KeyError: +# raise resolver.FetcherError() +# fetcher_mock.cached_mock_fetch_returns.append(dtdl) +# return dtdl + +# fetcher_mock.fetch.side_effect = mocked_fetch +# return fetcher_mock + +# @pytest.fixture +# def dtmi_resolver(self, mocker, fetcher): +# return resolver.DtmiResolver(fetcher) + +# @pytest.mark.it("Raises a ValueError if the provided DTMI is invalid") +# @pytest.mark.parametrize( +# "dtmi", +# [ +# pytest.param("", id="Empty string"), +# pytest.param("not a dtmi", id="Not a DTMI"), +# pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), +# pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), +# pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), +# ], +# ) +# def test_invalid_dtmi(self, dtmi_resolver, dtmi): +# with pytest.raises(ValueError): +# dtmi_resolver.resolve(dtmi) + +# @pytest.mark.it("Raises a ResolverError if the Fetcher is unable to fetch a DTDL") +# def test_fetcher_failure(self, dtmi_resolver, foo_dtmi): +# my_fetcher_error = resolver.FetcherError("Some arbitrary fetcher error") +# dtmi_resolver.fetcher.fetch.side_effect = my_fetcher_error +# with pytest.raises(resolver.ResolverError) as e_info: +# dtmi_resolver.resolve(foo_dtmi) +# assert e_info.value.__cause__ is my_fetcher_error + +# # @pytest.mark.it("Raises a ResolverError if provided an invalid resolve mode") +# # def test_invalid_resolve_mode(self, dtmi_resolver, dtmi): +# # with pytest.raises(resolver.ResolverError): +# # dtmi_resolver.resolve(dtmi=dtmi, resolve_mode="invalid_mode") + + +# @pytest.mark.describe("DtmiResolver = .resolve() -- Dependency Mode: Disabled") +# class TestDtmiResolverResolveDependencyModeDisabled(DtmiResolverResolveSharedTests): +# @pytest.mark.it( +# "Uses the Fetcher to fetch a model DTDL from a path that corresponds to the provided DTMI" +# ) +# def test_fetcher(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): +# dtmi_resolver.resolve(foo_dtmi) + +# assert dtmi_resolver.fetcher.fetch.call_count == 1 +# expected_path = dtmi_to_path_mappings[foo_dtmi] +# assert dtmi_resolver.fetcher.fetch.call_args == mocker.call(expected_path) + +# @pytest.mark.it( +# "Returns a dictionary mapping the provided DTMI the corresponding model DTDL that was returned by the Fetcher" +# ) +# def test_returned_dict(self, dtmi_resolver, foo_dtmi): +# d = dtmi_resolver.resolve(foo_dtmi) + +# assert isinstance(d, dict) +# assert len(d) == 1 == dtmi_resolver.fetcher.fetch.call_count +# assert d[foo_dtmi] == dtmi_resolver.fetcher.cached_mock_fetch_returns[0] + + +# @pytest.mark.describe("DtmiResolver - .resolve() -- Dependency Mode: Enabled") +# class TestDtmiResolverResolveDependencyModeEnabled(DtmiResolverResolveSharedTests): +# @pytest.mark.it( +# "Uses the Fetcher to fetch model DTDLs from paths corresponding to the provided DTMI, as well as the DTMIs of all its dependencies" +# ) +# def test_fetcher(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): +# dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_ENABLED) + +# # NOTE: there are 6 calls because we only fetch for each UNIQUE component or interface. +# # There are two components in FooDTDL that are of type BarDTDL, but that DTDL only has +# # to be fetched once. +# assert dtmi_resolver.fetcher.fetch.call_count == 6 +# expected_path1 = dtmi_to_path_mappings["dtmi:com:somedomain:example:FooDTDL;1"] +# expected_path2 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BarDTDL;1"] +# expected_path3 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BuzzDTDL;1"] +# expected_path4 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuxDTDL;1"] +# expected_path5 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuzDTDL;1"] +# expected_path6 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BazDTDL;1"] +# assert dtmi_resolver.fetcher.fetch.call_args_list[0] == mocker.call(expected_path1) +# assert dtmi_resolver.fetcher.fetch.call_args_list[1] == mocker.call(expected_path2) +# assert dtmi_resolver.fetcher.fetch.call_args_list[2] == mocker.call(expected_path3) +# assert dtmi_resolver.fetcher.fetch.call_args_list[3] == mocker.call(expected_path4) +# assert dtmi_resolver.fetcher.fetch.call_args_list[4] == mocker.call(expected_path5) +# assert dtmi_resolver.fetcher.fetch.call_args_list[5] == mocker.call(expected_path6) + +# @pytest.mark.it( +# "Returns a dictionary mapping DTMIs to model DTDLs returned by the Fetcher, for the provided DTMI, as well as the DTMIs of all dependencies" +# ) +# def test_returned_dict( +# self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, path_to_dtdl_mappings +# ): +# d = dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_ENABLED) + +# assert isinstance(d, dict) +# assert len(d) == 6 == dtmi_resolver.fetcher.fetch.call_count +# for model in dtmi_resolver.fetcher.cached_mock_fetch_returns: +# dtmi = model["@id"] +# assert dtmi in d.keys() +# assert d[dtmi] == model + + +# @pytest.mark.describe("DtmiResolver - .resolve() -- Dependency Mode: Try From Expanded") +# class TestDtmiResolverResolveDependencyModeTryFromExpanded(DtmiResolverResolveSharedTests): +# @pytest.mark.it( +# "Attempts to use the Fetcher to fetch an expanded model DTDL from a path that corresponds to the provided DTMI" +# ) +# def test_fetcher(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): +# dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_TRY_FROM_EXPANDED) + +# assert dtmi_resolver.fetcher.fetch.call_count == 1 +# expected_path = dtmi_to_path_mappings[foo_dtmi].replace(".json", ".expanded.json") +# assert dtmi_resolver.fetcher.fetch.call_args == mocker.call(expected_path) + +# @pytest.mark.it( +# "Returns a dictionary mapping DTMIs to model DTDLs for all models contained within the expanded DTDL that was returned by the Fetcher" +# ) +# def test_returned_dict_expanded(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings): +# d = dtmi_resolver.resolve( +# foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_TRY_FROM_EXPANDED +# ) + +# assert isinstance(d, dict) +# assert len(d) == 6 +# assert dtmi_resolver.fetcher.fetch.call_count == 1 +# expanded_dtdl = dtmi_resolver.fetcher.cached_mock_fetch_returns[0] +# assert len(expanded_dtdl) == 6 +# for model in expanded_dtdl: +# dtmi = model["@id"] +# assert dtmi in d.keys() +# assert d[dtmi] == model + +# @pytest.mark.it( +# "Uses the Fetcher to fetch model DTDLs from paths corresponding to the provided DTMI, as well as the DTMIs of all its dependencies, for each expanded DTDL that cannot be fetched" +# ) +# def test_fetcher_fallback(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): +# dtmi_resolver.fetcher.fail_expanded = True +# dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_TRY_FROM_EXPANDED) + +# # NOTE: There are 7 calls. 1 attempted expanded fetch + 6 regular fetches. The expanded +# # fetch will fail, and then as a fallback it will do the regular fetch procedure. +# # +# # There are 6 regular fetch calls because we only fetch for each UNIQUE component or +# # interface. There are two components in FooDTDL that are of type BarDTDL, but that +# # DTDL only has to be fetched once. +# assert dtmi_resolver.fetcher.fetch.call_count == 12 +# expected_path1 = dtmi_to_path_mappings["dtmi:com:somedomain:example:FooDTDL;1"] +# expected_path2 = dtmi_to_path_mappings["dtmi:com:somedomain:example:FooDTDL;1"] +# expected_path3 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BarDTDL;1"] +# expected_path4 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BuzzDTDL;1"] +# expected_path5 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuxDTDL;1"] +# expected_path6 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuzDTDL;1"] +# expected_path7 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BazDTDL;1"] +# assert dtmi_resolver.fetcher.fetch.call_args_list[0] == mocker.call(expected_path1) +# assert dtmi_resolver.fetcher.fetch.call_args_list[1] == mocker.call(expected_path2) +# assert dtmi_resolver.fetcher.fetch.call_args_list[2] == mocker.call(expected_path3) +# assert dtmi_resolver.fetcher.fetch.call_args_list[3] == mocker.call(expected_path4) +# assert dtmi_resolver.fetcher.fetch.call_args_list[4] == mocker.call(expected_path5) +# assert dtmi_resolver.fetcher.fetch.call_args_list[5] == mocker.call(expected_path6) +# assert dtmi_resolver.fetcher.fetch.call_args_list[6] == mocker.call(expected_path7) + + +@pytest.mark.describe("HttpFetcher - .fetch()") +class TestHttpFetcherFetch(object): + @pytest.fixture + def fetcher(self, mocker, dtdl): + mock_http_client = mocker.MagicMock() + mock_response = mock_http_client._pipeline.run.return_value.http_response + mock_response.status_code = 200 + mock_response.text.return_value = json.dumps(dtdl) + mock_http_client._pipeline + return resolver.HttpFetcher(mock_http_client) + + @pytest.mark.it( + "Sends an HTTP GET request for the provided path, using the fetcher's HTTP client" + ) + def test_request(self, fetcher, path, mocker): + fetcher.fetch(path) + + assert fetcher.client.get.call_count == 1 + assert fetcher.client.get.call_args == mocker.call(url=path) + request = fetcher.client.get.return_value + assert fetcher.client._pipeline.run.call_count == 1 + assert fetcher.client._pipeline.run.call_args == mocker.call(request) + + @pytest.mark.it("Returns the GET response in JSON format, if the GET request is successful") + def test_response_success(self, fetcher, path): + dtdl_json = fetcher.fetch(path) + + assert isinstance(dtdl_json, dict) + client_response = fetcher.client._pipeline.run.return_value.http_response + assert client_response.status_code == 200 + assert dtdl_json == json.loads(client_response.text()) + + @pytest.mark.it("Raises a FetcherError if the GET request is unsuccessful") + def test_response_failure(self, fetcher, path): + fetcher.client._pipeline.run.return_value.http_response.status_code = 400 + with pytest.raises(resolver.FetcherError): + fetcher.fetch(path) + + +@pytest.mark.describe("FilesystemFetcher - .fetch()") +class TestFilesystemFetcherFetch(object): + @pytest.fixture + def mock_open(self, mocker, dtdl): + return mocker.patch("builtins.open", mocker.mock_open(read_data=json.dumps(dtdl))) + + @pytest.fixture + def fetcher(self, mock_open, mocker): + base_path = "C:/some/base/path" + return resolver.FilesystemFetcher(base_path) + + @pytest.mark.it( + "Formats and normalizes syntax of provided path to fetch, then opens and reads the file at that location" + ) + def test_open_read_path(self, fetcher, path, mock_open, mocker): + mocker.spy(os.path, "join") + mocker.spy(os.path, "normcase") + mocker.spy(os.path, "normpath") + + fetcher.fetch(path) + + # These three functions being called ensure that the path will be formatted correctly for + # all cases (e.g. trailing slash, no trailing slash, leading slash, etc.) + # Because we know how these builtin functions work, there's no need to explicitly test + # these input variants - the logic is handled externally, and is not part of this unit + assert os.path.join.call_count == 1 + assert os.path.normcase.call_count == 1 + assert os.path.normpath.call_count == 1 + + # The expected formatted path was passed to the 'open()' function + expected_absolute_path = os.path.normpath( + os.path.normcase(os.path.join(fetcher.base_path, path)) + ) + assert mock_open.call_count == 1 + assert mock_open.call_args == mocker.call(expected_absolute_path) + + # The data was read from the file + assert mock_open.return_value.read.call_count == 1 + assert mock_open.return_value.read.call_args == mocker.call() + + @pytest.mark.it( + "Returns the data returned by the read operation in JSON format, if the read is successful" + ) + def test_open_read_success(self, fetcher, path, mock_open, dtdl, mocker): + dtdl_json = fetcher.fetch(path) + + assert isinstance(dtdl, dict) + # Unfortunately, there isn't really a way to show that the returned value comes from the + # file read due to how the mock of open/read builtins work. Best I can do is show that it + # has the expected value (with the assumption that the mock returned that value) + assert dtdl_json == dtdl + + @pytest.mark.it( + "Raises a FetcherError if there is an error while opening the file at the provided path" + ) + def test_open_failure(self, fetcher, mock_open, path, arbitrary_exception): + mock_open.side_effect = arbitrary_exception + with pytest.raises(resolver.FetcherError) as e_info: + fetcher.fetch(path) + assert e_info.value.__cause__ == arbitrary_exception + + @pytest.mark.it( + "Raises a FetcherError if there is an error while reading the file at the provided path" + ) + def test_read_failure(self, fetcher, mock_open, path, arbitrary_exception): + mock_open.return_value.read.side_effect = arbitrary_exception + with pytest.raises(resolver.FetcherError) as e_info: + fetcher.fetch(path) + assert e_info.value.__cause__ == arbitrary_exception diff --git a/sdk/iot/azure-iot-modelsrepository/tox.ini b/sdk/iot/azure-iot-modelsrepository/tox.ini index bd1501900709..633164a750c5 100644 --- a/sdk/iot/azure-iot-modelsrepository/tox.ini +++ b/sdk/iot/azure-iot-modelsrepository/tox.ini @@ -6,4 +6,4 @@ deps = pytest pytest-mock pytest-testdox -commands = pytest \ No newline at end of file +commands = pytest From a7cd8bfd439cbc53d1eafd94609e708074ec7202 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Thu, 11 Mar 2021 10:08:00 -0800 Subject: [PATCH 04/25] API adjustments --- .../azure/iot/modelsrepository/dtmi_utils.py | 10 +- .../samples/README.md | 4 +- .../samples/dtmi_util_sample.py | 35 ++ .../tests/test_client.py | 43 -- .../tests/test_resolver.py | 558 ------------------ 5 files changed, 43 insertions(+), 607 deletions(-) create mode 100644 sdk/iot/azure-iot-modelsrepository/samples/dtmi_util_sample.py delete mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_client.py delete mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py index 7dafe2f6f2ea..4f8a50083411 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py @@ -39,12 +39,10 @@ def get_model_uri(dtmi, repository_uri, expanded=False): if not repository_uri.endswith("/"): repository_uri += "/" model_uri = repository_uri + _convert_dtmi_to_path(dtmi) - if expanded: - model_uri.replace(".json", ".expanded.json") return model_uri -def _convert_dtmi_to_path(dtmi): +def _convert_dtmi_to_path(dtmi, expanded=False): """Returns the DTMI path for a DTMI E.g: dtmi:com:example:Thermostat;1 -> dtmi/com/example/thermostat-1.json @@ -60,5 +58,7 @@ def _convert_dtmi_to_path(dtmi): ) if not pattern.match(dtmi): raise ValueError("Invalid DTMI") - else: - return dtmi.lower().replace(":", "/").replace(";", "-") + ".json" + dtmi_path = dtmi.lower().replace(":", "/").replace(";", "-") + ".json" + if expanded: + dtmi_path = dtmi.replace(".json", ".expanded.json") + return dtmi_path diff --git a/sdk/iot/azure-iot-modelsrepository/samples/README.md b/sdk/iot/azure-iot-modelsrepository/samples/README.md index 38e3759fba2f..9c155ec6c881 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/README.md +++ b/sdk/iot/azure-iot-modelsrepository/samples/README.md @@ -7,4 +7,6 @@ The pre-configured endpoints and DTMIs within the sampmles refer to example DTDL ## Resolver Samples * [get_models_sample.py](get_models_sample.py) - Retrieve a model/models (and possibly dependencies) from a Model Repository, given a DTMI or DTMIs -* [client_configuration_sample.py](client_configuration_sample.py) - Configure the client to work with local or remote repositories, as well as custom policies and transports \ No newline at end of file +* [client_configuration_sample.py](client_configuration_sample.py) - Configure the client to work with local or remote repositories, as well as custom policies and transports + +* [dtmi_utils_sample.py](dtmi_utils_sample.py) - Use the dtmi_utils module to manipulate and check DTMIs \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/samples/dtmi_util_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/dtmi_util_sample.py new file mode 100644 index 000000000000..2a314984c5e4 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/samples/dtmi_util_sample.py @@ -0,0 +1,35 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from azure.iot.modelsrepository import dtmi_utils + + +def sample_is_valid_dtmi(): + """Check if a DTMI is valid or not using the .is_valid_dtmi() function""" + # Returns True - this is a valid DTMI + dtmi_utils.is_valid_dtmi("dtmi:com:example:Thermostat;1") + + # Returns False - this is NOT a valid DTMI + dtmi_utils.is_valid_dtmi("dtmi:com:example:Thermostat") + + +def sample_get_model_uri(): + """Get a URI for a model in a Models Repository using the .get_model_uri() function""" + dtmi = "dtmi:com:example:Thermostat;1" + + # Local repository example + repo_uri = "file:///path/to/repository" + print(dtmi_utils.get_model_uri(dtmi, repo_uri)) + # Prints: "file:///path/to/repository/dtmi/com/example/thermostat-1.json" + print(dtmi_utils.get_model_uri(dtmi, repo_uri, expanded=True)) + # Prints: "file:///path/to/repository/dtmi/com/example/thermostat-1.expanded.json" + + + # Remote repository example + repo_uri = "https://contoso.com/models/" + print(dtmi_utils.get_model_uri(dtmi, repo_uri)) + # Prints: "https://contoso/com/models/dtmi/com/example/thermostat-1.json" + print(dtmi_utils.get_model_uri(dtmi, repo_uri, expanded=True)) + # Prints: "https://contoso/com/models/dtmi/com/example/thermostat-1.expanded.json" diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_client.py b/sdk/iot/azure-iot-modelsrepository/tests/test_client.py deleted file mode 100644 index 2d08448c0983..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_client.py +++ /dev/null @@ -1,43 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import pytest -import logging -from azure.iot.modelsrepository import ModelsRepositoryClient, DEPENDENCY_MODE_ENABLED, DEPENDENCY_MODE_TRY_FROM_EXPANDED -from azure.iot.modelsrepository import resolver - -logging.basicConfig(level=logging.DEBUG) - - -@pytest.mark.describe("ModelsRepositoryClient -- Instantiation (Remote Repository)") -class TestModelsRepositoryClientInstantiation(object): - @pytest.fixture - def location(self): - return "https://my.fake.respository.com" - - @pytest.mark.it("Instantiates a PipelineClient that uses the provided repository location") - @pytest.mark.parametrize("location, expected_pl_base_url", [ - pytest.param("http://repository.mydomain.com/", "http://repository.mydomain.com/", id="HTTP URL"), - pytest.param("https://repository.mydomain.com/", "https://repository.mydomain.com/", id="HTTPS URL"), - pytest.param("repository.mydomain.com", "https://repository.mydomain.com", id="No protocol specified on URL (defaults to HTTPS)"), - pytest.param("https://repository.mydomain.com", "https://repository.mydomain.com", id="No trailing '/' on URL") - ]) - def test_pipeline_client(self, location, expected_pl_base_url): - client = ModelsRepositoryClient(repository_location=location) - assert client.fetcher.client._base_url == expected_pl_base_url - - @pytest.mark.it("Configures the PipelineClient ") - - # @pytest.mark.it("Instantiates a DtmiResolver from an HttpFetcher if the client is being created from a remote repository location") - # def test_resolver_fetcher(self): - # location = "https://my.fake.respository.com" - # client = ModelsRepositoryClient(repository_location=location) - - # assert isinstance(client.resolver, resolver.DtmiResolver) - # assert isinstance(client.fetcher, resolver.HttpFetcher) - # assert client.resolver.fetcher is client.fetcher - # assert client.fetcher.client._base_url == location - - # @pytest.mark.it("Instantiates a PseudoParser from the DtmiResolver") \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py b/sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py deleted file mode 100644 index 62d7299d2e08..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_resolver.py +++ /dev/null @@ -1,558 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import pytest -import logging -import json -import os -from azure.iot.modelsrepository import resolver - -logging.basicConfig(level=logging.DEBUG) - - -@pytest.fixture -def dtmi(): - return "dtmi:com:somedomain:example:FooDTDL;1" - - -@pytest.fixture -def dtdl(): - return { - "@context": "dtmi:dtdl:context;1", - "@id": "dtmi:com:somedomain:example:FooDTDL;1", - "@type": "Interface", - "displayName": "Foo", - "contents": [ - { - "@type": "Property", - "name": "fooproperty", - "displayName": "Foo Property", - "schema": "string", - "description": "A string representing some value. This isn't real", - }, - ], - } - - -# NOTE: maybe move this to a fetcher specific class -@pytest.fixture -def path(): - return "some/path/to/a/dtdl.json" - - -################################################################# - - -@pytest.fixture -def foo_dtmi(): - return "dtmi:com:somedomain:example:FooDTDL;1" - - -# @pytest.fixture -# def foo_dtdl_json(): -# # Testing Notes: -# # - Contains a single property -# # - Contains multiple components -# # - Contains an extension of an interface -# # - Contains two different components with the same model -# return { -# "@context": "dtmi:dtdl:context;1", -# "@id": "dtmi:com:somedomain:example:FooDTDL;1", -# "@type": "Interface", -# "displayName": "Foo", -# "extends": "dtmi:com:somedomain:example:BazDTDL;1", -# "contents": [ -# { -# "@type": "Property", -# "name": "fooproperty", -# "displayName": "Foo Property", -# "schema": "string", -# "description": "A string representing some value. This isn't real", -# }, -# { -# "@type": "Component", -# "name": "bar", -# "displayName": "Bar 1", -# "schema": "dtmi:com:somedomain:example:BarDTDL;1", -# "description": "Bar component 1", -# }, -# { -# "@type": "Component", -# "name": "bar", -# "displayName": "Bar 2", -# "schema": "dtmi:com:somedomain:example:BarDTDL;1", -# "description": "Bar component 2", -# }, -# { -# "@type": "Component", -# "name": "buzz", -# "displayName": "Buzz", -# "schema": "dtmi:com:somedomain:example:BuzzDTDL;1", -# }, -# ], -# } - - -# @pytest.fixture -# def bar_dtdl_json(): -# # Testing Notes: -# # - Contains a telemetry -# return { -# "@context": "dtmi:dtdl:context;1", -# "@id": "dtmi:com:somedomain:example:BarDTDL;1", -# "@type": "Interface", -# "displayName": "Bar", -# "contents": [ -# { -# "@type": "Property", -# "name": "barproperty", -# "displayName": "Bar Property", -# "schema": "string", -# "description": "A string representing some value. This isn't real", -# }, -# {"@type": "Telemetry", "name": "bartelemetry", "schema": "double"}, -# ], -# } - - -# @pytest.fixture -# def buzz_dtdl_json(): -# # Testing Notes: -# # - Contains two extensions of interfaces (maximum value) -# # - Contains a single property -# return { -# "@context": "dtmi:dtdl:context;1", -# "@id": "dtmi:com:somedomain:example:BuzzDTDL;1", -# "@type": "Interface", -# "displayName": "Buzz", -# "extends": [ -# "dtmi:com:somedomain:example:QuxDTDL;1", -# "dtmi:com:somedomain:example:QuzDTDL;1", -# ], -# "contents": [ -# { -# "@type": "Property", -# "name": "buzzproperty", -# "displayName": "Buzz Property", -# "schema": "string", -# "description": "A string representing some value. This isn't real", -# }, -# ], -# } - - -# @pytest.fixture -# def baz_dtdl_json(): -# # Testing Notes: -# # - Contains multiple properties -# return { -# "@context": "dtmi:dtdl:context;1", -# "@id": "dtmi:com:somedomain:example:BazDTDL;1", -# "@type": "Interface", -# "displayName": "Baz", -# "contents": [ -# { -# "@type": "Property", -# "name": "bazproperty1", -# "displayName": "Baz Property 1", -# "schema": "string", -# "description": "A string representing some value. This isn't real", -# }, -# { -# "@type": "Property", -# "name": "bazproperty2", -# "displayName": "Baz Property 2", -# "schema": "string", -# "description": "A string representing some value. This isn't real", -# }, -# ], -# } - - -# @pytest.fixture -# def qux_dtdl_json(): -# # Testing Notes: -# # - Contains a Command -# return { -# "@context": "dtmi:dtdl:context;1", -# "@id": "dtmi:com:somedomain:example:QuxDTDL;1", -# "@type": "Interface", -# "displayName": "Qux", -# "contents": [ -# { -# "@type": "Command", -# "name": "quxcommand", -# "request": { -# "name": "quxcommandtime", -# "displayName": "Qux Command Time", -# "description": "It's a command. For Qux.", -# "schema": "dateTime", -# }, -# "response": {"name": "quxresponsetime", "schema": "dateTime"}, -# } -# ], -# } - - -# @pytest.fixture -# def quz_dtdl_json(): -# # Testing Notes: -# # - Contains no contents (doesn't make much sense, but an edge case to test nontheless) -# return { -# "@context": "dtmi:dtdl:context;1", -# "@id": "dtmi:com:somedomain:example:QuzDTDL;1", -# "@type": "Interface", -# "displayName": "Quz", -# } - - -# @pytest.fixture -# def foo_dtdl_expanded_json( -# foo_dtdl_json, bar_dtdl_json, buzz_dtdl_json, qux_dtdl_json, quz_dtdl_json, baz_dtdl_json -# ): -# return [ -# foo_dtdl_json, -# bar_dtdl_json, -# buzz_dtdl_json, -# qux_dtdl_json, -# quz_dtdl_json, -# baz_dtdl_json, -# ] - - -# @pytest.fixture -# def dtmi_to_path_mappings(): -# # NOTE: Does not include .exapnded.json paths. -# # Manually replace .json with .expanded.json if necessary -# path_map = {} -# path_map["dtmi:com:somedomain:example:FooDTDL;1"] = "dtmi/com/somedomain/example/foodtdl-1.json" -# path_map["dtmi:com:somedomain:example:BarDTDL;1"] = "dtmi/com/somedomain/example/bardtdl-1.json" -# path_map[ -# "dtmi:com:somedomain:example:BuzzDTDL;1" -# ] = "dtmi/com/somedomain/example/buzzdtdl-1.json" -# path_map["dtmi:com:somedomain:example:QuxDTDL;1"] = "dtmi/com/somedomain/example/quxdtdl-1.json" -# path_map["dtmi:com:somedomain:example:QuzDTDL;1"] = "dtmi/com/somedomain/example/quzdtdl-1.json" -# path_map["dtmi:com:somedomain:example:BazDTDL;1"] = "dtmi/com/somedomain/example/bazdtdl-1.json" -# return path_map - - -# @pytest.fixture -# def path_to_dtdl_mappings( -# foo_dtdl_json, -# bar_dtdl_json, -# buzz_dtdl_json, -# qux_dtdl_json, -# quz_dtdl_json, -# baz_dtdl_json, -# foo_dtdl_expanded_json, -# dtmi_to_path_mappings, -# ): -# # NOTE: Keep this fixture updated with any new models added for testing -# dtdl_map = {} -# dtdl_list = [ -# # (Regular DTDL, Expanded DTDL) -# (foo_dtdl_json, foo_dtdl_expanded_json), -# (bar_dtdl_json, None), -# (buzz_dtdl_json, None), -# (qux_dtdl_json, None), -# (quz_dtdl_json, None), -# (baz_dtdl_json, None), -# ] -# for dtdl_tuple in dtdl_list: -# dtdl = dtdl_tuple[0] -# expanded_dtdl = dtdl_tuple[1] -# path = dtmi_to_path_mappings[dtdl["@id"]] -# dtdl_map[path] = dtdl -# if expanded_dtdl: -# expanded_path = path.replace(".json", ".expanded.json") -# dtdl_map[expanded_path] = expanded_dtdl -# return dtdl_map - - -# class DtmiResolverResolveSharedTests(object): -# @pytest.fixture -# def fetcher(self, mocker, path_to_dtdl_mappings): -# fetcher_mock = mocker.MagicMock() -# fetcher_mock.cached_mock_fetch_returns = [] -# fetcher_mock.fail_expanded = False - -# def mocked_fetch(path): -# if path.endswith(".expanded.json") and fetcher_mock.fail_expanded: -# raise resolver.FetcherError() -# try: -# dtdl = path_to_dtdl_mappings[path] -# except KeyError: -# raise resolver.FetcherError() -# fetcher_mock.cached_mock_fetch_returns.append(dtdl) -# return dtdl - -# fetcher_mock.fetch.side_effect = mocked_fetch -# return fetcher_mock - -# @pytest.fixture -# def dtmi_resolver(self, mocker, fetcher): -# return resolver.DtmiResolver(fetcher) - -# @pytest.mark.it("Raises a ValueError if the provided DTMI is invalid") -# @pytest.mark.parametrize( -# "dtmi", -# [ -# pytest.param("", id="Empty string"), -# pytest.param("not a dtmi", id="Not a DTMI"), -# pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), -# pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), -# pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), -# ], -# ) -# def test_invalid_dtmi(self, dtmi_resolver, dtmi): -# with pytest.raises(ValueError): -# dtmi_resolver.resolve(dtmi) - -# @pytest.mark.it("Raises a ResolverError if the Fetcher is unable to fetch a DTDL") -# def test_fetcher_failure(self, dtmi_resolver, foo_dtmi): -# my_fetcher_error = resolver.FetcherError("Some arbitrary fetcher error") -# dtmi_resolver.fetcher.fetch.side_effect = my_fetcher_error -# with pytest.raises(resolver.ResolverError) as e_info: -# dtmi_resolver.resolve(foo_dtmi) -# assert e_info.value.__cause__ is my_fetcher_error - -# # @pytest.mark.it("Raises a ResolverError if provided an invalid resolve mode") -# # def test_invalid_resolve_mode(self, dtmi_resolver, dtmi): -# # with pytest.raises(resolver.ResolverError): -# # dtmi_resolver.resolve(dtmi=dtmi, resolve_mode="invalid_mode") - - -# @pytest.mark.describe("DtmiResolver = .resolve() -- Dependency Mode: Disabled") -# class TestDtmiResolverResolveDependencyModeDisabled(DtmiResolverResolveSharedTests): -# @pytest.mark.it( -# "Uses the Fetcher to fetch a model DTDL from a path that corresponds to the provided DTMI" -# ) -# def test_fetcher(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): -# dtmi_resolver.resolve(foo_dtmi) - -# assert dtmi_resolver.fetcher.fetch.call_count == 1 -# expected_path = dtmi_to_path_mappings[foo_dtmi] -# assert dtmi_resolver.fetcher.fetch.call_args == mocker.call(expected_path) - -# @pytest.mark.it( -# "Returns a dictionary mapping the provided DTMI the corresponding model DTDL that was returned by the Fetcher" -# ) -# def test_returned_dict(self, dtmi_resolver, foo_dtmi): -# d = dtmi_resolver.resolve(foo_dtmi) - -# assert isinstance(d, dict) -# assert len(d) == 1 == dtmi_resolver.fetcher.fetch.call_count -# assert d[foo_dtmi] == dtmi_resolver.fetcher.cached_mock_fetch_returns[0] - - -# @pytest.mark.describe("DtmiResolver - .resolve() -- Dependency Mode: Enabled") -# class TestDtmiResolverResolveDependencyModeEnabled(DtmiResolverResolveSharedTests): -# @pytest.mark.it( -# "Uses the Fetcher to fetch model DTDLs from paths corresponding to the provided DTMI, as well as the DTMIs of all its dependencies" -# ) -# def test_fetcher(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): -# dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_ENABLED) - -# # NOTE: there are 6 calls because we only fetch for each UNIQUE component or interface. -# # There are two components in FooDTDL that are of type BarDTDL, but that DTDL only has -# # to be fetched once. -# assert dtmi_resolver.fetcher.fetch.call_count == 6 -# expected_path1 = dtmi_to_path_mappings["dtmi:com:somedomain:example:FooDTDL;1"] -# expected_path2 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BarDTDL;1"] -# expected_path3 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BuzzDTDL;1"] -# expected_path4 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuxDTDL;1"] -# expected_path5 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuzDTDL;1"] -# expected_path6 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BazDTDL;1"] -# assert dtmi_resolver.fetcher.fetch.call_args_list[0] == mocker.call(expected_path1) -# assert dtmi_resolver.fetcher.fetch.call_args_list[1] == mocker.call(expected_path2) -# assert dtmi_resolver.fetcher.fetch.call_args_list[2] == mocker.call(expected_path3) -# assert dtmi_resolver.fetcher.fetch.call_args_list[3] == mocker.call(expected_path4) -# assert dtmi_resolver.fetcher.fetch.call_args_list[4] == mocker.call(expected_path5) -# assert dtmi_resolver.fetcher.fetch.call_args_list[5] == mocker.call(expected_path6) - -# @pytest.mark.it( -# "Returns a dictionary mapping DTMIs to model DTDLs returned by the Fetcher, for the provided DTMI, as well as the DTMIs of all dependencies" -# ) -# def test_returned_dict( -# self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, path_to_dtdl_mappings -# ): -# d = dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_ENABLED) - -# assert isinstance(d, dict) -# assert len(d) == 6 == dtmi_resolver.fetcher.fetch.call_count -# for model in dtmi_resolver.fetcher.cached_mock_fetch_returns: -# dtmi = model["@id"] -# assert dtmi in d.keys() -# assert d[dtmi] == model - - -# @pytest.mark.describe("DtmiResolver - .resolve() -- Dependency Mode: Try From Expanded") -# class TestDtmiResolverResolveDependencyModeTryFromExpanded(DtmiResolverResolveSharedTests): -# @pytest.mark.it( -# "Attempts to use the Fetcher to fetch an expanded model DTDL from a path that corresponds to the provided DTMI" -# ) -# def test_fetcher(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): -# dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_TRY_FROM_EXPANDED) - -# assert dtmi_resolver.fetcher.fetch.call_count == 1 -# expected_path = dtmi_to_path_mappings[foo_dtmi].replace(".json", ".expanded.json") -# assert dtmi_resolver.fetcher.fetch.call_args == mocker.call(expected_path) - -# @pytest.mark.it( -# "Returns a dictionary mapping DTMIs to model DTDLs for all models contained within the expanded DTDL that was returned by the Fetcher" -# ) -# def test_returned_dict_expanded(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings): -# d = dtmi_resolver.resolve( -# foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_TRY_FROM_EXPANDED -# ) - -# assert isinstance(d, dict) -# assert len(d) == 6 -# assert dtmi_resolver.fetcher.fetch.call_count == 1 -# expanded_dtdl = dtmi_resolver.fetcher.cached_mock_fetch_returns[0] -# assert len(expanded_dtdl) == 6 -# for model in expanded_dtdl: -# dtmi = model["@id"] -# assert dtmi in d.keys() -# assert d[dtmi] == model - -# @pytest.mark.it( -# "Uses the Fetcher to fetch model DTDLs from paths corresponding to the provided DTMI, as well as the DTMIs of all its dependencies, for each expanded DTDL that cannot be fetched" -# ) -# def test_fetcher_fallback(self, dtmi_resolver, foo_dtmi, dtmi_to_path_mappings, mocker): -# dtmi_resolver.fetcher.fail_expanded = True -# dtmi_resolver.resolve(foo_dtmi, dependency_mode=constant.DEPENDENCY_MODE_TRY_FROM_EXPANDED) - -# # NOTE: There are 7 calls. 1 attempted expanded fetch + 6 regular fetches. The expanded -# # fetch will fail, and then as a fallback it will do the regular fetch procedure. -# # -# # There are 6 regular fetch calls because we only fetch for each UNIQUE component or -# # interface. There are two components in FooDTDL that are of type BarDTDL, but that -# # DTDL only has to be fetched once. -# assert dtmi_resolver.fetcher.fetch.call_count == 12 -# expected_path1 = dtmi_to_path_mappings["dtmi:com:somedomain:example:FooDTDL;1"] -# expected_path2 = dtmi_to_path_mappings["dtmi:com:somedomain:example:FooDTDL;1"] -# expected_path3 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BarDTDL;1"] -# expected_path4 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BuzzDTDL;1"] -# expected_path5 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuxDTDL;1"] -# expected_path6 = dtmi_to_path_mappings["dtmi:com:somedomain:example:QuzDTDL;1"] -# expected_path7 = dtmi_to_path_mappings["dtmi:com:somedomain:example:BazDTDL;1"] -# assert dtmi_resolver.fetcher.fetch.call_args_list[0] == mocker.call(expected_path1) -# assert dtmi_resolver.fetcher.fetch.call_args_list[1] == mocker.call(expected_path2) -# assert dtmi_resolver.fetcher.fetch.call_args_list[2] == mocker.call(expected_path3) -# assert dtmi_resolver.fetcher.fetch.call_args_list[3] == mocker.call(expected_path4) -# assert dtmi_resolver.fetcher.fetch.call_args_list[4] == mocker.call(expected_path5) -# assert dtmi_resolver.fetcher.fetch.call_args_list[5] == mocker.call(expected_path6) -# assert dtmi_resolver.fetcher.fetch.call_args_list[6] == mocker.call(expected_path7) - - -@pytest.mark.describe("HttpFetcher - .fetch()") -class TestHttpFetcherFetch(object): - @pytest.fixture - def fetcher(self, mocker, dtdl): - mock_http_client = mocker.MagicMock() - mock_response = mock_http_client._pipeline.run.return_value.http_response - mock_response.status_code = 200 - mock_response.text.return_value = json.dumps(dtdl) - mock_http_client._pipeline - return resolver.HttpFetcher(mock_http_client) - - @pytest.mark.it( - "Sends an HTTP GET request for the provided path, using the fetcher's HTTP client" - ) - def test_request(self, fetcher, path, mocker): - fetcher.fetch(path) - - assert fetcher.client.get.call_count == 1 - assert fetcher.client.get.call_args == mocker.call(url=path) - request = fetcher.client.get.return_value - assert fetcher.client._pipeline.run.call_count == 1 - assert fetcher.client._pipeline.run.call_args == mocker.call(request) - - @pytest.mark.it("Returns the GET response in JSON format, if the GET request is successful") - def test_response_success(self, fetcher, path): - dtdl_json = fetcher.fetch(path) - - assert isinstance(dtdl_json, dict) - client_response = fetcher.client._pipeline.run.return_value.http_response - assert client_response.status_code == 200 - assert dtdl_json == json.loads(client_response.text()) - - @pytest.mark.it("Raises a FetcherError if the GET request is unsuccessful") - def test_response_failure(self, fetcher, path): - fetcher.client._pipeline.run.return_value.http_response.status_code = 400 - with pytest.raises(resolver.FetcherError): - fetcher.fetch(path) - - -@pytest.mark.describe("FilesystemFetcher - .fetch()") -class TestFilesystemFetcherFetch(object): - @pytest.fixture - def mock_open(self, mocker, dtdl): - return mocker.patch("builtins.open", mocker.mock_open(read_data=json.dumps(dtdl))) - - @pytest.fixture - def fetcher(self, mock_open, mocker): - base_path = "C:/some/base/path" - return resolver.FilesystemFetcher(base_path) - - @pytest.mark.it( - "Formats and normalizes syntax of provided path to fetch, then opens and reads the file at that location" - ) - def test_open_read_path(self, fetcher, path, mock_open, mocker): - mocker.spy(os.path, "join") - mocker.spy(os.path, "normcase") - mocker.spy(os.path, "normpath") - - fetcher.fetch(path) - - # These three functions being called ensure that the path will be formatted correctly for - # all cases (e.g. trailing slash, no trailing slash, leading slash, etc.) - # Because we know how these builtin functions work, there's no need to explicitly test - # these input variants - the logic is handled externally, and is not part of this unit - assert os.path.join.call_count == 1 - assert os.path.normcase.call_count == 1 - assert os.path.normpath.call_count == 1 - - # The expected formatted path was passed to the 'open()' function - expected_absolute_path = os.path.normpath( - os.path.normcase(os.path.join(fetcher.base_path, path)) - ) - assert mock_open.call_count == 1 - assert mock_open.call_args == mocker.call(expected_absolute_path) - - # The data was read from the file - assert mock_open.return_value.read.call_count == 1 - assert mock_open.return_value.read.call_args == mocker.call() - - @pytest.mark.it( - "Returns the data returned by the read operation in JSON format, if the read is successful" - ) - def test_open_read_success(self, fetcher, path, mock_open, dtdl, mocker): - dtdl_json = fetcher.fetch(path) - - assert isinstance(dtdl, dict) - # Unfortunately, there isn't really a way to show that the returned value comes from the - # file read due to how the mock of open/read builtins work. Best I can do is show that it - # has the expected value (with the assumption that the mock returned that value) - assert dtdl_json == dtdl - - @pytest.mark.it( - "Raises a FetcherError if there is an error while opening the file at the provided path" - ) - def test_open_failure(self, fetcher, mock_open, path, arbitrary_exception): - mock_open.side_effect = arbitrary_exception - with pytest.raises(resolver.FetcherError) as e_info: - fetcher.fetch(path) - assert e_info.value.__cause__ == arbitrary_exception - - @pytest.mark.it( - "Raises a FetcherError if there is an error while reading the file at the provided path" - ) - def test_read_failure(self, fetcher, mock_open, path, arbitrary_exception): - mock_open.return_value.read.side_effect = arbitrary_exception - with pytest.raises(resolver.FetcherError) as e_info: - fetcher.fetch(path) - assert e_info.value.__cause__ == arbitrary_exception From 0dd9439ede9653a0522a4506c4b2bf2b704bc3c8 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Fri, 12 Mar 2021 17:29:31 -0800 Subject: [PATCH 05/25] Updated pseudoparser --- .../azure/iot/modelsrepository/client.py | 1 + .../iot/modelsrepository/pseudo_parser.py | 47 +++++++++++++------ .../azure/iot/modelsrepository/resolver.py | 2 +- .../tests/test_integration_client.py | 18 +++++++ 4 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index faeb14e2b9e7..1621b7142909 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -79,6 +79,7 @@ def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): :returns: Dictionary mapping DTMIs to models :rtype: dict """ + # TODO: If not ResolverError, then what? if dependency_resolution == DEPENDENCY_MODE_DISABLED: # Simply retrieve the model(s) model_map = self.resolver.resolve(dtmis) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py index d76911c6eb75..e18c3ef95b0c 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py @@ -15,10 +15,21 @@ class PseudoParser(object): def __init__(self, resolver): + """ + :param resolver: The resolver for the parser to use to resolve model dependencies + :type resolver: :class:`azure.iot.modelsrepository.resolver.DtmiResolver` + """ self.resolver = resolver def expand(self, models): - """""" + """Return a dictionary containing all the provided models, as well as their dependencies, + indexed by DTMI + + :param models list[str]: List of models + + :returns: Dictionary containing models and their dependencies, indexed by DTMI + :rtype: dict + """ expanded_map = {} for model in models: expanded_map[model["@id"]] = model @@ -26,7 +37,7 @@ def expand(self, models): return expanded_map def _expand(self, model, model_map): - dependencies = get_dependency_list(model) + dependencies = _get_dependency_list(model) dependencies_to_resolve = [ dependency for dependency in dependencies if dependency not in model_map ] @@ -38,21 +49,29 @@ def _expand(self, model, model_map): self._expand(dependency_model, model_map) -def get_dependency_list(model): +def _get_dependency_list(model): """Return a list of DTMIs for model dependencies""" + dependencies = [] + if "contents" in model: components = [item["schema"] for item in model["contents"] if item["@type"] == "Component"] - else: - components = [] + dependencies += components if "extends" in model: - # Models defined in a DTDL can implement extensions of up to two interfaces - if isinstance(model["extends"], list): - interfaces = model["extends"] - else: - interfaces = [model["extends"]] - else: - interfaces = [] - - dependencies = components + interfaces + # Models defined in a DTDL can implement extensions of up to two interfaces. + # These interfaces can be in the form of a DTMI reference, or a nested model (which would + # have it's own dependencies) + if isinstance(model["extends"], str): + # If it's just a string, that's a single DTMI reference, so just add that to our list + dependencies.append(model["extends"]) + elif isinstance(model["extends"], list): + # If it's a list, could have DTMIs or nested models + for item in model["extends"]: + if isinstance(item, str): + # If there are strings in the list, that's a DTMI reference, so add it + dependencies.append(item) + elif isinstance(item, dict): + # This is a nested model. Now go get its dependencies and add them + dependencies += _get_dependency_list(item) + return dependencies diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py index 7ef99a7658bf..9a05a04381a5 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py @@ -23,7 +23,7 @@ class DtmiResolver(object): def __init__(self, fetcher): """ :param fetcher: A Fetcher configured to an endpoint to resolve DTMIs from - :type fetcher: :class:`azure.iot.modelsrepository.resolver` + :type fetcher: :class:`azure.iot.modelsrepository.resolver.Fetcher` """ self.fetcher = fetcher diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py b/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py new file mode 100644 index 000000000000..6ecd956f668d --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +import logging +from azure.iot.modelsrepository import ModelsRepositoryClient + +logging.basicConfig(level=logging.DEBUG) + +@pytest.mark.describe("ModelsRepositoryClient - .get_models() [INTEGRATION]") +class TestModelsRepositoryClientGetModels(object): + + @pytest.mark.it("test recordings") + def test_simple(self): + c = ModelsRepositoryClient() + c.get_models(["dtmi:com:example:TemperatureController;1"]) \ No newline at end of file From ff956d218acb9c84a88e436418a60c4869a59840 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Mon, 15 Mar 2021 12:27:18 -0700 Subject: [PATCH 06/25] Updated for CR --- .../azure/iot/modelsrepository/__init__.py | 3 +++ .../azure/iot/modelsrepository/client.py | 13 +++++++------ .../{dtmi_utils.py => dtmi_conventions.py} | 10 +++++----- .../azure/iot/modelsrepository/resolver.py | 4 ++-- sdk/iot/azure-iot-modelsrepository/setup.py | 2 +- 5 files changed, 18 insertions(+), 14 deletions(-) rename sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/{dtmi_utils.py => dtmi_conventions.py} (91%) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py index fc4b5ce647bb..42d2158cbfb1 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py @@ -7,6 +7,9 @@ # Main Client from .client import ModelsRepositoryClient +# Modules +from . import dtmi_conventions + # Constants from .client import ( DEPENDENCY_MODE_DISABLED, diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index 1621b7142909..4ad6209391f4 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -20,7 +20,6 @@ ) from . import resolver from . import pseudo_parser -from .chainable_exception import ChainableException # Public constants exposed to consumers @@ -44,6 +43,8 @@ def __init__(self, repository_location=None, api_version=None, **kwargs): :param str repository_location: Location of the Models Repository you wish to access. This location can be a remote HTTP/HTTPS URL, or a local filesystem path. If omitted, will default to using "https://devicemodels.azure.com". + :param str api_version: The API version for the Models Repository Service you wish to + access. :raises: ValueError if repository_location is invalid """ @@ -65,7 +66,7 @@ def __init__(self, repository_location=None, api_version=None, **kwargs): def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): """Retrieve a model from the Models Repository. - :param list[str]: The DTMIs for the models you wish to retrieve + :param list[str] dtmis: The DTMIs for the models you wish to retrieve :param str dependency_resolution : Dependency resolution mode. Possible values: - "disabled": Do not resolve model dependencies - "enabled": Resolve model dependencies from the repository @@ -85,8 +86,8 @@ def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): model_map = self.resolver.resolve(dtmis) elif dependency_resolution == DEPENDENCY_MODE_ENABLED: # Manually resolve dependencies using pseudo-parser - base_model_map = model_map = self.resolver.resolve(dtmis) - base_model_list = [model for model in base_model_map.values()] + base_model_map = self.resolver.resolve(dtmis) + base_model_list = list(base_model_map.values()) model_map = self.pseudo_parser.expand(base_model_list) elif dependency_resolution == DEPENDENCY_MODE_TRY_FROM_EXPANDED: # Try to use an expanded DTDL to resolve dependencies @@ -94,8 +95,8 @@ def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): model_map = self.resolver.resolve(dtmis, expanded_model=True) except resolver.ResolverError: # Fallback to manual dependency resolution - base_model_map = model_map = self.resolver.resolve(dtmis) - base_model_list = [model for model in base_model_map.items()] + base_model_map = self.resolver.resolve(dtmis) + base_model_list = list(base_model_map.items()) model_map = self.pseudo_parser.expand(base_model_list) else: raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py similarity index 91% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py rename to sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py index 4f8a50083411..67d13063bdb6 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_utils.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py @@ -4,7 +4,6 @@ # license information. # -------------------------------------------------------------------------- import re -import pathlib def is_valid_dtmi(dtmi): @@ -25,8 +24,8 @@ def is_valid_dtmi(dtmi): def get_model_uri(dtmi, repository_uri, expanded=False): - """Get the URI representing the absolute location of a model in a Models Repository - + """Get the URI representing the absolute location of a model in a Models Repository + :param dtmi str: DTMI for a model :param repository_uri str: URI for a Models Repository :param expanded bool: Indicates if the URI should be for an expanded model (Default: False) @@ -38,7 +37,7 @@ def get_model_uri(dtmi, repository_uri, expanded=False): """ if not repository_uri.endswith("/"): repository_uri += "/" - model_uri = repository_uri + _convert_dtmi_to_path(dtmi) + model_uri = repository_uri + _convert_dtmi_to_path(dtmi, expanded) return model_uri @@ -51,7 +50,8 @@ def _convert_dtmi_to_path(dtmi, expanded=False): :raises ValueError if DTMI is invalid - :returns: Relative path + :returns: Relative path of the model in a Models Repository + :rtype: str """ pattern = re.compile( "^dtmi:[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?(?::[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?)*;[1-9][0-9]{0,8}$" diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py index 9a05a04381a5..2b09bb7e1c43 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py @@ -9,7 +9,7 @@ import abc import re import os -from . import dtmi_utils +from . import dtmi_conventions from .chainable_exception import ChainableException logger = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def resolve(self, dtmis, expanded_model=False): """ model_map = {} for dtmi in dtmis: - dtdl_path = dtmi_utils._convert_dtmi_to_path(dtmi) + dtdl_path = dtmi_conventions._convert_dtmi_to_path(dtmi) if expanded_model: dtdl_path = dtdl_path.replace(".json", ".expanded.json") diff --git a/sdk/iot/azure-iot-modelsrepository/setup.py b/sdk/iot/azure-iot-modelsrepository/setup.py index 5969dee9b52f..fe5bf96fb828 100644 --- a/sdk/iot/azure-iot-modelsrepository/setup.py +++ b/sdk/iot/azure-iot-modelsrepository/setup.py @@ -51,7 +51,7 @@ "Programming Language :: Python :: 3.9", ], install_requires=[ - "azure-core" + "azure-core", "six", ], extras_require={":python_version<'3.0'": ["azure-iot-nspkg>=1.0.1"]}, From f2e1da7712b0d6eb9e986b0f13f8232203018583 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Mon, 15 Mar 2021 12:59:22 -0700 Subject: [PATCH 07/25] Doc updates, API surface changes --- .../azure/iot/modelsrepository/__init__.py | 5 +--- ...e_exception.py => _chainable_exception.py} | 0 .../{pseudo_parser.py => _pseudo_parser.py} | 0 .../{resolver.py => _resolver.py} | 5 ++-- .../azure/iot/modelsrepository/client.py | 27 +++++++++---------- .../iot/modelsrepository/dtmi_conventions.py | 8 +++--- 6 files changed, 20 insertions(+), 25 deletions(-) rename sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/{chainable_exception.py => _chainable_exception.py} (100%) rename sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/{pseudo_parser.py => _pseudo_parser.py} (100%) rename sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/{resolver.py => _resolver.py} (98%) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py index 42d2158cbfb1..e02998c06115 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py @@ -7,9 +7,6 @@ # Main Client from .client import ModelsRepositoryClient -# Modules -from . import dtmi_conventions - # Constants from .client import ( DEPENDENCY_MODE_DISABLED, @@ -18,4 +15,4 @@ ) # Error handling -from .resolver import ResolverError +from ._resolver import ResolverError diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/chainable_exception.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_chainable_exception.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/chainable_exception.py rename to sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_chainable_exception.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/pseudo_parser.py rename to sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py similarity index 98% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py rename to sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index 2b09bb7e1c43..929eeaa824ac 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -4,13 +4,12 @@ # license information. # -------------------------------------------------------------------------- import logging -import six import json import abc -import re import os +import six from . import dtmi_conventions -from .chainable_exception import ChainableException +from ._chainable_exception import ChainableException logger = logging.getLogger(__name__) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index 4ad6209391f4..dfa7b141e567 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -13,13 +13,12 @@ HeadersPolicy, RetryPolicy, RedirectPolicy, - BearerTokenCredentialPolicy, ContentDecodePolicy, NetworkTraceLoggingPolicy, ProxyPolicy, ) -from . import resolver -from . import pseudo_parser +from . import _resolver +from . import _pseudo_parser # Public constants exposed to consumers @@ -60,14 +59,14 @@ def __init__(self, repository_location=None, api_version=None, **kwargs): self.fetcher = _create_fetcher( location=repository_location, api_version=api_version, **kwargs ) - self.resolver = resolver.DtmiResolver(self.fetcher) - self.pseudo_parser = pseudo_parser.PseudoParser(self.resolver) + self.resolver = _resolver.DtmiResolver(self.fetcher) + self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): """Retrieve a model from the Models Repository. :param list[str] dtmis: The DTMIs for the models you wish to retrieve - :param str dependency_resolution : Dependency resolution mode. Possible values: + :param str dependency_resolution: Dependency resolution mode. Possible values: - "disabled": Do not resolve model dependencies - "enabled": Resolve model dependencies from the repository - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded DTDL @@ -88,16 +87,16 @@ def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): # Manually resolve dependencies using pseudo-parser base_model_map = self.resolver.resolve(dtmis) base_model_list = list(base_model_map.values()) - model_map = self.pseudo_parser.expand(base_model_list) + model_map = self._psuedo_parser.expand(base_model_list) elif dependency_resolution == DEPENDENCY_MODE_TRY_FROM_EXPANDED: # Try to use an expanded DTDL to resolve dependencies try: model_map = self.resolver.resolve(dtmis, expanded_model=True) - except resolver.ResolverError: + except _resolver.ResolverError: # Fallback to manual dependency resolution base_model_map = self.resolver.resolve(dtmis) base_model_list = list(base_model_map.items()) - model_map = self.pseudo_parser.expand(base_model_list) + model_map = self._psuedo_parser.expand(base_model_list) else: raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) return model_map @@ -122,22 +121,22 @@ def _create_fetcher(location, **kwargs): if scheme in _REMOTE_PROTOCOLS: # HTTP/HTTPS URL client = _create_pipeline_client(base_url=location, **kwargs) - fetcher = resolver.HttpFetcher(client) + fetcher = _resolver.HttpFetcher(client) elif scheme == "file": # Filesystem URI location = location[len("file://") :] - fetcher = resolver.FilesystemFetcher(location) + fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and location.startswith("/"): # POSIX filesystem path - fetcher = resolver.FilesystemFetcher(location) + fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and re.search(r"\.[a-zA-z]{2,63}$", location[: location.find("/")]): # Web URL with protocol unspecified - default to HTTPS location = "https://" + location client = _create_pipeline_client(base_url=location, **kwargs) - fetcher = resolver.HttpFetcher(client) + fetcher = _resolver.HttpFetcher(client) elif scheme != "" and len(scheme) == 1 and scheme.isalpha(): # Filesystem path using drive letters (e.g. "C:", "D:", etc.) - fetcher = resolver.FilesystemFetcher(location) + fetcher = _resolver.FilesystemFetcher(location) else: raise ValueError("Unable to identify location: {}".format(location)) return fetcher diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py index 67d13063bdb6..6f29428a40ab 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py @@ -9,7 +9,7 @@ def is_valid_dtmi(dtmi): """Checks validity of a DTMI - :param dtmi str: DTMI + :param str dtmi: DTMI :returns: Boolean indicating if DTMI is valid :rtype: bool @@ -26,9 +26,9 @@ def is_valid_dtmi(dtmi): def get_model_uri(dtmi, repository_uri, expanded=False): """Get the URI representing the absolute location of a model in a Models Repository - :param dtmi str: DTMI for a model - :param repository_uri str: URI for a Models Repository - :param expanded bool: Indicates if the URI should be for an expanded model (Default: False) + :param str dtmi: DTMI for a model + :param str repository_uri: URI for a Models Repository + :param bool expanded: Indicates if the URI should be for an expanded model (Default: False) :raises: ValueError if given an invalid DTMI From cde66a08275fc286ab2473eb7a203519ecdd9add Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Mon, 15 Mar 2021 13:30:45 -0700 Subject: [PATCH 08/25] Updated tests/samples/naming --- .../iot/modelsrepository/dtmi_conventions.py | 7 +++--- ...l_sample.py => dtmi_conventions_sample.py} | 14 +++++------ ...dtmi_utils.py => test_dtmi_conventions.py} | 24 +++++++++---------- 3 files changed, 23 insertions(+), 22 deletions(-) rename sdk/iot/azure-iot-modelsrepository/samples/{dtmi_util_sample.py => dtmi_conventions_sample.py} (71%) rename sdk/iot/azure-iot-modelsrepository/tests/{test_dtmi_utils.py => test_dtmi_conventions.py} (82%) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py index 6f29428a40ab..06aeb4092f78 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py @@ -42,11 +42,12 @@ def get_model_uri(dtmi, repository_uri, expanded=False): def _convert_dtmi_to_path(dtmi, expanded=False): - """Returns the DTMI path for a DTMI + """Returns the relative path for a model given a DTMI E.g: dtmi:com:example:Thermostat;1 -> dtmi/com/example/thermostat-1.json - :param dtmi str: DTMI + :param str dtmi : DTMI for a model + :param bool expanded: Indicates if the relative path should be for an exapnded model :raises ValueError if DTMI is invalid @@ -60,5 +61,5 @@ def _convert_dtmi_to_path(dtmi, expanded=False): raise ValueError("Invalid DTMI") dtmi_path = dtmi.lower().replace(":", "/").replace(";", "-") + ".json" if expanded: - dtmi_path = dtmi.replace(".json", ".expanded.json") + dtmi_path = dtmi_path.replace(".json", ".expanded.json") return dtmi_path diff --git a/sdk/iot/azure-iot-modelsrepository/samples/dtmi_util_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py similarity index 71% rename from sdk/iot/azure-iot-modelsrepository/samples/dtmi_util_sample.py rename to sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py index 2a314984c5e4..f81840de6bd0 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/dtmi_util_sample.py +++ b/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py @@ -3,16 +3,16 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from azure.iot.modelsrepository import dtmi_utils +from azure.iot.modelsrepository import dtmi_conventions def sample_is_valid_dtmi(): """Check if a DTMI is valid or not using the .is_valid_dtmi() function""" # Returns True - this is a valid DTMI - dtmi_utils.is_valid_dtmi("dtmi:com:example:Thermostat;1") + dtmi_conventions.is_valid_dtmi("dtmi:com:example:Thermostat;1") # Returns False - this is NOT a valid DTMI - dtmi_utils.is_valid_dtmi("dtmi:com:example:Thermostat") + dtmi_conventions.is_valid_dtmi("dtmi:com:example:Thermostat") def sample_get_model_uri(): @@ -21,15 +21,15 @@ def sample_get_model_uri(): # Local repository example repo_uri = "file:///path/to/repository" - print(dtmi_utils.get_model_uri(dtmi, repo_uri)) + print(dtmi_conventions.get_model_uri(dtmi, repo_uri)) # Prints: "file:///path/to/repository/dtmi/com/example/thermostat-1.json" - print(dtmi_utils.get_model_uri(dtmi, repo_uri, expanded=True)) + print(dtmi_conventions.get_model_uri(dtmi, repo_uri, expanded=True)) # Prints: "file:///path/to/repository/dtmi/com/example/thermostat-1.expanded.json" # Remote repository example repo_uri = "https://contoso.com/models/" - print(dtmi_utils.get_model_uri(dtmi, repo_uri)) + print(dtmi_conventions.get_model_uri(dtmi, repo_uri)) # Prints: "https://contoso/com/models/dtmi/com/example/thermostat-1.json" - print(dtmi_utils.get_model_uri(dtmi, repo_uri, expanded=True)) + print(dtmi_conventions.get_model_uri(dtmi, repo_uri, expanded=True)) # Prints: "https://contoso/com/models/dtmi/com/example/thermostat-1.expanded.json" diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_utils.py b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py similarity index 82% rename from sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_utils.py rename to sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py index 34629cc61ba8..fdfa6d614cdc 100644 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_utils.py +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest import logging -import azure.iot.modelsrepository.dtmi_utils as dtmi_utils +from azure.iot.modelsrepository import dtmi_conventions logging.basicConfig(level=logging.DEBUG) @@ -17,7 +17,7 @@ class TestIsValidDTMI(object): pytest.param("dtmi:com:somedomain:example:FooDTDL;1", id="Long DTMI") ]) def test_valid_dtmi(self, dtmi): - assert dtmi_utils.is_valid_dtmi(dtmi) + assert dtmi_conventions.is_valid_dtmi(dtmi) @pytest.mark.it("Returns False if given an invalid DTMI") @pytest.mark.parametrize("dtmi", [ @@ -28,7 +28,7 @@ def test_valid_dtmi(self, dtmi): pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), ]) def test_invalid_dtmi(self, dtmi): - assert not dtmi_utils.is_valid_dtmi(dtmi) + assert not dtmi_conventions.is_valid_dtmi(dtmi) @pytest.mark.describe(".get_model_uri()") @@ -42,19 +42,19 @@ class TestGetModelURI(object): pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Repository URI without trailing '/'") ]) def test_uri(self, dtmi, repository_uri, expected_model_uri): - model_uri = dtmi_utils.get_model_uri(dtmi, repository_uri) + model_uri = dtmi_conventions.get_model_uri(dtmi, repository_uri) assert model_uri == expected_model_uri @pytest.mark.it("Returns the URI for a specified expanded model at a specified repository") @pytest.mark.parametrize("dtmi, repository_uri, expected_model_uri", [ - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "https://myrepository/", "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTPS repository URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository/", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTP repository URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file://localhost/myrepository/", "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI w/ host"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Repository URI without trailing '/'") + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "https://myrepository/", "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="HTTPS repository URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository/", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="HTTP repository URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="Filesystem URI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file://localhost/myrepository/", "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="Filesystem URI w/ host"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="Repository URI without trailing '/'") ]) - def test_uri(self, dtmi, repository_uri, expected_model_uri): - model_uri = dtmi_utils.get_model_uri(dtmi, repository_uri, expanded=True) + def test_uri_expanded(self, dtmi, repository_uri, expected_model_uri): + model_uri = dtmi_conventions.get_model_uri(dtmi, repository_uri, expanded=True) assert model_uri == expected_model_uri @pytest.mark.it("Raises ValueError if given an invalid DTMI") @@ -67,4 +67,4 @@ def test_uri(self, dtmi, repository_uri, expected_model_uri): ]) def test_invalid_dtmi(self, dtmi): with pytest.raises(ValueError): - dtmi_utils.get_model_uri(dtmi, "https://myrepository/") \ No newline at end of file + dtmi_conventions.get_model_uri(dtmi, "https://myrepository/") \ No newline at end of file From f7398a85d62f20ffacf985ccfafd127b9dc0025e Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Mon, 15 Mar 2021 13:32:00 -0700 Subject: [PATCH 09/25] doc fix --- sdk/iot/azure-iot-modelsrepository/samples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/iot/azure-iot-modelsrepository/samples/README.md b/sdk/iot/azure-iot-modelsrepository/samples/README.md index 9c155ec6c881..3d047f99cc62 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/README.md +++ b/sdk/iot/azure-iot-modelsrepository/samples/README.md @@ -9,4 +9,4 @@ The pre-configured endpoints and DTMIs within the sampmles refer to example DTDL * [client_configuration_sample.py](client_configuration_sample.py) - Configure the client to work with local or remote repositories, as well as custom policies and transports -* [dtmi_utils_sample.py](dtmi_utils_sample.py) - Use the dtmi_utils module to manipulate and check DTMIs \ No newline at end of file +* [dtmi_conventions_sample.py](dtmi_conventions_sample.py) - Use the `dtmi_conventions` module to manipulate and check DTMIs \ No newline at end of file From 4f5e79e1dceaafa3917ecedf01e5935d3976d96e Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Mon, 15 Mar 2021 19:19:38 -0700 Subject: [PATCH 10/25] Added DTMI checks to resolver --- .../iot/modelsrepository/_pseudo_parser.py | 12 ++++- .../azure/iot/modelsrepository/_resolver.py | 13 +++++ .../azure/iot/modelsrepository/client.py | 50 +++++++++++++++---- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py index e18c3ef95b0c..0fce3147c451 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py @@ -11,6 +11,10 @@ Note that this implementation is not representative of what an eventual full parser implementation would necessarily look like from an API perspective """ +from ._chainable_exception import ChainableException + +class ParsingError(ChainableException): + pass class PseudoParser(object): @@ -25,8 +29,8 @@ def expand(self, models): """Return a dictionary containing all the provided models, as well as their dependencies, indexed by DTMI - :param models list[str]: List of models - + :param list[str] models: List of models + :returns: Dictionary containing models and their dependencies, indexed by DTMI :rtype: dict """ @@ -73,5 +77,9 @@ def _get_dependency_list(model): elif isinstance(item, dict): # This is a nested model. Now go get its dependencies and add them dependencies += _get_dependency_list(item) + else: + raise ParsingError("Invalid DTDL") + else: + raise ParsingError("Invalid DTDL") return dependencies diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index 929eeaa824ac..f23714cf5499 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -50,9 +50,22 @@ def resolve(self, dtmis, expanded_model=False): raise ResolverError("Failed to resolve dtmi: {}".format(dtmi), cause=e) if expanded_model: + # Verify that the DTMI of the "root" model (i.e. the model we requested the + # expanded DTDL for) within the expanded DTDL matches the DTMI of the request + if True not in (model["@id"] == dtmi for model in dtdl): + raise ResolverError("DTMI mismatch on expanded DTDL - Request: {}".format( + dtmi + )) + # Add all the models in the expanded DTDL to the map for model in dtdl: model_map[model["@id"]] = model else: + # Verify that the DTMI of the fetched model matches the DTMI of the request + if model["@id"] != dtmi: + raise ResolverError("DTMI mismatch - Request: {}, Response: {}".format( + dtmi, model["@id"] + )) + # Add the model to the map model_map[dtmi] = dtdl return model_map diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index dfa7b141e567..7d888f8a8f99 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -36,22 +36,45 @@ class ModelsRepositoryClient(object): """Client providing APIs for Models Repository operations""" - # TODO: Should api_version be a kwarg? - def __init__(self, repository_location=None, api_version=None, **kwargs): + def __init__(self, repository_location=None, dependency_resolution=None, api_version=None, **kwargs): """ :param str repository_location: Location of the Models Repository you wish to access. This location can be a remote HTTP/HTTPS URL, or a local filesystem path. If omitted, will default to using "https://devicemodels.azure.com". :param str api_version: The API version for the Models Repository Service you wish to access. + :param str dependency_resolution: Dependency resolution mode. + Possible values: + - "disabled": Do not resolve model dependencies + - "enabled": Resolve model dependencies from the repository + - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded + DTDL document in the repository. If this is not successful, will fall back + on manually resolving dependencies in the repository + If using the default repository location, the default dependency resolution mode will + be "tryFromExpanded". If using a custom repository location, the default dependency + resolution mode will be "enabled". :raises: ValueError if repository_location is invalid + :raises: ValueError if dependency_resolution is invalid """ repository_location = ( _DEFAULT_LOCATION if repository_location is None else repository_location ) - # api_version = _DEFAULT_API_VERSION if api_version is None else api_version + if dependency_resolution is None: + # If using the default repository location, the resolution mode should default to + # expanded mode because the defeault repo guarantees the existence of expanded DTDLs + if repository_location == _DEFAULT_LOCATION: + self.resolution_mode = DEPENDENCY_MODE_TRY_FROM_EXPANDED + else: + self.resolution_mode = DEPENDENCY_MODE_ENABLED + else: + if dependency_resolution not in [DEPENDENCY_MODE_ENABLED, DEPENDENCY_MODE_DISABLED, DEPENDENCY_MODE_TRY_FROM_EXPANDED]: + raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) + self.resolution_mode = dependency_resolution + + # TODO: Should api_version be a kwarg in the API surface? + # api_version = _DEFAULT_API_VERSION if api_version is None else api_version kwargs.setdefault("api_verison", api_version) # NOTE: depending on how this class develops over time, may need to adjust relationship @@ -62,16 +85,18 @@ def __init__(self, repository_location=None, api_version=None, **kwargs): self.resolver = _resolver.DtmiResolver(self.fetcher) self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) - def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): + def get_models(self, dtmis, dependency_resolution=None): """Retrieve a model from the Models Repository. :param list[str] dtmis: The DTMIs for the models you wish to retrieve - :param str dependency_resolution: Dependency resolution mode. Possible values: - - "disabled": Do not resolve model dependencies - - "enabled": Resolve model dependencies from the repository - - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded DTDL - document in the repository. If this is not successful, will fall back on - manually resolving dependencies in the repository + :param str dependency_resolution: Dependency resolution mode override. This value takes + precedence over the value set on the client. + Possible values: + - "disabled": Do not resolve model dependencies + - "enabled": Resolve model dependencies from the repository + - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded DTDL + document in the repository. If this is not successful, will fall back on + manually resolving dependencies in the repository :raises: ValueError if given an invalid dependency resolution mode :raises: ResolverError if there is an error retrieving a model @@ -79,7 +104,10 @@ def get_models(self, dtmis, dependency_resolution=DEPENDENCY_MODE_DISABLED): :returns: Dictionary mapping DTMIs to models :rtype: dict """ - # TODO: If not ResolverError, then what? + # TODO: Use better error surface than the custom ResolverError + if dependency_resolution is None: + dependency_resolution = self.resolution_mode + if dependency_resolution == DEPENDENCY_MODE_DISABLED: # Simply retrieve the model(s) model_map = self.resolver.resolve(dtmis) From 9c87cf68fb1787e4b420efb859dbbf27645f23b7 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Mon, 15 Mar 2021 19:20:35 -0700 Subject: [PATCH 11/25] removed unnecessary parsing error more doc fixes --- .../azure/iot/modelsrepository/_pseudo_parser.py | 9 +-------- .../azure/iot/modelsrepository/_resolver.py | 2 +- sdk/iot/azure-iot-modelsrepository/samples/scratch.py | 7 +++++++ 3 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 sdk/iot/azure-iot-modelsrepository/samples/scratch.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py index 0fce3147c451..06ca702de688 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py @@ -13,15 +13,12 @@ """ from ._chainable_exception import ChainableException -class ParsingError(ChainableException): - pass - class PseudoParser(object): def __init__(self, resolver): """ :param resolver: The resolver for the parser to use to resolve model dependencies - :type resolver: :class:`azure.iot.modelsrepository.resolver.DtmiResolver` + :type resolver: :class:`azure.iot.modelsrepository._resolver.DtmiResolver` """ self.resolver = resolver @@ -77,9 +74,5 @@ def _get_dependency_list(model): elif isinstance(item, dict): # This is a nested model. Now go get its dependencies and add them dependencies += _get_dependency_list(item) - else: - raise ParsingError("Invalid DTDL") - else: - raise ParsingError("Invalid DTDL") return dependencies diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index f23714cf5499..44f6f748fe4f 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -22,7 +22,7 @@ class DtmiResolver(object): def __init__(self, fetcher): """ :param fetcher: A Fetcher configured to an endpoint to resolve DTMIs from - :type fetcher: :class:`azure.iot.modelsrepository.resolver.Fetcher` + :type fetcher: :class:`azure.iot.modelsrepository._resolver.Fetcher` """ self.fetcher = fetcher diff --git a/sdk/iot/azure-iot-modelsrepository/samples/scratch.py b/sdk/iot/azure-iot-modelsrepository/samples/scratch.py new file mode 100644 index 000000000000..cca8d2ecbbf5 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/samples/scratch.py @@ -0,0 +1,7 @@ +from azure.iot.modelsrepository import ModelsRepositoryClient +import pprint + +client = ModelsRepositoryClient(repository_location="https://devicemodels.azure.com") +models = client.get_models(["dtmi:com:example:TemperatureController;1"]) +pprint.pprint(models) + From d80d6ca2d7eb54d3e3376494cf6e40c22e58ef92 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Wed, 17 Mar 2021 12:29:05 -0700 Subject: [PATCH 12/25] Added distributed tracing --- .../azure/iot/modelsrepository/_resolver.py | 10 +- .../azure/iot/modelsrepository/client.py | 23 ++- .../tests/test_dtmi_conventions.py | 132 +++++++++++++----- .../tests/test_integration_client.py | 4 +- 4 files changed, 123 insertions(+), 46 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index 44f6f748fe4f..6ea0fcdc6eb8 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -53,18 +53,16 @@ def resolve(self, dtmis, expanded_model=False): # Verify that the DTMI of the "root" model (i.e. the model we requested the # expanded DTDL for) within the expanded DTDL matches the DTMI of the request if True not in (model["@id"] == dtmi for model in dtdl): - raise ResolverError("DTMI mismatch on expanded DTDL - Request: {}".format( - dtmi - )) + raise ResolverError("DTMI mismatch on expanded DTDL - Request: {}".format(dtmi)) # Add all the models in the expanded DTDL to the map for model in dtdl: model_map[model["@id"]] = model else: # Verify that the DTMI of the fetched model matches the DTMI of the request if model["@id"] != dtmi: - raise ResolverError("DTMI mismatch - Request: {}, Response: {}".format( - dtmi, model["@id"] - )) + raise ResolverError( + "DTMI mismatch - Request: {}, Response: {}".format(dtmi, model["@id"]) + ) # Add the model to the map model_map[dtmi] = dtdl return model_map diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index 7d888f8a8f99..39f7aceca2eb 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -6,6 +6,7 @@ import six.moves.urllib as urllib import re from azure.core import PipelineClient +from azure.core.tracing.decorator import distributed_trace from azure.core.pipeline.transport import RequestsTransport from azure.core.configuration import Configuration from azure.core.pipeline.policies import ( @@ -17,8 +18,10 @@ NetworkTraceLoggingPolicy, ProxyPolicy, ) -from . import _resolver -from . import _pseudo_parser +from . import ( + _resolver, + _pseudo_parser, +) # Public constants exposed to consumers @@ -31,12 +34,15 @@ _DEFAULT_LOCATION = "https://devicemodels.azure.com" _DEFAULT_API_VERSION = "2021-02-11" _REMOTE_PROTOCOLS = ["http", "https"] +_TRACE_NAMESPACE = "modelsrepository" class ModelsRepositoryClient(object): """Client providing APIs for Models Repository operations""" - def __init__(self, repository_location=None, dependency_resolution=None, api_version=None, **kwargs): + def __init__( + self, repository_location=None, dependency_resolution=None, api_version=None, **kwargs + ): """ :param str repository_location: Location of the Models Repository you wish to access. This location can be a remote HTTP/HTTPS URL, or a local filesystem path. @@ -69,8 +75,14 @@ def __init__(self, repository_location=None, dependency_resolution=None, api_ver else: self.resolution_mode = DEPENDENCY_MODE_ENABLED else: - if dependency_resolution not in [DEPENDENCY_MODE_ENABLED, DEPENDENCY_MODE_DISABLED, DEPENDENCY_MODE_TRY_FROM_EXPANDED]: - raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) + if dependency_resolution not in [ + DEPENDENCY_MODE_ENABLED, + DEPENDENCY_MODE_DISABLED, + DEPENDENCY_MODE_TRY_FROM_EXPANDED, + ]: + raise ValueError( + "Invalid dependency resolution mode: {}".format(dependency_resolution) + ) self.resolution_mode = dependency_resolution # TODO: Should api_version be a kwarg in the API surface? @@ -85,6 +97,7 @@ def __init__(self, repository_location=None, dependency_resolution=None, api_ver self.resolver = _resolver.DtmiResolver(self.fetcher) self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) + @distributed_trace(name_of_span=_TRACE_NAMESPACE + "/get_models") def get_models(self, dtmis, dependency_resolution=None): """Retrieve a model from the Models Repository. diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py index fdfa6d614cdc..25dc7a03f38d 100644 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py @@ -9,24 +9,31 @@ logging.basicConfig(level=logging.DEBUG) + @pytest.mark.describe(".is_valid_dtmi()") class TestIsValidDTMI(object): @pytest.mark.it("Returns True if given a valid DTMI") - @pytest.mark.parametrize("dtmi", [ - pytest.param("dtmi:FooDTDL;1", id="Short DTMI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", id="Long DTMI") - ]) + @pytest.mark.parametrize( + "dtmi", + [ + pytest.param("dtmi:FooDTDL;1", id="Short DTMI"), + pytest.param("dtmi:com:somedomain:example:FooDTDL;1", id="Long DTMI"), + ], + ) def test_valid_dtmi(self, dtmi): assert dtmi_conventions.is_valid_dtmi(dtmi) @pytest.mark.it("Returns False if given an invalid DTMI") - @pytest.mark.parametrize("dtmi", [ - pytest.param("", id="Empty string"), - pytest.param("not a dtmi", id="Not a DTMI"), - pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), - pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), - pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), - ]) + @pytest.mark.parametrize( + "dtmi", + [ + pytest.param("", id="Empty string"), + pytest.param("not a dtmi", id="Not a DTMI"), + pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), + pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), + pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), + ], + ) def test_invalid_dtmi(self, dtmi): assert not dtmi_conventions.is_valid_dtmi(dtmi) @@ -34,37 +41,96 @@ def test_invalid_dtmi(self, dtmi): @pytest.mark.describe(".get_model_uri()") class TestGetModelURI(object): @pytest.mark.it("Returns the URI for a specified model at a specified repository") - @pytest.mark.parametrize("dtmi, repository_uri, expected_model_uri", [ - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "https://myrepository/", "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTPS repository URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository/", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="HTTP repository URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file://localhost/myrepository/", "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Filesystem URI w/ host"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", id="Repository URI without trailing '/'") - ]) + @pytest.mark.parametrize( + "dtmi, repository_uri, expected_model_uri", + [ + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "https://myrepository/", + "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", + id="HTTPS repository URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "http://myrepository/", + "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", + id="HTTP repository URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file:///myrepository/", + "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.json", + id="Filesystem URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file://localhost/myrepository/", + "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.json", + id="Filesystem URI w/ host", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "http://myrepository", + "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.json", + id="Repository URI without trailing '/'", + ), + ], + ) def test_uri(self, dtmi, repository_uri, expected_model_uri): model_uri = dtmi_conventions.get_model_uri(dtmi, repository_uri) assert model_uri == expected_model_uri @pytest.mark.it("Returns the URI for a specified expanded model at a specified repository") - @pytest.mark.parametrize("dtmi, repository_uri, expected_model_uri", [ - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "https://myrepository/", "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="HTTPS repository URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository/", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="HTTP repository URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="Filesystem URI"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "file://localhost/myrepository/", "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="Filesystem URI w/ host"), - pytest.param("dtmi:com:somedomain:example:FooDTDL;1", "http://myrepository", "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="Repository URI without trailing '/'") - ]) + @pytest.mark.parametrize( + "dtmi, repository_uri, expected_model_uri", + [ + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "https://myrepository/", + "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + id="HTTPS repository URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "http://myrepository/", + "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + id="HTTP repository URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file:///myrepository/", + "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + id="Filesystem URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file://localhost/myrepository/", + "file://localhost/myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + id="Filesystem URI w/ host", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "http://myrepository", + "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + id="Repository URI without trailing '/'", + ), + ], + ) def test_uri_expanded(self, dtmi, repository_uri, expected_model_uri): model_uri = dtmi_conventions.get_model_uri(dtmi, repository_uri, expanded=True) assert model_uri == expected_model_uri @pytest.mark.it("Raises ValueError if given an invalid DTMI") - @pytest.mark.parametrize("dtmi", [ - pytest.param("", id="Empty string"), - pytest.param("not a dtmi", id="Not a DTMI"), - pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), - pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), - pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), - ]) + @pytest.mark.parametrize( + "dtmi", + [ + pytest.param("", id="Empty string"), + pytest.param("not a dtmi", id="Not a DTMI"), + pytest.param("com:somedomain:example:FooDTDL;1", id="DTMI missing scheme"), + pytest.param("dtmi:com:somedomain:example:FooDTDL", id="DTMI missing version"), + pytest.param("dtmi:foo_bar:_16:baz33:qux;12", id="System DTMI"), + ], + ) def test_invalid_dtmi(self, dtmi): with pytest.raises(ValueError): - dtmi_conventions.get_model_uri(dtmi, "https://myrepository/") \ No newline at end of file + dtmi_conventions.get_model_uri(dtmi, "https://myrepository/") diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py b/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py index 6ecd956f668d..6cf72c76cb48 100644 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py @@ -9,10 +9,10 @@ logging.basicConfig(level=logging.DEBUG) + @pytest.mark.describe("ModelsRepositoryClient - .get_models() [INTEGRATION]") class TestModelsRepositoryClientGetModels(object): - @pytest.mark.it("test recordings") def test_simple(self): c = ModelsRepositoryClient() - c.get_models(["dtmi:com:example:TemperatureController;1"]) \ No newline at end of file + c.get_models(["dtmi:com:example:TemperatureController;1"]) From 50e5b4bc646eb633d0a5b0a77cadcdfefd4696fc Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Wed, 17 Mar 2021 13:22:12 -0700 Subject: [PATCH 13/25] UserAgent added --- .../azure/iot/modelsrepository/_constants.py | 12 ++++++++++ .../azure/iot/modelsrepository/_tracing.py | 23 +++++++++++++++++++ .../azure/iot/modelsrepository/client.py | 11 ++++----- .../samples/dtmi_conventions_sample.py | 1 - sdk/iot/azure-iot-modelsrepository/setup.py | 13 ++++++++++- 5 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py create mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py new file mode 100644 index 000000000000..e8eca692c6ec --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import platform + +VERSION = "0.0.0-preview" +USER_AGENT = "azsdk-python-modelsrepository/{pkg_version} Python/{py_version} ({platform})".format( + pkg_version=VERSION, py_version=(platform.python_version()), platform=platform.platform() +) +DEFAULT_API_VERSION = "2021-02-11" diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py new file mode 100644 index 000000000000..568cef76e107 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py @@ -0,0 +1,23 @@ +# ------------------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from contextlib import contextmanager +from azure.core.settings import settings +from azure.core.tracing import SpanKind + + +TRACE_NAMESPACE = "modelsrepository" + + +@contextmanager +def trace_context_manager(span_name): + span_impl_type = settings.tracing_implementation() + + if span_impl_type is not None: + with span_impl_type(name=span_name) as child: + child.kind = SpanKind.CLIENT + yield child + else: + yield None diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index 39f7aceca2eb..5b0f38f9b770 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -21,6 +21,7 @@ from . import ( _resolver, _pseudo_parser, + _constants, ) @@ -32,7 +33,6 @@ # Convention-private constants _DEFAULT_LOCATION = "https://devicemodels.azure.com" -_DEFAULT_API_VERSION = "2021-02-11" _REMOTE_PROTOCOLS = ["http", "https"] _TRACE_NAMESPACE = "modelsrepository" @@ -86,7 +86,6 @@ def __init__( self.resolution_mode = dependency_resolution # TODO: Should api_version be a kwarg in the API surface? - # api_version = _DEFAULT_API_VERSION if api_version is None else api_version kwargs.setdefault("api_verison", api_version) # NOTE: depending on how this class develops over time, may need to adjust relationship @@ -153,7 +152,7 @@ def __init__(self, **kwargs): # the default repository's api version stored. Keep this in mind when expanding the # scope of the client in the future - perhaps there may need to eventually be unique # configs for default repository vs. custom repository endpoints - self._api_version = kwargs.get("api_version", _DEFAULT_API_VERSION) + self._api_version = kwargs.get("api_version", _constants.DEFAULT_API_VERSION) def _create_fetcher(location, **kwargs): @@ -203,11 +202,9 @@ def _create_pipeline_client(base_url, **kwargs): def _create_config(**kwargs): """Creates and returns a ModelsRepositoryConfiguration object""" config = ModelsRepositoryClientConfiguration(**kwargs) - config.headers_policy = kwargs.get( - "headers_policy", HeadersPolicy({"CustomHeader": "Value"}, **kwargs) - ) + config.headers_policy = kwargs.get("headers_policy", HeadersPolicy(**kwargs)) config.user_agent_policy = kwargs.get( - "user_agent_policy", UserAgentPolicy("ServiceUserAgentValue", **kwargs) + "user_agent_policy", UserAgentPolicy(_constants.USER_AGENT, **kwargs) ) config.authentication_policy = kwargs.get("authentication_policy") config.retry_policy = kwargs.get("retry_policy", RetryPolicy(**kwargs)) diff --git a/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py index f81840de6bd0..c6b5c7cbb236 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py +++ b/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py @@ -26,7 +26,6 @@ def sample_get_model_uri(): print(dtmi_conventions.get_model_uri(dtmi, repo_uri, expanded=True)) # Prints: "file:///path/to/repository/dtmi/com/example/thermostat-1.expanded.json" - # Remote repository example repo_uri = "https://contoso.com/models/" print(dtmi_conventions.get_model_uri(dtmi, repo_uri)) diff --git a/sdk/iot/azure-iot-modelsrepository/setup.py b/sdk/iot/azure-iot-modelsrepository/setup.py index fe5bf96fb828..3c526d112747 100644 --- a/sdk/iot/azure-iot-modelsrepository/setup.py +++ b/sdk/iot/azure-iot-modelsrepository/setup.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------- from setuptools import setup, find_packages +import re # azure v0.x is not compatible with this package # azure v0.x used to have a __version__ attribute (newer versions don't) @@ -22,12 +23,22 @@ except ImportError: pass + +# Fetch description with open("README.md", "r") as fh: _long_description = fh.read() + +# Fetch version +with open("azure/iot/modelsrepository/_constants.py", "r") as fh: + VERSION = re.search(r"^VERSION\s=\s*[\"']([^\"']*)", fh.read(), re.MULTILINE).group(1) +if not VERSION: + raise RuntimeError("Cannot find version information") + + setup( name="azure-iot-modelsrepository", - version="0.0.0-preview", + version=VERSION, description="Microsoft Azure IoT Models Repository Library", license="MIT License", author="Microsoft Corporation", From 0ee53d87ef6044c23940bc528d6ae4c901d0b411 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Thu, 18 Mar 2021 10:59:18 -0700 Subject: [PATCH 14/25] Added logging --- .../iot/modelsrepository/_pseudo_parser.py | 10 ++++++++- .../azure/iot/modelsrepository/_resolver.py | 7 +++++- .../azure/iot/modelsrepository/client.py | 22 ++++++++++++++++++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py index 06ca702de688..b3ebd974c0d0 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py @@ -11,8 +11,10 @@ Note that this implementation is not representative of what an eventual full parser implementation would necessarily look like from an API perspective """ +import logging from ._chainable_exception import ChainableException +_LOGGER = logging.getLogger(__name__) class PseudoParser(object): def __init__(self, resolver): @@ -38,16 +40,20 @@ def expand(self, models): return expanded_map def _expand(self, model, model_map): + _LOGGER.debug("Expanding model: %s", model["@id"]) dependencies = _get_dependency_list(model) dependencies_to_resolve = [ dependency for dependency in dependencies if dependency not in model_map ] if dependencies_to_resolve: + _LOGGER.debug("Outstanding dependencies found: %s", dependencies_to_resolve) resolved_dependency_map = self.resolver.resolve(dependencies_to_resolve) model_map.update(resolved_dependency_map) - for dependency_model in resolved_dependency_map.items(): + for dependency_model in resolved_dependency_map.values(): self._expand(dependency_model, model_map) + else: + _LOGGER.debug("No outstanding dependencies found") def _get_dependency_list(model): @@ -75,4 +81,6 @@ def _get_dependency_list(model): # This is a nested model. Now go get its dependencies and add them dependencies += _get_dependency_list(item) + # Remove duplicate dependencies + dependencies = list(set(dependencies)) return dependencies diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index 6ea0fcdc6eb8..b0f874e96c72 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -11,7 +11,7 @@ from . import dtmi_conventions from ._chainable_exception import ChainableException -logger = logging.getLogger(__name__) +_LOGGER = logging.getLogger(__name__) class ResolverError(ChainableException): @@ -43,6 +43,7 @@ def resolve(self, dtmis, expanded_model=False): dtdl_path = dtmi_conventions._convert_dtmi_to_path(dtmi) if expanded_model: dtdl_path = dtdl_path.replace(".json", ".expanded.json") + _LOGGER.debug("Model %s located in repository at %s", dtmi, dtdl_path) try: dtdl = self.fetcher.fetch(dtdl_path) @@ -58,6 +59,7 @@ def resolve(self, dtmis, expanded_model=False): for model in dtdl: model_map[model["@id"]] = model else: + model = dtdl # Verify that the DTMI of the fetched model matches the DTMI of the request if model["@id"] != dtmi: raise ResolverError( @@ -101,6 +103,7 @@ def fetch(self, path): :returns: JSON data at the path :rtype: JSON object """ + _LOGGER.debug("Fetching %s from remote endpoint", path) request = self.client.get(url=path) response = self.client._pipeline.run(request).http_response if response.status_code != 200: @@ -127,6 +130,7 @@ def fetch(self, path): :returns: JSON data at the path :rtype: JSON object """ + _LOGGER.debug("Fetching %s from local filesystem", path) # Format path path = os.path.join(self.base_path, path) path = os.path.normcase(path) @@ -137,6 +141,7 @@ def fetch(self, path): # Fetch try: + _LOGGER.debug("File open on %s", path) with open(path) as f: file_str = f.read() except Exception as e: diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index 5b0f38f9b770..4b1113d94cfa 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------- import six.moves.urllib as urllib import re +import logging from azure.core import PipelineClient from azure.core.tracing.decorator import distributed_trace from azure.core.pipeline.transport import RequestsTransport @@ -24,6 +25,8 @@ _constants, ) +_LOGGER = logging.getLogger(__name__) + # Public constants exposed to consumers DEPENDENCY_MODE_TRY_FROM_EXPANDED = "tryFromExpanded" @@ -66,6 +69,7 @@ def __init__( repository_location = ( _DEFAULT_LOCATION if repository_location is None else repository_location ) + _LOGGER.debug("Client configured for respository location %s", repository_location) if dependency_resolution is None: # If using the default repository location, the resolution mode should default to @@ -84,6 +88,7 @@ def __init__( "Invalid dependency resolution mode: {}".format(dependency_resolution) ) self.resolution_mode = dependency_resolution + _LOGGER.debug("Client configured for dependency mode %s", self.resolution_mode) # TODO: Should api_version be a kwarg in the API surface? kwargs.setdefault("api_verison", api_version) @@ -96,7 +101,7 @@ def __init__( self.resolver = _resolver.DtmiResolver(self.fetcher) self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) - @distributed_trace(name_of_span=_TRACE_NAMESPACE + "/get_models") + @distributed_trace def get_models(self, dtmis, dependency_resolution=None): """Retrieve a model from the Models Repository. @@ -122,20 +127,30 @@ def get_models(self, dtmis, dependency_resolution=None): if dependency_resolution == DEPENDENCY_MODE_DISABLED: # Simply retrieve the model(s) + _LOGGER.debug("Getting models w/ dependency resolution mode: disabled") + _LOGGER.debug("Retreiving model(s): %s...", dtmis) model_map = self.resolver.resolve(dtmis) elif dependency_resolution == DEPENDENCY_MODE_ENABLED: # Manually resolve dependencies using pseudo-parser + _LOGGER.debug("Getting models w/ dependency resolution mode: enabled") + _LOGGER.debug("Retreiving model(s): %s...", dtmis) base_model_map = self.resolver.resolve(dtmis) base_model_list = list(base_model_map.values()) + _LOGGER.debug("Retreiving model dependencies for %s...", dtmis) model_map = self._psuedo_parser.expand(base_model_list) elif dependency_resolution == DEPENDENCY_MODE_TRY_FROM_EXPANDED: + _LOGGER.debug("Getting models w/ dependency resolution mode: tryFromExpanded") # Try to use an expanded DTDL to resolve dependencies try: + _LOGGER.debug("Retreiving expanded model(s): %s...", dtmis) model_map = self.resolver.resolve(dtmis, expanded_model=True) except _resolver.ResolverError: # Fallback to manual dependency resolution + _LOGGER.debug("Could not retreive model(s) from expanded DTDL - fallback to manual dependency resolution mode") + _LOGGER.debug("Retreiving model(s): %s...", dtmis) base_model_map = self.resolver.resolve(dtmis) base_model_list = list(base_model_map.items()) + _LOGGER.debug("Retreiving model dependencies for %s...", dtmis) model_map = self._psuedo_parser.expand(base_model_list) else: raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) @@ -160,22 +175,27 @@ def _create_fetcher(location, **kwargs): scheme = urllib.parse.urlparse(location).scheme if scheme in _REMOTE_PROTOCOLS: # HTTP/HTTPS URL + _LOGGER.debug("Repository Location identified as HTTP/HTTPS endpoint - using HttpFetcher") client = _create_pipeline_client(base_url=location, **kwargs) fetcher = _resolver.HttpFetcher(client) elif scheme == "file": # Filesystem URI + _LOGGER.debug("Repository Location identified as filesystem URI - using FilesystemFetcher") location = location[len("file://") :] fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and location.startswith("/"): # POSIX filesystem path + _LOGGER.debug("Repository Location identified as POSIX fileystem path - using FilesystemFetcher") fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and re.search(r"\.[a-zA-z]{2,63}$", location[: location.find("/")]): # Web URL with protocol unspecified - default to HTTPS + _LOGGER.debug("Repository Location identified as remote endpoint without protocol specified - using HttpFetcher") location = "https://" + location client = _create_pipeline_client(base_url=location, **kwargs) fetcher = _resolver.HttpFetcher(client) elif scheme != "" and len(scheme) == 1 and scheme.isalpha(): # Filesystem path using drive letters (e.g. "C:", "D:", etc.) + _LOGGER.debug("Repository Location identified as drive letter fileystem path - using FilesystemFetcher") fetcher = _resolver.FilesystemFetcher(location) else: raise ValueError("Unable to identify location: {}".format(location)) From 1345f86248e67a3ab33f77bda662ae5bb6a14a4d Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Tue, 6 Apr 2021 14:57:08 -0700 Subject: [PATCH 15/25] switched pipelineclient to pipeline --- .../azure/iot/modelsrepository/_resolver.py | 17 +++-- .../azure/iot/modelsrepository/client.py | 66 +++++++------------ 2 files changed, 33 insertions(+), 50 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index b0f874e96c72..450901d27c8a 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -8,6 +8,7 @@ import abc import os import six +from azure.core.pipeline.transport import HttpRequest from . import dtmi_conventions from ._chainable_exception import ChainableException @@ -86,12 +87,15 @@ def fetch(self, path): class HttpFetcher(Fetcher): """Fetches JSON data from a web endpoint""" - def __init__(self, http_client): + def __init__(self, base_url, pipeline): """ - :param http_client: PipelineClient that has been configured for an endpoint - :type http_client: :class:`azure.core.PipelineClient` + :param pipeline: Pipeline (pre-configured) + :type pipeline: :class:`azure.core.pipeline.Pipeline` """ - self.client = http_client + self.pipeline = pipeline + self.base_url = base_url + if not self.base_url.endswith("/"): + self.base_url += "/" def fetch(self, path): """Fetch and return the contents of a JSON file at a given web path. @@ -104,8 +108,9 @@ def fetch(self, path): :rtype: JSON object """ _LOGGER.debug("Fetching %s from remote endpoint", path) - request = self.client.get(url=path) - response = self.client._pipeline.run(request).http_response + url = self.base_url + path + request = HttpRequest("GET", url) + response = self.pipeline.run(request).http_response if response.status_code != 200: raise FetcherError("Failed to fetch from remote endpoint") json_response = json.loads(response.text()) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index 4b1113d94cfa..687204c58d75 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -6,7 +6,7 @@ import six.moves.urllib as urllib import re import logging -from azure.core import PipelineClient +from azure.core.pipeline import Pipeline from azure.core.tracing.decorator import distributed_trace from azure.core.pipeline.transport import RequestsTransport from azure.core.configuration import Configuration @@ -157,27 +157,14 @@ def get_models(self, dtmis, dependency_resolution=None): return model_map -class ModelsRepositoryClientConfiguration(Configuration): - """ModelsRepositoryClient-specific variant of the Azure Core Configuration for Pipelines""" - - def __init__(self, **kwargs): - super(ModelsRepositoryClientConfiguration, self).__init__(**kwargs) - # NOTE: There might be some further organization to do here as it's kind of weird that - # the generic config (which could be used for any remote repository) always will have - # the default repository's api version stored. Keep this in mind when expanding the - # scope of the client in the future - perhaps there may need to eventually be unique - # configs for default repository vs. custom repository endpoints - self._api_version = kwargs.get("api_version", _constants.DEFAULT_API_VERSION) - - def _create_fetcher(location, **kwargs): """Return a Fetcher based upon the type of location""" scheme = urllib.parse.urlparse(location).scheme if scheme in _REMOTE_PROTOCOLS: # HTTP/HTTPS URL _LOGGER.debug("Repository Location identified as HTTP/HTTPS endpoint - using HttpFetcher") - client = _create_pipeline_client(base_url=location, **kwargs) - fetcher = _resolver.HttpFetcher(client) + pipeline = _create_pipeline(**kwargs) + fetcher = _resolver.HttpFetcher(location, pipeline) elif scheme == "file": # Filesystem URI _LOGGER.debug("Repository Location identified as filesystem URI - using FilesystemFetcher") @@ -191,8 +178,8 @@ def _create_fetcher(location, **kwargs): # Web URL with protocol unspecified - default to HTTPS _LOGGER.debug("Repository Location identified as remote endpoint without protocol specified - using HttpFetcher") location = "https://" + location - client = _create_pipeline_client(base_url=location, **kwargs) - fetcher = _resolver.HttpFetcher(client) + pipeline = _create_pipeline(**kwargs) + fetcher = _resolver.HttpFetcher(location, pipeline) elif scheme != "" and len(scheme) == 1 and scheme.isalpha(): # Filesystem path using drive letters (e.g. "C:", "D:", etc.) _LOGGER.debug("Repository Location identified as drive letter fileystem path - using FilesystemFetcher") @@ -202,33 +189,24 @@ def _create_fetcher(location, **kwargs): return fetcher -def _create_pipeline_client(base_url, **kwargs): +def _create_pipeline(**kwargs): """Creates and returns a PipelineClient configured for the provided base_url and kwargs""" transport = kwargs.get("transport", RequestsTransport(**kwargs)) - config = _create_config(**kwargs) + policies = _create_policies_list(**kwargs) + return Pipeline(policies=policies, transport=transport) + + +def _create_policies_list(**kwargs): + """Creates and returns a list of policies based on provided kwargs (or default policies)""" policies = [ - config.user_agent_policy, - config.headers_policy, - config.authentication_policy, - ContentDecodePolicy(), - config.proxy_policy, - config.redirect_policy, - config.retry_policy, - config.logging_policy, + kwargs.get("user_agent_policy", UserAgentPolicy(_constants.USER_AGENT, **kwargs)), + kwargs.get("headers_policy", HeadersPolicy(**kwargs)), + kwargs.get("retry_policy", RetryPolicy(**kwargs)), + kwargs.get("redirect_policy", RedirectPolicy(**kwargs)), + kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)), + kwargs.get("proxy_policy", ProxyPolicy(**kwargs)), ] - return PipelineClient(base_url=base_url, config=config, policies=policies, transport=transport) - - -def _create_config(**kwargs): - """Creates and returns a ModelsRepositoryConfiguration object""" - config = ModelsRepositoryClientConfiguration(**kwargs) - config.headers_policy = kwargs.get("headers_policy", HeadersPolicy(**kwargs)) - config.user_agent_policy = kwargs.get( - "user_agent_policy", UserAgentPolicy(_constants.USER_AGENT, **kwargs) - ) - config.authentication_policy = kwargs.get("authentication_policy") - config.retry_policy = kwargs.get("retry_policy", RetryPolicy(**kwargs)) - config.redirect_policy = kwargs.get("redirect_policy", RedirectPolicy(**kwargs)) - config.logging_policy = kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)) - config.proxy_policy = kwargs.get("proxy_policy", ProxyPolicy(**kwargs)) - return config + auth_policy = kwargs.get("authentication_policy") + if auth_policy: + policies.append(auth_policy) + return policies From 6f45544fb7dec536782f5cb05be937f0e70bad50 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Tue, 6 Apr 2021 15:24:57 -0700 Subject: [PATCH 16/25] Kwarg adjustments --- .../azure/iot/modelsrepository/client.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py index 687204c58d75..8f87f97dd863 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py @@ -44,7 +44,7 @@ class ModelsRepositoryClient(object): """Client providing APIs for Models Repository operations""" def __init__( - self, repository_location=None, dependency_resolution=None, api_version=None, **kwargs + self, repository_location=None, dependency_resolution=None, **kwargs ): """ :param str repository_location: Location of the Models Repository you wish to access. @@ -63,6 +63,9 @@ def __init__( be "tryFromExpanded". If using a custom repository location, the default dependency resolution mode will be "enabled". + There are additional keyword arguments you can provide, which are documented here: + https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md#configurations + :raises: ValueError if repository_location is invalid :raises: ValueError if dependency_resolution is invalid """ @@ -90,9 +93,6 @@ def __init__( self.resolution_mode = dependency_resolution _LOGGER.debug("Client configured for dependency mode %s", self.resolution_mode) - # TODO: Should api_version be a kwarg in the API surface? - kwargs.setdefault("api_verison", api_version) - # NOTE: depending on how this class develops over time, may need to adjust relationship # between some of these objects self.fetcher = _create_fetcher( @@ -101,11 +101,15 @@ def __init__( self.resolver = _resolver.DtmiResolver(self.fetcher) self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) + # Store api version here (for now). Currently doesn't do anything + self._api_version = kwargs.get("api_version", _constants.DEFAULT_API_VERSION) + @distributed_trace def get_models(self, dtmis, dependency_resolution=None): """Retrieve a model from the Models Repository. - :param list[str] dtmis: The DTMIs for the models you wish to retrieve + :param dtmis: The DTMI(s) for the model(s) you wish to retrieve + :type dtmis: str or list[str] :param str dependency_resolution: Dependency resolution mode override. This value takes precedence over the value set on the client. Possible values: @@ -121,6 +125,9 @@ def get_models(self, dtmis, dependency_resolution=None): :returns: Dictionary mapping DTMIs to models :rtype: dict """ + if type(dtmis) is str: + dtmis = [dtmis] + # TODO: Use better error surface than the custom ResolverError if dependency_resolution is None: dependency_resolution = self.resolution_mode From 1611bcac3f1633e515c013791ea8d6131840dba3 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Tue, 6 Apr 2021 15:27:50 -0700 Subject: [PATCH 17/25] packaging --- .../azure/iot/modelsrepository/__init__.py | 4 ++-- .../azure/iot/modelsrepository/{client.py => _client.py} | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) rename sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/{client.py => _client.py} (99%) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py index e02998c06115..285f59567a8d 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py @@ -5,10 +5,10 @@ # -------------------------------------------------------------------------- # Main Client -from .client import ModelsRepositoryClient +from ._client import ModelsRepositoryClient # Constants -from .client import ( +from ._client import ( DEPENDENCY_MODE_DISABLED, DEPENDENCY_MODE_ENABLED, DEPENDENCY_MODE_TRY_FROM_EXPANDED, diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py similarity index 99% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py rename to sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py index 8f87f97dd863..1ba84580911d 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py @@ -96,7 +96,7 @@ def __init__( # NOTE: depending on how this class develops over time, may need to adjust relationship # between some of these objects self.fetcher = _create_fetcher( - location=repository_location, api_version=api_version, **kwargs + location=repository_location, **kwargs ) self.resolver = _resolver.DtmiResolver(self.fetcher) self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) @@ -104,6 +104,7 @@ def __init__( # Store api version here (for now). Currently doesn't do anything self._api_version = kwargs.get("api_version", _constants.DEFAULT_API_VERSION) + @distributed_trace def get_models(self, dtmis, dependency_resolution=None): """Retrieve a model from the Models Repository. From 56667bf93707c9b97973246866a100550dc762eb Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Tue, 6 Apr 2021 15:30:29 -0700 Subject: [PATCH 18/25] removed custom transport + policies --- .../azure/iot/modelsrepository/_client.py | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py index 1ba84580911d..5674f014932d 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py @@ -199,22 +199,13 @@ def _create_fetcher(location, **kwargs): def _create_pipeline(**kwargs): """Creates and returns a PipelineClient configured for the provided base_url and kwargs""" - transport = kwargs.get("transport", RequestsTransport(**kwargs)) - policies = _create_policies_list(**kwargs) - return Pipeline(policies=policies, transport=transport) - - -def _create_policies_list(**kwargs): - """Creates and returns a list of policies based on provided kwargs (or default policies)""" + transport = RequestsTransport(**kwargs) policies = [ - kwargs.get("user_agent_policy", UserAgentPolicy(_constants.USER_AGENT, **kwargs)), - kwargs.get("headers_policy", HeadersPolicy(**kwargs)), - kwargs.get("retry_policy", RetryPolicy(**kwargs)), - kwargs.get("redirect_policy", RedirectPolicy(**kwargs)), - kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)), - kwargs.get("proxy_policy", ProxyPolicy(**kwargs)), + UserAgentPolicy(_constants.USER_AGENT, **kwargs), + HeadersPolicy(**kwargs), + RetryPolicy(**kwargs), + RedirectPolicy(**kwargs), + NetworkTraceLoggingPolicy(**kwargs), + ProxyPolicy(**kwargs), ] - auth_policy = kwargs.get("authentication_policy") - if auth_policy: - policies.append(auth_policy) - return policies + return Pipeline(policies=policies, transport=transport) From 6d8b03f5273ae20c15e49ce4a9337036eac1bb73 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Tue, 6 Apr 2021 20:42:50 -0700 Subject: [PATCH 19/25] enforced kwargs --- .../azure/iot/modelsrepository/_client.py | 76 +++++++++---------- .../iot/modelsrepository/_pseudo_parser.py | 1 + 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py index 5674f014932d..b1537aeeeaad 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py @@ -43,16 +43,12 @@ class ModelsRepositoryClient(object): """Client providing APIs for Models Repository operations""" - def __init__( - self, repository_location=None, dependency_resolution=None, **kwargs - ): + def __init__(self, **kwargs): """ - :param str repository_location: Location of the Models Repository you wish to access. + :keyword str repository_location: Location of the Models Repository you wish to access. This location can be a remote HTTP/HTTPS URL, or a local filesystem path. If omitted, will default to using "https://devicemodels.azure.com". - :param str api_version: The API version for the Models Repository Service you wish to - access. - :param str dependency_resolution: Dependency resolution mode. + :keyword str dependency_resolution: Dependency resolution mode. Possible values: - "disabled": Do not resolve model dependencies - "enabled": Resolve model dependencies from the repository @@ -62,56 +58,47 @@ def __init__( If using the default repository location, the default dependency resolution mode will be "tryFromExpanded". If using a custom repository location, the default dependency resolution mode will be "enabled". + :keyword str api_version: The API version for the Models Repository Service you wish to + access. - There are additional keyword arguments you can provide, which are documented here: - https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/README.md#configurations + For additional request configuration options, please see [core options](https://aka.ms/azsdk/python/options). :raises: ValueError if repository_location is invalid :raises: ValueError if dependency_resolution is invalid """ - repository_location = ( - _DEFAULT_LOCATION if repository_location is None else repository_location - ) + repository_location = kwargs.get("repository_location", _DEFAULT_LOCATION) _LOGGER.debug("Client configured for respository location %s", repository_location) - if dependency_resolution is None: - # If using the default repository location, the resolution mode should default to - # expanded mode because the defeault repo guarantees the existence of expanded DTDLs - if repository_location == _DEFAULT_LOCATION: - self.resolution_mode = DEPENDENCY_MODE_TRY_FROM_EXPANDED - else: - self.resolution_mode = DEPENDENCY_MODE_ENABLED - else: - if dependency_resolution not in [ - DEPENDENCY_MODE_ENABLED, - DEPENDENCY_MODE_DISABLED, - DEPENDENCY_MODE_TRY_FROM_EXPANDED, - ]: - raise ValueError( - "Invalid dependency resolution mode: {}".format(dependency_resolution) - ) - self.resolution_mode = dependency_resolution + self.resolution_mode = kwargs.get( + "dependency_resolution", + DEPENDENCY_MODE_TRY_FROM_EXPANDED + if repository_location == _DEFAULT_LOCATION + else DEPENDENCY_MODE_ENABLED, + ) + if self.resolution_mode not in [ + DEPENDENCY_MODE_ENABLED, + DEPENDENCY_MODE_DISABLED, + DEPENDENCY_MODE_TRY_FROM_EXPANDED, + ]: + raise ValueError("Invalid dependency resolution mode: {}".format(self.resolution_mode)) _LOGGER.debug("Client configured for dependency mode %s", self.resolution_mode) # NOTE: depending on how this class develops over time, may need to adjust relationship # between some of these objects - self.fetcher = _create_fetcher( - location=repository_location, **kwargs - ) + self.fetcher = _create_fetcher(location=repository_location, **kwargs) self.resolver = _resolver.DtmiResolver(self.fetcher) self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) # Store api version here (for now). Currently doesn't do anything self._api_version = kwargs.get("api_version", _constants.DEFAULT_API_VERSION) - @distributed_trace - def get_models(self, dtmis, dependency_resolution=None): + def get_models(self, dtmis, **kwargs): """Retrieve a model from the Models Repository. :param dtmis: The DTMI(s) for the model(s) you wish to retrieve :type dtmis: str or list[str] - :param str dependency_resolution: Dependency resolution mode override. This value takes + :keyword str dependency_resolution: Dependency resolution mode override. This value takes precedence over the value set on the client. Possible values: - "disabled": Do not resolve model dependencies @@ -130,8 +117,7 @@ def get_models(self, dtmis, dependency_resolution=None): dtmis = [dtmis] # TODO: Use better error surface than the custom ResolverError - if dependency_resolution is None: - dependency_resolution = self.resolution_mode + dependency_resolution = kwargs.get("dependency_resolution", self.resolution_mode) if dependency_resolution == DEPENDENCY_MODE_DISABLED: # Simply retrieve the model(s) @@ -154,7 +140,9 @@ def get_models(self, dtmis, dependency_resolution=None): model_map = self.resolver.resolve(dtmis, expanded_model=True) except _resolver.ResolverError: # Fallback to manual dependency resolution - _LOGGER.debug("Could not retreive model(s) from expanded DTDL - fallback to manual dependency resolution mode") + _LOGGER.debug( + "Could not retreive model(s) from expanded DTDL - fallback to manual dependency resolution mode" + ) _LOGGER.debug("Retreiving model(s): %s...", dtmis) base_model_map = self.resolver.resolve(dtmis) base_model_list = list(base_model_map.items()) @@ -180,17 +168,23 @@ def _create_fetcher(location, **kwargs): fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and location.startswith("/"): # POSIX filesystem path - _LOGGER.debug("Repository Location identified as POSIX fileystem path - using FilesystemFetcher") + _LOGGER.debug( + "Repository Location identified as POSIX fileystem path - using FilesystemFetcher" + ) fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and re.search(r"\.[a-zA-z]{2,63}$", location[: location.find("/")]): # Web URL with protocol unspecified - default to HTTPS - _LOGGER.debug("Repository Location identified as remote endpoint without protocol specified - using HttpFetcher") + _LOGGER.debug( + "Repository Location identified as remote endpoint without protocol specified - using HttpFetcher" + ) location = "https://" + location pipeline = _create_pipeline(**kwargs) fetcher = _resolver.HttpFetcher(location, pipeline) elif scheme != "" and len(scheme) == 1 and scheme.isalpha(): # Filesystem path using drive letters (e.g. "C:", "D:", etc.) - _LOGGER.debug("Repository Location identified as drive letter fileystem path - using FilesystemFetcher") + _LOGGER.debug( + "Repository Location identified as drive letter fileystem path - using FilesystemFetcher" + ) fetcher = _resolver.FilesystemFetcher(location) else: raise ValueError("Unable to identify location: {}".format(location)) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py index b3ebd974c0d0..809339ccde58 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py @@ -16,6 +16,7 @@ _LOGGER = logging.getLogger(__name__) + class PseudoParser(object): def __init__(self, resolver): """ From dc326b868e9475f656ab76bd16de9f30635ae1b0 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Tue, 6 Apr 2021 22:41:20 -0700 Subject: [PATCH 20/25] Normalized filepaths and urls --- .../azure/iot/modelsrepository/_client.py | 12 ++++++++++ .../azure/iot/modelsrepository/_resolver.py | 23 ++++++++----------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py index b1537aeeeaad..d2d6583f6c39 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py @@ -6,6 +6,7 @@ import six.moves.urllib as urllib import re import logging +import os from azure.core.pipeline import Pipeline from azure.core.tracing.decorator import distributed_trace from azure.core.pipeline.transport import RequestsTransport @@ -165,12 +166,14 @@ def _create_fetcher(location, **kwargs): # Filesystem URI _LOGGER.debug("Repository Location identified as filesystem URI - using FilesystemFetcher") location = location[len("file://") :] + location = _sanitize_filesystem_path(location) fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and location.startswith("/"): # POSIX filesystem path _LOGGER.debug( "Repository Location identified as POSIX fileystem path - using FilesystemFetcher" ) + location = _sanitize_filesystem_path(location) fetcher = _resolver.FilesystemFetcher(location) elif scheme == "" and re.search(r"\.[a-zA-z]{2,63}$", location[: location.find("/")]): # Web URL with protocol unspecified - default to HTTPS @@ -185,6 +188,7 @@ def _create_fetcher(location, **kwargs): _LOGGER.debug( "Repository Location identified as drive letter fileystem path - using FilesystemFetcher" ) + location = _sanitize_filesystem_path(location) fetcher = _resolver.FilesystemFetcher(location) else: raise ValueError("Unable to identify location: {}".format(location)) @@ -203,3 +207,11 @@ def _create_pipeline(**kwargs): ProxyPolicy(**kwargs), ] return Pipeline(policies=policies, transport=transport) + + +# TODO: Ensure support for relative and absolute paths +# TODO: Need robust suite of testing for different types of paths +def _sanitize_filesystem_path(path): + path = os.path.normcase(path) + path = os.path.normpath(path) + return path diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index 450901d27c8a..d51e0dc2694a 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -8,6 +8,7 @@ import abc import os import six +import six.moves.urllib as urllib from azure.core.pipeline.transport import HttpRequest from . import dtmi_conventions from ._chainable_exception import ChainableException @@ -94,8 +95,6 @@ def __init__(self, base_url, pipeline): """ self.pipeline = pipeline self.base_url = base_url - if not self.base_url.endswith("/"): - self.base_url += "/" def fetch(self, path): """Fetch and return the contents of a JSON file at a given web path. @@ -108,11 +107,15 @@ def fetch(self, path): :rtype: JSON object """ _LOGGER.debug("Fetching %s from remote endpoint", path) - url = self.base_url + path + url = urllib.parse.urljoin(self.base_url, path) + + # Fetch request = HttpRequest("GET", url) response = self.pipeline.run(request).http_response if response.status_code != 200: - raise FetcherError("Failed to fetch from remote endpoint") + raise FetcherError("Failed to fetch from remote endpoint. Status code: {}".format( + response.status_code + )) json_response = json.loads(response.text()) return json_response @@ -136,18 +139,12 @@ def fetch(self, path): :rtype: JSON object """ _LOGGER.debug("Fetching %s from local filesystem", path) - # Format path - path = os.path.join(self.base_path, path) - path = os.path.normcase(path) - path = os.path.normpath(path) - - # TODO: Ensure support for relative and absolute paths - # TODO: Need robust suite of testing for different types of paths + abs_path = os.path.join(self.base_path, path) # Fetch try: - _LOGGER.debug("File open on %s", path) - with open(path) as f: + _LOGGER.debug("File open on %s", abs_path) + with open(abs_path) as f: file_str = f.read() except Exception as e: raise FetcherError("Failed to fetch from Filesystem", e) From 89d18518370903f6a0fbb202c9829b4b0ad2fe9c Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Tue, 6 Apr 2021 22:52:22 -0700 Subject: [PATCH 21/25] doc updates --- .../azure/iot/modelsrepository/_client.py | 2 -- .../azure/iot/modelsrepository/_resolver.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py index d2d6583f6c39..9f85f9008a3a 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py @@ -209,8 +209,6 @@ def _create_pipeline(**kwargs): return Pipeline(policies=policies, transport=transport) -# TODO: Ensure support for relative and absolute paths -# TODO: Need robust suite of testing for different types of paths def _sanitize_filesystem_path(path): path = os.path.normcase(path) path = os.path.normpath(path) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index d51e0dc2694a..a3f2363de194 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -98,8 +98,8 @@ def __init__(self, base_url, pipeline): def fetch(self, path): """Fetch and return the contents of a JSON file at a given web path. - The path can be relative to the path configured in the Fetcher's HttpClient, - or it can be an absolute path. + + :param str path: Path to JSON file (relative to the base_filepath of the Fetcher) :raises: FetcherError if data cannot be fetched @@ -123,15 +123,16 @@ def fetch(self, path): class FilesystemFetcher(Fetcher): """Fetches JSON data from a local filesystem endpoint""" - def __init__(self, base_path): + def __init__(self, base_filepath): """ - :param str base_path: The base filepath for fetching from + :param str base_filepath: The base filepath for fetching from """ - self.base_path = base_path + self.base_filepath = base_filepath def fetch(self, path): """Fetch and return the contents of a JSON file at a given filesystem path. - The path can be relative to the Fetcher's base_path, or it can be an absolute path. + + :param str path: Path to JSON file (relative to the base_filepath of the Fetcher) :raises: FetcherError if data cannot be fetched @@ -139,7 +140,7 @@ def fetch(self, path): :rtype: JSON object """ _LOGGER.debug("Fetching %s from local filesystem", path) - abs_path = os.path.join(self.base_path, path) + abs_path = os.path.join(self.base_filepath, path) # Fetch try: From ae38e3865a3b5e42cf399d35656b89858b979d3b Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Thu, 8 Apr 2021 10:07:20 -0700 Subject: [PATCH 22/25] Restored user supplied policy and transport --- .../azure/iot/modelsrepository/_client.py | 16 +++++++++------- .../iot/modelsrepository/_pseudo_parser.py | 8 ++++---- .../samples/client_configuration_sample.py | 19 +------------------ .../samples/get_models_sample.py | 2 +- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py index 9f85f9008a3a..8fe015ddc45b 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py @@ -197,19 +197,21 @@ def _create_fetcher(location, **kwargs): def _create_pipeline(**kwargs): """Creates and returns a PipelineClient configured for the provided base_url and kwargs""" - transport = RequestsTransport(**kwargs) + transport = kwargs.get("transport", RequestsTransport(**kwargs)) policies = [ - UserAgentPolicy(_constants.USER_AGENT, **kwargs), - HeadersPolicy(**kwargs), - RetryPolicy(**kwargs), - RedirectPolicy(**kwargs), - NetworkTraceLoggingPolicy(**kwargs), - ProxyPolicy(**kwargs), + kwargs.get("user_agent_policy", UserAgentPolicy(_constants.USER_AGENT, **kwargs)), + kwargs.get("headers_policy", HeadersPolicy(**kwargs)), + kwargs.get("authentication_policy"), + kwargs.get("retry_policy", RetryPolicy(**kwargs)), + kwargs.get("redirect_policy", RedirectPolicy(**kwargs)), + kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs), + kwargs.get("proxy_policy", ProxyPolicy(**kwargs)) ] return Pipeline(policies=policies, transport=transport) def _sanitize_filesystem_path(path): + """Sanitize the filesystem path to be formatted correctly for the current OS""" path = os.path.normcase(path) path = os.path.normpath(path) return path diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py index 809339ccde58..321cefe4bf01 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py @@ -42,7 +42,7 @@ def expand(self, models): def _expand(self, model, model_map): _LOGGER.debug("Expanding model: %s", model["@id"]) - dependencies = _get_dependency_list(model) + dependencies = _get_model_dependencies(model) dependencies_to_resolve = [ dependency for dependency in dependencies if dependency not in model_map ] @@ -57,8 +57,8 @@ def _expand(self, model, model_map): _LOGGER.debug("No outstanding dependencies found") -def _get_dependency_list(model): - """Return a list of DTMIs for model dependencies""" +def _get_model_dependencies(model): + """Return a list of dependency DTMIs for a given model""" dependencies = [] if "contents" in model: @@ -80,7 +80,7 @@ def _get_dependency_list(model): dependencies.append(item) elif isinstance(item, dict): # This is a nested model. Now go get its dependencies and add them - dependencies += _get_dependency_list(item) + dependencies += _get_model_dependencies(item) # Remove duplicate dependencies dependencies = list(set(dependencies)) diff --git a/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py index 734f582866a8..5f0f6dcbeab7 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py +++ b/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py @@ -22,21 +22,4 @@ def use_remote_repository(): def use_local_repository(): # You can also specify a custom local filesystem path where your Models Repository is located. # Paths can be specified as relative or absolute, as well as in URI format - client = ModelsRepositoryClient(repository_location="file://home/fake/myrepository") - - -def custom_policies(): - # You can customize policies for remote operations in the client by passing through kwargs. - # Please refer to documentation for the azure.core libraries for reference on all the possible - # policy options. - user_agent_policy = UserAgentPolicy("myuseragent") - retry_policy = RetryPolicy(retry_total=10) - client = ModelsRepositoryClient(user_agent_policy=user_agent_policy, retry_policy=retry_policy) - - -def custom_transport(): - # You can supply your own transport for remote operations in the client by passing through - # kwargs. Please refer to documentation for the azure.core libraries for reference on the - # possible transport options. - transport = RequestsTransport(use_env_settings=False) - client = ModelsRepositoryClient(transport=transport) + client = ModelsRepositoryClient(repository_location="file:///home/fake/myrepository") diff --git a/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py index 4408c1ff98ff..b6fe5b6958eb 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py +++ b/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py @@ -23,7 +23,7 @@ def get_model(): # This API call will return a dictionary mapping DTMI to its corresponding model from # a DTDL document at the specified endpoint # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.json - model_map = client.get_models([dtmi]) + model_map = client.get_models(dtmi) pprint.pprint(model_map) From f2b26fbcfdc8e7261c009182166fdbdb0a91cfd8 Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Thu, 8 Apr 2021 10:30:33 -0700 Subject: [PATCH 23/25] Added type annotations Added yml infrastructure Commented links to pass CI removed artifact? Basic integration tests (local only) Added context manager support + integration tests Integration tests Added unittests Black formatting Infrastructure + py27 fixes Removed unnecessary file Addressed PR updated readme Added required fields to README packaging adjustments Added MANIFEST and updated version string updated logging details Version string updated to use b instead of rc Addressing PR DTDL -> model added greater requirement specificity remove duplicate pytest requirement from local dev_requirements.txt. update readme to correct match capitalization and add missing examples section. Address PR comments nspkg naming? Linting fixes More linting Linter suppression + fixes Fixed formatting line error that broke pylint Addressing PR feedback --- .github/CODEOWNERS | 3 + .../azure-iot-modelsrepository/CHANGELOG.md | 5 + .../azure-iot-modelsrepository/MANIFEST.in | 5 + sdk/iot/azure-iot-modelsrepository/README.md | 62 +- .../azure/iot/modelsrepository/__init__.py | 14 +- .../modelsrepository/_chainable_exception.py | 24 - .../azure/iot/modelsrepository/_client.py | 75 ++- .../azure/iot/modelsrepository/_constants.py | 2 +- .../iot/modelsrepository/_pseudo_parser.py | 6 +- .../azure/iot/modelsrepository/_resolver.py | 86 ++- .../azure/iot/modelsrepository/_tracing.py | 23 - .../iot/modelsrepository/dtmi_conventions.py | 8 +- .../iot/modelsrepository/exceptions.py} | 9 +- .../dev_requirements.txt | 6 + .../requirements.txt | 7 - .../samples/README.md | 10 +- .../samples/get_models_sample.py | 55 +- .../samples/scratch.py | 7 - sdk/iot/azure-iot-modelsrepository/setup.cfg | 2 + sdk/iot/azure-iot-modelsrepository/setup.py | 14 +- .../devicemanagement/deviceinformation-1.json | 64 +++ .../devicemanagement/deviceinformation-2.json | 16 + .../dtmi/com/example/base-1.json | 56 ++ .../dtmi/com/example/base-2.json | 57 ++ .../dtmi/com/example/building-1.json | 19 + .../dtmi/com/example/camera-3.json | 13 + .../dtmi/com/example/coldstorage-1.json | 13 + .../dtmi/com/example/conferenceroom-1.json | 13 + .../example/danglingexpanded-1.expanded.json | 215 +++++++ .../dtmi/com/example/freezer-1.json | 12 + .../incompleteexpanded-1.expanded.json | 151 +++++ .../dtmi/com/example/invalidmodel-1.json | 13 + .../dtmi/com/example/invalidmodel-2.json | 23 + .../dtmi/com/example/phone-2.json | 23 + .../dtmi/com/example/room-1.json | 12 + .../temperaturecontroller-1.expanded.json | 215 +++++++ .../com/example/temperaturecontroller-1.json | 60 ++ .../dtmi/com/example/thermostat-1.json | 19 + .../dtmi/company/demodevice-1.json | 31 + .../dtmi/company/demodevice-2.json | 31 + .../dtmi/strict/badfilepath-1.json | 12 + .../dtmi/strict/emptyarray-1.json | 1 + .../dtmi/strict/namespaceconflict-1.json | 37 ++ .../dtmi/strict/nondtdl-1.json | 1 + .../dtmi/strict/unsupportedrootarray-1.json | 91 +++ .../tests/test_client.py | 139 +++++ .../tests/test_dtmi_conventions.py | 43 +- .../tests/test_integration_client.py | 538 +++++++++++++++++- sdk/iot/azure-iot-modelsrepository/tox.ini | 9 - sdk/iot/ci.yml | 4 +- sdk/iot/tests.yml | 10 + 51 files changed, 2170 insertions(+), 194 deletions(-) create mode 100644 sdk/iot/azure-iot-modelsrepository/CHANGELOG.md create mode 100644 sdk/iot/azure-iot-modelsrepository/MANIFEST.in delete mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_chainable_exception.py delete mode 100644 sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py rename sdk/iot/azure-iot-modelsrepository/{tests/conftest.py => azure/iot/modelsrepository/exceptions.py} (62%) create mode 100644 sdk/iot/azure-iot-modelsrepository/dev_requirements.txt delete mode 100644 sdk/iot/azure-iot-modelsrepository/requirements.txt delete mode 100644 sdk/iot/azure-iot-modelsrepository/samples/scratch.py create mode 100644 sdk/iot/azure-iot-modelsrepository/setup.cfg create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json create mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_client.py delete mode 100644 sdk/iot/azure-iot-modelsrepository/tox.ini create mode 100644 sdk/iot/tests.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32e16186815b..d131175718a4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -71,6 +71,9 @@ # PRLabel: %HDInsight /sdk/hdinsight/ @idear1203 +# PRLabel: %Models repository +/sdk/iot/azure-iot-modelsrepository @cartertinney @digimaun + # PRLabel: %Machine Learning Compute /sdk/machinelearningcompute/ @shutchings diff --git a/sdk/iot/azure-iot-modelsrepository/CHANGELOG.md b/sdk/iot/azure-iot-modelsrepository/CHANGELOG.md new file mode 100644 index 000000000000..adbb33a3ee52 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/CHANGELOG.md @@ -0,0 +1,5 @@ +# Release History + +## 1.0.0b1 (Unreleased) + +* Initial (Preview) Release diff --git a/sdk/iot/azure-iot-modelsrepository/MANIFEST.in b/sdk/iot/azure-iot-modelsrepository/MANIFEST.in new file mode 100644 index 000000000000..1071130923d5 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/MANIFEST.in @@ -0,0 +1,5 @@ +include *.md +include azure/__init__.py +include azure/iot/__init__.py +recursive-include samples *.py +recursive-include tests *.py \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/README.md b/sdk/iot/azure-iot-modelsrepository/README.md index 28e32b7ce465..5406e9498e99 100644 --- a/sdk/iot/azure-iot-modelsrepository/README.md +++ b/sdk/iot/azure-iot-modelsrepository/README.md @@ -1,16 +1,64 @@ -# Azure IoT Models Repository Library +# Azure IoT Models Repository client library for Python The Azure IoT Models Repository Library for Python provides functionality for working with the Azure IoT Models Repository -## Installation +## Getting started -This package is not yet available on pip. Please install locally from source: +### Install package + +Install the Azure IoT Models Repository library for Python with [pip][pip]: ```Shell -python -m pip install -e +pip install azure-iot-modelsrepository ``` -## Features +### Prerequisites +* A models repository following [repo_conventions][Azure IoT conventions] + * The models repository can be hosted on the local filesystem or hosted on a webserver + * Azure IoT hosts the global [global_azure_repo][Azure IoT Models Repository] which the client will use if no custom location is provided + +### Authentication +Currently, no authentication mechanisms are supported. The global endpoint is not tied to an Azure subscription and does not support authentication. All models published are meant for anonymous public consumption. + +## Key concepts + +The Azure IoT Models Repository enables builders to manage and share digital twin models. The models are [json_ld][JSON-LD] documents defined using the Digital Twins Definition Language ([dtdl_spec][DTDL]). + +The repository defines a pattern to store DTDL interfaces in a directory structure based on the Digital Twin Model Identifier (DTMI). You can locate an interface in the repository by converting the DTMI to a relative path. For example, the DTMI `dtmi:com:example:Thermostat;1` translates to `/dtmi/com/example/thermostat-1.json`. + +## Examples + +## Troubleshooting + +### General +Models Repository clients raise exceptions defined in [azure_core_exceptions][azure-core]. + +### Logging +This library uses the standard [logging_doc][logging] library for logging. Information about HTTP sessions (URLs, headers, etc.) is logged at `DEBUG` level. + +## Next steps + +Several samples are available in the Azure SDK for Python GitHub repository. These provide example code for Models Repository Client scenarios: + +* [client_configuration_sample][client_configuration_sample] - Configure a ModelsRepositoryClient for a local or remote repository +* [get_models_sample][get_models_sample] - Retrieve models from a repository +* [dtmi_conventions_sample][dtmi_conventions_sample] - Use utility functions to generate and validate DTMIs + + +[azure_core_exceptions]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/core/azure-core#azure-core-library-exceptions +[client_configuration_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py +[dtdl_spec]: https://github.com/Azure/opendigitaltwins-dtdl/blob/master/DTDL/v2/dtdlv2.md +[dtmi_conventions_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py +[get_models_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py +[global_azure_repo]: https://devicemodels.azure.com/ +[json_ld]: https://json-ld.org/ +[logging_doc]: https://docs.python.org/3.5/library/logging.html +[pip]: https://pypi.org/project/pip/ +[repo_conventions]: https://github.com/Azure/iot-plugandplay-models-tools/wiki + +## Contributing +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. -* ### ModelsRepositoryClient - * Allows retrieval of model DTDLs from remote URLs or local filesystems +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py index 285f59567a8d..01149e8f128e 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py @@ -15,4 +15,16 @@ ) # Error handling -from ._resolver import ResolverError +from .exceptions import ModelError + +__all__ = [ + "ModelsRepositoryClient", + "ModelError", + "DEPENDENCY_MODE_DISABLED", + "DEPENDENCY_MODE_ENABLED", + "DEPENDENCY_MODE_TRY_FROM_EXPANDED", +] + +from ._constants import VERSION + +__version__ = VERSION diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_chainable_exception.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_chainable_exception.py deleted file mode 100644 index bad9523e3a45..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_chainable_exception.py +++ /dev/null @@ -1,24 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - - -class ChainableException(Exception): - """This exception stores a reference to a previous exception which has caused - the current one""" - - def __init__(self, message=None, cause=None): - # By using .__cause__, this will allow typical stack trace behavior in Python 3, - # while still being able to operate in Python 2. - self.__cause__ = cause - super(ChainableException, self).__init__(message) - - def __str__(self): - if self.__cause__: - return "{} caused by {}".format( - super(ChainableException, self).__repr__(), self.__cause__.__repr__() - ) - else: - return super(ChainableException, self).__repr__() diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py index 8fe015ddc45b..19f46f1cc52a 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py @@ -10,13 +10,12 @@ from azure.core.pipeline import Pipeline from azure.core.tracing.decorator import distributed_trace from azure.core.pipeline.transport import RequestsTransport -from azure.core.configuration import Configuration +from azure.core.exceptions import ResourceNotFoundError from azure.core.pipeline.policies import ( UserAgentPolicy, HeadersPolicy, RetryPolicy, RedirectPolicy, - ContentDecodePolicy, NetworkTraceLoggingPolicy, ProxyPolicy, ) @@ -44,7 +43,8 @@ class ModelsRepositoryClient(object): """Client providing APIs for Models Repository operations""" - def __init__(self, **kwargs): + def __init__(self, **kwargs): # pylint: disable=missing-client-constructor-parameter-credential + # type: (Any) -> None """ :keyword str repository_location: Location of the Models Repository you wish to access. This location can be a remote HTTP/HTTPS URL, or a local filesystem path. @@ -54,8 +54,8 @@ def __init__(self, **kwargs): - "disabled": Do not resolve model dependencies - "enabled": Resolve model dependencies from the repository - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded - DTDL document in the repository. If this is not successful, will fall back - on manually resolving dependencies in the repository + model DTDL document in the repository. If this is not successful, will fall + back on manually resolving dependencies in the repository If using the default repository location, the default dependency resolution mode will be "tryFromExpanded". If using a custom repository location, the default dependency resolution mode will be "enabled". @@ -64,8 +64,7 @@ def __init__(self, **kwargs): For additional request configuration options, please see [core options](https://aka.ms/azsdk/python/options). - :raises: ValueError if repository_location is invalid - :raises: ValueError if dependency_resolution is invalid + :raises: ValueError if an invalid argument is provided """ repository_location = kwargs.get("repository_location", _DEFAULT_LOCATION) _LOGGER.debug("Client configured for respository location %s", repository_location) @@ -88,13 +87,26 @@ def __init__(self, **kwargs): # between some of these objects self.fetcher = _create_fetcher(location=repository_location, **kwargs) self.resolver = _resolver.DtmiResolver(self.fetcher) - self._psuedo_parser = _pseudo_parser.PseudoParser(self.resolver) + self._pseudo_parser = _pseudo_parser.PseudoParser(self.resolver) # Store api version here (for now). Currently doesn't do anything self._api_version = kwargs.get("api_version", _constants.DEFAULT_API_VERSION) + def __enter__(self): + self.fetcher.__enter__() + return self + + def __exit__(self, *exc_details): + self.fetcher.__exit__(*exc_details) + + def close(self): + # type: () -> None + """Close the client, preventing future operations""" + self.__exit__() + @distributed_trace def get_models(self, dtmis, **kwargs): + # type: (Union[List[str], str], Any) -> Dict[str, Any] """Retrieve a model from the Models Repository. :param dtmis: The DTMI(s) for the model(s) you wish to retrieve @@ -104,51 +116,55 @@ def get_models(self, dtmis, **kwargs): Possible values: - "disabled": Do not resolve model dependencies - "enabled": Resolve model dependencies from the repository - - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded DTDL - document in the repository. If this is not successful, will fall back on - manually resolving dependencies in the repository + - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded + model DTDL document in the repository. If this is not successful, will fall + back on manually resolving dependencies in the repository :raises: ValueError if given an invalid dependency resolution mode - :raises: ResolverError if there is an error retrieving a model + :raises: ~azure.iot.modelsrepository.ModelError if there is an error parsing the retrieved model(s) + :raises: ~azure.core.exceptions.ResourceNotFoundError if the model(s) cannot be found in the repository + :raises: ~azure.core.exceptions.ServiceRequestError if there is an error sending a request for the model(s) + :raises: ~azure.core.exceptions.ServiceResponseError if the model(s) cannot be retrieved + :raises: ~azure.core.exceptions.HttpResponseError if a failure response is received :returns: Dictionary mapping DTMIs to models :rtype: dict """ - if type(dtmis) is str: + if isinstance(dtmis, str): dtmis = [dtmis] - # TODO: Use better error surface than the custom ResolverError dependency_resolution = kwargs.get("dependency_resolution", self.resolution_mode) if dependency_resolution == DEPENDENCY_MODE_DISABLED: # Simply retrieve the model(s) _LOGGER.debug("Getting models w/ dependency resolution mode: disabled") - _LOGGER.debug("Retreiving model(s): %s...", dtmis) + _LOGGER.debug("Retrieving model(s): %s...", dtmis) model_map = self.resolver.resolve(dtmis) elif dependency_resolution == DEPENDENCY_MODE_ENABLED: # Manually resolve dependencies using pseudo-parser _LOGGER.debug("Getting models w/ dependency resolution mode: enabled") - _LOGGER.debug("Retreiving model(s): %s...", dtmis) + _LOGGER.debug("Retrieving model(s): %s...", dtmis) base_model_map = self.resolver.resolve(dtmis) base_model_list = list(base_model_map.values()) - _LOGGER.debug("Retreiving model dependencies for %s...", dtmis) - model_map = self._psuedo_parser.expand(base_model_list) + _LOGGER.debug("Retrieving model dependencies for %s...", dtmis) + model_map = self._pseudo_parser.expand(base_model_list) elif dependency_resolution == DEPENDENCY_MODE_TRY_FROM_EXPANDED: _LOGGER.debug("Getting models w/ dependency resolution mode: tryFromExpanded") # Try to use an expanded DTDL to resolve dependencies try: - _LOGGER.debug("Retreiving expanded model(s): %s...", dtmis) + _LOGGER.debug("Retrieving expanded model(s): %s...", dtmis) model_map = self.resolver.resolve(dtmis, expanded_model=True) - except _resolver.ResolverError: + except ResourceNotFoundError: # Fallback to manual dependency resolution _LOGGER.debug( - "Could not retreive model(s) from expanded DTDL - fallback to manual dependency resolution mode" + "Could not retrieve model(s) from expanded model DTDL - " + "fallback to manual dependency resolution mode" ) - _LOGGER.debug("Retreiving model(s): %s...", dtmis) + _LOGGER.debug("Retrieving model(s): %s...", dtmis) base_model_map = self.resolver.resolve(dtmis) - base_model_list = list(base_model_map.items()) - _LOGGER.debug("Retreiving model dependencies for %s...", dtmis) - model_map = self._psuedo_parser.expand(base_model_list) + base_model_list = list(base_model_map.values()) + _LOGGER.debug("Retrieving model dependencies for %s...", dtmis) + model_map = self._pseudo_parser.expand(base_model_list) else: raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) return model_map @@ -175,7 +191,10 @@ def _create_fetcher(location, **kwargs): ) location = _sanitize_filesystem_path(location) fetcher = _resolver.FilesystemFetcher(location) - elif scheme == "" and re.search(r"\.[a-zA-z]{2,63}$", location[: location.find("/")]): + elif scheme == "" and re.search( + r"\.[a-zA-z]{2,63}$", + location[: location.find("/") if location.find("/") >= 0 else len(location)], + ): # Web URL with protocol unspecified - default to HTTPS _LOGGER.debug( "Repository Location identified as remote endpoint without protocol specified - using HttpFetcher" @@ -204,8 +223,8 @@ def _create_pipeline(**kwargs): kwargs.get("authentication_policy"), kwargs.get("retry_policy", RetryPolicy(**kwargs)), kwargs.get("redirect_policy", RedirectPolicy(**kwargs)), - kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs), - kwargs.get("proxy_policy", ProxyPolicy(**kwargs)) + kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)), + kwargs.get("proxy_policy", ProxyPolicy(**kwargs)), ] return Pipeline(policies=policies, transport=transport) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py index e8eca692c6ec..fef30b347020 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import platform -VERSION = "0.0.0-preview" +VERSION = "1.0.0b1" USER_AGENT = "azsdk-python-modelsrepository/{pkg_version} Python/{py_version} ({platform})".format( pkg_version=VERSION, py_version=(platform.python_version()), platform=platform.platform() ) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py index 321cefe4bf01..00e8996b86e7 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py @@ -12,7 +12,7 @@ parser implementation would necessarily look like from an API perspective """ import logging -from ._chainable_exception import ChainableException +import six _LOGGER = logging.getLogger(__name__) @@ -69,13 +69,13 @@ def _get_model_dependencies(model): # Models defined in a DTDL can implement extensions of up to two interfaces. # These interfaces can be in the form of a DTMI reference, or a nested model (which would # have it's own dependencies) - if isinstance(model["extends"], str): + if isinstance(model["extends"], six.text_type): # If it's just a string, that's a single DTMI reference, so just add that to our list dependencies.append(model["extends"]) elif isinstance(model["extends"], list): # If it's a list, could have DTMIs or nested models for item in model["extends"]: - if isinstance(item, str): + if isinstance(item, six.text_type): # If there are strings in the list, that's a DTMI reference, so add it dependencies.append(item) elif isinstance(item, dict): diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py index a3f2363de194..a2e3d22c7a57 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py @@ -7,19 +7,22 @@ import json import abc import os -import six +import io import six.moves.urllib as urllib +import six from azure.core.pipeline.transport import HttpRequest +from azure.core.exceptions import ( + map_error, + ResourceNotFoundError, + HttpResponseError, + raise_with_traceback, +) +from azure.iot.modelsrepository.exceptions import ModelError from . import dtmi_conventions -from ._chainable_exception import ChainableException _LOGGER = logging.getLogger(__name__) -class ResolverError(ChainableException): - pass - - class DtmiResolver(object): def __init__(self, fetcher): """ @@ -34,29 +37,28 @@ def resolve(self, dtmis, expanded_model=False): :param list[str] dtmis: DTMIs to resolve :param bool expanded_model: Indicates whether to resolve a regular or expanded model - :raises: ValueError if the DTMI is invalid - :raises: ResolverError if the DTMI cannot be resolved to a model + :raises: ValueError if the DTMI is invalid. + :raises: ModelError if there is an error with the contents of the JSON model. :returns: A dictionary mapping DTMIs to models :rtype: dict """ model_map = {} for dtmi in dtmis: + # pylint: disable=protected-access dtdl_path = dtmi_conventions._convert_dtmi_to_path(dtmi) if expanded_model: dtdl_path = dtdl_path.replace(".json", ".expanded.json") _LOGGER.debug("Model %s located in repository at %s", dtmi, dtdl_path) - try: - dtdl = self.fetcher.fetch(dtdl_path) - except FetcherError as e: - raise ResolverError("Failed to resolve dtmi: {}".format(dtmi), cause=e) + # Errors raised here bubble up + dtdl = self.fetcher.fetch(dtdl_path) if expanded_model: # Verify that the DTMI of the "root" model (i.e. the model we requested the # expanded DTDL for) within the expanded DTDL matches the DTMI of the request if True not in (model["@id"] == dtmi for model in dtdl): - raise ResolverError("DTMI mismatch on expanded DTDL - Request: {}".format(dtmi)) + raise ModelError("DTMI mismatch on expanded DTDL - Request: {}".format(dtmi)) # Add all the models in the expanded DTDL to the map for model in dtdl: model_map[model["@id"]] = model @@ -64,7 +66,7 @@ def resolve(self, dtmis, expanded_model=False): model = dtdl # Verify that the DTMI of the fetched model matches the DTMI of the request if model["@id"] != dtmi: - raise ResolverError( + raise ModelError( "DTMI mismatch - Request: {}, Response: {}".format(dtmi, model["@id"]) ) # Add the model to the map @@ -72,10 +74,6 @@ def resolve(self, dtmis, expanded_model=False): return model_map -class FetcherError(ChainableException): - pass - - @six.add_metaclass(abc.ABCMeta) class Fetcher(object): """Interface for fetching from a generic location""" @@ -84,10 +82,20 @@ class Fetcher(object): def fetch(self, path): pass + @abc.abstractmethod + def __enter__(self): + pass + + @abc.abstractmethod + def __exit__(self, *exc_details): + pass + class HttpFetcher(Fetcher): """Fetches JSON data from a web endpoint""" + error_map = {404: ResourceNotFoundError} + def __init__(self, base_url, pipeline): """ :param pipeline: Pipeline (pre-configured) @@ -96,12 +104,22 @@ def __init__(self, base_url, pipeline): self.pipeline = pipeline self.base_url = base_url + def __enter__(self): + self.pipeline.__enter__() + return self + + def __exit__(self, *exc_details): + self.pipeline.__exit__(*exc_details) + def fetch(self, path): """Fetch and return the contents of a JSON file at a given web path. :param str path: Path to JSON file (relative to the base_filepath of the Fetcher) - :raises: FetcherError if data cannot be fetched + :raises: ServiceRequestError if there is an error sending the request + :raises: ServiceResponseError if no response was received for the request + :raises: ResourceNotFoundError if the JSON file cannot be found + :raises: HttpResponseError if there is some other failure during fetch :returns: JSON data at the path :rtype: JSON object @@ -111,11 +129,14 @@ def fetch(self, path): # Fetch request = HttpRequest("GET", url) + _LOGGER.debug("GET %s", url) response = self.pipeline.run(request).http_response if response.status_code != 200: - raise FetcherError("Failed to fetch from remote endpoint. Status code: {}".format( - response.status_code - )) + map_error(status_code=response.status_code, response=response, error_map=self.error_map) + raise HttpResponseError( + "Failed to fetch from remote endpoint. Status code: {}".format(response.status_code) + ) + json_response = json.loads(response.text()) return json_response @@ -129,24 +150,37 @@ def __init__(self, base_filepath): """ self.base_filepath = base_filepath + def __enter__(self): + # Nothing is required here for filesystem + return self + + def __exit__(self, *exc_details): + # Nothing is required here for filesystem + pass + def fetch(self, path): """Fetch and return the contents of a JSON file at a given filesystem path. :param str path: Path to JSON file (relative to the base_filepath of the Fetcher) - :raises: FetcherError if data cannot be fetched + :raises: ResourceNotFoundError if the JSON file cannot be found :returns: JSON data at the path :rtype: JSON object """ _LOGGER.debug("Fetching %s from local filesystem", path) abs_path = os.path.join(self.base_filepath, path) + abs_path = os.path.normpath(abs_path) # Fetch try: _LOGGER.debug("File open on %s", abs_path) - with open(abs_path) as f: + with io.open(abs_path, encoding="utf-8-sig") as f: file_str = f.read() - except Exception as e: - raise FetcherError("Failed to fetch from Filesystem", e) + except (OSError, IOError): + # In Python 3 a FileNotFoundError is raised when a file doesn't exist. + # In Python 2 an IOError is raised when a file doesn't exist. + # Both of these errors are inherited from OSError, so we use this to catch them both. + # The semantics would ideally be better, but this is the price of supporting both. + raise_with_traceback(ResourceNotFoundError, message="Could not open file") return json.loads(file_str) diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py deleted file mode 100644 index 568cef76e107..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_tracing.py +++ /dev/null @@ -1,23 +0,0 @@ -# ------------------------------------------------------------------------ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ------------------------------------------------------------------------- -from contextlib import contextmanager -from azure.core.settings import settings -from azure.core.tracing import SpanKind - - -TRACE_NAMESPACE = "modelsrepository" - - -@contextmanager -def trace_context_manager(span_name): - span_impl_type = settings.tracing_implementation() - - if span_impl_type is not None: - with span_impl_type(name=span_name) as child: - child.kind = SpanKind.CLIENT - yield child - else: - yield None diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py index 06aeb4092f78..228d09545ee1 100644 --- a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py @@ -19,8 +19,7 @@ def is_valid_dtmi(dtmi): ) if not pattern.match(dtmi): return False - else: - return True + return True def get_model_uri(dtmi, repository_uri, expanded=False): @@ -54,10 +53,7 @@ def _convert_dtmi_to_path(dtmi, expanded=False): :returns: Relative path of the model in a Models Repository :rtype: str """ - pattern = re.compile( - "^dtmi:[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?(?::[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?)*;[1-9][0-9]{0,8}$" - ) - if not pattern.match(dtmi): + if not is_valid_dtmi(dtmi): raise ValueError("Invalid DTMI") dtmi_path = dtmi.lower().replace(":", "/").replace(";", "-") + ".json" if expanded: diff --git a/sdk/iot/azure-iot-modelsrepository/tests/conftest.py b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py similarity index 62% rename from sdk/iot/azure-iot-modelsrepository/tests/conftest.py rename to sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py index 3e49705aa942..d96158ffa434 100644 --- a/sdk/iot/azure-iot-modelsrepository/tests/conftest.py +++ b/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py @@ -3,12 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import pytest -@pytest.fixture -def arbitrary_exception(): - class ArbitraryException(Exception): - pass - - return ArbitraryException("This exception is completely arbitrary") +class ModelError(Exception): + pass diff --git a/sdk/iot/azure-iot-modelsrepository/dev_requirements.txt b/sdk/iot/azure-iot-modelsrepository/dev_requirements.txt new file mode 100644 index 000000000000..362326dd0248 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/dev_requirements.txt @@ -0,0 +1,6 @@ +-e ../../../tools/azure-devtools +-e ../../../tools/azure-sdk-tools +pytest-mock +pytest-testdox +parameterized +flake8 diff --git a/sdk/iot/azure-iot-modelsrepository/requirements.txt b/sdk/iot/azure-iot-modelsrepository/requirements.txt deleted file mode 100644 index 7c35e7d07437..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -pytest -pytest-mock -pytest-testdox -flake8 - -azure-core -six diff --git a/sdk/iot/azure-iot-modelsrepository/samples/README.md b/sdk/iot/azure-iot-modelsrepository/samples/README.md index 3d047f99cc62..eaf8e279f84a 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/README.md +++ b/sdk/iot/azure-iot-modelsrepository/samples/README.md @@ -2,11 +2,11 @@ This directory contains samples showing how to use the features of the Azure IoT Models Repository Library. -The pre-configured endpoints and DTMIs within the sampmles refer to example DTDL documents that can be found on [devicemodels.azure.com](https://devicemodels.azure.com/). These values can be replaced to reflect the locations of your own DTDLs, wherever they may be. +The pre-configured endpoints and DTMIs within the samples refer to example models that can be found on [devicemodels.azure.com](https://devicemodels.azure.com/). These values can be replaced to reflect the locations of your own models, wherever they may be. -## Resolver Samples -* [get_models_sample.py](get_models_sample.py) - Retrieve a model/models (and possibly dependencies) from a Model Repository, given a DTMI or DTMIs +## ModelsRepositoryClient Samples +* [get_models_sample.py] - Retrieve a model/models (and possibly dependencies) from a Model Repository, given a DTMI or DTMIs -* [client_configuration_sample.py](client_configuration_sample.py) - Configure the client to work with local or remote repositories, as well as custom policies and transports +* [client_configuration_sample.py] - Configure the client to work with local or remote repositories, as well as custom policies and transports -* [dtmi_conventions_sample.py](dtmi_conventions_sample.py) - Use the `dtmi_conventions` module to manipulate and check DTMIs \ No newline at end of file +* [dtmi_conventions_sample.py] - Use the `dtmi_conventions` module to manipulate and check DTMIs \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py b/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py index b6fe5b6958eb..1cb5a64c6379 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py +++ b/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py @@ -8,23 +8,29 @@ DEPENDENCY_MODE_TRY_FROM_EXPANDED, DEPENDENCY_MODE_ENABLED, ) +from azure.core.exceptions import ( + ResourceNotFoundError, + ServiceRequestError, + ServiceResponseError, + HttpResponseError, +) import pprint dtmi = "dtmi:com:example:TemperatureController;1" dtmi2 = "dtmi:com:example:TemperatureController;2" -# By default this client will use the Azure Device Models Repository endpoint +# By default the clients in this sample will use the Azure Device Models Repository endpoint # i.e. https://devicemodels.azure.com/ # See client_configuration_sample.py for examples of alternate configurations -client = ModelsRepositoryClient() def get_model(): # This API call will return a dictionary mapping DTMI to its corresponding model from # a DTDL document at the specified endpoint # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.json - model_map = client.get_models(dtmi) - pprint.pprint(model_map) + with ModelsRepositoryClient() as client: + model_map = client.get_models(dtmi) + pprint.pprint(model_map) def get_models(): @@ -32,24 +38,47 @@ def get_models(): # DTDL documents at the specified endpoint # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.json # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-2.json - model_map = client.get_models([dtmi, dtmi2]) - pprint.pprint(model_map) + with ModelsRepositoryClient() as client: + model_map = client.get_models([dtmi, dtmi2]) + pprint.pprint(model_map) def get_model_expanded_dtdl(): # This API call will return a dictionary mapping DTMIs to corresponding models for all elements # of an expanded DTDL document at the specified endpoint # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.expanded.json - model_map = client.get_models( - dtmis=[dtmi], dependency_resolution=DEPENDENCY_MODE_TRY_FROM_EXPANDED - ) - pprint.pprint(model_map) + with ModelsRepositoryClient() as client: + model_map = client.get_models( + dtmis=[dtmi], dependency_resolution=DEPENDENCY_MODE_TRY_FROM_EXPANDED + ) + pprint.pprint(model_map) def get_model_and_dependencies(): - # This API call will return a dictionary mapping the specified DTMI to it's corresponding model, + # This API call will return a dictionary mapping the specified DTMI to its corresponding model, # from a DTDL document at the specified endpoint, as well as the DTMIs and models for all # dependencies on components and interfaces # i.e. https://devicemodels.azure.com/dtmi/com/example/temperaturecontroller-1.json - model_map = client.get_models(dtmis=[dtmi], dependency_resolution=DEPENDENCY_MODE_ENABLED) - pprint.pprint(model_map) + with ModelsRepositoryClient() as client: + model_map = client.get_models(dtmis=[dtmi], dependency_resolution=DEPENDENCY_MODE_ENABLED) + pprint.pprint(model_map) + + +def get_model_error_handling(): + # Various errors that can be raised when fetching models + try: + with ModelsRepositoryClient() as client: + model_map = client.get_models(dtmi) + pprint.pprint(model_map) + except ResourceNotFoundError as e: + print("The model could not be found") + print("{}".format(e.message)) + except ServiceRequestError as e: + print("There was an error sending the request") + print("{}".format(e.message)) + except ServiceResponseError as e: + print("No response was received") + print("{}".format(e.message)) + except HttpResponseError as e: + print("HTTP Error Response received") + print("{}".format(e.message)) diff --git a/sdk/iot/azure-iot-modelsrepository/samples/scratch.py b/sdk/iot/azure-iot-modelsrepository/samples/scratch.py deleted file mode 100644 index cca8d2ecbbf5..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/samples/scratch.py +++ /dev/null @@ -1,7 +0,0 @@ -from azure.iot.modelsrepository import ModelsRepositoryClient -import pprint - -client = ModelsRepositoryClient(repository_location="https://devicemodels.azure.com") -models = client.get_models(["dtmi:com:example:TemperatureController;1"]) -pprint.pprint(models) - diff --git a/sdk/iot/azure-iot-modelsrepository/setup.cfg b/sdk/iot/azure-iot-modelsrepository/setup.cfg new file mode 100644 index 000000000000..3c6e79cf31da --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/sdk/iot/azure-iot-modelsrepository/setup.py b/sdk/iot/azure-iot-modelsrepository/setup.py index 3c526d112747..dfa86d6cb0ca 100644 --- a/sdk/iot/azure-iot-modelsrepository/setup.py +++ b/sdk/iot/azure-iot-modelsrepository/setup.py @@ -4,8 +4,8 @@ # license information. # -------------------------------------------------------------------------- -from setuptools import setup, find_packages import re +from setuptools import setup, find_packages # azure v0.x is not compatible with this package # azure v0.x used to have a __version__ attribute (newer versions don't) @@ -44,10 +44,14 @@ author="Microsoft Corporation", author_email="azpysdkhelp@microsoft.com", url="https://github.com/Azure/azure-sdk-for-python", + project_urls={ + "Bug Tracker": "https://github.com/Azure/azure-sdk-for-python/issues", + "Source": "https://github.com/Azure/azure-sdk-for-python", + }, long_description=_long_description, long_description_content_type="text/markdown", classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: MIT License", @@ -62,10 +66,10 @@ "Programming Language :: Python :: 3.9", ], install_requires=[ - "azure-core", - "six", + "azure-core<2.0.0,>=1.2.2", + "six>=1.11.0", ], - extras_require={":python_version<'3.0'": ["azure-iot-nspkg>=1.0.1"]}, + extras_require={":python_version<'3.0'": ["azure-iot-nspkg"]}, python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3*, !=3.4.*", packages=find_packages( exclude=[ diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json new file mode 100644 index 000000000000..8a37e6d2c2c3 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json @@ -0,0 +1,64 @@ +{ + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "@type": "Interface", + "displayName": "Device Information", + "contents": [ + { + "@type": "Property", + "name": "manufacturer", + "displayName": "Manufacturer", + "schema": "string", + "description": "Company name of the device manufacturer. This could be the same as the name of the original equipment manufacturer (OEM). Ex. Contoso." + }, + { + "@type": "Property", + "name": "model", + "displayName": "Device model", + "schema": "string", + "description": "Device model name or ID. Ex. Surface Book 2." + }, + { + "@type": "Property", + "name": "swVersion", + "displayName": "Software version", + "schema": "string", + "description": "Version of the software on your device. This could be the version of your firmware. Ex. 1.3.45" + }, + { + "@type": "Property", + "name": "osName", + "displayName": "Operating system name", + "schema": "string", + "description": "Name of the operating system on the device. Ex. Windows 10 IoT Core." + }, + { + "@type": "Property", + "name": "processorArchitecture", + "displayName": "Processor architecture", + "schema": "string", + "description": "Architecture of the processor on the device. Ex. x64 or ARM." + }, + { + "@type": "Property", + "name": "processorManufacturer", + "displayName": "Processor manufacturer", + "schema": "string", + "description": "Name of the manufacturer of the processor on the device. Ex. Intel." + }, + { + "@type": "Property", + "name": "totalStorage", + "displayName": "Total storage", + "schema": "double", + "description": "Total available storage on the device in kilobytes. Ex. 2048000 kilobytes." + }, + { + "@type": "Property", + "name": "totalMemory", + "displayName": "Total memory", + "schema": "double", + "description": "Total available memory on the device in kilobytes. Ex. 256000 kilobytes." + } + ] +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json new file mode 100644 index 000000000000..d35b8a3e3a1d --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json @@ -0,0 +1,16 @@ +{ + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:azure:DeviceManagement:DeviceInformation;2", + "@type": "Interface", + "extends": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "displayName": "Device Information", + "contents": [ + { + "@type": "Property", + "name": "osKernelVersion", + "displayName": "OS Kernel Version", + "schema": "string", + "description": "OS Kernel Version. Ex. Linux 4.15.0-54-generic x86_64." + } + ] +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json new file mode 100644 index 000000000000..85424e8a229e --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json @@ -0,0 +1,56 @@ +{ + "@id": "dtmi:com:example:base;1", + "@type": "Interface", + "contents": [ + { + "@type": "Property", + "name": "baseSerialNumber", + "schema": "string" + } + ], + "displayName": { + "en": "mybaseProp" + }, + "extends": [ + { + "@id": "dtmi:com:example:basic;1", + "@type": "Interface", + "contents": [ + { + "@type": "Property", + "name": "serialNumber", + "schema": "string", + "writable": false + }, + { + "@type": [ + "Telemetry", + "Temperature" + ], + "displayName": { + "en": "temperature" + }, + "name": "temperature", + "schema": "double", + "unit": "degreeCelsius" + }, + { + "@type": "Property", + "displayName": { + "en": "targetTemperature" + }, + "name": "targetTemperature", + "schema": "double", + "writable": true + } + ], + "displayName": { + "en": "Basic" + } + } + ], + "@context": [ + "dtmi:iotcentral:context;2", + "dtmi:dtdl:context;2" + ] +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json new file mode 100644 index 000000000000..29accafcc252 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json @@ -0,0 +1,57 @@ +{ + "@id": "dtmi:com:example:base;2", + "@type": "Interface", + "contents": [ + { + "@type": "Property", + "name": "baseSerialNumber", + "schema": "string" + } + ], + "displayName": { + "en": "mybaseProp" + }, + "extends": [ + { + "@id": "dtmi:com:example:basic;1", + "@type": "Interface", + "contents": [ + { + "@type": "Property", + "name": "serialNumber", + "schema": "string", + "writable": false + }, + { + "@type": [ + "Telemetry", + "Temperature" + ], + "displayName": { + "en": "temperature" + }, + "name": "temperature", + "schema": "double", + "unit": "degreeCelsius" + }, + { + "@type": "Property", + "displayName": { + "en": "targetTemperature" + }, + "name": "targetTemperature", + "schema": "double", + "writable": true + } + ], + "displayName": { + "en": "Basic" + } + }, + "dtmi:com:example:Freezer;1" + ], + "@context": [ + "dtmi:iotcentral:context;2", + "dtmi:dtdl:context;2" + ] +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json new file mode 100644 index 000000000000..c8b6bc6f7375 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json @@ -0,0 +1,19 @@ +{ + "@id": "dtmi:com:example:Building;1", + "@type": "Interface", + "displayName": "Building", + "contents": [ + { + "@type": "Property", + "name": "name", + "schema": "string", + "writable": true + }, + { + "@type": "Relationship", + "name": "contains", + "target": "dtmi:com:example:Room;1" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json new file mode 100644 index 000000000000..f912746c0040 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json @@ -0,0 +1,13 @@ +{ + "@id": "dtmi:com:example:Camera;3", + "@type": "Interface", + "displayName": "Phone", + "contents": [ + { + "@type": "Component", + "name": "deviceInfo", + "schema": "dtmi:azure:DeviceManagement:DeviceInformation;1" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json new file mode 100644 index 000000000000..a3b8466118a9 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json @@ -0,0 +1,13 @@ +{ + "@id": "dtmi:com:example:ColdStorage;1", + "@type": "Interface", + "extends": ["dtmi:com:example:Room;1", "dtmi:com:example:Freezer;1"], + "contents": [ + { + "@type": "Property", + "name": "capacity", + "schema": "integer" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json new file mode 100644 index 000000000000..2e756ee73b6e --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json @@ -0,0 +1,13 @@ +{ + "@id": "dtmi:com:example:ConferenceRoom;1", + "@type": "Interface", + "extends": "dtmi:com:example:Room;1", + "contents": [ + { + "@type": "Property", + "name": "capacity", + "schema": "integer" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json new file mode 100644 index 000000000000..93126c749ce9 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json @@ -0,0 +1,215 @@ +[ + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:com:example:DanglingExpanded;1", + "@type": "Interface", + "displayName": "Valid expanded model with no root model.", + "description": "Device with two thermostats and remote reboot.", + "contents": [ + { + "@type": [ + "Telemetry", + "DataSize" + ], + "name": "workingSet", + "displayName": "Working Set", + "description": "Current working set of the device memory in KiB.", + "schema": "double", + "unit": "kibibyte" + }, + { + "@type": "Property", + "name": "serialNumber", + "displayName": "Serial Number", + "description": "Serial number of the device.", + "schema": "string" + }, + { + "@type": "Command", + "name": "reboot", + "displayName": "Reboot", + "description": "Reboots the device after waiting the number of seconds specified.", + "request": { + "name": "delay", + "displayName": "Delay", + "description": "Number of seconds to wait before rebooting the device.", + "schema": "integer" + } + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat1", + "displayName": "Thermostat One", + "description": "Thermostat One of Two." + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat2", + "displayName": "Thermostat Two", + "description": "Thermostat Two of Two." + }, + { + "@type": "Component", + "schema": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "name": "deviceInformation", + "displayName": "Device Information interface", + "description": "Optional interface with basic device hardware information." + } + ] + }, + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:com:example:Thermostat;1", + "@type": "Interface", + "displayName": "Thermostat", + "description": "Reports current temperature and provides desired temperature control.", + "contents": [ + { + "@type": [ + "Telemetry", + "Temperature" + ], + "name": "temperature", + "displayName": "Temperature", + "description": "Temperature in degrees Celsius.", + "schema": "double", + "unit": "degreeCelsius" + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "targetTemperature", + "schema": "double", + "displayName": "Target Temperature", + "description": "Allows to remotely specify the desired target temperature.", + "unit": "degreeCelsius", + "writable": true + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "maxTempSinceLastReboot", + "schema": "double", + "unit": "degreeCelsius", + "displayName": "Max temperature since last reboot.", + "description": "Returns the max temperature since last device reboot." + }, + { + "@type": "Command", + "name": "getMaxMinReport", + "displayName": "Get Max-Min report.", + "description": "This command returns the max, min and average temperature from the specified time to the current time.", + "request": { + "name": "since", + "displayName": "Since", + "description": "Period to return the max-min report.", + "schema": "dateTime" + }, + "response": { + "name": "tempReport", + "displayName": "Temperature Report", + "schema": { + "@type": "Object", + "fields": [ + { + "name": "maxTemp", + "displayName": "Max temperature", + "schema": "double" + }, + { + "name": "minTemp", + "displayName": "Min temperature", + "schema": "double" + }, + { + "name": "avgTemp", + "displayName": "Average Temperature", + "schema": "double" + }, + { + "name": "startTime", + "displayName": "Start Time", + "schema": "dateTime" + }, + { + "name": "endTime", + "displayName": "End Time", + "schema": "dateTime" + } + ] + } + } + } + ] + }, + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "@type": "Interface", + "displayName": "Device Information", + "contents": [ + { + "@type": "Property", + "name": "manufacturer", + "displayName": "Manufacturer", + "schema": "string", + "description": "Company name of the device manufacturer. This could be the same as the name of the original equipment manufacturer (OEM). Ex. Contoso." + }, + { + "@type": "Property", + "name": "model", + "displayName": "Device model", + "schema": "string", + "description": "Device model name or ID. Ex. Surface Book 2." + }, + { + "@type": "Property", + "name": "swVersion", + "displayName": "Software version", + "schema": "string", + "description": "Version of the software on your device. This could be the version of your firmware. Ex. 1.3.45" + }, + { + "@type": "Property", + "name": "osName", + "displayName": "Operating system name", + "schema": "string", + "description": "Name of the operating system on the device. Ex. Windows 10 IoT Core." + }, + { + "@type": "Property", + "name": "processorArchitecture", + "displayName": "Processor architecture", + "schema": "string", + "description": "Architecture of the processor on the device. Ex. x64 or ARM." + }, + { + "@type": "Property", + "name": "processorManufacturer", + "displayName": "Processor manufacturer", + "schema": "string", + "description": "Name of the manufacturer of the processor on the device. Ex. Intel." + }, + { + "@type": "Property", + "name": "totalStorage", + "displayName": "Total storage", + "schema": "double", + "description": "Total available storage on the device in kilobytes. Ex. 2048000 kilobytes." + }, + { + "@type": "Property", + "name": "totalMemory", + "displayName": "Total memory", + "schema": "double", + "description": "Total available memory on the device in kilobytes. Ex. 256000 kilobytes." + } + ] + } +] diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json new file mode 100644 index 000000000000..6006b6673299 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json @@ -0,0 +1,12 @@ +{ + "@id": "dtmi:com:example:Freezer;1", + "@type": "Interface", + "contents": [ + { + "@type": "Component", + "name": "deviceInfo", + "schema": "dtmi:com:example:Thermostat;1" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json new file mode 100644 index 000000000000..1688ef4c0e3c --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json @@ -0,0 +1,151 @@ +[ + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:com:example:IncompleteExpanded;1", + "@type": "Interface", + "displayName": "Incomplete expanded Temperature Controller", + "description": "Device with two thermostats and remote reboot.", + "contents": [ + { + "@type": [ + "Telemetry", + "DataSize" + ], + "name": "workingSet", + "displayName": "Working Set", + "description": "Current working set of the device memory in KiB.", + "schema": "double", + "unit": "kibibyte" + }, + { + "@type": "Property", + "name": "serialNumber", + "displayName": "Serial Number", + "description": "Serial number of the device.", + "schema": "string" + }, + { + "@type": "Command", + "name": "reboot", + "displayName": "Reboot", + "description": "Reboots the device after waiting the number of seconds specified.", + "request": { + "name": "delay", + "displayName": "Delay", + "description": "Number of seconds to wait before rebooting the device.", + "schema": "integer" + } + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat1", + "displayName": "Thermostat One", + "description": "Thermostat One of Two." + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat2", + "displayName": "Thermostat Two", + "description": "Thermostat Two of Two." + }, + { + "@type": "Component", + "schema": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "name": "deviceInformation", + "displayName": "Device Information interface", + "description": "Optional interface with basic device hardware information." + } + ] + }, + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:com:example:Thermostat;1", + "@type": "Interface", + "displayName": "Thermostat", + "description": "Reports current temperature and provides desired temperature control.", + "contents": [ + { + "@type": [ + "Telemetry", + "Temperature" + ], + "name": "temperature", + "displayName": "Temperature", + "description": "Temperature in degrees Celsius.", + "schema": "double", + "unit": "degreeCelsius" + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "targetTemperature", + "schema": "double", + "displayName": "Target Temperature", + "description": "Allows to remotely specify the desired target temperature.", + "unit": "degreeCelsius", + "writable": true + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "maxTempSinceLastReboot", + "schema": "double", + "unit": "degreeCelsius", + "displayName": "Max temperature since last reboot.", + "description": "Returns the max temperature since last device reboot." + }, + { + "@type": "Command", + "name": "getMaxMinReport", + "displayName": "Get Max-Min report.", + "description": "This command returns the max, min and average temperature from the specified time to the current time.", + "request": { + "name": "since", + "displayName": "Since", + "description": "Period to return the max-min report.", + "schema": "dateTime" + }, + "response": { + "name": "tempReport", + "displayName": "Temperature Report", + "schema": { + "@type": "Object", + "fields": [ + { + "name": "maxTemp", + "displayName": "Max temperature", + "schema": "double" + }, + { + "name": "minTemp", + "displayName": "Min temperature", + "schema": "double" + }, + { + "name": "avgTemp", + "displayName": "Average Temperature", + "schema": "double" + }, + { + "name": "startTime", + "displayName": "Start Time", + "schema": "dateTime" + }, + { + "name": "endTime", + "displayName": "End Time", + "schema": "dateTime" + } + ] + } + } + } + ] + } +] diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json new file mode 100644 index 000000000000..4f18d7b17658 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json @@ -0,0 +1,13 @@ +{ + "@id": "dtmi:com:example:invalidmodel;1", + "@type": "Interface", + "displayName": "Phone", + "contents": [ + { + "@type": "Component", + "name": "deviceInfo", + "schema": "dtmi:azure:fakeDeviceManagement:FakeDeviceInformation;2" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json new file mode 100644 index 000000000000..61443734cd90 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json @@ -0,0 +1,23 @@ +{ + "@id": "dtmi:com:example:Phone;2", + "@type": "Interfacez", + "displayName": "Phone", + "contentsz": [ + { + "@type": "Component", + "name": "frontCamera", + "schema": "dtmi:com:example:Camera;3" + }, + { + "@type": "Component", + "name": "backCamera", + "schema": "dtmi:com:example:Camera;3" + }, + { + "@type": "Component", + "name": "deviceInfo", + "schema": "dtmi:azure:deviceManagement:DeviceInformation;2" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json new file mode 100644 index 000000000000..26c7efbdedc0 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json @@ -0,0 +1,23 @@ +{ + "@id": "dtmi:com:example:Phone;2", + "@type": "Interface", + "displayName": "Phone", + "contents": [ + { + "@type": "Component", + "name": "frontCamera", + "schema": "dtmi:com:example:Camera;3" + }, + { + "@type": "Component", + "name": "backCamera", + "schema": "dtmi:com:example:Camera;3" + }, + { + "@type": "Component", + "name": "deviceInfo", + "schema": "dtmi:azure:DeviceManagement:DeviceInformation;2" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json new file mode 100644 index 000000000000..1a07edec4d98 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json @@ -0,0 +1,12 @@ +{ + "@id": "dtmi:com:example:Room;1", + "@type": "Interface", + "contents": [ + { + "@type": "Property", + "name": "occupied", + "schema": "boolean" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json new file mode 100644 index 000000000000..14e8e294189e --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json @@ -0,0 +1,215 @@ +[ + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:com:example:TemperatureController;1", + "@type": "Interface", + "displayName": "Temperature Controller", + "description": "Device with two thermostats and remote reboot.", + "contents": [ + { + "@type": [ + "Telemetry", + "DataSize" + ], + "name": "workingSet", + "displayName": "Working Set", + "description": "Current working set of the device memory in KiB.", + "schema": "double", + "unit": "kibibyte" + }, + { + "@type": "Property", + "name": "serialNumber", + "displayName": "Serial Number", + "description": "Serial number of the device.", + "schema": "string" + }, + { + "@type": "Command", + "name": "reboot", + "displayName": "Reboot", + "description": "Reboots the device after waiting the number of seconds specified.", + "request": { + "name": "delay", + "displayName": "Delay", + "description": "Number of seconds to wait before rebooting the device.", + "schema": "integer" + } + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat1", + "displayName": "Thermostat One", + "description": "Thermostat One of Two." + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat2", + "displayName": "Thermostat Two", + "description": "Thermostat Two of Two." + }, + { + "@type": "Component", + "schema": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "name": "deviceInformation", + "displayName": "Device Information interface", + "description": "Optional interface with basic device hardware information." + } + ] + }, + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:com:example:Thermostat;1", + "@type": "Interface", + "displayName": "Thermostat", + "description": "Reports current temperature and provides desired temperature control.", + "contents": [ + { + "@type": [ + "Telemetry", + "Temperature" + ], + "name": "temperature", + "displayName": "Temperature", + "description": "Temperature in degrees Celsius.", + "schema": "double", + "unit": "degreeCelsius" + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "targetTemperature", + "schema": "double", + "displayName": "Target Temperature", + "description": "Allows to remotely specify the desired target temperature.", + "unit": "degreeCelsius", + "writable": true + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "maxTempSinceLastReboot", + "schema": "double", + "unit": "degreeCelsius", + "displayName": "Max temperature since last reboot.", + "description": "Returns the max temperature since last device reboot." + }, + { + "@type": "Command", + "name": "getMaxMinReport", + "displayName": "Get Max-Min report.", + "description": "This command returns the max, min and average temperature from the specified time to the current time.", + "request": { + "name": "since", + "displayName": "Since", + "description": "Period to return the max-min report.", + "schema": "dateTime" + }, + "response": { + "name": "tempReport", + "displayName": "Temperature Report", + "schema": { + "@type": "Object", + "fields": [ + { + "name": "maxTemp", + "displayName": "Max temperature", + "schema": "double" + }, + { + "name": "minTemp", + "displayName": "Min temperature", + "schema": "double" + }, + { + "name": "avgTemp", + "displayName": "Average Temperature", + "schema": "double" + }, + { + "name": "startTime", + "displayName": "Start Time", + "schema": "dateTime" + }, + { + "name": "endTime", + "displayName": "End Time", + "schema": "dateTime" + } + ] + } + } + } + ] + }, + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "@type": "Interface", + "displayName": "Device Information", + "contents": [ + { + "@type": "Property", + "name": "manufacturer", + "displayName": "Manufacturer", + "schema": "string", + "description": "Company name of the device manufacturer. This could be the same as the name of the original equipment manufacturer (OEM). Ex. Contoso." + }, + { + "@type": "Property", + "name": "model", + "displayName": "Device model", + "schema": "string", + "description": "Device model name or ID. Ex. Surface Book 2." + }, + { + "@type": "Property", + "name": "swVersion", + "displayName": "Software version", + "schema": "string", + "description": "Version of the software on your device. This could be the version of your firmware. Ex. 1.3.45" + }, + { + "@type": "Property", + "name": "osName", + "displayName": "Operating system name", + "schema": "string", + "description": "Name of the operating system on the device. Ex. Windows 10 IoT Core." + }, + { + "@type": "Property", + "name": "processorArchitecture", + "displayName": "Processor architecture", + "schema": "string", + "description": "Architecture of the processor on the device. Ex. x64 or ARM." + }, + { + "@type": "Property", + "name": "processorManufacturer", + "displayName": "Processor manufacturer", + "schema": "string", + "description": "Name of the manufacturer of the processor on the device. Ex. Intel." + }, + { + "@type": "Property", + "name": "totalStorage", + "displayName": "Total storage", + "schema": "double", + "description": "Total available storage on the device in kilobytes. Ex. 2048000 kilobytes." + }, + { + "@type": "Property", + "name": "totalMemory", + "displayName": "Total memory", + "schema": "double", + "description": "Total available memory on the device in kilobytes. Ex. 256000 kilobytes." + } + ] + } +] \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json new file mode 100644 index 000000000000..c455ddf8bae6 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json @@ -0,0 +1,60 @@ +{ + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:com:example:TemperatureController;1", + "@type": "Interface", + "displayName": "Temperature Controller", + "description": "Device with two thermostats and remote reboot.", + "contents": [ + { + "@type": [ + "Telemetry", + "DataSize" + ], + "name": "workingSet", + "displayName": "Working Set", + "description": "Current working set of the device memory in KiB.", + "schema": "double", + "unit": "kibibyte" + }, + { + "@type": "Property", + "name": "serialNumber", + "displayName": "Serial Number", + "description": "Serial number of the device.", + "schema": "string" + }, + { + "@type": "Command", + "name": "reboot", + "displayName": "Reboot", + "description": "Reboots the device after waiting the number of seconds specified.", + "request": { + "name": "delay", + "displayName": "Delay", + "description": "Number of seconds to wait before rebooting the device.", + "schema": "integer" + } + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat1", + "displayName": "Thermostat One", + "description": "Thermostat One of Two." + }, + { + "@type": "Component", + "schema": "dtmi:com:example:Thermostat;1", + "name": "thermostat2", + "displayName": "Thermostat Two", + "description": "Thermostat Two of Two." + }, + { + "@type": "Component", + "schema": "dtmi:azure:DeviceManagement:DeviceInformation;1", + "name": "deviceInformation", + "displayName": "Device Information interface", + "description": "Optional interface with basic device hardware information." + } + ] +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json new file mode 100644 index 000000000000..315a307bbcb3 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json @@ -0,0 +1,19 @@ +{ + "@id": "dtmi:com:example:Thermostat;1", + "@type": "Interface", + "displayName": "Thermostat", + "contents": [ + { + "@type": "Telemetry", + "name": "temp", + "schema": "double" + }, + { + "@type": "Property", + "name": "setPointTemp", + "writable": true, + "schema": "double" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json new file mode 100644 index 000000000000..33c9554664a6 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json @@ -0,0 +1,31 @@ +{ + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:company:demodevice;1", + "@type": "Interface", + "displayName": "demodevice", + "contents": [ + { + "@type": "Component", + "name": "c1", + "schema": "dtmi:azure:deviceManagement:DeviceInformation;1" + }, + { + "@type": "Telemetry", + "name": "temperature", + "schema": "double" + }, + { + "@type": "Property", + "name": "deviceStatus", + "schema": "string" + }, + { + "@type": "Command", + "name": "reboot", + "request": { + "name": "delay", + "schema": "integer" + } + } + ] +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json new file mode 100644 index 000000000000..9d9b3bd2322a --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json @@ -0,0 +1,31 @@ +{ + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:company:demodevice;1", + "@type": "Interface", + "displayName": "demodevice", + "contents": [ + { + "@type": "Component", + "name": "c1", + "schema": "dtmi:azure:DeviceManagement:DeviceInformation;1" + }, + { + "@type": "Telemetry", + "name": "temperature", + "schema": "double" + }, + { + "@type": "Property", + "name": "deviceStatus", + "schema": "string" + }, + { + "@type": "Command", + "name": "reboot", + "request": { + "name": "delay", + "schema": "integer" + } + } + ] +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json new file mode 100644 index 000000000000..6006b6673299 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json @@ -0,0 +1,12 @@ +{ + "@id": "dtmi:com:example:Freezer;1", + "@type": "Interface", + "contents": [ + { + "@type": "Component", + "name": "deviceInfo", + "schema": "dtmi:com:example:Thermostat;1" + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json new file mode 100644 index 000000000000..0637a088a01e --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json new file mode 100644 index 000000000000..6f2df812a67f --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json @@ -0,0 +1,37 @@ +{ + "@id": "dtmi:strict:namespaceconflict;1", + "@type": "Interface", + "contents": [ + { + "@type": "Telemetry", + "name": "accelerometer1", + "schema": "dtmi:com:example:acceleration;1" + }, + { + "@type": "Telemetry", + "name": "accelerometer2", + "schema": "dtmi:com:example:acceleration;1" + } + ], + "schemas": [ + { + "@id": "dtmi:com:example:acceleration;1", + "@type": "Object", + "fields": [ + { + "name": "x", + "schema": "double" + }, + { + "name": "y", + "schema": "double" + }, + { + "name": "z", + "schema": "double" + } + ] + } + ], + "@context": "dtmi:dtdl:context;2" +} \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json new file mode 100644 index 000000000000..b25ba2ac3af6 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json @@ -0,0 +1 @@ +"content" \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json new file mode 100644 index 000000000000..1f282307a8c3 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json @@ -0,0 +1,91 @@ +[ + { + "@context": "dtmi:dtdl:context;2", + "@id": "dtmi:strict:unsupportedrootarray;1", + "@type": "Interface", + "displayName": "Thermostat", + "description": "Reports current temperature and provides desired temperature control.", + "contents": [ + { + "@type": [ + "Telemetry", + "Temperature" + ], + "name": "temperature", + "displayName": "Temperature", + "description": "Temperature in degrees Celsius.", + "schema": "double", + "unit": "degreeCelsius" + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "targetTemperature", + "schema": "double", + "displayName": "Target Temperature", + "description": "Allows to remotely specify the desired target temperature.", + "unit": "degreeCelsius", + "writable": true + }, + { + "@type": [ + "Property", + "Temperature" + ], + "name": "maxTempSinceLastReboot", + "schema": "double", + "unit": "degreeCelsius", + "displayName": "Max temperature since last reboot.", + "description": "Returns the max temperature since last device reboot." + }, + { + "@type": "Command", + "name": "getMaxMinReport", + "displayName": "Get Max-Min report.", + "description": "This command returns the max, min and average temperature from the specified time to the current time.", + "request": { + "name": "since", + "displayName": "Since", + "description": "Period to return the max-min report.", + "schema": "dateTime" + }, + "response": { + "name": "tempReport", + "displayName": "Temperature Report", + "schema": { + "@type": "Object", + "fields": [ + { + "name": "maxTemp", + "displayName": "Max temperature", + "schema": "double" + }, + { + "name": "minTemp", + "displayName": "Min temperature", + "schema": "double" + }, + { + "name": "avgTemp", + "displayName": "Average Temperature", + "schema": "double" + }, + { + "name": "startTime", + "displayName": "Start Time", + "schema": "dateTime" + }, + { + "name": "endTime", + "displayName": "End Time", + "schema": "dateTime" + } + ] + } + } + } + ] + } +] \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_client.py b/sdk/iot/azure-iot-modelsrepository/tests/test_client.py new file mode 100644 index 000000000000..880a8cdfc976 --- /dev/null +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_client.py @@ -0,0 +1,139 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from azure.iot.modelsrepository._client import ( + ModelsRepositoryClient, + DEPENDENCY_MODE_TRY_FROM_EXPANDED, + DEPENDENCY_MODE_ENABLED, + DEPENDENCY_MODE_DISABLED, +) +from azure.iot.modelsrepository._resolver import HttpFetcher, FilesystemFetcher + + +@pytest.mark.describe("ModelsRepositoryClient - Instantiation") +class TestModelsRepositoryClientInstantiation(object): + @pytest.mark.it( + "Defaults to using 'https://devicemodels.azure.com' as the repository location if one is not provided" + ) + def test_default_repository_location(self): + client = ModelsRepositoryClient() + assert client.fetcher.base_url == "https://devicemodels.azure.com" + + @pytest.mark.it( + "Is configured for HTTP/HTTPS REST operations if provided a repository location in URL format" + ) + @pytest.mark.parametrize( + "url", + [ + pytest.param("http://myfakerepository.com/", id="HTTP URL, with trailing '/'"), + pytest.param("http://myfakerepository.com", id="HTTP URL, no trailing '/'"), + pytest.param("https://myfakerepository.com/", id="HTTPS URL, with trailing '/'"), + pytest.param("https://myfakerepository.com", id="HTTPS URL, no trailing '/'"), + pytest.param( + "myfakerepository.com/", id="Web URL, no protocol specified, with trailing '/'" + ), + pytest.param( + "myfakerepository.com", id="Web URL, no protocol specified, no trailing '/'" + ), + ], + ) + def test_repository_location_remote(self, url): + client = ModelsRepositoryClient(repository_location=url) + assert isinstance(client.fetcher, HttpFetcher) + + @pytest.mark.it( + "Is configured for local filesystem operations if provided a repository location in filesystem format" + ) + @pytest.mark.parametrize( + "path", + [ + pytest.param( + "F:/repos/myrepo/", + id="Drive letter filesystem path, '/' separators, with trailing '/'", + ), + pytest.param( + "F:/repos/myrepo", + id="Drive letter filesystem path, '/' separators, no trailing '/'", + ), + pytest.param( + "F:\\repos\\myrepo\\", + id="Drive letter filesystem path, '\\' separators, with trailing '\\'", + ), + pytest.param( + "F:\\repos\\myrepo", + id="Drive letter filesystem path, '\\' separators, no trailing '\\'", + ), + pytest.param( + "F:\\repos/myrepo/", + id="Drive letter filesystem path, mixed separators, with trailing separator", + ), + pytest.param( + "F:\\repos/myrepo", + id="Drive letter filesystem path, mixed separators, no trailing separator", + ), + pytest.param("/repos/myrepo/", id="POSIX filesystem path, with trailing '/'"), + pytest.param("/repos/myrepo", id="POSIX filesystem path, no trailing '/'"), + pytest.param( + "file:///f:/repos/myrepo/", + id="URI scheme, drive letter filesystem path, with trailing '/'", + ), + pytest.param( + "file:///f:/repos/myrepo", + id="URI scheme, drive letter filesystem path, no trailing '/'", + ), + pytest.param( + "file:///repos/myrepo/", id="URI scheme, POSIX filesystem path, with trailing '/'" + ), + pytest.param( + "file:///repos/myrepo", id="URI schem, POSIX filesystem path, no trailing '/'" + ), + ], + ) + def test_repository_location_local(self, path): + client = ModelsRepositoryClient(repository_location=path) + assert isinstance(client.fetcher, FilesystemFetcher) + + @pytest.mark.it("Raises ValueError if provided a repository location that cannot be identified") + def test_invalid_repository_location(self): + with pytest.raises(ValueError): + ModelsRepositoryClient(repository_location="not a location") + + @pytest.mark.it( + "Defaults to using 'tryFromExpanded' as the dependency resolution mode if one is not specified while using the default repository location" + ) + def test_default_dependency_resolution_mode_default_location(self): + client = ModelsRepositoryClient() + assert client.resolution_mode == DEPENDENCY_MODE_TRY_FROM_EXPANDED + + @pytest.mark.it( + "Defaults to using 'enabled' as the dependency resolution mode if one is not specified while using a custom repository location" + ) + @pytest.mark.parametrize( + "location", + [ + pytest.param("https://myfakerepository.com", id="Remote repository location"), + pytest.param("/repos/myrepo", id="Local repository location"), + ], + ) + def test_default_dependency_resolution_mode_custom_location(self, location): + client = ModelsRepositoryClient(repository_location=location) + assert client.resolution_mode == DEPENDENCY_MODE_ENABLED + + @pytest.mark.it( + "Is configured with the provided dependency resolution mode if one is specified" + ) + @pytest.mark.parametrize( + "dependency_mode", + [DEPENDENCY_MODE_ENABLED, DEPENDENCY_MODE_DISABLED, DEPENDENCY_MODE_TRY_FROM_EXPANDED], + ) + def test_dependency_mode(self, dependency_mode): + client = ModelsRepositoryClient(dependency_resolution=dependency_mode) + assert client.resolution_mode == dependency_mode + + @pytest.mark.it("Raises ValueError if provided an unrecognized dependency resolution mode") + def test_invalid_dependency_mode(self): + with pytest.raises(ValueError): + ModelsRepositoryClient(dependency_resolution="not a mode") diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py index 25dc7a03f38d..1ad81a500752 100644 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py @@ -4,11 +4,8 @@ # license information. # -------------------------------------------------------------------------- import pytest -import logging from azure.iot.modelsrepository import dtmi_conventions -logging.basicConfig(level=logging.DEBUG) - @pytest.mark.describe(".is_valid_dtmi()") class TestIsValidDTMI(object): @@ -60,7 +57,19 @@ class TestGetModelURI(object): "dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.json", - id="Filesystem URI", + id="POSIX Filesystem URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file://c:/myrepository/", + "file://c:/myrepository/dtmi/com/somedomain/example/foodtdl-1.json", + id="Drive Letter Filesystem URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file://server/myrepository", + "file://server/myrepository/dtmi/com/somedomain/example/foodtdl-1.json", + id="Windows UNC Filesystem URI", ), pytest.param( "dtmi:com:somedomain:example:FooDTDL;1", @@ -86,21 +95,33 @@ def test_uri(self, dtmi, repository_uri, expected_model_uri): [ pytest.param( "dtmi:com:somedomain:example:FooDTDL;1", - "https://myrepository/", - "https://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + "https://myfakerepository.com/", + "https://myfakerepository.com/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="HTTPS repository URI", ), pytest.param( "dtmi:com:somedomain:example:FooDTDL;1", - "http://myrepository/", - "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + "http://myfakerepository.com/", + "http://myfakerepository.com/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="HTTP repository URI", ), pytest.param( "dtmi:com:somedomain:example:FooDTDL;1", "file:///myrepository/", "file:///myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", - id="Filesystem URI", + id="POSIX Filesystem URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file://c:/myrepository/", + "file://c:/myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + id="Drive Letter Filesystem URI", + ), + pytest.param( + "dtmi:com:somedomain:example:FooDTDL;1", + "file://server/myrepository", + "file://server/myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + id="Windows UNC Filesystem URI", ), pytest.param( "dtmi:com:somedomain:example:FooDTDL;1", @@ -110,8 +131,8 @@ def test_uri(self, dtmi, repository_uri, expected_model_uri): ), pytest.param( "dtmi:com:somedomain:example:FooDTDL;1", - "http://myrepository", - "http://myrepository/dtmi/com/somedomain/example/foodtdl-1.expanded.json", + "http://myrepository.com", + "http://myrepository.com/dtmi/com/somedomain/example/foodtdl-1.expanded.json", id="Repository URI without trailing '/'", ), ], diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py b/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py index 6cf72c76cb48..34ee17c071db 100644 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py +++ b/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py @@ -3,16 +3,534 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import pytest -import logging -from azure.iot.modelsrepository import ModelsRepositoryClient +import os +from parameterized import parameterized +from devtools_testutils import AzureTestCase +from azure.core.exceptions import ResourceNotFoundError +from azure.iot.modelsrepository import ( + ModelsRepositoryClient, + DEPENDENCY_MODE_ENABLED, + DEPENDENCY_MODE_DISABLED, + DEPENDENCY_MODE_TRY_FROM_EXPANDED, + ModelError, +) -logging.basicConfig(level=logging.DEBUG) +LOCAL_REPO = "local repo" +REMOTE_REPO = "remote_repo" -@pytest.mark.describe("ModelsRepositoryClient - .get_models() [INTEGRATION]") -class TestModelsRepositoryClientGetModels(object): - @pytest.mark.it("test recordings") - def test_simple(self): - c = ModelsRepositoryClient() - c.get_models(["dtmi:com:example:TemperatureController;1"]) +################################ +# Client Fixture Mixin Classes # +################################ + + +class RemoteRepositoryMixin(object): + def setUp(self): + self.client = ModelsRepositoryClient() + self.client_type = REMOTE_REPO + + def tearDown(self): + self.client.close() + + +class LocalRepositoryMixin(object): + def setUp(self): + test_dir = os.path.dirname(os.path.abspath(__file__)) + local_repo = os.path.join(test_dir, "local_repository") + self.client = ModelsRepositoryClient(repository_location=local_repo) + self.client_type = LOCAL_REPO + + def tearDown(self): + self.client.close() + + +########################### +# Test Case Mixin Classes # +########################### + + +class GetModelsDependencyModeEnabledIntegrationTestCaseMixin(object): + def test_dtmi_mismatch_casing(self): + dtmi = "dtmi:com:example:thermostat;1" + with self.assertRaises(ModelError): + self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + @parameterized.expand( + [ + ("No semicolon", "dtmi:com:example:Thermostat:1"), + ("Double colon", "dtmi:com:example::Thermostat;1"), + ("No DTMI prefix", "com:example:Thermostat;1"), + ] + ) + def test_invalid_dtmi_format(self, _, dtmi): + with self.assertRaises(ValueError): + self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + def test_nonexistant_dtdl_doc(self): + dtmi = "dtmi:com:example:thermojax;999" + with self.assertRaises(ResourceNotFoundError): + self.client.get_models(dtmi) + + def test_nonexistent_dependency_dtdl_doc(self): + dtmi = "dtmi:com:example:invalidmodel;1" + with self.assertRaises(ResourceNotFoundError): + self.client.get_models(dtmi) + + def test_single_dtmi_no_components_no_extends(self): + dtmi = "dtmi:com:example:Thermostat;1" + model_map = self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_no_components_no_extends(self): + dtmi1 = "dtmi:com:example:Thermostat;1" + dtmi2 = "dtmi:azure:DeviceManagement:DeviceInformation;1" + model_map = self.client.get_models( + [dtmi1, dtmi2], dependency_resolution=DEPENDENCY_MODE_ENABLED + ) + + self.assertTrue(len(model_map) == 2) + self.assertTrue(dtmi1 in model_map.keys()) + self.assertTrue(dtmi2 in model_map.keys()) + model1 = model_map[dtmi1] + model2 = model_map[dtmi2] + self.assertTrue(model1["@id"] == dtmi1) + self.assertTrue(model2["@id"] == dtmi2) + + def test_single_dtmi_with_component_deps(self): + root_dtmi = "dtmi:com:example:TemperatureController;1" + expected_deps = [ + "dtmi:com:example:Thermostat;1", + "dtmi:azure:DeviceManagement:DeviceInformation;1", + ] + expected_dtmis = [root_dtmi] + expected_deps + model_map = self.client.get_models(root_dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_with_component_deps(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + root_dtmi1 = "dtmi:com:example:Phone;2" + root_dtmi2 = "dtmi:com:example:TemperatureController;1" + expected_deps = [ + "dtmi:com:example:Thermostat;1", + "dtmi:azure:DeviceManagement:DeviceInformation;1", + "dtmi:azure:DeviceManagement:DeviceInformation;2", + "dtmi:com:example:Camera;3", + ] + expected_dtmis = [root_dtmi1, root_dtmi2] + expected_deps + model_map = self.client.get_models( + [root_dtmi1, root_dtmi2], dependency_resolution=DEPENDENCY_MODE_ENABLED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_with_extends_deps_single_dtmi(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + root_dtmi1 = "dtmi:com:example:TemperatureController;1" + root_dtmi2 = "dtmi:com:example:ConferenceRoom;1" + expected_deps = [ + "dtmi:com:example:Thermostat;1", + "dtmi:azure:DeviceManagement:DeviceInformation;1", + "dtmi:com:example:Room;1", + ] + expected_dtmis = [root_dtmi1, root_dtmi2] + expected_deps + model_map = self.client.get_models( + [root_dtmi1, root_dtmi2], dependency_resolution=DEPENDENCY_MODE_ENABLED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_with_extends_deps_multiple_dtmi(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + root_dtmi1 = "dtmi:com:example:TemperatureController;1" + root_dtmi2 = "dtmi:com:example:ColdStorage;1" + expected_deps = [ + "dtmi:com:example:Thermostat;1", + "dtmi:azure:DeviceManagement:DeviceInformation;1", + "dtmi:com:example:Room;1", + "dtmi:com:example:Freezer;1", + ] + expected_dtmis = [root_dtmi1, root_dtmi2] + expected_deps + model_map = self.client.get_models( + [root_dtmi1, root_dtmi2], dependency_resolution=DEPENDENCY_MODE_ENABLED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_single_dtmi_with_extends_single_model_inline(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + dtmi = "dtmi:com:example:base;1" + model_map = self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_single_dtmi_with_extends_mixed_inline_and_dtmi(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + root_dtmi = "dtmi:com:example:base;2" + expected_deps = ["dtmi:com:example:Freezer;1", "dtmi:com:example:Thermostat;1"] + expected_dtmis = [root_dtmi] + expected_deps + model_map = self.client.get_models(root_dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_duplicate_dtmi(self): + dtmi1 = "dtmi:azure:DeviceManagement:DeviceInformation;1" + dtmi2 = "dtmi:azure:DeviceManagement:DeviceInformation;1" + model_map = self.client.get_models( + [dtmi1, dtmi1], dependency_resolution=DEPENDENCY_MODE_ENABLED + ) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi1 in model_map.keys()) + self.assertTrue(dtmi2 in model_map.keys()) + model = model_map[dtmi1] + self.assertTrue(model["@id"] == dtmi1 == dtmi2) + + +class GetModelsDependencyModeDisabledIntegrationTestCaseMixin(object): + def test_dtmi_mismatch_casing(self): + dtmi = "dtmi:com:example:thermostat;1" + with self.assertRaises(ModelError): + self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_DISABLED) + + @parameterized.expand( + [ + ("No semicolon", "dtmi:com:example:Thermostat:1"), + ("Double colon", "dtmi:com:example::Thermostat;1"), + ("No DTMI prefix", "com:example:Thermostat;1"), + ] + ) + def test_invalid_dtmi_format(self, _, dtmi): + with self.assertRaises(ValueError): + self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_DISABLED) + + def test_nonexistant_dtdl_doc(self): + dtmi = "dtmi:com:example:thermojax;999" + with self.assertRaises(ResourceNotFoundError): + self.client.get_models(dtmi) + + def test_nonexistent_dependency_dtdl_doc(self): + dtmi = "dtmi:com:example:invalidmodel;1" + with self.assertRaises(ResourceNotFoundError): + self.client.get_models(dtmi) + + def test_single_dtmi_no_components_no_extends(self): + dtmi = "dtmi:com:example:Thermostat;1" + model_map = self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_DISABLED) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_no_components_no_extends(self): + dtmi1 = "dtmi:com:example:Thermostat;1" + dtmi2 = "dtmi:azure:DeviceManagement:DeviceInformation;1" + model_map = self.client.get_models( + [dtmi1, dtmi2], dependency_resolution=DEPENDENCY_MODE_DISABLED + ) + + self.assertTrue(len(model_map) == 2) + self.assertTrue(dtmi1 in model_map.keys()) + self.assertTrue(dtmi2 in model_map.keys()) + model1 = model_map[dtmi1] + model2 = model_map[dtmi2] + self.assertTrue(model1["@id"] == dtmi1) + self.assertTrue(model2["@id"] == dtmi2) + + def test_single_dtmi_with_component_deps(self): + dtmi = "dtmi:com:example:TemperatureController;1" + model_map = self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_DISABLED) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_with_component_deps(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + dtmi1 = "dtmi:com:example:Phone;2" + dtmi2 = "dtmi:com:example:TemperatureController;1" + model_map = self.client.get_models( + [dtmi1, dtmi2], dependency_resolution=DEPENDENCY_MODE_DISABLED + ) + + self.assertTrue(len(model_map) == 2) + self.assertTrue(dtmi1 in model_map.keys()) + self.assertTrue(dtmi2 in model_map.keys()) + model1 = model_map[dtmi1] + model2 = model_map[dtmi2] + self.assertTrue(model1["@id"] == dtmi1) + self.assertTrue(model2["@id"] == dtmi2) + + def test_multiple_dtmis_with_extends_deps_single_dtmi(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + dtmi1 = "dtmi:com:example:TemperatureController;1" + dtmi2 = "dtmi:com:example:ConferenceRoom;1" + model_map = self.client.get_models( + [dtmi1, dtmi2], dependency_resolution=DEPENDENCY_MODE_DISABLED + ) + + self.assertTrue(len(model_map) == 2) + self.assertTrue(dtmi1 in model_map.keys()) + self.assertTrue(dtmi2 in model_map.keys()) + model1 = model_map[dtmi1] + model2 = model_map[dtmi2] + self.assertTrue(model1["@id"] == dtmi1) + self.assertTrue(model2["@id"] == dtmi2) + + def test_multiple_dtmis_with_extends_deps_multiple_dtmi(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + dtmi1 = "dtmi:com:example:TemperatureController;1" + dtmi2 = "dtmi:com:example:ColdStorage;1" + model_map = self.client.get_models( + [dtmi1, dtmi2], dependency_resolution=DEPENDENCY_MODE_DISABLED + ) + + self.assertTrue(len(model_map) == 2) + self.assertTrue(dtmi1 in model_map.keys()) + self.assertTrue(dtmi2 in model_map.keys()) + model1 = model_map[dtmi1] + model2 = model_map[dtmi2] + self.assertTrue(model1["@id"] == dtmi1) + self.assertTrue(model2["@id"] == dtmi2) + + def test_single_dtmi_with_extends_single_model_inline(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + dtmi = "dtmi:com:example:base;1" + model_map = self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_DISABLED) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_single_dtmi_with_extends_mixed_inline_and_dtmi(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + dtmi = "dtmi:com:example:base;2" + model_map = self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_DISABLED) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_duplicate_dtmi(self): + dtmi1 = "dtmi:azure:DeviceManagement:DeviceInformation;1" + dtmi2 = "dtmi:azure:DeviceManagement:DeviceInformation;1" + model_map = self.client.get_models( + [dtmi1, dtmi1], dependency_resolution=DEPENDENCY_MODE_DISABLED + ) + + self.assertTrue(len(model_map) == 1) + self.assertTrue(dtmi1 in model_map.keys()) + self.assertTrue(dtmi2 in model_map.keys()) + model = model_map[dtmi1] + self.assertTrue(model["@id"] == dtmi1 == dtmi2) + + +class GetModelsDependencyModeTryFromExpandedIntegrationTestCaseMixin(object): + def test_dtmi_mismatch_casing(self): + dtmi = "dtmi:com:example:thermostat;1" + with self.assertRaises(ModelError): + self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + @parameterized.expand( + [ + ("No semicolon", "dtmi:com:example:Thermostat:1"), + ("Double colon", "dtmi:com:example::Thermostat;1"), + ("No DTMI prefix", "com:example:Thermostat;1"), + ] + ) + def test_invalid_dtmi_format(self, _, dtmi): + with self.assertRaises(ValueError): + self.client.get_models(dtmi, dependency_resolution=DEPENDENCY_MODE_ENABLED) + + def test_nonexistant_dtdl_doc(self): + dtmi = "dtmi:com:example:thermojax;999" + with self.assertRaises(ResourceNotFoundError): + self.client.get_models(dtmi) + + def test_single_dtmi_with_component_deps_expanded_json(self): + root_dtmi = "dtmi:com:example:TemperatureController;1" + expected_deps = [ + "dtmi:com:example:Thermostat;1", + "dtmi:azure:DeviceManagement:DeviceInformation;1", + ] + expected_dtmis = [root_dtmi] + expected_deps + model_map = self.client.get_models( + root_dtmi, dependency_resolution=DEPENDENCY_MODE_TRY_FROM_EXPANDED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_single_dtmi_with_component_deps_no_expanded_json(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + # this tests the fallback procedure when no expanded doc exists + root_dtmi = "dtmi:com:example:Phone;2" + expected_deps = [ + "dtmi:azure:DeviceManagement:DeviceInformation;1", + "dtmi:azure:DeviceManagement:DeviceInformation;2", + "dtmi:com:example:Camera;3", + ] + expected_dtmis = [root_dtmi] + expected_deps + model_map = self.client.get_models( + root_dtmi, dependency_resolution=DEPENDENCY_MODE_TRY_FROM_EXPANDED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_with_component_deps_no_expanded_json(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + # this tests the fallback procedure when no expanded doc exists + root_dtmi1 = "dtmi:com:example:Phone;2" + root_dtmi2 = "dtmi:com:example:Freezer;1" + expected_deps = [ + "dtmi:azure:DeviceManagement:DeviceInformation;1", + "dtmi:azure:DeviceManagement:DeviceInformation;2", + "dtmi:com:example:Camera;3", + "dtmi:com:example:Thermostat;1", + ] + expected_dtmis = [root_dtmi1, root_dtmi2] + expected_deps + model_map = self.client.get_models( + [root_dtmi1, root_dtmi2], dependency_resolution=DEPENDENCY_MODE_TRY_FROM_EXPANDED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_multiple_dtmis_with_component_deps_extends_deps_mixed_expanded_json(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + root_dtmi1 = "dtmi:com:example:ColdStorage;1" # no expanded doc + root_dtmi2 = "dtmi:com:example:TemperatureController;1" # has expanded doc + expected_deps = [ + "dtmi:com:example:Thermostat;1", + "dtmi:azure:DeviceManagement:DeviceInformation;1", + "dtmi:com:example:Room;1", + "dtmi:com:example:Freezer;1", + ] + expected_dtmis = [root_dtmi1, root_dtmi2] + expected_deps + model_map = self.client.get_models( + [root_dtmi1, root_dtmi2], dependency_resolution=DEPENDENCY_MODE_ENABLED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + def test_dangling_expanded(self): + if self.client_type == REMOTE_REPO: + self.skipTest("Insufficient data") + root_dtmi = "dtmi:com:example:DanglingExpanded;1" + expected_deps = [ + "dtmi:com:example:Thermostat;1", + "dtmi:azure:DeviceManagement:DeviceInformation;1", + ] + expected_dtmis = [root_dtmi] + expected_deps + model_map = self.client.get_models( + root_dtmi, dependency_resolution=DEPENDENCY_MODE_TRY_FROM_EXPANDED + ) + + self.assertTrue(len(model_map) == len(expected_dtmis)) + for dtmi in expected_dtmis: + self.assertTrue(dtmi in model_map.keys()) + model = model_map[dtmi] + self.assertTrue(model["@id"] == dtmi) + + +####################### +# Actual Test Classes # +####################### + + +class TestIntegrationGetModelsDependencyModeEnabledLocalRepository( + GetModelsDependencyModeEnabledIntegrationTestCaseMixin, LocalRepositoryMixin, AzureTestCase +): + pass + + +class TestIntegrationGetModelsDependencyModeDisabledLocalRepository( + GetModelsDependencyModeDisabledIntegrationTestCaseMixin, LocalRepositoryMixin, AzureTestCase +): + pass + + +class TestIntegrationGetModelsDependencyModeTryFromExpandedLocalRepository( + GetModelsDependencyModeTryFromExpandedIntegrationTestCaseMixin, + LocalRepositoryMixin, + AzureTestCase, +): + pass + + +class TestIntegrationGetModelsDependencyModeEnabledRemoteRepository( + GetModelsDependencyModeEnabledIntegrationTestCaseMixin, RemoteRepositoryMixin, AzureTestCase +): + pass + + +class TestIntegrationGetModelsDependencyModeDisabledRemoteRepository( + GetModelsDependencyModeDisabledIntegrationTestCaseMixin, RemoteRepositoryMixin, AzureTestCase +): + pass + + +class TestIntegrationGetModelsDependencyModeTryFromExpandedRemoteRepository( + GetModelsDependencyModeTryFromExpandedIntegrationTestCaseMixin, + RemoteRepositoryMixin, + AzureTestCase, +): + pass diff --git a/sdk/iot/azure-iot-modelsrepository/tox.ini b/sdk/iot/azure-iot-modelsrepository/tox.ini deleted file mode 100644 index 633164a750c5..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/tox.ini +++ /dev/null @@ -1,9 +0,0 @@ -[tox] -envlist = py27, py35, py36, py37, py38, py39 - -[testenv] -deps = - pytest - pytest-mock - pytest-testdox -commands = pytest diff --git a/sdk/iot/ci.yml b/sdk/iot/ci.yml index 1232c8c695a9..296d141d92ce 100644 --- a/sdk/iot/ci.yml +++ b/sdk/iot/ci.yml @@ -30,5 +30,7 @@ extends: parameters: ServiceDirectory: iot Artifacts: - - name: azure_iot_nspkg + - name: azure-iot-modelsrepository + safeName: azureiotmodelsrepository + - name: azure-iot-nspkg safeName: azureiotnspkg diff --git a/sdk/iot/tests.yml b/sdk/iot/tests.yml new file mode 100644 index 000000000000..da9493dc1f04 --- /dev/null +++ b/sdk/iot/tests.yml @@ -0,0 +1,10 @@ +trigger: none + +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml + parameters: + AllocateResourceGroup: 'false' + BuildTargetingString: $(BuildTargetingString) + ServiceDirectory: iot + EnvVars: + TEST_MODE: 'RunLiveNoRecord' From 0ebc3e700a3148c525861d441d936efdba420f6b Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Thu, 22 Apr 2021 18:02:28 -0700 Subject: [PATCH 24/25] Moved directory yml fixes removed tests.yml from iot CODEOWNERS change restored original iot ci.yml --- .github/CODEOWNERS | 2 +- .../tests/test_client.py | 139 ------------------ sdk/iot/ci.yml | 4 +- .../azure-iot-modelsrepository/.flake8 | 0 .../azure-iot-modelsrepository/CHANGELOG.md | 0 .../azure-iot-modelsrepository/MANIFEST.in | 0 .../azure-iot-modelsrepository/README.md | 6 +- .../azure/__init__.py | 0 .../azure/iot/__init__.py | 0 .../azure/iot/modelsrepository/__init__.py | 0 .../azure/iot/modelsrepository/_client.py | 0 .../azure/iot/modelsrepository/_constants.py | 0 .../iot/modelsrepository/_pseudo_parser.py | 0 .../azure/iot/modelsrepository/_resolver.py | 0 .../iot/modelsrepository/dtmi_conventions.py | 0 .../azure/iot/modelsrepository/exceptions.py | 0 .../dev_requirements.txt | 0 .../azure-iot-modelsrepository/pytest.ini | 0 .../samples/README.md | 6 +- .../samples/client_configuration_sample.py | 0 .../samples/dtmi_conventions_sample.py | 0 .../samples/get_models_sample.py | 0 .../azure-iot-modelsrepository/setup.cfg | 0 .../azure-iot-modelsrepository/setup.py | 0 .../tests/__init__.py | 0 .../devicemanagement/deviceinformation-1.json | 0 .../devicemanagement/deviceinformation-2.json | 0 .../dtmi/com/example/base-1.json | 0 .../dtmi/com/example/base-2.json | 0 .../dtmi/com/example/building-1.json | 0 .../dtmi/com/example/camera-3.json | 0 .../dtmi/com/example/coldstorage-1.json | 0 .../dtmi/com/example/conferenceroom-1.json | 0 .../example/danglingexpanded-1.expanded.json | 0 .../dtmi/com/example/freezer-1.json | 0 .../incompleteexpanded-1.expanded.json | 0 .../dtmi/com/example/invalidmodel-1.json | 0 .../dtmi/com/example/invalidmodel-2.json | 0 .../dtmi/com/example/phone-2.json | 0 .../dtmi/com/example/room-1.json | 0 .../temperaturecontroller-1.expanded.json | 0 .../com/example/temperaturecontroller-1.json | 0 .../dtmi/com/example/thermostat-1.json | 0 .../dtmi/company/demodevice-1.json | 0 .../dtmi/company/demodevice-2.json | 0 .../dtmi/strict/badfilepath-1.json | 0 .../dtmi/strict/emptyarray-1.json | 0 .../dtmi/strict/namespaceconflict-1.json | 0 .../dtmi/strict/nondtdl-1.json | 0 .../dtmi/strict/unsupportedrootarray-1.json | 0 .../tests/test_dtmi_conventions.py | 0 .../tests/test_integration_client.py | 0 sdk/modelsrepository/ci.yml | 34 +++++ sdk/{iot => modelsrepository}/tests.yml | 2 +- 54 files changed, 43 insertions(+), 150 deletions(-) delete mode 100644 sdk/iot/azure-iot-modelsrepository/tests/test_client.py rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/.flake8 (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/CHANGELOG.md (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/MANIFEST.in (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/README.md (92%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/__init__.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/__init__.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/dev_requirements.txt (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/pytest.ini (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/samples/README.md (50%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/samples/client_configuration_sample.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/samples/get_models_sample.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/setup.cfg (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/setup.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/__init__.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/test_dtmi_conventions.py (100%) rename sdk/{iot => modelsrepository}/azure-iot-modelsrepository/tests/test_integration_client.py (100%) create mode 100644 sdk/modelsrepository/ci.yml rename sdk/{iot => modelsrepository}/tests.yml (86%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d131175718a4..bcd796aea969 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -72,7 +72,7 @@ /sdk/hdinsight/ @idear1203 # PRLabel: %Models repository -/sdk/iot/azure-iot-modelsrepository @cartertinney @digimaun +/sdk/modelsrepository/ @cartertinney @digimaun # PRLabel: %Machine Learning Compute /sdk/machinelearningcompute/ @shutchings diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_client.py b/sdk/iot/azure-iot-modelsrepository/tests/test_client.py deleted file mode 100644 index 880a8cdfc976..000000000000 --- a/sdk/iot/azure-iot-modelsrepository/tests/test_client.py +++ /dev/null @@ -1,139 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import pytest -from azure.iot.modelsrepository._client import ( - ModelsRepositoryClient, - DEPENDENCY_MODE_TRY_FROM_EXPANDED, - DEPENDENCY_MODE_ENABLED, - DEPENDENCY_MODE_DISABLED, -) -from azure.iot.modelsrepository._resolver import HttpFetcher, FilesystemFetcher - - -@pytest.mark.describe("ModelsRepositoryClient - Instantiation") -class TestModelsRepositoryClientInstantiation(object): - @pytest.mark.it( - "Defaults to using 'https://devicemodels.azure.com' as the repository location if one is not provided" - ) - def test_default_repository_location(self): - client = ModelsRepositoryClient() - assert client.fetcher.base_url == "https://devicemodels.azure.com" - - @pytest.mark.it( - "Is configured for HTTP/HTTPS REST operations if provided a repository location in URL format" - ) - @pytest.mark.parametrize( - "url", - [ - pytest.param("http://myfakerepository.com/", id="HTTP URL, with trailing '/'"), - pytest.param("http://myfakerepository.com", id="HTTP URL, no trailing '/'"), - pytest.param("https://myfakerepository.com/", id="HTTPS URL, with trailing '/'"), - pytest.param("https://myfakerepository.com", id="HTTPS URL, no trailing '/'"), - pytest.param( - "myfakerepository.com/", id="Web URL, no protocol specified, with trailing '/'" - ), - pytest.param( - "myfakerepository.com", id="Web URL, no protocol specified, no trailing '/'" - ), - ], - ) - def test_repository_location_remote(self, url): - client = ModelsRepositoryClient(repository_location=url) - assert isinstance(client.fetcher, HttpFetcher) - - @pytest.mark.it( - "Is configured for local filesystem operations if provided a repository location in filesystem format" - ) - @pytest.mark.parametrize( - "path", - [ - pytest.param( - "F:/repos/myrepo/", - id="Drive letter filesystem path, '/' separators, with trailing '/'", - ), - pytest.param( - "F:/repos/myrepo", - id="Drive letter filesystem path, '/' separators, no trailing '/'", - ), - pytest.param( - "F:\\repos\\myrepo\\", - id="Drive letter filesystem path, '\\' separators, with trailing '\\'", - ), - pytest.param( - "F:\\repos\\myrepo", - id="Drive letter filesystem path, '\\' separators, no trailing '\\'", - ), - pytest.param( - "F:\\repos/myrepo/", - id="Drive letter filesystem path, mixed separators, with trailing separator", - ), - pytest.param( - "F:\\repos/myrepo", - id="Drive letter filesystem path, mixed separators, no trailing separator", - ), - pytest.param("/repos/myrepo/", id="POSIX filesystem path, with trailing '/'"), - pytest.param("/repos/myrepo", id="POSIX filesystem path, no trailing '/'"), - pytest.param( - "file:///f:/repos/myrepo/", - id="URI scheme, drive letter filesystem path, with trailing '/'", - ), - pytest.param( - "file:///f:/repos/myrepo", - id="URI scheme, drive letter filesystem path, no trailing '/'", - ), - pytest.param( - "file:///repos/myrepo/", id="URI scheme, POSIX filesystem path, with trailing '/'" - ), - pytest.param( - "file:///repos/myrepo", id="URI schem, POSIX filesystem path, no trailing '/'" - ), - ], - ) - def test_repository_location_local(self, path): - client = ModelsRepositoryClient(repository_location=path) - assert isinstance(client.fetcher, FilesystemFetcher) - - @pytest.mark.it("Raises ValueError if provided a repository location that cannot be identified") - def test_invalid_repository_location(self): - with pytest.raises(ValueError): - ModelsRepositoryClient(repository_location="not a location") - - @pytest.mark.it( - "Defaults to using 'tryFromExpanded' as the dependency resolution mode if one is not specified while using the default repository location" - ) - def test_default_dependency_resolution_mode_default_location(self): - client = ModelsRepositoryClient() - assert client.resolution_mode == DEPENDENCY_MODE_TRY_FROM_EXPANDED - - @pytest.mark.it( - "Defaults to using 'enabled' as the dependency resolution mode if one is not specified while using a custom repository location" - ) - @pytest.mark.parametrize( - "location", - [ - pytest.param("https://myfakerepository.com", id="Remote repository location"), - pytest.param("/repos/myrepo", id="Local repository location"), - ], - ) - def test_default_dependency_resolution_mode_custom_location(self, location): - client = ModelsRepositoryClient(repository_location=location) - assert client.resolution_mode == DEPENDENCY_MODE_ENABLED - - @pytest.mark.it( - "Is configured with the provided dependency resolution mode if one is specified" - ) - @pytest.mark.parametrize( - "dependency_mode", - [DEPENDENCY_MODE_ENABLED, DEPENDENCY_MODE_DISABLED, DEPENDENCY_MODE_TRY_FROM_EXPANDED], - ) - def test_dependency_mode(self, dependency_mode): - client = ModelsRepositoryClient(dependency_resolution=dependency_mode) - assert client.resolution_mode == dependency_mode - - @pytest.mark.it("Raises ValueError if provided an unrecognized dependency resolution mode") - def test_invalid_dependency_mode(self): - with pytest.raises(ValueError): - ModelsRepositoryClient(dependency_resolution="not a mode") diff --git a/sdk/iot/ci.yml b/sdk/iot/ci.yml index 296d141d92ce..1232c8c695a9 100644 --- a/sdk/iot/ci.yml +++ b/sdk/iot/ci.yml @@ -30,7 +30,5 @@ extends: parameters: ServiceDirectory: iot Artifacts: - - name: azure-iot-modelsrepository - safeName: azureiotmodelsrepository - - name: azure-iot-nspkg + - name: azure_iot_nspkg safeName: azureiotnspkg diff --git a/sdk/iot/azure-iot-modelsrepository/.flake8 b/sdk/modelsrepository/azure-iot-modelsrepository/.flake8 similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/.flake8 rename to sdk/modelsrepository/azure-iot-modelsrepository/.flake8 diff --git a/sdk/iot/azure-iot-modelsrepository/CHANGELOG.md b/sdk/modelsrepository/azure-iot-modelsrepository/CHANGELOG.md similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/CHANGELOG.md rename to sdk/modelsrepository/azure-iot-modelsrepository/CHANGELOG.md diff --git a/sdk/iot/azure-iot-modelsrepository/MANIFEST.in b/sdk/modelsrepository/azure-iot-modelsrepository/MANIFEST.in similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/MANIFEST.in rename to sdk/modelsrepository/azure-iot-modelsrepository/MANIFEST.in diff --git a/sdk/iot/azure-iot-modelsrepository/README.md b/sdk/modelsrepository/azure-iot-modelsrepository/README.md similarity index 92% rename from sdk/iot/azure-iot-modelsrepository/README.md rename to sdk/modelsrepository/azure-iot-modelsrepository/README.md index 5406e9498e99..a2c08c24f127 100644 --- a/sdk/iot/azure-iot-modelsrepository/README.md +++ b/sdk/modelsrepository/azure-iot-modelsrepository/README.md @@ -46,10 +46,10 @@ Several samples are available in the Azure SDK for Python GitHub repository. The [azure_core_exceptions]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/core/azure-core#azure-core-library-exceptions -[client_configuration_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py +[client_configuration_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/modelsrepository/azure-iot-modelsrepository/samples/client_configuration_sample.py [dtdl_spec]: https://github.com/Azure/opendigitaltwins-dtdl/blob/master/DTDL/v2/dtdlv2.md -[dtmi_conventions_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py -[get_models_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py +[dtmi_conventions_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/modelsrepository/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py +[get_models_sample]: https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/modelsrepository/azure-iot-modelsrepository/samples/get_models_sample.py [global_azure_repo]: https://devicemodels.azure.com/ [json_ld]: https://json-ld.org/ [logging_doc]: https://docs.python.org/3.5/library/logging.html diff --git a/sdk/iot/azure-iot-modelsrepository/azure/__init__.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/__init__.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/__init__.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/__init__.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/__init__.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/__init__.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/__init__.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/__init__.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/__init__.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_client.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_constants.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_pseudo_parser.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/_resolver.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/dtmi_conventions.py diff --git a/sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py b/sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py rename to sdk/modelsrepository/azure-iot-modelsrepository/azure/iot/modelsrepository/exceptions.py diff --git a/sdk/iot/azure-iot-modelsrepository/dev_requirements.txt b/sdk/modelsrepository/azure-iot-modelsrepository/dev_requirements.txt similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/dev_requirements.txt rename to sdk/modelsrepository/azure-iot-modelsrepository/dev_requirements.txt diff --git a/sdk/iot/azure-iot-modelsrepository/pytest.ini b/sdk/modelsrepository/azure-iot-modelsrepository/pytest.ini similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/pytest.ini rename to sdk/modelsrepository/azure-iot-modelsrepository/pytest.ini diff --git a/sdk/iot/azure-iot-modelsrepository/samples/README.md b/sdk/modelsrepository/azure-iot-modelsrepository/samples/README.md similarity index 50% rename from sdk/iot/azure-iot-modelsrepository/samples/README.md rename to sdk/modelsrepository/azure-iot-modelsrepository/samples/README.md index eaf8e279f84a..0852c58f6982 100644 --- a/sdk/iot/azure-iot-modelsrepository/samples/README.md +++ b/sdk/modelsrepository/azure-iot-modelsrepository/samples/README.md @@ -5,8 +5,8 @@ This directory contains samples showing how to use the features of the Azure IoT The pre-configured endpoints and DTMIs within the samples refer to example models that can be found on [devicemodels.azure.com](https://devicemodels.azure.com/). These values can be replaced to reflect the locations of your own models, wherever they may be. ## ModelsRepositoryClient Samples -* [get_models_sample.py] - Retrieve a model/models (and possibly dependencies) from a Model Repository, given a DTMI or DTMIs +* [get_models_sample.py] - Retrieve a model/models (and possibly dependencies) from a Model Repository, given a DTMI or DTMIs -* [client_configuration_sample.py] - Configure the client to work with local or remote repositories, as well as custom policies and transports +* [client_configuration_sample.py] - Configure the client to work with local or remote repositories, as well as custom policies and transports -* [dtmi_conventions_sample.py] - Use the `dtmi_conventions` module to manipulate and check DTMIs \ No newline at end of file +* [dtmi_conventions_sample.py] - Use the `dtmi_conventions` module to manipulate and check DTMIs \ No newline at end of file diff --git a/sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py b/sdk/modelsrepository/azure-iot-modelsrepository/samples/client_configuration_sample.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/samples/client_configuration_sample.py rename to sdk/modelsrepository/azure-iot-modelsrepository/samples/client_configuration_sample.py diff --git a/sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py b/sdk/modelsrepository/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py rename to sdk/modelsrepository/azure-iot-modelsrepository/samples/dtmi_conventions_sample.py diff --git a/sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py b/sdk/modelsrepository/azure-iot-modelsrepository/samples/get_models_sample.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/samples/get_models_sample.py rename to sdk/modelsrepository/azure-iot-modelsrepository/samples/get_models_sample.py diff --git a/sdk/iot/azure-iot-modelsrepository/setup.cfg b/sdk/modelsrepository/azure-iot-modelsrepository/setup.cfg similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/setup.cfg rename to sdk/modelsrepository/azure-iot-modelsrepository/setup.cfg diff --git a/sdk/iot/azure-iot-modelsrepository/setup.py b/sdk/modelsrepository/azure-iot-modelsrepository/setup.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/setup.py rename to sdk/modelsrepository/azure-iot-modelsrepository/setup.py diff --git a/sdk/iot/azure-iot-modelsrepository/tests/__init__.py b/sdk/modelsrepository/azure-iot-modelsrepository/tests/__init__.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/__init__.py rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/__init__.py diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/azure/devicemanagement/deviceinformation-2.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/base-2.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/building-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/camera-3.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/coldstorage-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/conferenceroom-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/danglingexpanded-1.expanded.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/freezer-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/incompleteexpanded-1.expanded.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/invalidmodel-2.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/phone-2.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/room-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.expanded.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/temperaturecontroller-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/com/example/thermostat-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/company/demodevice-2.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/badfilepath-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/emptyarray-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/namespaceconflict-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/nondtdl-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json b/sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/local_repository/dtmi/strict/unsupportedrootarray-1.json diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py b/sdk/modelsrepository/azure-iot-modelsrepository/tests/test_dtmi_conventions.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/test_dtmi_conventions.py rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/test_dtmi_conventions.py diff --git a/sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py b/sdk/modelsrepository/azure-iot-modelsrepository/tests/test_integration_client.py similarity index 100% rename from sdk/iot/azure-iot-modelsrepository/tests/test_integration_client.py rename to sdk/modelsrepository/azure-iot-modelsrepository/tests/test_integration_client.py diff --git a/sdk/modelsrepository/ci.yml b/sdk/modelsrepository/ci.yml new file mode 100644 index 000000000000..c1c314fe102f --- /dev/null +++ b/sdk/modelsrepository/ci.yml @@ -0,0 +1,34 @@ +# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. + +trigger: + branches: + include: + - master + - main + - hotfix/* + - release/* + - restapi* + paths: + include: + - sdk/modelsrepository/ + +pr: + branches: + include: + - master + - main + - feature/* + - hotfix/* + - release/* + - restapi* + paths: + include: + - sdk/modelsrepository/ + +extends: + template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml + parameters: + ServiceDirectory: modelsrepository + Artifacts: + - name: azure-iot-modelsrepository + safeName: azureiotmodelsrepository diff --git a/sdk/iot/tests.yml b/sdk/modelsrepository/tests.yml similarity index 86% rename from sdk/iot/tests.yml rename to sdk/modelsrepository/tests.yml index da9493dc1f04..4ba4f62f9e98 100644 --- a/sdk/iot/tests.yml +++ b/sdk/modelsrepository/tests.yml @@ -5,6 +5,6 @@ stages: parameters: AllocateResourceGroup: 'false' BuildTargetingString: $(BuildTargetingString) - ServiceDirectory: iot + ServiceDirectory: modelsrepository EnvVars: TEST_MODE: 'RunLiveNoRecord' From fd5d35f0ceb16d440c58ed3d7a2ca6addbfff50c Mon Sep 17 00:00:00 2001 From: Carter Tinney Date: Fri, 23 Apr 2021 09:59:28 -0700 Subject: [PATCH 25/25] Update sdk/modelsrepository/tests.yml Co-authored-by: Sean Kane <68240067+seankane-msft@users.noreply.github.com> --- sdk/modelsrepository/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/modelsrepository/tests.yml b/sdk/modelsrepository/tests.yml index 4ba4f62f9e98..ecb958f6d6e9 100644 --- a/sdk/modelsrepository/tests.yml +++ b/sdk/modelsrepository/tests.yml @@ -8,3 +8,5 @@ stages: ServiceDirectory: modelsrepository EnvVars: TEST_MODE: 'RunLiveNoRecord' + AZURE_SKIP_LIVE_RECORDING: 'True' + AZURE_TEST_RUN_LIVE: 'true'